hadoop3自学入门笔记(3)-java 操作hdfs
程序员文章站
2022-06-10 07:55:38
1.core site.xml 2.pom.xml 3.测试代码 testDownloadFileToLocal 这里测试请注意,本地也要装hdfs才可以 "更多精彩请关注" 公众号【lovepythoncn】 ......
1.core-site.xml
<configuration> <property> <name>fs.defaultfs</name> <value>hdfs://192.168.3.61:9820</value> </property> <property> <name>hadoop.tmp.dir</name> <value>/opt/hadoopdata</value> </property> </configuration>
2.pom.xml
<?xml version="1.0" encoding="utf-8"?> <project xmlns="http://maven.apache.org/pom/4.0.0" xmlns:xsi="http://www.w3.org/2001/xmlschema-instance" xsi:schemalocation="http://maven.apache.org/pom/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelversion>4.0.0</modelversion> <groupid>com.qmkj</groupid> <artifactid>hdfsclienttest</artifactid> <version>0.1</version> <name>hdfsclienttest</name> <!-- fixme change it to the project's website --> <url>http://www.example.com</url> <properties> <project.build.sourceencoding>utf-8</project.build.sourceencoding> <maven.compiler.source>1.7</maven.compiler.source> <maven.compiler.target>1.7</maven.compiler.target> </properties> <dependencies> <dependency> <groupid>junit</groupid> <artifactid>junit</artifactid> <version>4.11</version> <scope>test</scope> </dependency> <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs-client --> <dependency> <groupid>org.apache.hadoop</groupid> <artifactid>hadoop-hdfs-client</artifactid> <version>3.2.1</version> <scope>provided</scope> </dependency> <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common --> <dependency> <groupid>org.apache.hadoop</groupid> <artifactid>hadoop-common</artifactid> <version>3.2.1</version> </dependency> <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs --> <dependency> <groupid>org.apache.hadoop</groupid> <artifactid>hadoop-hdfs</artifactid> <version>3.2.1</version> </dependency> </dependencies> <build> <pluginmanagement><!-- lock down plugins versions to avoid using maven defaults (may be moved to parent pom) --> <plugins> <!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_lifecycle --> <plugin> <artifactid>maven-clean-plugin</artifactid> <version>3.1.0</version> </plugin> <!-- default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#plugin_bindings_for_jar_packaging --> <plugin> <artifactid>maven-resources-plugin</artifactid> <version>3.0.2</version> </plugin> <plugin> <artifactid>maven-compiler-plugin</artifactid> <version>3.8.0</version> </plugin> <plugin> <artifactid>maven-surefire-plugin</artifactid> <version>2.22.1</version> </plugin> <plugin> <artifactid>maven-jar-plugin</artifactid> <version>3.0.2</version> </plugin> <plugin> <artifactid>maven-install-plugin</artifactid> <version>2.5.2</version> </plugin> <plugin> <artifactid>maven-deploy-plugin</artifactid> <version>2.8.2</version> </plugin> <!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_lifecycle --> <plugin> <artifactid>maven-site-plugin</artifactid> <version>3.7.1</version> </plugin> <plugin> <artifactid>maven-project-info-reports-plugin</artifactid> <version>3.0.0</version> </plugin> </plugins> </pluginmanagement> </build> </project>
3.测试代码
package com.qmkj; import org.apache.hadoop.conf.configuration; import org.apache.hadoop.fs.*; import org.junit.before; import org.junit.test; import java.io.filenotfoundexception; import java.io.ioexception; import java.net.uri; /** * unit test for simple app. */ public class apptest { filesystem fs = null; @before public void init() throws exception { configuration conf = new configuration(); //设立的设置url请注意,设置core-site.xml中配置fs.defaultfs的地址 fs = filesystem.get(new uri("hdfs://192.168.3.61:9820"), conf, "root"); } @test public void testadd() throws exception { fs.copyfromlocalfile(new path("d:\\kk_movies\\kk 2020-02-20 20-45-55.mp4"), new path("/zhanglei")); fs.close(); } /** * 从hdfs中复制文件到本地文件系统 * * @throws ioexception * @throws illegalargumentexception */ @test public void testdownloadfiletolocal() throws illegalargumentexception, ioexception { // fs.copytolocalfile(new path("/mysql-connector-java-5.1.28.jar"), new // path("d:/")); fs.copytolocalfile(false, new path("test.txt"), new path("e:/"), true); fs.close(); } /** * 目录操作 * * @throws illegalargumentexception * @throws ioexception */ @test public void testmkdiranddeleteandrename() throws illegalargumentexception, ioexception { // 创建目录 fs.mkdirs(new path("/zhanglei/b1/c1")); // 删除文件夹 ,如果是非空文件夹,参数2必须给值true ,删除所有子文件夹 fs.delete(new path("/b1"), true); // 重命名文件或文件夹 fs.rename(new path("/zhanglei"), new path("/qmkj")); } /** * 查看目录信息,只显示文件 * * @throws ioexception * @throws illegalargumentexception * @throws filenotfoundexception */ @test public void testlistfiles() throws filenotfoundexception, illegalargumentexception, ioexception { remoteiterator<locatedfilestatus> listfiles = fs.listfiles(new path("/"), true); while (listfiles.hasnext()) { locatedfilestatus filestatus = listfiles.next(); system.out.println(filestatus.getpath().getname()); system.out.println(filestatus.getblocksize()); system.out.println(filestatus.getpermission()); system.out.println(filestatus.getlen()); blocklocation[] blocklocations = filestatus.getblocklocations(); for (blocklocation bl : blocklocations) { system.out.println("block-length:" + bl.getlength() + "--" + "block-offset:" + bl.getoffset()); string[] hosts = bl.gethosts(); for (string host : hosts) { system.out.println(host); } } system.out.println("--------------打印的分割线--------------"); } } /** * 查看文件及文件夹信息 * * @throws ioexception * @throws illegalargumentexception * @throws filenotfoundexception */ @test public void testlistall() throws filenotfoundexception, illegalargumentexception, ioexception { //可以右击方法名,run 测试一下。 filestatus[] liststatus = fs.liststatus(new path("/")); string flag = ""; for (filestatus fstatus : liststatus) { if (fstatus.isfile()) { flag = "f-- "; } else { flag = "d-- "; } system.out.println(flag + fstatus.getpath().getname()); system.out.println(fstatus.getpermission()); } } }
testdownloadfiletolocal 这里测试请注意,本地也要装hdfs才可以
公众号【lovepythoncn】