HDFS Java api操作
程序员文章站
2024-03-23 08:01:21
...
新建一个maven工程
在pom.xml中添加相关依赖
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.lzc.hadoop</groupId>
<artifactId>hadoop-api</artifactId>
<version>1.0-SNAPSHOT</version>
<name>hadoop-api</name>
<!-- FIXME change it to the project's website -->
<url>http://www.example.com</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.7</maven.compiler.source>
<maven.compiler.target>1.7</maven.compiler.target>
<hadoop.version>2.6.0-cdh5.7.0</hadoop.version>
</properties>
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.10</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
基本操作
package com.lzc.hadoop.hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.*;
import java.net.URI;
/**
* hadoop hdfs java api操作
*/
public class HDFSApp {
public static final String HDFS_PATH = "hdfs://192.168.126.129:8020";
FileSystem fileSystem = null;
Configuration configuration = null;
@Before
public void setUp() throws Exception {
System.out.println("HDFSApp.setUp");
configuration = new Configuration();
configuration.set("dfs.replication", "1"); // 设置hdfs副本,如果不设置,默认采用的是hadoop自己的副本系数
fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, "root");
}
@After
public void tearDown() {
configuration = null;
fileSystem = null;
System.out.println("HDFSApp.tearDown");
}
/**
* 创建hdfs目录
*/
@Test
public void mkdir() throws Exception {
fileSystem.mkdirs(new Path("/hdfsapi/"));
}
/**
*创建文件
*/
@Test
public void create() throws Exception {
FSDataOutputStream outputStream = fileSystem.create(new Path("/hdfsapi/lzc.txt"));
outputStream.write("lzc hello".getBytes());
outputStream.flush();
outputStream.close();
}
/**
*查看hdfs文件内容
*/
@Test
public void cat() throws Exception {
FSDataInputStream inputStream = fileSystem.open(new Path("/hdfsapi/test.txt"));
IOUtils.copyBytes(inputStream, System.out, 1024);
inputStream.close();
}
/**
* 重命名文件
* @throws Exception
*/
@Test
public void rename() throws Exception {
Path oldPath = new Path("/hdfsapi/lzc.txt");
Path newPath = new Path("/hdfsapi/lizhencheng.txt");
fileSystem.rename(oldPath, newPath);
}
/**
*上传文件到hdfs
* @throws Exception
*/
@Test
public void copyFromLocal() throws Exception {
Path localPath = new Path("F:\\test.txt");
Path hdfsPath = new Path("/hdfsapi/");
fileSystem.copyFromLocalFile(localPath,hdfsPath);
}
/**
*上传文件到hdfs
* @throws Exception
*/
@Test
public void copyFromLocalWithProgress() throws Exception {
InputStream in = new BufferedInputStream(new FileInputStream(new File("F:\\jdk-8u162-windows-x64.exe")));
FSDataOutputStream output = fileSystem.create(new Path("/hdfsapi/jdk.exe"), new Progressable() {
@Override
public void progress() {
System.out.print("=>");
}
});
IOUtils.copyBytes(in, output, 4096);
}
/**
*下载hdfs文件
* @throws Exception
*/
@Test
public void copyToLocalFile() throws Exception {
Path localPath = new Path("F:\\hh.txt");
Path hdfsPath = new Path("/hdfsapi/test.txt");
fileSystem.copyToLocalFile(false, hdfsPath, localPath, true);
}
/**
*查看hdfs某个目录下的所有文件
* @throws Exception
*/
@Test
public void listFiles() throws Exception {
FileStatus[] fileStatus = fileSystem.listStatus(new Path("/"));
for (FileStatus f : fileStatus) {
String isDir = f.isDirectory()? "文件夹":"文件";
short replication = f.getReplication();
long len = f.getLen();
String path = f.getPath().toString();
System.out.println(isDir + "\t" + replication + "\t" + len + "\t" + path);
}
}
/**
*查看hdfs某个目录下的所有文件
* @throws Exception
*/
@Test
public void delete() throws Exception {
fileSystem.delete(new Path("/hdfsapi/jdk.exe"), true);
}
}
上一篇: java操作hdfs
下一篇: HDFS java API操作