HDFS文件系统操作文件或文件夹 博客分类： hadoop HDFS

package com.my.hdfs;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.text.SimpleDateFormat;
import java.util.Date;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

@SuppressWarnings("all")
public class HDFSTest {

private final static String HDFS_PATH = "hdfs://pc1:9000/hellow";

public static void main(String[] args) throws IOException, URISyntaxException {
//获取到HDFS文件系统对象
final FileSystem fileSystem = FileSystem.get(new URI(HDFS_PATH), new Configuration());


createFolder(fileSystem,new Path("/logs"));

fileUploadToHDFS(fileSystem,new Path("/logs/hadoop9.avi"),new File("D:/hadoop9.avi"));

downLoadHDFSFile(fileSystem,new Path("/logs/hadoop9.avi"),new File("D:/hadoop9_02.avi"));

deleteFileOrFolder(fileSystem,new Path("/logs"));

getHDFSDirList(fileSystem);

}

/**
* 创建文件夹
* @param fileSystem HDFS文件系统对象
* @param path 要创建的文件夹，注意带有"/"为文件系统的根目录下下，否则为当前用户所在的路径
* @throws IOException
*/
private static void createFolder(final FileSystem fileSystem,final Path path) throws IOException {
if (fileSystem.exists(path)) {
System.out.println(path.getName()+" folder exist!");
} else {
fileSystem.mkdirs(path);
}
}

/**
* 上传文件到HDFS文件系统
* @param fileSystem HDFS文件系统对象
* @param toPath HDFS文件系统目标路径
* @param srcFile 本地源文件
* @throws IOException
* @throws FileNotFoundException
*/
private static void fileUploadToHDFS(final FileSystem fileSystem,final Path toPath,final File srcFile)
throws IOException, FileNotFoundException {
final FSDataOutputStream out = fileSystem.create(toPath);
final BufferedInputStream in = new BufferedInputStream(new FileInputStream(srcFile));
IOUtils.copyBytes(in, out, 1024, true);
}

/**
* 删除HDFS文件系统上的文件或文件夹
* @param fileSystem HDFS文件系统对象
* @param fsPath HDFS文件系统上的文件或者是文件夹
* @throws IOException
*/
private static void deleteFileOrFolder(final FileSystem fileSystem,final Path fsPath)
throws IOException {
if (fileSystem.exists(fsPath)) {
System.out.println("folder exist,delete the folder!");
fileSystem.delete(fsPath, true);//注意如果是文件夹必须为true，如果是文件true和false无所谓
} else {
System.out.println("folder not exist!");
}
}

/**
* 从HDFS文件系统上下载文件
* @param fileSystem HDFS文件系统对象
* @param srcFile HDFS文件系统上的源文件
* @param toPath 下载到本地目标路径
* @throws IOException
* @throws FileNotFoundException
*/
private static void downLoadHDFSFile(final FileSystem fileSystem,final Path srcFile,final File toPath)
throws IOException, FileNotFoundException {
final FSDataInputStream in = fileSystem.open(srcFile);
final BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(toPath));
IOUtils.copyBytes(in, out, 1024, true);
}

/**
* 获取到HDFS文件系统上的根目录列表
* @param fileSystem HDFS文件系统对象
* @throws IOException
*/
private static void getHDFSDirList(FileSystem fileSystem) throws IOException {
final FileStatus[] listStatus = fileSystem.listStatus(new Path("/"));
for (FileStatus fileStatus : listStatus) {
String isDir = fileStatus.isDir()?"文件夹":"文件";
final String permission = fileStatus.getPermission().toString();
final short replication = fileStatus.getReplication();
final long len = fileStatus.getLen();
final String dateStr = new SimpleDateFormat("yyyy-MM-dd hh:MM:ss").format(new Date(fileStatus.getAccessTime()));
final String path = fileStatus.getPath().toString();
System.out.println(isDir+"\t"+permission+"\t"+replication+"\t"+len+"\t"+dateStr+"\t"+path);
}
}
}