java生成文件并上传本地文件至hdfs

项目整体结构说明：

1.创建一个maven项目，导入jar包（将下面代码放入pom.xml文件中，然后导入相应的jar包）

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>


    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.6.5</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>2.6.5</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.6.5</version>
        </dependency>
    </dependencies>
    <groupId>org.example</groupId>
    <artifactId>sc</artifactId>
    <version>1.0-SNAPSHOT</version>


</project>

2.生成本地文件代码（将下面代码放入createFile文件中）

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;

public class createFile {

    public static void main(String[] args) {
        // TODO Auto-generated method stub
        File f1 = new File("D:/test.txt");//本地文件地址
        try {
            FileOutputStream fos = new FileOutputStream(f1);
            if (!f1.exists()) {
                try {
                    f1.createNewFile();
                } catch (IOException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            }
            byte[] buffer = new byte[1024 * 640];
            long start = System.currentTimeMillis();
            for (int i = 0; i < 1024; i++) {
                fos.write(buffer);
                fos.flush();

            }
            long end = System.currentTimeMillis();
            long time = end - start;
            System.out.println("写入时间：" + time + "ms");
            fos.close();
        } catch (IOException e1) {
            // TODO Auto-generated catch block
            e1.printStackTrace();
        }
    }

}

3.上传文件代码（将下面代码放入HdfsTest文件中）

import java.io.FileInputStream;
import java.io.FileOutputStream;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class HdfsTest {
        public static void main (String[]args) throws Exception {
            //1 创建连接
            Configuration conf = new Configuration();
            //2 连接端口
            conf.set("fs.defaultFS", "hdfs://192.168.43.190:9870");//主节点的ip地址
            //3 获取连接对象
            FileSystem fs = FileSystem.get(conf);
            //本地文件上传到 hdfs
            System.out.println("开始上传");
            fs.copyFromLocalFile(new Path("D://test.txt"), new Path("usr/local/hadoop/input"));//第一个路径是本地文件路径，第二个路径是hdfs下路径
            fs.close();
            System.out.println("上传完毕");
        }
    }

4、使用流上传文件

//流上传文件
        FileInputStream in=new FileInputStream("D://test.txt");//读取本地文件
        FSDataOutputStream out = fs.create(new Path("usr/local/hadoop/input"));//在hdfs上创建路径
        byte[] b = new byte[1024*1024];
        int read = 0;
        while((read = in.read(b)) > 0){
            out.write(b, 0, read);
        }

5.将hdfs文件下载到本地

//hdfs文件复制到本地(流)
System.out.println("------开始下载------");
FSDataInputStream in = fs.open(new Path("usr/local/hadoop/input1"));
FileOutputStream out = new FileOutputStream("D:/xx.txt");
IOUtils.copyBytes(in, out, conf);

注：所有任务做完，记得关闭流

整个项目打包：链接：https://pan.baidu.com/s/1Mpz74-qBOIzZkSFJUg9Rug
提取码：mnay （百度网盘）