[Hadoop]HDFS：java API

1、使用URL读取数据

package com.demo.hellohadoop;

import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.apache.hadoop.io.IOUtils;

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;

/**
 * 使用URL读取数据
 */
public class HDFSTest01 {

    static {
        URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
    }

    public static void main(String[] args) {

        final String hdfsURL = "hdfs://192.168.236.135:9000/tmp/README.txt";
        InputStream inputStream = null;
        try {
            inputStream = new URL(hdfsURL).openStream();
            IOUtils.copyBytes(inputStream, System.out, 4096, false);

        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            IOUtils.closeStream(inputStream);
        }
    }
}

2、通过FileSystem API读取数据

package com.demo.hellohadoop;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

import java.io.IOException;
import java.net.URI;

/**
 * 通过FileSystem API读取数据
 * */
public class HDFSTest02 {

    public static void main(String[] args) {
        final String hdfs_address = "hdfs://192.168.236.135:9000";
        final String uri = hdfs_address + "/tmp/README.txt";
        Configuration config = new Configuration();
        FSDataInputStream in = null;
        try {

            FileSystem fileSystem = FileSystem.get(URI.create(hdfs_address), config);
            in = fileSystem.open(new Path(uri));
            IOUtils.copyBytes(in, System.out, 4096, false);

        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            IOUtils.closeStream(in);
        }

    }
}

3、创建目录

package com.demo.hellohadoop;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.omg.PortableInterceptor.SYSTEM_EXCEPTION;

import java.io.IOException;
import java.net.URI;

/*
* 创建目录
* */
public class HDFSTest03 {

    public static void main(String[] args) {
        final String hdfsAddress = "hdfs://192.168.236.135:9000";

        Configuration config = new Configuration();
        try {
            FileSystem fileSystem = FileSystem.get(URI.create(hdfsAddress), config, "hadoop");
            boolean flag = fileSystem.mkdirs(new Path(hdfsAddress + "/tmp/tmpDir1"));
            System.out.println("make directory " + (flag ? "successfully" : "failed"));

        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }

    }
}

4、将本地文件复制到hadoop文件系统并显示进度

package com.demo.hellohadoop;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URI;

/*
*  将本地文件复制到hadoop文件系统并显示进度
* */
public class HDFSTest04 {

    private static final String LOCAL_FILE = "D:\\workspace_idea\\hello-hadoop\\src\\main\\resources\\test.txt";
    private static final String HDFS_ADDRESS = "hdfs://192.168.236.135:9000";
    private static final String REMOTE_FILE = HDFS_ADDRESS + "/tmp/test.txt";

    public static void main(String[] args) {
        BufferedInputStream in = null;
        FSDataOutputStream out = null;
        try {
            in = new BufferedInputStream(new FileInputStream(LOCAL_FILE));

            FileSystem fileSystem = FileSystem.get(URI.create(HDFS_ADDRESS), new Configuration(), "hadoop");

            out = fileSystem.create(new Path(REMOTE_FILE), () -> System.out.print("*"));
            IOUtils.copyBytes(in, out, 8, false);
        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        } finally {
            IOUtils.closeStream(in);
            IOUtils.closeStream(out);
        }

    }
}

5、从tmp目录下载所有txt文件到本地

6、删除数据

7、列出目录文件