在eclipse中编写HDFS的Java程序

1.开始我们需要配置jdk+eclipse+maven,文章参考链接https://blog.csdn.net/qq_44240521/article/details/90574089
2.在Eclipse里新建Maven Project
【file】-【new】-【Maven project】


如下图自动生成Maven项目，新建包hdfs.files，包里新建class：4个java文件。

3.编写程序
在pom.xml里添加

<dependencies>
  <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-hdfs</artifactId>
      <version>2.7.3</version>
  </dependency>
  <dependency>  
      <groupId>org.apache.hadoop</groupId>  
      <artifactId>hadoop-client</artifactId>  
      <version>2.7.3</version>  
  </dependency> 
  <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <version>2.7.3</version>
  </dependency>

</dependencies>

在HDFSDownload.java（下载）里编写(hdfs上需要有word.txt)

package hdfs.files;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class HDFSDownload {
	private static InputStream input;
	private static OutputStream output;
	public static void main(String[] args) throws IOException{
		System.setProperty("HADOOP_USER_NAME", "root");
		Configuration conf=new Configuration();
		conf.set("fs.defaultFS", "hdfs://centos01:9000");
        FileSystem client=FileSystem.get(conf);
        output = new FileOutputStream("e:\\eclipse\\word.txt");
        input =client.open(new Path("/word.txt"));
        byte[] buffer=new byte[1024];
        int len=0;
        while((len=input.read(buffer))!=-1) {
        	output.write(buffer,0,len);
        }
        output.flush();
        input.close();
        output.close();              
	}
}

在HDFSFilelfExist.java（判断hdfs文件是否存在）里编写

package hdfs.files;
import java.io.IOException;
import org.apache.hadoop. conf.Configuration;
import org.apache.hadoop. fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class HDFSFilelfExist{
      public static void main(String[] args) throws IOException{
      System. setProperty( "HADOOP_ USER_ NAME", "root");
      Configuration conf=new Configuration();
      conf . set("fs.defaultFS", "hdfs://centos01:9000");
      FileSystem client=FileSystem. get(conf);
      String fileName="/aadir";
      if(client.exists(new Path(fileName))) { 
    	  System.out.println("文件存在! ");
      }
      else {
    	  System.out.println(" 文件不存在! "); }
      }
}

在HDFSMKdir.java（创建目录）里编写

package hdfs.files;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class HDFSMKdir{
    public static void main(String[] args) throws IOException {
        System.setProperty( "HADOOP_ USER_ NAME", "root");
        Configuration conf=new Configuration();
        conf.set("fs.defaultFS", "hdfs://centos01:9000");
        FileSystem client=FileSystem.get(conf);
        client.mkdirs( new Path("/aadir"));
        client.close();
        System.out.println(" successfully!");
    }
 }

在HDFSUpload .java（上传本地文件到hdfs）里编写

package hdfs.files;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class HDFSUpload {
	private static InputStream input;
	private static OutputStream output;
	public static void main(String[] args) throws IOException{
		Configuration conf=new Configuration();
		conf.set("fs.defaultFS", "hdfs://centos01:9000");
        FileSystem client=FileSystem.get(conf);
        input = new FileInputStream("e:\\eclipse\\test.txt");
        output =client.create(new Path("/aadir/test.txt"));
        byte[] buffer=new byte[1024];
        int len=0;
        while((len=input.read(buffer))!=-1) {
        	output.write(buffer,0,len);
        }
        output.flush();
        input.close();
        output.close();              
	}
}

4.运行程序
运行下载


运行判断文件是否存在


好，就演示到这
其中可能要用到winutils，下载地址https://github.com/rucyang/hadoop.dll-and-winutils.exe-for-hadoop2.7.3-on-windows_X64/tree/master，然后配置环境变量