欢迎您访问程序员文章站本站旨在为大家提供分享程序员计算机编程知识!
您现在的位置是: 首页

hadoop客户端提交job的java代码 博客分类: java  

程序员文章站 2024-02-24 20:16:46
...
package com.xhb1.test;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import com.test.UserBroserCount;

public class RunBroserCount {
	public static void main(String[] args) throws Exception {
		String inputPath = "/logdata/userLogFile.log";
		String outputPath = "/oozie/test1";
		new RunBroserCount().runBroserCount(inputPath, outputPath);
	}

	public void runBroserCount(String inputPath, String outputPath)
			throws IOException, InterruptedException, ClassNotFoundException {
		Configuration conf = new Configuration();
		conf.set("fs.default.name", "hdfs://192.168.0.170:4310");
		//conf.set("hadoop.job.user", "root");
		conf.set("mapred.job.tracker", "192.168.0.170:4311");
		Job job = new Job(conf, "browserCount");
		((JobConf) job.getConfiguration()).setJar("E:/workspaces/pmp/BrowserCount/target/pr.jar");  
		job.setJarByClass(UserBroserCount.class);
		job.setMapOutputValueClass(IntWritable.class);
		job.setMapperClass(UserBroserCount.UserBroserCountMapper.class);
		job.setReducerClass(UserBroserCount.UserBroserCountReducer.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(IntWritable.class);
		FileInputFormat.addInputPath(job, new Path(inputPath));
		FileOutputFormat.setOutputPath(job, new Path(outputPath));
		job.waitForCompletion(true);

	}

}