intelij创建MapReduce工程

1、创建一个maven工程

2、pom文件

<?xml version="1.0" encoding="utf-8"?>
<project xmlns="http://maven.apache.org/pom/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/xmlschema-instance"
         xsi:schemalocation="http://maven.apache.org/pom/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelversion>4.0.0</modelversion>

    <groupid>com.sogou</groupid>
    <artifactid>teemo-dc-etl</artifactid>
    <version>1.0.0</version>
    <packaging>jar</packaging>

    <name>teemo-dc-etl</name>
    <url>http://maven.apache.org</url>

    <properties>
        <project.build.sourceencoding>utf-8</project.build.sourceencoding>
        <mahout.version>0.5</mahout.version>
        <mahout.groupid>org.apache.mahout</mahout.groupid>
        <spring.version>3.0.6.release</spring.version>
    </properties>

    <repositories>
    <repository>
        <id>maven-ali</id>
        <url>http://maven.twttr.com/</url>
        <releases>
            <enabled>true</enabled>
        </releases>
        <snapshots>
            <enabled>true</enabled>
            <updatepolicy>always</updatepolicy>
            <checksumpolicy>fail</checksumpolicy>
        </snapshots>
    </repository>
    </repositories>

    <dependencies>
        <dependency>
            <groupid>junit</groupid>
            <artifactid>junit</artifactid>
            <version>3.8.1</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupid>org.apache.hadoop</groupid>
            <artifactid>hadoop-hdfs</artifactid>
            <version>2.5.0</version>
        </dependency>

        <dependency>
            <groupid>org.apache.hadoop</groupid>
            <artifactid>hadoop-client</artifactid>
            <version>2.5.1</version>
        </dependency>

        <dependency>
            <groupid>org.apache.hadoop</groupid>
            <artifactid>hadoop-common</artifactid>
            <version>2.5.0</version>
        </dependency>

        <dependency>
            <groupid>com.hadoop.gplcompression</groupid>
            <artifactid>hadoop-lzo</artifactid>
            <version>0.4.19</version>
        </dependency>

        <dependency>
            <groupid>org.apache.hadoop</groupid>
            <artifactid>hadoop-yarn-common</artifactid>
            <version>2.5.2</version>
        </dependency>

        <dependency>
            <groupid>com.alibaba</groupid>
            <artifactid>fastjson</artifactid>
            <version>1.2.4</version>
        </dependency>

    </dependencies>


    <build>
        <plugins>
            <!--
            bind the maven-assembly-plugin to the package phase
            this will create a jar file without the storm dependencies
            suitable for deployment to a cluster.
             -->
            <plugin>
                <artifactid>maven-assembly-plugin</artifactid>
                <configuration>
                    <archive>
                        <manifest>
                            <mainclass></mainclass>
                        </manifest>
                    </archive>
                    <descriptorrefs>
                        <descriptorref>jar-with-dependencies</descriptorref>
                    </descriptorrefs>
                </configuration>
                <executions>
                    <execution>
                        <id>make-assembly</id>
                        <phase>package</phase> <!-- packaging phase -->
                        <goals>
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupid>org.apache.maven.plugins</groupid>
                <artifactid>maven-compiler-plugin</artifactid>
                <configuration>
                    <source>1.6</source>
                    <target>1.6</target>
                    <encoding>utf-8</encoding>
                </configuration>
            </plugin>

            <plugin>
                <groupid>org.apache.maven.plugins</groupid>
                <artifactid>maven-surefire-plugin</artifactid>
                <version>2.14.1</version>
                <configuration>
                    <argline>-xmx2048m</argline>
                </configuration>
            </plugin>
        </plugins>
    </build>
</project>
这里有个lzo包，需要增加twiter的资源库
3、mapreduce文件写法

package com.sogou.teemo.test;

import org.apache.hadoop.conf.configuration;
import org.apache.hadoop.fs.path;
import org.apache.hadoop.io.intwritable;
import org.apache.hadoop.io.text;
import org.apache.hadoop.mapreduce.job;
import org.apache.hadoop.mapreduce.mapper;
import org.apache.hadoop.mapreduce.reducer;
import org.apache.hadoop.mapreduce.lib.input.fileinputformat;
import org.apache.hadoop.mapreduce.lib.output.fileoutputformat;

import java.io.ioexception;
import java.util.stringtokenizer;

public class wordcount {
    /* mapper */
    public static class tokenizermapper extends mapper<object, text, text, intwritable> {
        private final static intwritable one = new intwritable(1);
        private text word = new text();
        @override
        public void map(object key, text value, context context) throws ioexception, interruptedexception{
            stringtokenizer itr = new stringtokenizer(value.tostring());
            while(itr.hasmoretokens()){
                word.set(itr.nexttoken());
                context.write(word, one);
            }
        }
    }

    /* reducer */
    public static class intsumreducer extends reducer<text,intwritable,text,intwritable>{
        private intwritable result = new intwritable();
        @override
        public void reduce(text key, iterable<intwritable> values, context context) throws ioexception,interruptedexception{
            int sum = 0;
            for(intwritable val : values){
                sum += val.get();
            }
            result.set(sum);
            context.write(key,result);
        }
    }

    /* 启动 mapreduce job */
    public static void main(string[] args) throws exception{
        system.setproperty("hadoop.home.dir","d:/hadoop-2.6.5" );
        configuration conf = new configuration();
        /*if(args.length != 2){
            system.err.println("usage: wordcount <int> <out>");
            system.exit(2);
        }*/
        string arg1 = "input";
        string arg2 = "output";
        job job = new job(conf, "word count");
        job.setjarbyclass(wordcount.class);
        job.setmapperclass(tokenizermapper.class);
        job.setreducerclass(intsumreducer.class);
        job.setoutputkeyclass(text.class);
        job.setoutputvalueclass(intwritable.class);
        fileinputformat.addinputpath(job,new path(arg1));
        fileoutputformat.setoutputpath(job,new path(arg2));
        system.exit(job.waitforcompletion(true)?0:1);
    }
}