mapReduce(3)---入门示例WordCount-白红宇

mapReduce(3)---入门示例WordCount

阅读量：4086 次

发布时间：2019-05-25

本文共 37121 字，大约阅读时间需要 123 分钟。

package com.mapReduce;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;import java.io.IOException;/** * MapReduce程序入门示例：wordCount * */public class WordCountMR {    /**     * main()方法作为该mapReduce程序的入口     * */    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {        //指定hdfs相关的参数        Configuration conf=new Configuration();        conf.set("fs.defaultFS","hdfs://hadMaster01:9000");        System.setProperty("HADOOP_USER_NAME","hadoop");        //通过Configuration对象获取job对象,该job对象会组织所有的该MapReduce程序所需各种组件        Job job= Job.getInstance(conf);        //设置jar包所在路径        job.setJarByClass(WordCountMR.class);        //指定mapper类和reducer类        job.setMapperClass(WordCountMapper.class);        job.setReducerClass(WordCountReducer.class);        //Mapper的输入key-value类型,由MapReduce框架决定，下面指定mapTask的输出类型        job.setMapOutputKeyClass(Text.class);   //Text====>String        job.setMapOutputValueClass(IntWritable.class); //IntWritable===>Integer        //如果mapTask的输出key-value类型,同reduceTask的输出key-value类型一致,那么以上两句代码可以不用设置        //reduceTask的输入key-value类型,就是mapTask的输出key-value类型,所以不需要指定。        //指定reduceTask的输出类型        job.setOutputKeyClass(Text.class);        job.setOutputValueClass(IntWritable.class);        //为job指定输入和输出数据的组件,以下两个参数是默认的,所以不指定也可以        job.setInputFormatClass(TextInputFormat.class);        job.setOutputFormatClass(TextOutputFormat.class);        //为该mapreduce程序制定默认的数据分区组件,默认是HashPartitioner.class        job.setPartitionerClass(HashPartitioner.class);        //设置mapReduce程序的ReduceTask的个数        job.setNumReduceTasks(3);        //指定该mapReduce程序数据的输入和输出路径        Path inputPath=new Path("/input");        Path outputPath=new Path("/output");        //该段代码是用来判断输出路径存不存在,存在就删除,虽然方便使用,但是谨慎使用        FileSystem fs=FileSystem.get(conf);        if(fs.exists(outputPath)){            fs.delete(outputPath,true);        }        //设置wordcount程序的输入路径        FileInputFormat.setInputPaths(job,inputPath);        //设置wordcount程序的输出路径        FileOutputFormat.setOutputPath(job,outputPath);        //最后提交任务,（verbose布尔值,决定要不要将运行进度信息输出给用户)        boolean waitForCompletion=job.waitForCompletion(true);        System.exit(waitForCompletion?0:1);    }    /**     * Mapper
   
         * keyIn:指框架读取到的数据的key的类型,在默认的InputFormat下,读到的key是一行文本的其实偏移量,所以key的类型是Long     * valueIn：指框架读取到的数据的value的类型,在默认的InputFormat下,读到的value是一行文本的内容,所以value的类型是String     * keyOut:指用户自定义逻辑方法返回的数据中key的类型,由用户业务逻辑决定,在此wordcount程序中,我们输出的key是单词,所以是String     * valueOut:指用户自定义逻辑方法返回的数据中value的类型,由用户业务逻辑决定,在此wordcount程序中，我们输出的value是单词的数量,所以是Integer     * 但是String,Long等jdk中自带的数据类型,在序列化时,效率比较低,hadoop为了提高序列化效率,自定义了一套序列化框架     * 所以,在hadoop的程序中,如果该数据需要进行序列化(读写磁盘或者网络传输),就一定要用实现了hadoop序列化框架的数据类型     * Long--->LongWritable     * String--->Text     * Integer-->IntWritable     * Null----->NullWritable     * */    static class WordCountMapper extends Mapper
    
     {        /**         * LongWritable key：该key就是value在文件当中的起始偏移量         * Text value：就是mapReduce框架默认的数据读取组件TextInputFormat读取文件当中的一行文本         * */        @Override        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {            //切分单词            String[] words=value.toString().split("");            for(String word:words){                //每个单词计数一次,也就是把单词组织成
     
      这样的key-value对 往外写出                context.write(new Text(word),new IntWritable(1));            }        }    }    /**     *Reducer类也有输入和输出,输入就是map阶段的处理结果,输出就是Reduce最后的输出     * reduceTask在调用我们写的reduce方法,reduceTask应该收到了前一阶段(map阶段）中所有maptask输出的数据中的一部分     * (数据的key.hashcode%reducetask数==本reducetask号),所以reducetasks的输入类型必须和maptask的输出类型一样     *     * reducetask将这些收到kv数据拿来处理时,是这样调用我们的reduce方法的：     * 先将自己收到的所有的kv对按照k分组(根据k是否相同)     * 将某一组kv中的第一个kv中的k传给reduce方法的key变量,把这一组kv中的所有的v用一个迭代器传给reduce方法的变量value     * */    static class WordCountReducer extends Reducer
      
       {        /**         * Text key:mapTask输出的key值         * Iterable
       
         values: key对应的value的集合(该key只是相同的一个key)         * reduce方法接收key值相同的一组key-value进行汇总计算         * */        @Override        protected void reduce(Text key, Iterable
        
          values, Context context) throws IOException, InterruptedException { //结果汇总 int sum=0; for(IntWritable v:values){ sum +=v.get(); } //汇总的结果往外输出 context.write(key,new IntWritable(sum)); } }}

程序执行结果：

D:\developTool\jdk\jdk8\bin\java "-javaagent:D:\developTool\idea2017\IntelliJ IDEA 2017.3\lib\idea_rt.jar=53284:D:\developTool\idea2017\IntelliJ IDEA 2017.3\bin" -Dfile.encoding=UTF-8 -classpath F:\downloads\demo-provider\hadoop_example\target\classes;D:\developTool\jdk\jdk8\jre\lib\charsets.jar;D:\developTool\jdk\jdk8\jre\lib\deploy.jar;D:\developTool\jdk\jdk8\jre\lib\ext\access-bridge-64.jar;D:\developTool\jdk\jdk8\jre\lib\ext\cldrdata.jar;D:\developTool\jdk\jdk8\jre\lib\ext\dnsns.jar;D:\developTool\jdk\jdk8\jre\lib\ext\jaccess.jar;D:\developTool\jdk\jdk8\jre\lib\ext\jfxrt.jar;D:\developTool\jdk\jdk8\jre\lib\ext\localedata.jar;D:\developTool\jdk\jdk8\jre\lib\ext\nashorn.jar;D:\developTool\jdk\jdk8\jre\lib\ext\sunec.jar;D:\developTool\jdk\jdk8\jre\lib\ext\sunjce_provider.jar;D:\developTool\jdk\jdk8\jre\lib\ext\sunmscapi.jar;D:\developTool\jdk\jdk8\jre\lib\ext\sunpkcs11.jar;D:\developTool\jdk\jdk8\jre\lib\ext\zipfs.jar;D:\developTool\jdk\jdk8\jre\lib\javaws.jar;D:\developTool\jdk\jdk8\jre\lib\jce.jar;D:\developTool\jdk\jdk8\jre\lib\jfr.jar;D:\developTool\jdk\jdk8\jre\lib\jfxswt.jar;D:\developTool\jdk\jdk8\jre\lib\jsse.jar;D:\developTool\jdk\jdk8\jre\lib\management-agent.jar;D:\developTool\jdk\jdk8\jre\lib\plugin.jar;D:\developTool\jdk\jdk8\jre\lib\resources.jar;D:\developTool\jdk\jdk8\jre\lib\rt.jar;D:\developTool\hadoop-2.9.1\share\hadoop\httpfs;D:\developTool\hadoop-2.9.1\share\hadoop\common\hadoop-nfs-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\hadoop-common-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\hadoop-common-2.9.1-tests.jar;D:\developTool\hadoop-2.9.1\share\hadoop\hdfs\hadoop-hdfs-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\hdfs\hadoop-hdfs-nfs-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\hdfs\hadoop-hdfs-rbf-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\hdfs\hadoop-hdfs-2.9.1-tests.jar;D:\developTool\hadoop-2.9.1\share\hadoop\hdfs\hadoop-hdfs-client-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\hdfs\hadoop-hdfs-rbf-2.9.1-tests.jar;D:\developTool\hadoop-2.9.1\share\hadoop\hdfs\hadoop-hdfs-client-2.9.1-tests.jar;D:\developTool\hadoop-2.9.1\share\hadoop\hdfs\hadoop-hdfs-native-client-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\hdfs\hadoop-hdfs-native-client-2.9.1-tests.jar;D:\developTool\hadoop-2.9.1\share\hadoop\mapreduce\hadoop-mapreduce-examples-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\mapreduce\hadoop-mapreduce-client-hs-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\mapreduce\hadoop-mapreduce-client-app-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\mapreduce\hadoop-mapreduce-client-core-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\mapreduce\hadoop-mapreduce-client-common-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\mapreduce\hadoop-mapreduce-client-shuffle-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\mapreduce\hadoop-mapreduce-client-jobclient-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\mapreduce\hadoop-mapreduce-client-hs-plugins-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\mapreduce\hadoop-mapreduce-client-jobclient-2.9.1-tests.jar;D:\developTool\hadoop-2.9.1\share\hadoop\yarn\hadoop-yarn-ui-2.9.1.war;D:\developTool\hadoop-2.9.1\share\hadoop\yarn\hadoop-yarn-api-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\yarn\hadoop-yarn-client-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\yarn\hadoop-yarn-common-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\yarn\hadoop-yarn-registry-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\yarn\hadoop-yarn-server-tests-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\yarn\hadoop-yarn-server-common-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\yarn\hadoop-yarn-server-router-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\yarn\hadoop-yarn-server-web-proxy-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\yarn\hadoop-yarn-server-nodemanager-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\yarn\hadoop-yarn-server-resourcemanager-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\yarn\hadoop-yarn-server-sharedcachemanager-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\yarn\hadoop-yarn-applications-distributedshell-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\yarn\hadoop-yarn-server-timeline-pluginstorage-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\yarn\hadoop-yarn-server-applicationhistoryservice-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\yarn\hadoop-yarn-applications-unmanaged-am-launcher-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\woodstox-core-5.0.3.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\slf4j-api-1.7.25.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\slf4j-log4j12-1.7.25.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\log4j-1.2.17.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\stax2-api-3.1.4.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\activation-1.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\apacheds-i18n-2.0.0-M15.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\apacheds-kerberos-codec-2.0.0-M15.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\api-asn1-api-1.0.0-M20.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\api-util-1.0.0-M20.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\asm-3.2.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\avro-1.7.7.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\commons-beanutils-1.7.0.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\commons-beanutils-core-1.8.0.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\commons-cli-1.2.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\commons-codec-1.4.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\commons-collections-3.2.2.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\commons-compress-1.4.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\commons-configuration-1.6.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\commons-digester-1.8.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\commons-io-2.4.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\commons-lang-2.6.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\commons-lang3-3.4.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\commons-logging-1.1.3.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\commons-math3-3.1.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\commons-net-3.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\curator-client-2.7.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\curator-framework-2.7.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\curator-recipes-2.7.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\gson-2.2.4.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\guava-11.0.2.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\hadoop-annotations-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\hadoop-auth-2.9.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\hamcrest-core-1.3.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\htrace-core-3.0.4.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\htrace-core4-4.1.0-incubating.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\httpclient-4.5.2.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\httpcore-4.4.4.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\jackson-core-asl-1.9.13.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\jackson-jaxrs-1.9.13.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\jackson-mapper-asl-1.9.13.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\jackson-xc-1.9.13.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\java-xmlbuilder-0.4.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\jaxb-api-2.2.2.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\jaxb-impl-2.2.3-1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\jcip-annotations-1.0-1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\jersey-core-1.9.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\jersey-json-1.9.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\jersey-server-1.9.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\jets3t-0.9.0.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\jettison-1.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\jetty-6.1.26.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\jetty-sslengine-6.1.26.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\jetty-util-6.1.26.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\jsch-0.1.54.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\json-smart-1.3.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\jsp-api-2.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\jsr305-3.0.0.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\junit-4.11.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\mockito-all-1.8.5.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\netty-3.6.2.Final.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\nimbus-jose-jwt-4.41.1.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\paranamer-2.3.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\protobuf-java-2.5.0.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\servlet-api-2.5.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\snappy-java-1.0.5.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\stax-api-1.0-2.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\xmlenc-0.52.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\xz-1.0.jar;D:\developTool\hadoop-2.9.1\share\hadoop\common\lib\zookeeper-3.4.6.jar com.mapReduce.WordCountMR20/07/13 17:06:47 INFO Configuration.deprecation: session.id is deprecated. Instead, use dfs.metrics.session-id20/07/13 17:06:47 INFO jvm.JvmMetrics: Initializing JVM Metrics with processName=JobTracker, sessionId=20/07/13 17:06:57 WARN mapreduce.JobResourceUploader: Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this.20/07/13 17:06:58 WARN mapreduce.JobResourceUploader: No job jar file set.  User classes may not be found. See Job or Job#setJar(String).20/07/13 17:07:00 INFO input.FileInputFormat: Total input files to process : 320/07/13 17:07:01 INFO mapreduce.JobSubmitter: number of splits:320/07/13 17:07:04 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_local508204586_000120/07/13 17:07:05 INFO mapreduce.Job: The url to track the job: http://localhost:8080/20/07/13 17:07:05 INFO mapreduce.Job: Running job: job_local508204586_000120/07/13 17:07:05 INFO mapred.LocalJobRunner: OutputCommitter set in config null20/07/13 17:07:05 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 120/07/13 17:07:05 INFO output.FileOutputCommitter: FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false20/07/13 17:07:05 INFO mapred.LocalJobRunner: OutputCommitter is org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter20/07/13 17:07:06 INFO mapreduce.Job: Job job_local508204586_0001 running in uber mode : false20/07/13 17:07:06 INFO mapreduce.Job:  map 0% reduce 0%20/07/13 17:07:07 INFO mapred.LocalJobRunner: Waiting for map tasks20/07/13 17:07:07 INFO mapred.LocalJobRunner: Starting task: attempt_local508204586_0001_m_000000_020/07/13 17:07:07 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 120/07/13 17:07:07 INFO output.FileOutputCommitter: FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false20/07/13 17:07:07 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.20/07/13 17:07:07 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@114770b020/07/13 17:07:07 INFO mapred.MapTask: Processing split: hdfs://hadMaster01:9000/input/wc.txt:0+8320/07/13 17:07:07 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)20/07/13 17:07:07 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 10020/07/13 17:07:07 INFO mapred.MapTask: soft limit at 8388608020/07/13 17:07:07 INFO mapred.MapTask: bufstart = 0; bufvoid = 10485760020/07/13 17:07:07 INFO mapred.MapTask: kvstart = 26214396; length = 655360020/07/13 17:07:07 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer20/07/13 17:07:18 INFO mapred.LocalJobRunner: 20/07/13 17:07:18 INFO mapred.MapTask: Starting flush of map output20/07/13 17:07:18 INFO mapred.MapTask: Spilling map output20/07/13 17:07:18 INFO mapred.MapTask: bufstart = 0; bufend = 450; bufvoid = 10485760020/07/13 17:07:18 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 26214100(104856400); length = 297/655360020/07/13 17:07:19 INFO mapred.MapTask: Finished spill 020/07/13 17:07:19 INFO mapred.Task: Task:attempt_local508204586_0001_m_000000_0 is done. And is in the process of committing20/07/13 17:07:19 INFO mapred.LocalJobRunner: map20/07/13 17:07:19 INFO mapred.Task: Task 'attempt_local508204586_0001_m_000000_0' done.20/07/13 17:07:19 INFO mapred.LocalJobRunner: Finishing task: attempt_local508204586_0001_m_000000_020/07/13 17:07:19 INFO mapred.LocalJobRunner: Starting task: attempt_local508204586_0001_m_000001_020/07/13 17:07:19 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 120/07/13 17:07:19 INFO output.FileOutputCommitter: FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false20/07/13 17:07:19 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.20/07/13 17:07:19 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@656d878120/07/13 17:07:19 INFO mapred.MapTask: Processing split: hdfs://hadMaster01:9000/input/hello.txt:0+1420/07/13 17:07:19 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)20/07/13 17:07:19 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 10020/07/13 17:07:19 INFO mapred.MapTask: soft limit at 8388608020/07/13 17:07:19 INFO mapred.MapTask: bufstart = 0; bufvoid = 10485760020/07/13 17:07:19 INFO mapred.MapTask: kvstart = 26214396; length = 655360020/07/13 17:07:19 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer20/07/13 17:07:19 INFO mapred.LocalJobRunner: 20/07/13 17:07:19 INFO mapred.MapTask: Starting flush of map output20/07/13 17:07:19 INFO mapred.MapTask: Spilling map output20/07/13 17:07:19 INFO mapred.MapTask: bufstart = 0; bufend = 82; bufvoid = 10485760020/07/13 17:07:19 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 26214352(104857408); length = 45/655360020/07/13 17:07:19 INFO mapred.MapTask: Finished spill 020/07/13 17:07:19 INFO mapred.Task: Task:attempt_local508204586_0001_m_000001_0 is done. And is in the process of committing20/07/13 17:07:19 INFO mapred.LocalJobRunner: map20/07/13 17:07:19 INFO mapred.Task: Task 'attempt_local508204586_0001_m_000001_0' done.20/07/13 17:07:19 INFO mapred.LocalJobRunner: Finishing task: attempt_local508204586_0001_m_000001_020/07/13 17:07:19 INFO mapred.LocalJobRunner: Starting task: attempt_local508204586_0001_m_000002_020/07/13 17:07:19 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 120/07/13 17:07:19 INFO output.FileOutputCommitter: FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false20/07/13 17:07:19 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.20/07/13 17:07:19 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@75c726f020/07/13 17:07:19 INFO mapred.MapTask: Processing split: hdfs://hadMaster01:9000/input/hw.txt:0+1420/07/13 17:07:19 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)20/07/13 17:07:19 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 10020/07/13 17:07:19 INFO mapred.MapTask: soft limit at 8388608020/07/13 17:07:19 INFO mapred.MapTask: bufstart = 0; bufvoid = 10485760020/07/13 17:07:19 INFO mapred.MapTask: kvstart = 26214396; length = 655360020/07/13 17:07:19 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer20/07/13 17:07:19 INFO mapred.LocalJobRunner: 20/07/13 17:07:19 INFO mapred.MapTask: Starting flush of map output20/07/13 17:07:19 INFO mapred.MapTask: Spilling map output20/07/13 17:07:19 INFO mapred.MapTask: bufstart = 0; bufend = 82; bufvoid = 10485760020/07/13 17:07:19 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 26214352(104857408); length = 45/655360020/07/13 17:07:19 INFO mapreduce.Job:  map 67% reduce 0%20/07/13 17:07:19 INFO mapred.MapTask: Finished spill 020/07/13 17:07:20 INFO mapred.Task: Task:attempt_local508204586_0001_m_000002_0 is done. And is in the process of committing20/07/13 17:07:20 INFO mapred.LocalJobRunner: map20/07/13 17:07:20 INFO mapred.Task: Task 'attempt_local508204586_0001_m_000002_0' done.20/07/13 17:07:20 INFO mapred.LocalJobRunner: Finishing task: attempt_local508204586_0001_m_000002_020/07/13 17:07:20 INFO mapred.LocalJobRunner: map task executor complete.20/07/13 17:07:20 INFO mapred.LocalJobRunner: Waiting for reduce tasks20/07/13 17:07:20 INFO mapred.LocalJobRunner: Starting task: attempt_local508204586_0001_r_000000_020/07/13 17:07:20 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 120/07/13 17:07:20 INFO output.FileOutputCommitter: FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false20/07/13 17:07:20 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.20/07/13 17:07:20 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@3b089a8920/07/13 17:07:20 INFO mapred.ReduceTask: Using ShuffleConsumerPlugin: org.apache.hadoop.mapreduce.task.reduce.Shuffle@1f1b703820/07/13 17:07:20 INFO reduce.MergeManagerImpl: MergerManager: memoryLimit=1311663744, maxSingleShuffleLimit=327915936, mergeThreshold=865698112, ioSortFactor=10, memToMemMergeOutputsThreshold=1020/07/13 17:07:20 INFO reduce.EventFetcher: attempt_local508204586_0001_r_000000_0 Thread started: EventFetcher for fetching Map Completion Events20/07/13 17:07:20 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local508204586_0001_m_000000_0 decomp: 218 len: 222 to MEMORY20/07/13 17:07:20 INFO reduce.InMemoryMapOutput: Read 218 bytes from map-output for attempt_local508204586_0001_m_000000_020/07/13 17:07:20 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 218, inMemoryMapOutputs.size() -> 1, commitMemory -> 0, usedMemory ->21820/07/13 17:07:20 INFO mapreduce.Job:  map 100% reduce 0%20/07/13 17:07:21 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local508204586_0001_m_000002_0 decomp: 10 len: 14 to MEMORY20/07/13 17:07:21 INFO reduce.InMemoryMapOutput: Read 10 bytes from map-output for attempt_local508204586_0001_m_000002_020/07/13 17:07:21 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 10, inMemoryMapOutputs.size() -> 2, commitMemory -> 218, usedMemory ->22820/07/13 17:07:21 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local508204586_0001_m_000001_0 decomp: 10 len: 14 to MEMORY20/07/13 17:07:21 INFO reduce.InMemoryMapOutput: Read 10 bytes from map-output for attempt_local508204586_0001_m_000001_020/07/13 17:07:21 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 10, inMemoryMapOutputs.size() -> 3, commitMemory -> 228, usedMemory ->23820/07/13 17:07:21 INFO reduce.EventFetcher: EventFetcher is interrupted.. Returning20/07/13 17:07:21 INFO mapred.LocalJobRunner: 3 / 3 copied.20/07/13 17:07:21 INFO reduce.MergeManagerImpl: finalMerge called with 3 in-memory map-outputs and 0 on-disk map-outputs20/07/13 17:07:21 INFO mapred.Merger: Merging 3 sorted segments20/07/13 17:07:21 INFO mapred.Merger: Down to the last merge-pass, with 3 segments left of total size: 226 bytes20/07/13 17:07:21 INFO reduce.MergeManagerImpl: Merged 3 segments, 238 bytes to disk to satisfy reduce memory limit20/07/13 17:07:21 INFO reduce.MergeManagerImpl: Merging 1 files, 238 bytes from disk20/07/13 17:07:21 INFO reduce.MergeManagerImpl: Merging 0 segments, 0 bytes from memory into reduce20/07/13 17:07:21 INFO mapred.Merger: Merging 1 sorted segments20/07/13 17:07:21 INFO mapred.Merger: Down to the last merge-pass, with 1 segments left of total size: 230 bytes20/07/13 17:07:21 INFO mapred.LocalJobRunner: 3 / 3 copied.20/07/13 17:07:22 INFO Configuration.deprecation: mapred.skip.on is deprecated. Instead, use mapreduce.job.skiprecords20/07/13 17:07:32 INFO mapred.LocalJobRunner: reduce > reduce20/07/13 17:07:32 INFO mapreduce.Job:  map 100% reduce 33%20/07/13 17:07:39 INFO mapred.Task: Task:attempt_local508204586_0001_r_000000_0 is done. And is in the process of committing20/07/13 17:07:39 INFO mapred.LocalJobRunner: reduce > reduce20/07/13 17:07:39 INFO mapred.Task: Task attempt_local508204586_0001_r_000000_0 is allowed to commit now20/07/13 17:07:39 INFO output.FileOutputCommitter: Saved output of task 'attempt_local508204586_0001_r_000000_0' to hdfs://hadMaster01:9000/output/_temporary/0/task_local508204586_0001_r_00000020/07/13 17:07:39 INFO mapred.LocalJobRunner: reduce > reduce20/07/13 17:07:39 INFO mapred.Task: Task 'attempt_local508204586_0001_r_000000_0' done.20/07/13 17:07:39 INFO mapred.LocalJobRunner: Finishing task: attempt_local508204586_0001_r_000000_020/07/13 17:07:39 INFO mapred.LocalJobRunner: Starting task: attempt_local508204586_0001_r_000001_020/07/13 17:07:39 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 120/07/13 17:07:39 INFO output.FileOutputCommitter: FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false20/07/13 17:07:39 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.20/07/13 17:07:40 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@4c441e3620/07/13 17:07:40 INFO mapred.ReduceTask: Using ShuffleConsumerPlugin: org.apache.hadoop.mapreduce.task.reduce.Shuffle@1c438e7520/07/13 17:07:40 INFO reduce.MergeManagerImpl: MergerManager: memoryLimit=1311663744, maxSingleShuffleLimit=327915936, mergeThreshold=865698112, ioSortFactor=10, memToMemMergeOutputsThreshold=1020/07/13 17:07:40 INFO reduce.EventFetcher: attempt_local508204586_0001_r_000001_0 Thread started: EventFetcher for fetching Map Completion Events20/07/13 17:07:40 INFO reduce.LocalFetcher: localfetcher#2 about to shuffle output of map attempt_local508204586_0001_m_000000_0 decomp: 250 len: 254 to MEMORY20/07/13 17:07:40 INFO reduce.InMemoryMapOutput: Read 250 bytes from map-output for attempt_local508204586_0001_m_000000_020/07/13 17:07:40 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 250, inMemoryMapOutputs.size() -> 1, commitMemory -> 0, usedMemory ->25020/07/13 17:07:40 INFO reduce.LocalFetcher: localfetcher#2 about to shuffle output of map attempt_local508204586_0001_m_000002_0 decomp: 36 len: 40 to MEMORY20/07/13 17:07:40 INFO reduce.InMemoryMapOutput: Read 36 bytes from map-output for attempt_local508204586_0001_m_000002_020/07/13 17:07:40 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 36, inMemoryMapOutputs.size() -> 2, commitMemory -> 250, usedMemory ->28620/07/13 17:07:40 INFO reduce.LocalFetcher: localfetcher#2 about to shuffle output of map attempt_local508204586_0001_m_000001_0 decomp: 36 len: 40 to MEMORY20/07/13 17:07:40 INFO reduce.InMemoryMapOutput: Read 36 bytes from map-output for attempt_local508204586_0001_m_000001_020/07/13 17:07:40 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 36, inMemoryMapOutputs.size() -> 3, commitMemory -> 286, usedMemory ->32220/07/13 17:07:40 INFO reduce.EventFetcher: EventFetcher is interrupted.. Returning20/07/13 17:07:40 INFO mapred.LocalJobRunner: 3 / 3 copied.20/07/13 17:07:40 INFO reduce.MergeManagerImpl: finalMerge called with 3 in-memory map-outputs and 0 on-disk map-outputs20/07/13 17:07:40 INFO mapred.Merger: Merging 3 sorted segments20/07/13 17:07:40 INFO mapred.Merger: Down to the last merge-pass, with 3 segments left of total size: 310 bytes20/07/13 17:07:40 INFO reduce.MergeManagerImpl: Merged 3 segments, 322 bytes to disk to satisfy reduce memory limit20/07/13 17:07:40 INFO reduce.MergeManagerImpl: Merging 1 files, 322 bytes from disk20/07/13 17:07:40 INFO reduce.MergeManagerImpl: Merging 0 segments, 0 bytes from memory into reduce20/07/13 17:07:40 INFO mapred.Merger: Merging 1 sorted segments20/07/13 17:07:40 INFO mapred.Merger: Down to the last merge-pass, with 1 segments left of total size: 314 bytes20/07/13 17:07:40 INFO mapred.LocalJobRunner: 3 / 3 copied.20/07/13 17:07:49 INFO mapred.Task: Task:attempt_local508204586_0001_r_000001_0 is done. And is in the process of committing20/07/13 17:07:49 INFO mapred.LocalJobRunner: 3 / 3 copied.20/07/13 17:07:49 INFO mapred.Task: Task attempt_local508204586_0001_r_000001_0 is allowed to commit now20/07/13 17:07:49 INFO output.FileOutputCommitter: Saved output of task 'attempt_local508204586_0001_r_000001_0' to hdfs://hadMaster01:9000/output/_temporary/0/task_local508204586_0001_r_00000120/07/13 17:07:49 INFO mapred.LocalJobRunner: reduce > reduce20/07/13 17:07:49 INFO mapred.Task: Task 'attempt_local508204586_0001_r_000001_0' done.20/07/13 17:07:49 INFO mapred.LocalJobRunner: Finishing task: attempt_local508204586_0001_r_000001_020/07/13 17:07:49 INFO mapred.LocalJobRunner: Starting task: attempt_local508204586_0001_r_000002_020/07/13 17:07:49 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 120/07/13 17:07:49 INFO output.FileOutputCommitter: FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false20/07/13 17:07:49 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.20/07/13 17:07:50 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@2db8228f20/07/13 17:07:50 INFO mapred.ReduceTask: Using ShuffleConsumerPlugin: org.apache.hadoop.mapreduce.task.reduce.Shuffle@2d88316b20/07/13 17:07:50 INFO reduce.MergeManagerImpl: MergerManager: memoryLimit=1311663744, maxSingleShuffleLimit=327915936, mergeThreshold=865698112, ioSortFactor=10, memToMemMergeOutputsThreshold=1020/07/13 17:07:50 INFO reduce.EventFetcher: attempt_local508204586_0001_r_000002_0 Thread started: EventFetcher for fetching Map Completion Events20/07/13 17:07:50 INFO reduce.LocalFetcher: localfetcher#3 about to shuffle output of map attempt_local508204586_0001_m_000000_0 decomp: 138 len: 142 to MEMORY20/07/13 17:07:50 INFO reduce.InMemoryMapOutput: Read 138 bytes from map-output for attempt_local508204586_0001_m_000000_020/07/13 17:07:50 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 138, inMemoryMapOutputs.size() -> 1, commitMemory -> 0, usedMemory ->13820/07/13 17:07:50 INFO reduce.LocalFetcher: localfetcher#3 about to shuffle output of map attempt_local508204586_0001_m_000002_0 decomp: 66 len: 70 to MEMORY20/07/13 17:07:50 INFO reduce.InMemoryMapOutput: Read 66 bytes from map-output for attempt_local508204586_0001_m_000002_020/07/13 17:07:50 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 66, inMemoryMapOutputs.size() -> 2, commitMemory -> 138, usedMemory ->20420/07/13 17:07:50 INFO mapreduce.Job:  map 100% reduce 100%20/07/13 17:07:50 INFO reduce.LocalFetcher: localfetcher#3 about to shuffle output of map attempt_local508204586_0001_m_000001_0 decomp: 66 len: 70 to MEMORY20/07/13 17:07:50 INFO reduce.InMemoryMapOutput: Read 66 bytes from map-output for attempt_local508204586_0001_m_000001_020/07/13 17:07:50 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 66, inMemoryMapOutputs.size() -> 3, commitMemory -> 204, usedMemory ->27020/07/13 17:07:50 INFO reduce.EventFetcher: EventFetcher is interrupted.. Returning20/07/13 17:07:50 INFO mapred.LocalJobRunner: 3 / 3 copied.20/07/13 17:07:50 INFO reduce.MergeManagerImpl: finalMerge called with 3 in-memory map-outputs and 0 on-disk map-outputs20/07/13 17:07:51 INFO mapred.Merger: Merging 3 sorted segments20/07/13 17:07:51 INFO mapred.Merger: Down to the last merge-pass, with 3 segments left of total size: 258 bytes20/07/13 17:07:51 INFO reduce.MergeManagerImpl: Merged 3 segments, 270 bytes to disk to satisfy reduce memory limit20/07/13 17:07:51 INFO reduce.MergeManagerImpl: Merging 1 files, 270 bytes from disk20/07/13 17:07:51 INFO reduce.MergeManagerImpl: Merging 0 segments, 0 bytes from memory into reduce20/07/13 17:07:51 INFO mapred.Merger: Merging 1 sorted segments20/07/13 17:07:51 INFO mapred.Merger: Down to the last merge-pass, with 1 segments left of total size: 262 bytes20/07/13 17:07:51 INFO mapred.LocalJobRunner: 3 / 3 copied.20/07/13 17:07:51 INFO mapreduce.Job:  map 100% reduce 67%20/07/13 17:08:01 INFO mapred.LocalJobRunner: reduce > reduce20/07/13 17:08:02 INFO mapreduce.Job:  map 100% reduce 100%20/07/13 17:08:10 INFO mapred.Task: Task:attempt_local508204586_0001_r_000002_0 is done. And is in the process of committing20/07/13 17:08:10 INFO mapred.LocalJobRunner: reduce > reduce20/07/13 17:08:10 INFO mapred.Task: Task attempt_local508204586_0001_r_000002_0 is allowed to commit now20/07/13 17:08:10 INFO output.FileOutputCommitter: Saved output of task 'attempt_local508204586_0001_r_000002_0' to hdfs://hadMaster01:9000/output/_temporary/0/task_local508204586_0001_r_00000220/07/13 17:08:10 INFO mapred.LocalJobRunner: reduce > reduce20/07/13 17:08:10 INFO mapred.Task: Task 'attempt_local508204586_0001_r_000002_0' done.20/07/13 17:08:10 INFO mapred.LocalJobRunner: Finishing task: attempt_local508204586_0001_r_000002_020/07/13 17:08:10 INFO mapred.LocalJobRunner: reduce task executor complete.20/07/13 17:08:11 INFO mapreduce.Job: Job job_local508204586_0001 completed successfully20/07/13 17:08:12 INFO mapreduce.Job: Counters: 35	File System Counters		FILE: Number of bytes read=12661		FILE: Number of bytes written=2974574		FILE: Number of read operations=0		FILE: Number of large read operations=0		FILE: Number of write operations=0		HDFS: Number of bytes read=624		HDFS: Number of bytes written=197		HDFS: Number of read operations=72		HDFS: Number of large read operations=0		HDFS: Number of write operations=24	Map-Reduce Framework		Map input records=7		Map output records=99		Map output bytes=614		Map output materialized bytes=866		Input split bytes=306		Combine input records=0		Combine output records=0		Reduce input groups=24		Reduce shuffle bytes=866		Reduce input records=99		Reduce output records=24		Spilled Records=198		Shuffled Maps =9		Failed Shuffles=0		Merged Map outputs=9		GC time elapsed (ms)=12		Total committed heap usage (bytes)=2236612608	Shuffle Errors		BAD_ID=0		CONNECTION=0		IO_ERROR=0		WRONG_LENGTH=0		WRONG_MAP=0		WRONG_REDUCE=0	File Input Format Counters 		Bytes Read=111	File Output Format Counters 		Bytes Written=103Process finished with exit code 0

总结：

1）、该程序有一个main()方法,用来启动任务。其中job对象存储了该程序运行的必要信息，比如指定Mapper类和Reducer类。

比如：job.setMapperClass(WordCountMapper.class)；job.setReducerClass(WordCountReducer.class)

2）、MapReduce程序的业务编码分为两个部分。一部分配置程序的运行信息,一部分编写该MapReduce程序的业务逻辑。

该程序的业务逻辑分为map阶段和reduce阶段,代码分别继承Mapper类和Reducer类。

MapReduce程序的编写规范如下：

1）、用户编写的程序分为三个部分：Mapper、Reducer、Driver(提交运行MR程序的客户端)

2）、Mapper的输入和输出数据都是KV对的形式(KV的类型可以自定义)

3）、Reducer的输入数据类型对应Mapper的输出数据类型,也是KV对的形式

4）、Mapper中的业务逻辑写在map()方法中,Reducer的业务逻辑写在reduce()方法中

5）、map()方法(mapTask进程) 对每一个<k,v>调用一次

6）、reduceTask进程对每一组相同的<k,v>组调用一次reduce()方法