mapreduce demo实现详解大数据

mapreduce主要的实现需要三个类:主方法类、map类、reduce类。

1. 主方法类：

public class BreadPointDriver { 
    /** 
     * 
     * @param args 
     * @throws IOException 
     * @throws ClassNotFoundException 
     * @throws InterruptedException 
     */ 
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { 
        //配置项 
        Configuration conf = new Configuration(); 
 
	//定义任务，第一个参数设定配置项,第二个参数设置任务名称 
	Job job = new Job(conf, "bscreenUserCount"); 
	//设置输入文件格式化类 
	job.setInputFormatClass(FileInputFormat.class); 
	//设置输出文件格式化类 
	job.setOutputFormatClass(FileOutputFormat.class); 
	//设置启动类 
	job.setJarByClass(BreadPointDriver.class); 
	//设置mapper 
	job.setMapperClass(UserCountMapper.class); 
	//设置reducer 
	job.setReducerClass(UserCountReducer.class); 
	//设置reducer数量, 如果不设置默认为1,如果设置0则不执行reducer,设置1以上需要编写partition类 
	job.setNumReduceTasks(0); 
	job.setMapOutputKeyClass(Text.class);//map输出key数据类型 
	job.setMapOutputValueClass(Text.class);//map输出value数据类型 
	job.setOutputKeyClass(BscreenUseCount.class);//reducer输出key数据类型 
	job.setOutputValueClass(NullWritable.class);//reducer输出value数据类型 
        FileInputFormat.addInputPath(job, new Path(inputPath));//输入路径 
        FileOutputFormat.addInputPath(job, new Path(outputPath));//输入路径 
        job.waitForCompletion(true);//执行任务 
    } 
} 

2. map类

public class BreadPointMapper extends Mapper<LongWritable, Text, Text, Text> { 
    /** 
     *  
     * @param key 行索引 
     * @param value 行内容 
     * @param context 容器 
     * @throws IOException 
     * @throws InterruptedException 
     */ 
    @Override 
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { 
        String outKey = ""; 
        String outValue = ""; 
        context.write(new Text(outKey),new Text(outValue)); 
    } 
}

3. reducer类

public class BreadPointReducer extends Reducer<Text, Text, Text, NullWritable> { 
 
	/** 
     *  
     * @param key map中存储的key,每个key会执行一次reduce 
     * @param value 对应的value列表 
     * @param context 
     * @throws IOException 
     * @throws InterruptedException 
     */ 
    @Override 
    protected void reduce(Text key, Iterable<Text> value, Context context) throws IOException, InterruptedException { 
    	//处理key和value 
    	//... 
        String outKey = ""; 
        String outValue = ""; 
        context.write(new Text(outKey),new Text(outValue)); 
    } 
}