eclipse配置hadoop2.7.2开发环境并本地跑起来详解大数据

package com.wulinfeng.hadoop.wordsplit; 
import java.io.IOException; 
import java.io.StringReader; 
import org.apache.hadoop.conf.Configuration; 
import org.apache.hadoop.fs.FileSystem; 
import org.apache.hadoop.fs.Path; 
import org.apache.hadoop.io.IntWritable; 
import org.apache.hadoop.io.Text; 
import org.apache.hadoop.io.WritableComparable; 
import org.apache.hadoop.mapreduce.Job; 
import org.apache.hadoop.mapreduce.Mapper; 
import org.apache.hadoop.mapreduce.Reducer; 
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; 
import org.apache.hadoop.mapreduce.lib.map.InverseMapper; 
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; 
import org.wltea.analyzer.core.IKSegmenter; 
import org.wltea.analyzer.core.Lexeme; 
public class WordSplit { 
/** 
* map实现分词 
* @author Administrator 
* 
*/ 
public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> { 
private static final IntWritable one = new IntWritable(1); 
private Text word = new Text(); 
public void map(Object key, Text value, Mapper<Object, Text, Text, IntWritable>.Context context) 
throws IOException, InterruptedException { 
StringReader input = new StringReader(value.toString()); 
IKSegmenter ikSeg = new IKSegmenter(input, true); // 智能分词 
for (Lexeme lexeme = ikSeg.next(); lexeme != null; lexeme = ikSeg.next()) { 
this.word.set(lexeme.getLexemeText()); 
context.write(this.word, one); 
} 
} 
} 
/** 
* reduce实现分词累计 
* @author Administrator 
* 
*/ 
public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> { 
private IntWritable result = new IntWritable(); 
public void reduce(Text key, Iterable<IntWritable> values, 
Reducer<Text, IntWritable, Text, IntWritable>.Context context) 
throws IOException, InterruptedException { 
int sum = 0; 
for (IntWritable val : values) { 
sum += val.get(); 
} 
this.result.set(sum); 
context.write(key, this.result); 
} 
} 
public static void main(String[] args) throws Exception { 
Configuration conf = new Configuration(); 
String inputFile = "/input/people.txt"; // 输入文件 
Path outDir = new Path("/out"); // 输出目录 
Path tempDir = new Path("/tmp" + System.currentTimeMillis()); // 临时目录 
// 第一个任务：分词 
System.out.println("start task..."); 
Job job = Job.getInstance(conf, "word split"); 
job.setJarByClass(WordSplit.class); 
job.setMapperClass(TokenizerMapper.class); 
job.setCombinerClass(IntSumReducer.class); 
job.setReducerClass(IntSumReducer.class); 
job.setOutputKeyClass(Text.class); 
job.setOutputValueClass(IntWritable.class); 
FileInputFormat.addInputPath(job, new Path(inputFile)); 
FileOutputFormat.setOutputPath(job, tempDir); 
// 第一个任务结束，输出作为第二个任务的输入，开始排序任务 
job.setOutputFormatClass(SequenceFileOutputFormat.class); 
if (job.waitForCompletion(true)) { 
System.out.println("start sort..."); 
Job sortJob = Job.getInstance(conf, "word sort"); 
sortJob.setJarByClass(WordSplit.class); 
sortJob.setMapperClass(InverseMapper.class); 
sortJob.setInputFormatClass(SequenceFileInputFormat.class); 
// 反转map键值，计算词频并降序 
sortJob.setMapOutputKeyClass(IntWritable.class); 
sortJob.setMapOutputValueClass(Text.class); 
sortJob.setSortComparatorClass(IntWritableDecreasingComparator.class); 
sortJob.setNumReduceTasks(1); 
// 输出到out目录文件 
sortJob.setOutputKeyClass(IntWritable.class); 
sortJob.setOutputValueClass(Text.class); 
FileInputFormat.addInputPath(sortJob, tempDir); 
// 如果已经有out目录，先删再创建 
FileSystem fileSystem = outDir.getFileSystem(conf); 
if (fileSystem.exists(outDir)) { 
fileSystem.delete(outDir, true); 
} 
FileOutputFormat.setOutputPath(sortJob, outDir); 
if (sortJob.waitForCompletion(true)) { 
System.out.println("finish and quit...."); 
// 删掉临时目录 
fileSystem = tempDir.getFileSystem(conf); 
if (fileSystem.exists(tempDir)) { 
fileSystem.delete(tempDir, true); 
} 
System.exit(0); 
} 
} 
} 
/** 
* 实现降序 
*  
* @author Administrator 
* 
*/ 
private static class IntWritableDecreasingComparator extends IntWritable.Comparator { 
public int compare(WritableComparable a, WritableComparable b) { 
return -super.compare(a, b); 
} 
public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { 
return -super.compare(b1, s1, l1, b2, s2, l2); 
} 
} 
}

<?xml version="1.0" encoding="UTF-8"?> 
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd"> 
<properties> 
<comment>IK Analyzer 扩展配置</comment> 
<!--用户可以在这里配置自己的扩展字典 --> 
<entry key="ext_dict">myext.dic</entry> 
<!--用户可以在这里配置自己的扩展停止词字典 --> 
<entry key="ext_stopwords">mystopword.dic</entry> 
</properties>

高育良 
祁同伟 
陈海 
陈岩石 
侯亮平 
高小琴 
沙瑞金 
李达康 
蔡成功

你 
我 
他 
是 
的 
了 
啊 
说 
也 
和 
在 
就

        String inputFile = "hdfs://localhost:9000/input/people.txt"; // 输入文件 
Path outDir = new Path("hdfs://localhost:9000/out"); // 输出目录 
Path tempDir = new Path("hdfs://localhost:9000/tmp" + System.currentTimeMillis()); // 临时目录