程式設計實現單詞去重要用到NullWritable型別。

NullWritable：

NullWritable 是一種特殊的Writable 型別，由於它的序列化是零長度的，所以沒有位元組被寫入流或從流中讀出，可以用作佔位符。比如，在MapReduce 中，在不需要這個位置的時候，鍵或值能夠被宣告為NullWritable，從而有效儲存一個不變的空值。

通過呼叫NullWritable.get() 方法來檢索。

單詞去重我們最後要輸出的形式是<單詞>，所以值可以宣告為NullWritable。

程式碼如下：

 1 package org.apache.hadoop.examples;
 2      
 3     import java.io.IOException;
 4     import java.util.Iterator;
 5     import java.util.StringTokenizer;
 6     import org.apache.hadoop.conf.Configuration;
 7     import org.apache.hadoop.fs.Path;
 8     import org.apache.hadoop.io.IntWritable;
 
 9     import org.apache.hadoop.io.NullWritable;
10     import org.apache.hadoop.io.Text;
11     import org.apache.hadoop.mapreduce.Job;
12     import org.apache.hadoop.mapreduce.Mapper;
13     import org.apache.hadoop.mapreduce.Reducer;
14     import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 
15     import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
16      
17     public class DistinctWord{
18         public DistinctWord() {
19         }
20      
21         public static void main(String[] args) throws Exception {
22             Configuration conf = new Configuration();
23             
24             //String[] otherArgs = (new GenericOptionsParser(conf, args)).getRemainingArgs();
25             String[] otherArgs = new String[]{"input","output"};  //設定輸入和輸出
26             if(otherArgs.length < 2) {
27                 System.err.println("Usage: wordcount <in> [<in>...] <out>");
28                 System.exit(2);
29             }
30      
31             Job job = Job.getInstance(conf, "distinct word");
32             
33             job.setJarByClass(DistinctWord.class);  //設定jar包所在路徑
34             
35             //指定Mapper和Reducer類
36             job.setMapperClass(DistinctWord.DistinctWordMapper.class);  
37             job.setCombinerClass(DistinctWord.DistinctWordReducer.class);
38             job.setReducerClass(DistinctWord.DistinctWordReducer.class);
39             
40             //指定MapTask的輸出型別
41             job.setMapOutputKeyClass(Text.class);
42             job.setMapOutputValueClass(NullWritable.class);
43             
44             //指定ReduceTask的輸出型別
45             job.setOutputKeyClass(Text.class);
46             job.setOutputValueClass(NullWritable.class);
47      
48             //指定資料輸入路徑
49             for(int i = 0; i < otherArgs.length - 1; ++i) {
50                 FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
51             }
52             
53             //指定資料輸出路徑
54             FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1]));
55             
56             //提交任務
57             System.exit(job.waitForCompletion(true)?0:1);
58         }
59      
60       
61         //輸出型別定義為NullWritable
62         public static class DistinctWordMapper extends Mapper<Object, Text, Text, NullWritable> {
63             private Text word = new Text();
64      
65             public DistinctWordMapper() {
66             }
67      
68             public void map(Object key, Text value, Mapper<Object, Text, Text, NullWritable>.Context context) throws IOException, InterruptedException {
69                 StringTokenizer itr = new StringTokenizer(value.toString());  //分詞器
70      
71                 while(itr.hasMoreTokens()) {
72                     this.word.set(itr.nextToken());
73                     context.write(this.word, NullWritable.get());
74                 }
75      
76             }
77         }
78         
79         
80         
81         public static class DistinctWordReducer extends Reducer<Text, NullWritable, Text, NullWritable> {
82             
83             public DistinctWordReducer() {
84             }
85             
86             //reduce方法每呼叫一次，就接收到一組相同的單詞，所以直接輸出一次key即可。
87             public void reduce(Text key, Iterable<NullWritable> values, Reducer<Text, NullWritable, Text, NullWritable>.Context context) throws IOException, InterruptedException {
88                 context.write(key, NullWritable.get());
89             }
90         }
91         
92         
93     }