如何在開發環境中建立mapreduce程式
阿新 • • 發佈:2019-01-24
1. 首先建立開發環境(eclipse-hadoop),網上搭建部落格很多,不細說
2. 開發環境建立之後自己建立一個包,這個名字是隨便起的,可以起名為mapreduce。
3.然後建立一個.java檔案,可以是mapreduce檔案:mapreduce.java。
4.把下面程式碼放入放入mapreduce.java
5.建立mapreduce程式。
package mapreduce; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.output.*; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner; public class mapreduce { static final String INPUT_PATH = "hdfs://master:8020/input"; static final String OUT_PATH = "hdfs://master:8020/Output"; public static void main(String[] args) throws Exception { //主類 Configuration conf = new Configuration(); final Job job = new Job(conf, mapreduce.class.getSimpleName()); job.setJarByClass(mapreduce.class); // 尋找輸入 FileInputFormat.setInputPaths(job, INPUT_PATH); // 1.2對輸入資料進行格式化處理的類 job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(MyMapper.class); // 1.2指定map輸出型別<key,value>型別 job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); // 1.3指定分割槽 job.setPartitionerClass(HashPartitioner.class); job.setNumReduceTasks(1); // 1.4排序分組省略,使用預設 // 1.5規約省略,使用預設 job.setReducerClass(MyReduce.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(LongWritable.class); // 指定輸出路徑 FileOutputFormat.setOutputPath(job, new Path(OUT_PATH)); // 指定輸出的格式或則類 job.setOutputFormatClass(TextOutputFormat.class); // 把作業提交給jobtracer job.waitForCompletion(true); } //map類 static class MyMapper extends Mapper<LongWritable, Text, Text, LongWritable> { protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { final String[] splited = value.toString().split("\t"); for (String word : splited) { context.write(new Text(word), new LongWritable(1L)); } } } //reduce類 static class MyReduce extends Reducer<Text, LongWritable, Text, LongWritable> { @Override protected void reduce(Text k2, java.lang.Iterable<LongWritable> v2s, Context ctx) throws java.io.IOException, InterruptedException { long times = 0L; for (LongWritable count : v2s) { times += count.get(); ctx.write(k2, new LongWritable(times)); } } } }