1. 程式人生 > >如何在開發環境中建立mapreduce程式

如何在開發環境中建立mapreduce程式

1. 首先建立開發環境(eclipse-hadoop),網上搭建部落格很多,不細說

2. 開發環境建立之後自己建立一個包,這個名字是隨便起的,可以起名為mapreduce。

3.然後建立一個.java檔案,可以是mapreduce檔案:mapreduce.java

4.把下面程式碼放入放入mapreduce.java

5.建立mapreduce程式。

package mapreduce;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.output.*;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;

public class mapreduce {

static final String INPUT_PATH = "hdfs://master:8020/input";
static final String OUT_PATH = "hdfs://master:8020/Output";

public static void main(String[] args) throws Exception {
//主類
Configuration conf = new Configuration();
final Job job = new Job(conf, mapreduce.class.getSimpleName());
job.setJarByClass(mapreduce.class);
// 尋找輸入
FileInputFormat.setInputPaths(job, INPUT_PATH);
// 1.2對輸入資料進行格式化處理的類
job.setInputFormatClass(TextInputFormat.class);
job.setMapperClass(MyMapper.class);

// 1.2指定map輸出型別<key,value>型別
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(LongWritable.class);

// 1.3指定分割槽
job.setPartitionerClass(HashPartitioner.class);
job.setNumReduceTasks(1);

// 1.4排序分組省略,使用預設
// 1.5規約省略,使用預設
job.setReducerClass(MyReduce.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongWritable.class);
// 指定輸出路徑
FileOutputFormat.setOutputPath(job, new Path(OUT_PATH));
// 指定輸出的格式或則類
job.setOutputFormatClass(TextOutputFormat.class);

// 把作業提交給jobtracer
job.waitForCompletion(true);

}
//map類
static class MyMapper extends
Mapper<LongWritable, Text, Text, LongWritable> {
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
final String[] splited = value.toString().split("\t");
for (String word : splited) {
context.write(new Text(word), new LongWritable(1L));

}

}

}
//reduce類
static class MyReduce extends
Reducer<Text, LongWritable, Text, LongWritable> {
@Override
protected void reduce(Text k2, java.lang.Iterable<LongWritable> v2s,
Context ctx) throws java.io.IOException, InterruptedException {
long times = 0L;
for (LongWritable count : v2s) {
times += count.get();
ctx.write(k2, new LongWritable(times));
}

}

}

}