Windows下,在IDEA中搭建MapReduce開發環境(需基礎,非詳盡篇)
阿新 • • 發佈:2018-12-11
hadoop在官網下載,jdk在官網下載,自行解壓下載的hadoop包,安裝下載的jdk。
開啟IDEA,新建專案:
File->Project Structure:
選擇之前解壓的hadoop資料夾中的share/hadoop目錄下的common、hdfs、mapreduce、yarn:
重複上兩步操作,選擇share/hadoop/common/lib:
測試程式碼(WordCount類,可直接跑,內容不多,可自己修改除錯):
package com.company; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import java.io.IOException; import java.util.StringTokenizer; public class WordCount { /* Mapper */ public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> { private final static IntWritable one = new IntWritable(1); private Text word = new Text(); @Override public void map(Object key, Text value, Context context) throws IOException, InterruptedException{ StringTokenizer itr = new StringTokenizer(value.toString()); while(itr.hasMoreTokens()){ word.set(itr.nextToken()); context.write(word, one); } } } /* Reducer */ public static class IntSumReducer extends Reducer<Text,IntWritable,Text,IntWritable>{ private IntWritable result = new IntWritable(); @Override public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException,InterruptedException{ int sum = 0; for(IntWritable val : values){ sum += val.get(); } result.set(sum); context.write(key,result); } } /* 啟動 MapReduce Job */ public static void main(String[] args) throws Exception{ System.setProperty("hadoop.home.dir","E:/hadoop-2.7.7" ); Configuration conf = new Configuration(); /*if(args.length != 2){ System.err.println("Usage: wordcount <int> <out>"); System.exit(2); }*/ String arg1 = "input"; String arg2 = "output"; Job job = new Job(conf, "word count"); job.setJarByClass(WordCount.class); job.setMapperClass(TokenizerMapper.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job,new Path(arg1)); FileOutputFormat.setOutputPath(job,new Path(arg2)); System.exit(job.waitForCompletion(true)?0:1); } }
會報錯(非程式碼錯誤),檢視解決方案,主要是下三個檔案:winutils.exe、libwinutils.lib、hadoop.dll。