1. 程式人生 > >hadoop2執行MR異常解決-NoSuchMethodException:xxMapper.()

hadoop2執行MR異常解決-NoSuchMethodException:xxMapper.()

異常問題:

java.lang.Exception: java.lang.RuntimeException: java.lang.NoSuchMethodException: mrTest.TrafficApp$TrafficMapper.<init>()

        at org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:462)
        at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:522)
Caused by: java.lang.RuntimeException: java.lang.NoSuchMethodException: mrTest.TrafficApp$TrafficMapper.<init>()
        at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:131)
        at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:742)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
        at org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:243)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
        at java.util.concurrent.FutureTask.run(FutureTask.java:262)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NoSuchMethodException: mrTest.TrafficApp$TrafficMapper.<init>()
        at java.lang.Class.getConstructor0(Class.java:2892)
        at java.lang.Class.getDeclaredConstructor(Class.java:2058)
        at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:125)

        ... 8 more

MR原始碼

package mrTest;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class TrafficApp2 {
public class TrafficMapper2 extends Mapper<LongWritable, Text, Text, TrafficWritable>{
@Override
protected void map(LongWritable k1,Text v1,Mapper<LongWritable, Text, Text, TrafficWritable>.Context context)
throws IOException, InterruptedException {
String line=v1.toString();
String[] conArr=line.split("\t");
long upPakageNum=Long.parseLong(conArr[6]);
long downPakageNum=Long.parseLong(conArr[7]);
long upTrafficNum=Long.parseLong(conArr[8]);
long downTrafficNum=Long.parseLong(conArr[9]);
context.write(new Text(conArr[1]), new TrafficWritable(upPakageNum, downPakageNum, upTrafficNum, downTrafficNum));
}
}
public class TrafficReduce2 extends Reducer<Text, TrafficWritable, Text, TrafficWritable>{
@Override
protected void reduce(Text k2,Iterable<TrafficWritable> v2s,Reducer<Text, TrafficWritable, Text, TrafficWritable>.Context context)
throws IOException, InterruptedException {
long upPakageNum=0;
long downPakageNum=0;
long upTrafficNum=0;
long downTrafficNum=0;
for(TrafficWritable tw:v2s){
upPakageNum+=tw.getUpPakageNum();
downPakageNum+=tw.getDownPakageNum();
upTrafficNum+=tw.getUpTrafficNum();
downTrafficNum+=tw.getDownTrafficNum();
}
context.write(k2, new TrafficWritable(upPakageNum, downPakageNum, upTrafficNum, downTrafficNum));
}
}
public static void main(String[] args) throws Exception {
Configuration conf=new Configuration();
Job job=new Job(conf, TrafficApp2.class.getName());
job.setJarByClass(TrafficApp2.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
//FileInputFormat.addInputPath(job, new Path("hdfs://192.168.14.30:9000/input"));
job.setMapperClass(TrafficMapper2.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(TrafficWritable.class);

job.setReducerClass(TrafficReduce2.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(TrafficWritable.class);
FileOutputFormat.setOutputPath(job, new Path(args[1]));
//FileOutputFormat.setOutputPath(job, new Path("hdfs://192.168.14.30:9000/output"));
job.waitForCompletion(true);

}
public class TrafficWritable implements Writable{
long upPakageNum;
long downPakageNum;
long upTrafficNum;
long downTrafficNum;
public TrafficWritable(){}
public TrafficWritable(long upPakageNum,long downPakageNum,long upTrafficNum,long downTrafficNum){
set(upPakageNum, downPakageNum, upTrafficNum, downTrafficNum);
}
public void set(long upPakageNum,long downPakageNum,long upTrafficNum,long downTrafficNum){
this.upPakageNum=upPakageNum;
this.downPakageNum=downPakageNum;
this.upTrafficNum=upTrafficNum;
this.downTrafficNum=downTrafficNum;
}
@Override
public void write(DataOutput output) throws IOException {
output.writeLong(upPakageNum);
output.writeLong(downPakageNum);
output.writeLong(upTrafficNum);
output.writeLong(downTrafficNum);

}
@Override
public String toString() {
return upPakageNum+"\t"+downPakageNum+"\t"+upTrafficNum+"\t"+downTrafficNum;
}
@Override
public void readFields(DataInput input) throws IOException {
// TODO Auto-generated method stub
this.upPakageNum = input.readLong();
this.downPakageNum = input.readLong();
this.upTrafficNum = input.readLong();
this.downTrafficNum = input.readLong();
}
public long getUpPakageNum() {
return upPakageNum;
}
public long getDownPakageNum() {
return downPakageNum;
}
public long getUpTrafficNum() {
return upTrafficNum;
}
public long getDownTrafficNum() {
return downTrafficNum;
}
}
}

解決方式一:所有內部類都加上static

public class TrafficApp2 {
public static class TrafficMapper2 extends Mapper<LongWritable, Text, Text, TrafficWritable>{
……
}
public static class TrafficReduce2 extends Reducer<Text, TrafficWritable, Text, TrafficWritable>{
……
}
public static void main(String[] args) throws Exception {
……
}
public static

 class TrafficWritable implements Writable{
……
}
}

解決方式二:內部類放置與主類同級

public class TrafficApp2 {
public static void main(String[] args) throws Exception {
……
}

}
class TrafficMapper2 extends Mapper<LongWritable, Text, Text, TrafficWritable>{
……
}
class TrafficReduce2 extends Reducer<Text, TrafficWritable, Text, TrafficWritable>{
……
}
class TrafficWritable implements Writable{
……
}