HBase學習筆記(5)——MapReduce操作Hbase
阿新 • • 發佈:2018-12-10
1 實現方法
Hbase對MapReduce提供支援,它實現了TableMapper類和TableReducer類,我們只需要繼承這兩個類即可
1.1 寫個mapper繼承TableMapper<Text, IntWritable>
引數:Text:mapper的輸出key型別; IntWritable:mapper的輸出value型別。
map(ImmutableBytesWritable key, Result value,Context context)
注:immutable不可變的
引數:key:rowKey;value: Result ,一行資料; context上下文
1.2 寫個reduce繼承TableReducer<Text, IntWritable, ImmutableBytesWritable>
引數:Text:reduce的輸入key; IntWritable:reduce的輸入value;
ImmutableBytesWritable:reduce輸出到hbase中的rowKey型別。
其中的reduce方法如下:
reduce(Text key, Iterable<IntWritable> values,Context context)
引數: key:reduce的輸入key;values:reduce的輸入value;
2 測試程式碼
注意匯入hbase,mapreduce 相關的 jar 包
2.1 測試
package hbase_mr;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class HBaseMR {
static Configuration config = null;
static {
config = HBaseConfiguration.create();
config.set("hbase.zookeeper.quorum", "node1,node2,node3");
config.set("habse.zookeeper.property.clientPort", "2181");
}
public static final String tableName = "word";
public static final String colf = "content";
public static final String col = "info";
public static final String tableName2 = "stat";
public static void initTB() {
HTable table = null;
HBaseAdmin admin = null;
try {
admin = new HBaseAdmin(config);
if (admin.tableExists(tableName) || admin.tableExists(tableName2)) {
System.out.println("table has existed!!");
admin.disableTable(tableName);
admin.deleteTable(tableName);
admin.disableTable(tableName2);
admin.deleteTable(tableName2);
}
/*
* 建立表
* */
HTableDescriptor desc = new HTableDescriptor(tableName);
HColumnDescriptor family = new HColumnDescriptor(colf);
desc.addFamily(family);
admin.createTable(desc);
HTableDescriptor desc2 = new HTableDescriptor(tableName2);
HColumnDescriptor family2 = new HColumnDescriptor(colf);
desc2.addFamily(family2);
admin.createTable(desc2);
/*
* 插入資料
* */
table = new HTable(config, tableName);
table.setAutoFlush(false);
table.setWriteBufferSize(500);
List<Put> lp = new ArrayList<>();
Put p1 = new Put(Bytes.toBytes("1"));
p1.add(colf.getBytes(), col.getBytes(), ("The Apache Hadoop software library is a framework").getBytes());
lp.add(p1);
Put p2 = new Put(Bytes.toBytes("2"));p2.add(colf.getBytes(),col.getBytes(),("The common utilities that support the other Hadoop modules").getBytes());
lp.add(p2);
Put p3 = new Put(Bytes.toBytes("3"));
p3.add(colf.getBytes(), col.getBytes(),("Hadoop by reading the documentation").getBytes());
lp.add(p3);
Put p4 = new Put(Bytes.toBytes("4"));
p4.add(colf.getBytes(), col.getBytes(),("Hadoop from the release page").getBytes());
lp.add(p4);
Put p5 = new Put(Bytes.toBytes("5"));
p5.add(colf.getBytes(), col.getBytes(),("Hadoop on the mailing list").getBytes());
lp.add(p5);
table.put(lp);
table.flushCommits();
lp.clear();
} catch (IOException e) {
e.printStackTrace();
}finally {
if(table!=null){
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
/**
* MyMapper 繼承 TableMapper
* TableMapper<Text,IntWritable>
* Text:輸出的key型別,
* IntWritable:輸出的value型別
*/
public static class MyMapper extends TableMapper<Text, IntWritable> {
private static IntWritable one = new IntWritable(1);
private static Text word = new Text();
@Override
//輸入的型別為:key:rowKey; value:一行資料的結果集Result
protected void map(ImmutableBytesWritable key, Result value,
Context context) throws IOException, InterruptedException {
//獲取一行資料中的colf:col
String words = Bytes.toString(value.getValue(Bytes.toBytes(colf), Bytes.toBytes(col)));// 表裡面只有一個列族,所以我就直接獲取每一行的值
//按空格分割
String itr[] = words.toString().split(" ");
//迴圈輸出word和1
for (int i = 0; i < itr.length; i++) {
word.set(itr[i]);
context.write(word, one);
}
}
}
/**
* MyReducer 繼承 TableReducer
* TableReducer<Text,IntWritable>
* Text:輸入的key型別,
* IntWritable:輸入的value型別,
* ImmutableBytesWritable:輸出型別,表示rowkey的型別
*/
public static class MyReducer extends
TableReducer<Text, IntWritable, ImmutableBytesWritable> {
@Override
protected void reduce(Text key, Iterable<IntWritable> values,
Context context) throws IOException, InterruptedException {
//對mapper的資料求和
int sum = 0;
for (IntWritable val : values) {//疊加
sum += val.get();
}
// 建立put,設定rowkey為單詞
Put put = new Put(Bytes.toBytes(key.toString()));
// 封裝資料
put.add(Bytes.toBytes(colf), Bytes.toBytes(col),Bytes.toBytes(String.valueOf(sum)));
//寫到hbase,需要指定rowkey、put
context.write(new ImmutableBytesWritable(Bytes.toBytes(key.toString())),put);
}
}
public static void main(String[] args) throws IOException,
ClassNotFoundException, InterruptedException {
config.set("fs.defaultFS", "hdfs://node1:9000");//設定hdfs的預設路徑
//初始化表
initTB();//初始化表
//建立job
Job job = Job.getInstance(config,"HBaseMR");
job.setJarByClass(HBaseMR.class);//主類
//建立scan
Scan scan = new Scan();
//可以指定查詢某一列
scan.addColumn(Bytes.toBytes(colf), Bytes.toBytes(col));
//建立查詢hbase的mapper,設定表名、scan、mapper類、mapper的輸出key、mapper的輸出value
TableMapReduceUtil.initTableMapperJob(tableName, scan, MyMapper.class,Text.class, IntWritable.class, job);
//建立寫入hbase的reducer,指定表名、reducer類、job
TableMapReduceUtil.initTableReducerJob(tableName2, MyReducer.class, job);
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
執行發現報錯
Exception in thread "main" java.lang.IllegalArgumentException: Pathname /D:/Data/JavaProject/hbase/lib/netty-all-4.0.23.Final.jar from hdfs://node1:9000/D:/Data/JavaProject/hbase/lib/netty-all-4.0.23.Final.jar is not a valid DFS filename.
at org.apache.hadoop.hdfs.DistributedFileSystem.getPathName(DistributedFileSystem.java:187)
at org.apache.hadoop.hdfs.DistributedFileSystem.access$000(DistributedFileSystem.java:101)
at org.apache.hadoop.hdfs.DistributedFileSystem$17.doCall(DistributedFileSystem.java:1068)
at org.apache.hadoop.hdfs.DistributedFileSystem$17.doCall(DistributedFileSystem.java:1064)
at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1064)
at org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager.getFileStatus(ClientDistributedCacheManager.java:288)
at org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager.getFileStatus(ClientDistributedCacheManager.java:224)
at org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager.determineTimestamps(ClientDistributedCacheManager.java:93)
at org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager.determineTimestampsAndCacheVisibilities(ClientDistributedCacheManager.java:57)
at org.apache.hadoop.mapreduce.JobSubmitter.copyAndConfigureFiles(JobSubmitter.java:265)
at org.apache.hadoop.mapreduce.JobSubmitter.copyAndConfigureFiles(JobSubmitter.java:301)
at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:389)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1285)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1282)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1614)
at org.apache.hadoop.mapreduce.Job.submit(Job.java:1282)
at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1303)
at hbase_mr.HBaseMR.main(HBaseMR.java:172)
Process finished with exit code 1
2.2 錯誤原因
Hadoop檔案系統沒有檢查路徑時沒有區分是本地windows系統還是Hadoop叢集檔案系統 參考連結
修正方案
TableMapReduceUtil.initTableMapperJob(tableName, scan, MyMapper.class,Text.class, IntWritable.class, job,false);
//建立寫入hbase的reducer,指定表名、reducer類、job
TableMapReduceUtil.initTableReducerJob(tableName2, MyReducer.class, job,
null,null,null,null,false);
2.3 執行結果
日誌
D:\programs\Java\jdk1.8.0_181\bin\java "-javaagent:D:\programs\JetBrains\IntelliJ IDEA 2017.3.4\lib\idea_rt.jar=51278:D:\programs\JetBrains\IntelliJ IDEA 2017.3.4\bin" -Dfile.encoding=UTF-8 -classpath D:\programs\Java\jdk1.8.0_181\jre\lib\charsets.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\deploy.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\access-bridge-64.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\cldrdata.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\dnsns.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\jaccess.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\jfxrt.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\localedata.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\nashorn.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\sunec.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\sunjce_provider.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\sunmscapi.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\sunpkcs11.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\zipfs.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\javaws.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\jce.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\jfr.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\jfxswt.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\jsse.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\management-agent.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\plugin.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\resources.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\rt.jar;D:\Data\JavaProject\hbase\out\production\hbase;D:\Data\JavaProject\hbase\lib\hamcrest-core-1.3.jar;D:\Data\JavaProject\hbase\lib\xz-1.0.jar;D:\Data\JavaProject\hbase\lib\asm-3.1.jar;D:\Data\JavaProject\hbase\lib\guice-3.0.jar;D:\Data\JavaProject\hbase\lib\avro-1.7.4.jar;D:\Data\JavaProject\hbase\lib\joni-2.1.2.jar;D:\Data\JavaProject\hbase\lib\junit-4.12.jar;D:\Data\JavaProject\hbase\lib\jsch-0.1.42.jar;D:\Data\JavaProject\hbase\lib\xmlenc-0.52.jar;D:\Data\JavaProject\hbase\lib\guava-12.0.1.jar;D:\Data\JavaProject\hbase\lib\jets3t-0.9.0.jar;D:\Data\JavaProject\hbase\lib\jetty-6.1.26.jar;D:\Data\JavaProject\hbase\lib\log4j-1.2.17.jar;D:\Data\JavaProject\hbase\lib\paranamer-2.3.jar;D:\Data\JavaProject\hbase\lib\activation-1.1.jar;D:\Data\JavaProject\hbase\lib\commons-el-1.0.jar;D:\Data\JavaProject\hbase\lib\commons-io-2.4.jar;D:\Data\JavaProject\hbase\lib\hbase-it-1.2.6.jar;D:\Data\JavaProject\hbase\lib\httpcore-4.4.1.jar;D:\Data\JavaProject\hbase\lib\javax.inject-1.jar;D:\Data\JavaProject\hbase\lib\jaxb-api-2.2.2.jar;D:\Data\JavaProject\hbase\lib\jcodings-1.0.8.jar;D:\Data\JavaProject\hbase\lib\jettison-1.3.3.jar;D:\Data\JavaProject\hbase\lib\jsp-2.1-6.1.14.jar;D:\Data\JavaProject\hbase\lib\aopalliance-1.0.jar;D:\Data\JavaProject\hbase\lib\commons-cli-1.2.jar;D:\Data\JavaProject\hbase\lib\commons-net-3.1.jar;D:\Data\JavaProject\hbase\lib\disruptor-3.3.0.jar;D:\Data\JavaProject\hbase\lib\jersey-core-1.9.jar;D:\Data\JavaProject\hbase\lib\jersey-json-1.9.jar;D:\Data\JavaProject\hbase\lib\libthrift-0.9.3.jar;D:\Data\JavaProject\hbase\lib\servlet-api-2.5.jar;D:\Data\JavaProject\hbase\lib\slf4j-api-1.7.7.jar;D:\Data\JavaProject\hbase\lib\zookeeper-3.4.6.jar;D:\Data\JavaProject\hbase\lib\commons-lang-2.6.jar;D:\Data\JavaProject\hbase\lib\commons-math-2.2.jar;D:\Data\JavaProject\hbase\lib\hbase-rest-1.2.6.jar;D:\Data\JavaProject\hbase\lib\httpclient-4.2.5.jar;D:\Data\JavaProject\hbase\lib\jersey-guice-1.9.jar;D:\Data\JavaProject\hbase\lib\commons-codec-1.9.jar;D:\Data\JavaProject\hbase\lib\guice-servlet-3.0.jar;D:\Data\JavaProject\hbase\lib\hadoop-auth-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-hdfs-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hbase-shell-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jackson-xc-1.9.13.jar;D:\Data\JavaProject\hbase\lib\jaxb-impl-2.2.3-1.jar;D:\Data\JavaProject\hbase\lib\jersey-client-1.9.jar;D:\Data\JavaProject\hbase\lib\jersey-server-1.9.jar;D:\Data\JavaProject\hbase\lib\jetty-util-6.1.26.jar;D:\Data\JavaProject\hbase\lib\api-util-1.0.0-M20.jar;D:\Data\JavaProject\hbase\lib\hbase-client-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-common-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-server-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-thrift-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jsp-api-2.1-6.1.14.jar;D:\Data\JavaProject\hbase\lib\leveldbjni-all-1.8.jar;D:\Data\JavaProject\hbase\lib\metrics-core-2.2.0.jar;D:\Data\JavaProject\hbase\lib\commons-logging-1.2.jar;D:\Data\JavaProject\hbase\lib\commons-math3-3.1.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-client-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-common-2.5.1.jar;D:\Data\JavaProject\hbase\lib\jamon-runtime-2.4.1.jar;D:\Data\JavaProject\hbase\lib\java-xmlbuilder-0.4.jar;D:\Data\JavaProject\hbase\lib\protobuf-java-2.5.0.jar;D:\Data\JavaProject\hbase\lib\slf4j-log4j12-1.7.5.jar;D:\Data\JavaProject\hbase\lib\snappy-java-1.0.4.1.jar;D:\Data\JavaProject\hbase\lib\spymemcached-2.11.6.jar;D:\Data\JavaProject\hbase\lib\commons-digester-1.8.jar;D:\Data\JavaProject\hbase\lib\hbase-examples-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-it-1.2.6-tests.jar;D:\Data\JavaProject\hbase\lib\hbase-protocol-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jackson-jaxrs-1.9.13.jar;D:\Data\JavaProject\hbase\lib\jruby-complete-1.6.8.jar;D:\Data\JavaProject\hbase\lib\commons-daemon-1.0.13.jar;D:\Data\JavaProject\hbase\lib\hadoop-yarn-api-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hbase-procedure-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jasper-runtime-5.5.23.jar;D:\Data\JavaProject\hbase\lib\api-asn1-api-1.0.0-M20.jar;D:\Data\JavaProject\hbase\lib\commons-compress-1.4.1.jar;D:\Data\JavaProject\hbase\lib\commons-httpclient-3.1.jar;D:\Data\JavaProject\hbase\lib\jasper-compiler-5.5.23.jar;D:\Data\JavaProject\hbase\lib\jetty-sslengine-6.1.26.jar;D:\Data\JavaProject\hbase\lib\netty-all-4.0.23.Final.jar;D:\Data\JavaProject\hbase\lib\servlet-api-2.5-6.1.14.jar;D:\Data\JavaProject\hbase\lib\apacheds-i18n-2.0.0-M15.jar;D:\Data\JavaProject\hbase\lib\commons-beanutils-1.7.0.jar;D:\Data\JavaProject\hbase\lib\hbase-annotations-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-prefix-tree-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jackson-core-asl-1.9.13.jar;D:\Data\JavaProject\hbase\lib\hadoop-annotations-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-yarn-client-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-yarn-common-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hbase-common-1.2.6-tests.jar;D:\Data\JavaProject\hbase\lib\hbase-server-1.2.6-tests.jar;D:\Data\JavaProject\hbase\lib\commons-collections-3.2.2.jar;D:\Data\JavaProject\hbase\lib\commons-configuration-1.6.jar;D:\Data\JavaProject\hbase\lib\hbase-hadoop-compat-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jackson-mapper-asl-1.9.13.jar;D:\Data\JavaProject\hbase\lib\hbase-hadoop2-compat-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-resource-bundle-1.2.6.jar;D:\Data\JavaProject\hbase\lib\commons-beanutils-core-1.8.0.jar;D:\Data\JavaProject\hbase\lib\findbugs-annotations-1.3.9-1.jar;D:\Data\JavaProject\hbase\lib\htrace-core-3.1.0-incubating.jar;D:\Data\JavaProject\hbase\lib\hbase-annotations-1.2.6-tests.jar;D:\Data\JavaProject\hbase\lib\hadoop-yarn-server-common-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hbase-external-blockcache-1.2.6.jar;D:\Data\JavaProject\hbase\lib\apacheds-kerberos-codec-2.0.0-M15.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-app-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-core-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-common-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-shuffle-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-jobclient-2.5.1.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\xz-1.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\asm-3.2.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\guice-3.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\avro-1.7.4.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\junit-4.11.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\log4j-1.2.17.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\paranamer-2.3.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\commons-io-2.4.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\javax.inject-1.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\aopalliance-1.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jersey-core-1.9.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jersey-guice-1.9.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\guice-servlet-3.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\hamcrest-core-1.3.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jersey-server-1.9.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\netty-3.6.2.Final.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\leveldbjni-all-1.8.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\protobuf-java-2.5.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\snappy-java-1.0.4.1.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\commons-compress-1.4.1.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jackson-core-asl-1.9.13.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\hadoop-annotations-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jackson-mapper-asl-1.9.13.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-app-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-common-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-core-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-hs-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-hs-plugins-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-jobclient-2.7.6-tests.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-jobclient-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-shuffle-2.7.6.jar hbase_mr.HBaseMR
2018-09-19 14:47:15,343 INFO [main] zookeeper.RecoverableZooKeeper (RecoverableZooKeeper.java:<init>(120)) - Process identifier=hconnection-0x12028586 connecting to ZooKeeper ensemble=node1:2181,node2:2181,node3:2181
2018-09-19 14:47:15,352 INFO [main] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:zookeeper.version=3.4.6-1569965, built on 02/20/2014 09:09 GMT
2018-09-19 14:47:15,352 INFO [main] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:host.name=XPS-15-9570
2018-09-19 14:47:15,352 INFO [main] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:java.version=1.8.0_181
2018-09-19 14:47:15,352 INFO [main] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:java.vendor=Oracle Corporation
2018-09-19 14:47:15,352 INFO [main] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:java.home=D:\programs\Java\jdk1.8.0_181\jre
2018-09-19 14:47:15,352 INFO [main] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:java.class.path=D:\programs\Java\jdk1.8.0_181\jre\lib\charsets.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\deploy.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\access-bridge-64.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\cldrdata.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\dnsns.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\jaccess.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\jfxrt.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\localedata.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\nashorn.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\sunec.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\sunjce_provider.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\sunmscapi.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\sunpkcs11.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\zipfs.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\javaws.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\jce.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\jfr.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\jfxswt.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\jsse.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\management-agent.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\plugin.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\resources.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\rt.jar;D:\Data\JavaProject\hbase\out\production\hbase;D:\Data\JavaProject\hbase\lib\hamcrest-core-1.3.jar;D:\Data\JavaProject\hbase\lib\xz-1.0.jar;D:\Data\JavaProject\hbase\lib\asm-3.1.jar;D:\Data\JavaProject\hbase\lib\guice-3.0.jar;D:\Data\JavaProject\hbase\lib\avro-1.7.4.jar;D:\Data\JavaProject\hbase\lib\joni-2.1.2.jar;D:\Data\JavaProject\hbase\lib\junit-4.12.jar;D:\Data\JavaProject\hbase\lib\jsch-0.1.42.jar;D:\Data\JavaProject\hbase\lib\xmlenc-0.52.jar;D:\Data\JavaProject\hbase\lib\guava-12.0.1.jar;D:\Data\JavaProject\hbase\lib\jets3t-0.9.0.jar;D:\Data\JavaProject\hbase\lib\jetty-6.1.26.jar;D:\Data\JavaProject\hbase\lib\log4j-1.2.17.jar;D:\Data\JavaProject\hbase\lib\paranamer-2.3.jar;D:\Data\JavaProject\hbase\lib\activation-1.1.jar;D:\Data\JavaProject\hbase\lib\commons-el-1.0.jar;D:\Data\JavaProject\hbase\lib\commons-io-2.4.jar;D:\Data\JavaProject\hbase\lib\hbase-it-1.2.6.jar;D:\Data\JavaProject\hbase\lib\httpcore-4.4.1.jar;D:\Data\JavaProject\hbase\lib\javax.inject-1.jar;D:\Data\JavaProject\hbase\lib\jaxb-api-2.2.2.jar;D:\Data\JavaProject\hbase\lib\jcodings-1.0.8.jar;D:\Data\JavaProject\hbase\lib\jettison-1.3.3.jar;D:\Data\JavaProject\hbase\lib\jsp-2.1-6.1.14.jar;D:\Data\JavaProject\hbase\lib\aopalliance-1.0.jar;D:\Data\JavaProject\hbase\lib\commons-cli-1.2.jar;D:\Data\JavaProject\hbase\lib\commons-net-3.1.jar;D:\Data\JavaProject\hbase\lib\disruptor-3.3.0.jar;D:\Data\JavaProject\hbase\lib\jersey-core-1.9.jar;D:\Data\JavaProject\hbase\lib\jersey-json-1.9.jar;D:\Data\JavaProject\hbase\lib\libthrift-0.9.3.jar;D:\Data\JavaProject\hbase\lib\servlet-api-2.5.jar;D:\Data\JavaProject\hbase\lib\slf4j-api-1.7.7.jar;D:\Data\JavaProject\hbase\lib\zookeeper-3.4.6.jar;D:\Data\JavaProject\hbase\lib\commons-lang-2.6.jar;D:\Data\JavaProject\hbase\lib\commons-math-2.2.jar;D:\Data\JavaProject\hbase\lib\hbase-rest-1.2.6.jar;D:\Data\JavaProject\hbase\lib\httpclient-4.2.5.jar;D:\Data\JavaProject\hbase\lib\jersey-guice-1.9.jar;D:\Data\JavaProject\hbase\lib\commons-codec-1.9.jar;D:\Data\JavaProject\hbase\lib\guice-servlet-3.0.jar;D:\Data\JavaProject\hbase\lib\hadoop-auth-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-hdfs-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hbase-shell-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jackson-xc-1.9.13.jar;D:\Data\JavaProject\hbase\lib\jaxb-impl-2.2.3-1.jar;D:\Data\JavaProject\hbase\lib\jersey-client-1.9.jar;D:\Data\JavaProject\hbase\lib\jersey-server-1.9.jar;D:\Data\JavaProject\hbase\lib\jetty-util-6.1.26.jar;D:\Data\JavaProject\hbase\lib\api-util-1.0.0-M20.jar;D:\Data\JavaProject\hbase\lib\hbase-client-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-common-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-server-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-thrift-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jsp-api-2.1-6.1.14.jar;D:\Data\JavaProject\hbase\lib\leveldbjni-all-1.8.jar;D:\Data\JavaProject\hbase\lib\metrics-core-2.2.0.jar;D:\Data\JavaProject\hbase\lib\commons-logging-1.2.jar;D:\Data\JavaProject\hbase\lib\commons-math3-3.1.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-client-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-common-2.5.1.jar;D:\Data\JavaProject\hbase\lib\jamon-runtime-2.4.1.jar;D:\Data\JavaProject\hbase\lib\java-xmlbuilder-0.4.jar;D:\Data\JavaProject\hbase\lib\protobuf-java-2.5.0.jar;D:\Data\JavaProject\hbase\lib\slf4j-log4j12-1.7.5.jar;D:\Data\JavaProject\hbase\lib\snappy-java-1.0.4.1.jar;D:\Data\JavaProject\hbase\lib\spymemcached-2.11.6.jar;D:\Data\JavaProject\hbase\lib\commons-digester-1.8.jar;D:\Data\JavaProject\hbase\lib\hbase-examples-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-it-1.2.6-tests.jar;D:\Data\JavaProject\hbase\lib\hbase-protocol-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jackson-jaxrs-1.9.13.jar;D:\Data\JavaProject\hbase\lib\jruby-complete-1.6.8.jar;D:\Data\JavaProject\hbase\lib\commons-daemon-1.0.13.jar;D:\Data\JavaProject\hbase\lib\hadoop-yarn-api-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hbase-procedure-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jasper-runtime-5.5.23.jar;D:\Data\JavaProject\hbase\lib\api-asn1-api-1.0.0-M20.jar;D:\Data\JavaProject\hbase\lib\commons-compress-1.4.1.jar;D:\Data\JavaProject\hbase\lib\commons-httpclient-3.1.jar;D:\Data\JavaProject\hbase\lib\jasper-compiler-5.5.23.jar;D:\Data\JavaProject\hbase\lib\jetty-sslengine-6.1.26.jar;D:\Data\JavaProject\hbase\lib\netty-all-4.0.23.Final.jar;D:\Data\JavaProject\hbase\lib\servlet-api-2.5-6.1.14.jar;D:\Data\JavaProject\hbase\lib\apacheds-i18n-2.0.0-M15.jar;D:\Data\JavaProject\hbase\lib\commons-beanutils-1.7.0.jar;D:\Data\JavaProject\hbase\lib\hbase-annotations-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-prefix-tree-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jackson-core-asl-1.9.13.jar;D:\Data\JavaProject\hbase\lib\hadoop-annotations-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-yarn-client-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-yarn-common-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hbase-common-1.2.6-tests.jar;D:\Data\JavaProject\hbase\lib\hbase-server-1.2.6-tests.jar;D:\Data\JavaProject\hbase\lib\commons-collections-3.2.2.jar;D:\Data\JavaProject\hbase\lib\commons-configuration-1.6.jar;D:\Data\JavaProject\hbase\lib\hbase-hadoop-compat-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jackson-mapper-asl-1.9.13.jar;D:\Data\JavaProject\hbase\lib\hbase-hadoop2-compat-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-resource-bundle-1.2.6.jar;D:\Data\JavaProject\hbase\lib\commons-beanutils-core-1.8.0.jar;D:\Data\JavaProject\hbase\lib\findbugs-annotations-1.3.9-1.jar;D:\Data\JavaProject\hbase\lib\htrace-core-3.1.0-incubating.jar;D:\Data\JavaProject\hbase\lib\hbase-annotations-1.2.6-tests.jar;D:\Data\JavaProject\hbase\lib\hadoop-yarn-server-common-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hbase-external-blockcache-1.2.6.jar;D:\Data\JavaProject\hbase\lib\apacheds-kerberos-codec-2.0.0-M15.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-app-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-core-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-common-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-shuffle-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-jobclient-2.5.1.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\xz-1.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\asm-3.2.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\guice-3.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\avro-1.7.4.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\junit-4.11.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\log4j-1.2.17.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\paranamer-2.3.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\commons-io-2.4.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\javax.inject-1.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\aopalliance-1.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jersey-core-1.9.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jersey-guice-1.9.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\guice-servlet-3.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\hamcrest-core-1.3.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jersey-server-1.9.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\netty-3.6.2.Final.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\leveldbjni-all-1.8.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\protobuf-java-2.5.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\snappy-java-1.0.4.1.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\commons-compress-1.4.1.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jackson-core-asl-1.9.13.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\hadoop-annotations-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jackson-mapper-asl-1.9.13.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-app-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-common-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-core-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-hs-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-hs-plugins-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-jobclient-2.7.6-tests.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-jobclient-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-shuffle-2.7.6.jar;D:\programs\JetBrains\IntelliJ IDEA 2017.3.4\lib\idea_rt.jar
2018-09-19 14:47:15,353 INFO [main] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:java.library.path=D:\progra