1. 程式人生 > >HBase學習筆記(5)——MapReduce操作Hbase

HBase學習筆記(5)——MapReduce操作Hbase

1 實現方法

Hbase對MapReduce提供支援,它實現了TableMapper類和TableReducer類,我們只需要繼承這兩個類即可

1.1 寫個mapper繼承TableMapper<Text, IntWritable>

引數:Text:mapper的輸出key型別; IntWritable:mapper的輸出value型別。 map(ImmutableBytesWritable key, Result value,Context context) 注:immutable不可變的 引數:key:rowKey;value: Result ,一行資料; context上下文

1.2 寫個reduce繼承TableReducer<Text, IntWritable, ImmutableBytesWritable>

引數:Text:reduce的輸入key; IntWritable:reduce的輸入value; ImmutableBytesWritable:reduce輸出到hbase中的rowKey型別。 其中的reduce方法如下: reduce(Text key, Iterable<IntWritable> values,Context context) 引數: key:reduce的輸入key;values:reduce的輸入value;

2 測試程式碼

注意匯入hbase,mapreduce 相關的 jar 包

2.1 測試

package hbase_mr;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; import org.apache.hadoop.hbase.mapreduce.TableMapper; import org.apache.hadoop.hbase.mapreduce.TableReducer; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import java.io.IOException; import java.util.ArrayList; import java.util.List; public class HBaseMR { static Configuration config = null; static { config = HBaseConfiguration.create(); config.set("hbase.zookeeper.quorum", "node1,node2,node3"); config.set("habse.zookeeper.property.clientPort", "2181"); } public static final String tableName = "word"; public static final String colf = "content"; public static final String col = "info"; public static final String tableName2 = "stat"; public static void initTB() { HTable table = null; HBaseAdmin admin = null; try { admin = new HBaseAdmin(config); if (admin.tableExists(tableName) || admin.tableExists(tableName2)) { System.out.println("table has existed!!"); admin.disableTable(tableName); admin.deleteTable(tableName); admin.disableTable(tableName2); admin.deleteTable(tableName2); } /* * 建立表 * */ HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor family = new HColumnDescriptor(colf); desc.addFamily(family); admin.createTable(desc); HTableDescriptor desc2 = new HTableDescriptor(tableName2); HColumnDescriptor family2 = new HColumnDescriptor(colf); desc2.addFamily(family2); admin.createTable(desc2); /* * 插入資料 * */ table = new HTable(config, tableName); table.setAutoFlush(false); table.setWriteBufferSize(500); List<Put> lp = new ArrayList<>(); Put p1 = new Put(Bytes.toBytes("1")); p1.add(colf.getBytes(), col.getBytes(), ("The Apache Hadoop software library is a framework").getBytes()); lp.add(p1); Put p2 = new Put(Bytes.toBytes("2"));p2.add(colf.getBytes(),col.getBytes(),("The common utilities that support the other Hadoop modules").getBytes()); lp.add(p2); Put p3 = new Put(Bytes.toBytes("3")); p3.add(colf.getBytes(), col.getBytes(),("Hadoop by reading the documentation").getBytes()); lp.add(p3); Put p4 = new Put(Bytes.toBytes("4")); p4.add(colf.getBytes(), col.getBytes(),("Hadoop from the release page").getBytes()); lp.add(p4); Put p5 = new Put(Bytes.toBytes("5")); p5.add(colf.getBytes(), col.getBytes(),("Hadoop on the mailing list").getBytes()); lp.add(p5); table.put(lp); table.flushCommits(); lp.clear(); } catch (IOException e) { e.printStackTrace(); }finally { if(table!=null){ try { table.close(); } catch (IOException e) { e.printStackTrace(); } } } } /** * MyMapper 繼承 TableMapper * TableMapper<Text,IntWritable> * Text:輸出的key型別, * IntWritable:輸出的value型別 */ public static class MyMapper extends TableMapper<Text, IntWritable> { private static IntWritable one = new IntWritable(1); private static Text word = new Text(); @Override //輸入的型別為:key:rowKey; value:一行資料的結果集Result protected void map(ImmutableBytesWritable key, Result value, Context context) throws IOException, InterruptedException { //獲取一行資料中的colf:col String words = Bytes.toString(value.getValue(Bytes.toBytes(colf), Bytes.toBytes(col)));// 表裡面只有一個列族,所以我就直接獲取每一行的值 //按空格分割 String itr[] = words.toString().split(" "); //迴圈輸出word和1 for (int i = 0; i < itr.length; i++) { word.set(itr[i]); context.write(word, one); } } } /** * MyReducer 繼承 TableReducer * TableReducer<Text,IntWritable> * Text:輸入的key型別, * IntWritable:輸入的value型別, * ImmutableBytesWritable:輸出型別,表示rowkey的型別 */ public static class MyReducer extends TableReducer<Text, IntWritable, ImmutableBytesWritable> { @Override protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { //對mapper的資料求和 int sum = 0; for (IntWritable val : values) {//疊加 sum += val.get(); } // 建立put,設定rowkey為單詞 Put put = new Put(Bytes.toBytes(key.toString())); // 封裝資料 put.add(Bytes.toBytes(colf), Bytes.toBytes(col),Bytes.toBytes(String.valueOf(sum))); //寫到hbase,需要指定rowkey、put context.write(new ImmutableBytesWritable(Bytes.toBytes(key.toString())),put); } } public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { config.set("fs.defaultFS", "hdfs://node1:9000");//設定hdfs的預設路徑 //初始化表 initTB();//初始化表 //建立job Job job = Job.getInstance(config,"HBaseMR"); job.setJarByClass(HBaseMR.class);//主類 //建立scan Scan scan = new Scan(); //可以指定查詢某一列 scan.addColumn(Bytes.toBytes(colf), Bytes.toBytes(col)); //建立查詢hbase的mapper,設定表名、scan、mapper類、mapper的輸出key、mapper的輸出value TableMapReduceUtil.initTableMapperJob(tableName, scan, MyMapper.class,Text.class, IntWritable.class, job); //建立寫入hbase的reducer,指定表名、reducer類、job TableMapReduceUtil.initTableReducerJob(tableName2, MyReducer.class, job); System.exit(job.waitForCompletion(true) ? 0 : 1); } }

執行發現報錯

Exception in thread "main" java.lang.IllegalArgumentException: Pathname /D:/Data/JavaProject/hbase/lib/netty-all-4.0.23.Final.jar from hdfs://node1:9000/D:/Data/JavaProject/hbase/lib/netty-all-4.0.23.Final.jar is not a valid DFS filename.
	at org.apache.hadoop.hdfs.DistributedFileSystem.getPathName(DistributedFileSystem.java:187)
	at org.apache.hadoop.hdfs.DistributedFileSystem.access$000(DistributedFileSystem.java:101)
	at org.apache.hadoop.hdfs.DistributedFileSystem$17.doCall(DistributedFileSystem.java:1068)
	at org.apache.hadoop.hdfs.DistributedFileSystem$17.doCall(DistributedFileSystem.java:1064)
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
	at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1064)
	at org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager.getFileStatus(ClientDistributedCacheManager.java:288)
	at org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager.getFileStatus(ClientDistributedCacheManager.java:224)
	at org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager.determineTimestamps(ClientDistributedCacheManager.java:93)
	at org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager.determineTimestampsAndCacheVisibilities(ClientDistributedCacheManager.java:57)
	at org.apache.hadoop.mapreduce.JobSubmitter.copyAndConfigureFiles(JobSubmitter.java:265)
	at org.apache.hadoop.mapreduce.JobSubmitter.copyAndConfigureFiles(JobSubmitter.java:301)
	at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:389)
	at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1285)
	at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1282)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.security.auth.Subject.doAs(Subject.java:422)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1614)
	at org.apache.hadoop.mapreduce.Job.submit(Job.java:1282)
	at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1303)
	at hbase_mr.HBaseMR.main(HBaseMR.java:172)

Process finished with exit code 1

2.2 錯誤原因

Hadoop檔案系統沒有檢查路徑時沒有區分是本地windows系統還是Hadoop叢集檔案系統 參考連結

修正方案

TableMapReduceUtil.initTableMapperJob(tableName, scan, MyMapper.class,Text.class, IntWritable.class, job,false);
        //建立寫入hbase的reducer,指定表名、reducer類、job
        TableMapReduceUtil.initTableReducerJob(tableName2, MyReducer.class, job,
                null,null,null,null,false);

2.3 執行結果

日誌

D:\programs\Java\jdk1.8.0_181\bin\java "-javaagent:D:\programs\JetBrains\IntelliJ IDEA 2017.3.4\lib\idea_rt.jar=51278:D:\programs\JetBrains\IntelliJ IDEA 2017.3.4\bin" -Dfile.encoding=UTF-8 -classpath D:\programs\Java\jdk1.8.0_181\jre\lib\charsets.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\deploy.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\access-bridge-64.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\cldrdata.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\dnsns.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\jaccess.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\jfxrt.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\localedata.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\nashorn.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\sunec.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\sunjce_provider.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\sunmscapi.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\sunpkcs11.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\zipfs.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\javaws.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\jce.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\jfr.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\jfxswt.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\jsse.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\management-agent.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\plugin.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\resources.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\rt.jar;D:\Data\JavaProject\hbase\out\production\hbase;D:\Data\JavaProject\hbase\lib\hamcrest-core-1.3.jar;D:\Data\JavaProject\hbase\lib\xz-1.0.jar;D:\Data\JavaProject\hbase\lib\asm-3.1.jar;D:\Data\JavaProject\hbase\lib\guice-3.0.jar;D:\Data\JavaProject\hbase\lib\avro-1.7.4.jar;D:\Data\JavaProject\hbase\lib\joni-2.1.2.jar;D:\Data\JavaProject\hbase\lib\junit-4.12.jar;D:\Data\JavaProject\hbase\lib\jsch-0.1.42.jar;D:\Data\JavaProject\hbase\lib\xmlenc-0.52.jar;D:\Data\JavaProject\hbase\lib\guava-12.0.1.jar;D:\Data\JavaProject\hbase\lib\jets3t-0.9.0.jar;D:\Data\JavaProject\hbase\lib\jetty-6.1.26.jar;D:\Data\JavaProject\hbase\lib\log4j-1.2.17.jar;D:\Data\JavaProject\hbase\lib\paranamer-2.3.jar;D:\Data\JavaProject\hbase\lib\activation-1.1.jar;D:\Data\JavaProject\hbase\lib\commons-el-1.0.jar;D:\Data\JavaProject\hbase\lib\commons-io-2.4.jar;D:\Data\JavaProject\hbase\lib\hbase-it-1.2.6.jar;D:\Data\JavaProject\hbase\lib\httpcore-4.4.1.jar;D:\Data\JavaProject\hbase\lib\javax.inject-1.jar;D:\Data\JavaProject\hbase\lib\jaxb-api-2.2.2.jar;D:\Data\JavaProject\hbase\lib\jcodings-1.0.8.jar;D:\Data\JavaProject\hbase\lib\jettison-1.3.3.jar;D:\Data\JavaProject\hbase\lib\jsp-2.1-6.1.14.jar;D:\Data\JavaProject\hbase\lib\aopalliance-1.0.jar;D:\Data\JavaProject\hbase\lib\commons-cli-1.2.jar;D:\Data\JavaProject\hbase\lib\commons-net-3.1.jar;D:\Data\JavaProject\hbase\lib\disruptor-3.3.0.jar;D:\Data\JavaProject\hbase\lib\jersey-core-1.9.jar;D:\Data\JavaProject\hbase\lib\jersey-json-1.9.jar;D:\Data\JavaProject\hbase\lib\libthrift-0.9.3.jar;D:\Data\JavaProject\hbase\lib\servlet-api-2.5.jar;D:\Data\JavaProject\hbase\lib\slf4j-api-1.7.7.jar;D:\Data\JavaProject\hbase\lib\zookeeper-3.4.6.jar;D:\Data\JavaProject\hbase\lib\commons-lang-2.6.jar;D:\Data\JavaProject\hbase\lib\commons-math-2.2.jar;D:\Data\JavaProject\hbase\lib\hbase-rest-1.2.6.jar;D:\Data\JavaProject\hbase\lib\httpclient-4.2.5.jar;D:\Data\JavaProject\hbase\lib\jersey-guice-1.9.jar;D:\Data\JavaProject\hbase\lib\commons-codec-1.9.jar;D:\Data\JavaProject\hbase\lib\guice-servlet-3.0.jar;D:\Data\JavaProject\hbase\lib\hadoop-auth-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-hdfs-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hbase-shell-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jackson-xc-1.9.13.jar;D:\Data\JavaProject\hbase\lib\jaxb-impl-2.2.3-1.jar;D:\Data\JavaProject\hbase\lib\jersey-client-1.9.jar;D:\Data\JavaProject\hbase\lib\jersey-server-1.9.jar;D:\Data\JavaProject\hbase\lib\jetty-util-6.1.26.jar;D:\Data\JavaProject\hbase\lib\api-util-1.0.0-M20.jar;D:\Data\JavaProject\hbase\lib\hbase-client-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-common-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-server-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-thrift-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jsp-api-2.1-6.1.14.jar;D:\Data\JavaProject\hbase\lib\leveldbjni-all-1.8.jar;D:\Data\JavaProject\hbase\lib\metrics-core-2.2.0.jar;D:\Data\JavaProject\hbase\lib\commons-logging-1.2.jar;D:\Data\JavaProject\hbase\lib\commons-math3-3.1.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-client-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-common-2.5.1.jar;D:\Data\JavaProject\hbase\lib\jamon-runtime-2.4.1.jar;D:\Data\JavaProject\hbase\lib\java-xmlbuilder-0.4.jar;D:\Data\JavaProject\hbase\lib\protobuf-java-2.5.0.jar;D:\Data\JavaProject\hbase\lib\slf4j-log4j12-1.7.5.jar;D:\Data\JavaProject\hbase\lib\snappy-java-1.0.4.1.jar;D:\Data\JavaProject\hbase\lib\spymemcached-2.11.6.jar;D:\Data\JavaProject\hbase\lib\commons-digester-1.8.jar;D:\Data\JavaProject\hbase\lib\hbase-examples-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-it-1.2.6-tests.jar;D:\Data\JavaProject\hbase\lib\hbase-protocol-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jackson-jaxrs-1.9.13.jar;D:\Data\JavaProject\hbase\lib\jruby-complete-1.6.8.jar;D:\Data\JavaProject\hbase\lib\commons-daemon-1.0.13.jar;D:\Data\JavaProject\hbase\lib\hadoop-yarn-api-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hbase-procedure-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jasper-runtime-5.5.23.jar;D:\Data\JavaProject\hbase\lib\api-asn1-api-1.0.0-M20.jar;D:\Data\JavaProject\hbase\lib\commons-compress-1.4.1.jar;D:\Data\JavaProject\hbase\lib\commons-httpclient-3.1.jar;D:\Data\JavaProject\hbase\lib\jasper-compiler-5.5.23.jar;D:\Data\JavaProject\hbase\lib\jetty-sslengine-6.1.26.jar;D:\Data\JavaProject\hbase\lib\netty-all-4.0.23.Final.jar;D:\Data\JavaProject\hbase\lib\servlet-api-2.5-6.1.14.jar;D:\Data\JavaProject\hbase\lib\apacheds-i18n-2.0.0-M15.jar;D:\Data\JavaProject\hbase\lib\commons-beanutils-1.7.0.jar;D:\Data\JavaProject\hbase\lib\hbase-annotations-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-prefix-tree-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jackson-core-asl-1.9.13.jar;D:\Data\JavaProject\hbase\lib\hadoop-annotations-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-yarn-client-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-yarn-common-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hbase-common-1.2.6-tests.jar;D:\Data\JavaProject\hbase\lib\hbase-server-1.2.6-tests.jar;D:\Data\JavaProject\hbase\lib\commons-collections-3.2.2.jar;D:\Data\JavaProject\hbase\lib\commons-configuration-1.6.jar;D:\Data\JavaProject\hbase\lib\hbase-hadoop-compat-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jackson-mapper-asl-1.9.13.jar;D:\Data\JavaProject\hbase\lib\hbase-hadoop2-compat-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-resource-bundle-1.2.6.jar;D:\Data\JavaProject\hbase\lib\commons-beanutils-core-1.8.0.jar;D:\Data\JavaProject\hbase\lib\findbugs-annotations-1.3.9-1.jar;D:\Data\JavaProject\hbase\lib\htrace-core-3.1.0-incubating.jar;D:\Data\JavaProject\hbase\lib\hbase-annotations-1.2.6-tests.jar;D:\Data\JavaProject\hbase\lib\hadoop-yarn-server-common-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hbase-external-blockcache-1.2.6.jar;D:\Data\JavaProject\hbase\lib\apacheds-kerberos-codec-2.0.0-M15.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-app-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-core-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-common-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-shuffle-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-jobclient-2.5.1.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\xz-1.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\asm-3.2.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\guice-3.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\avro-1.7.4.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\junit-4.11.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\log4j-1.2.17.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\paranamer-2.3.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\commons-io-2.4.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\javax.inject-1.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\aopalliance-1.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jersey-core-1.9.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jersey-guice-1.9.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\guice-servlet-3.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\hamcrest-core-1.3.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jersey-server-1.9.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\netty-3.6.2.Final.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\leveldbjni-all-1.8.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\protobuf-java-2.5.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\snappy-java-1.0.4.1.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\commons-compress-1.4.1.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jackson-core-asl-1.9.13.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\hadoop-annotations-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jackson-mapper-asl-1.9.13.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-app-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-common-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-core-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-hs-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-hs-plugins-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-jobclient-2.7.6-tests.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-jobclient-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-shuffle-2.7.6.jar hbase_mr.HBaseMR
2018-09-19 14:47:15,343 INFO  [main] zookeeper.RecoverableZooKeeper (RecoverableZooKeeper.java:<init>(120)) - Process identifier=hconnection-0x12028586 connecting to ZooKeeper ensemble=node1:2181,node2:2181,node3:2181
2018-09-19 14:47:15,352 INFO  [main] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:zookeeper.version=3.4.6-1569965, built on 02/20/2014 09:09 GMT
2018-09-19 14:47:15,352 INFO  [main] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:host.name=XPS-15-9570
2018-09-19 14:47:15,352 INFO  [main] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:java.version=1.8.0_181
2018-09-19 14:47:15,352 INFO  [main] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:java.vendor=Oracle Corporation
2018-09-19 14:47:15,352 INFO  [main] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:java.home=D:\programs\Java\jdk1.8.0_181\jre
2018-09-19 14:47:15,352 INFO  [main] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:java.class.path=D:\programs\Java\jdk1.8.0_181\jre\lib\charsets.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\deploy.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\access-bridge-64.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\cldrdata.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\dnsns.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\jaccess.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\jfxrt.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\localedata.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\nashorn.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\sunec.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\sunjce_provider.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\sunmscapi.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\sunpkcs11.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\ext\zipfs.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\javaws.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\jce.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\jfr.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\jfxswt.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\jsse.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\management-agent.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\plugin.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\resources.jar;D:\programs\Java\jdk1.8.0_181\jre\lib\rt.jar;D:\Data\JavaProject\hbase\out\production\hbase;D:\Data\JavaProject\hbase\lib\hamcrest-core-1.3.jar;D:\Data\JavaProject\hbase\lib\xz-1.0.jar;D:\Data\JavaProject\hbase\lib\asm-3.1.jar;D:\Data\JavaProject\hbase\lib\guice-3.0.jar;D:\Data\JavaProject\hbase\lib\avro-1.7.4.jar;D:\Data\JavaProject\hbase\lib\joni-2.1.2.jar;D:\Data\JavaProject\hbase\lib\junit-4.12.jar;D:\Data\JavaProject\hbase\lib\jsch-0.1.42.jar;D:\Data\JavaProject\hbase\lib\xmlenc-0.52.jar;D:\Data\JavaProject\hbase\lib\guava-12.0.1.jar;D:\Data\JavaProject\hbase\lib\jets3t-0.9.0.jar;D:\Data\JavaProject\hbase\lib\jetty-6.1.26.jar;D:\Data\JavaProject\hbase\lib\log4j-1.2.17.jar;D:\Data\JavaProject\hbase\lib\paranamer-2.3.jar;D:\Data\JavaProject\hbase\lib\activation-1.1.jar;D:\Data\JavaProject\hbase\lib\commons-el-1.0.jar;D:\Data\JavaProject\hbase\lib\commons-io-2.4.jar;D:\Data\JavaProject\hbase\lib\hbase-it-1.2.6.jar;D:\Data\JavaProject\hbase\lib\httpcore-4.4.1.jar;D:\Data\JavaProject\hbase\lib\javax.inject-1.jar;D:\Data\JavaProject\hbase\lib\jaxb-api-2.2.2.jar;D:\Data\JavaProject\hbase\lib\jcodings-1.0.8.jar;D:\Data\JavaProject\hbase\lib\jettison-1.3.3.jar;D:\Data\JavaProject\hbase\lib\jsp-2.1-6.1.14.jar;D:\Data\JavaProject\hbase\lib\aopalliance-1.0.jar;D:\Data\JavaProject\hbase\lib\commons-cli-1.2.jar;D:\Data\JavaProject\hbase\lib\commons-net-3.1.jar;D:\Data\JavaProject\hbase\lib\disruptor-3.3.0.jar;D:\Data\JavaProject\hbase\lib\jersey-core-1.9.jar;D:\Data\JavaProject\hbase\lib\jersey-json-1.9.jar;D:\Data\JavaProject\hbase\lib\libthrift-0.9.3.jar;D:\Data\JavaProject\hbase\lib\servlet-api-2.5.jar;D:\Data\JavaProject\hbase\lib\slf4j-api-1.7.7.jar;D:\Data\JavaProject\hbase\lib\zookeeper-3.4.6.jar;D:\Data\JavaProject\hbase\lib\commons-lang-2.6.jar;D:\Data\JavaProject\hbase\lib\commons-math-2.2.jar;D:\Data\JavaProject\hbase\lib\hbase-rest-1.2.6.jar;D:\Data\JavaProject\hbase\lib\httpclient-4.2.5.jar;D:\Data\JavaProject\hbase\lib\jersey-guice-1.9.jar;D:\Data\JavaProject\hbase\lib\commons-codec-1.9.jar;D:\Data\JavaProject\hbase\lib\guice-servlet-3.0.jar;D:\Data\JavaProject\hbase\lib\hadoop-auth-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-hdfs-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hbase-shell-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jackson-xc-1.9.13.jar;D:\Data\JavaProject\hbase\lib\jaxb-impl-2.2.3-1.jar;D:\Data\JavaProject\hbase\lib\jersey-client-1.9.jar;D:\Data\JavaProject\hbase\lib\jersey-server-1.9.jar;D:\Data\JavaProject\hbase\lib\jetty-util-6.1.26.jar;D:\Data\JavaProject\hbase\lib\api-util-1.0.0-M20.jar;D:\Data\JavaProject\hbase\lib\hbase-client-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-common-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-server-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-thrift-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jsp-api-2.1-6.1.14.jar;D:\Data\JavaProject\hbase\lib\leveldbjni-all-1.8.jar;D:\Data\JavaProject\hbase\lib\metrics-core-2.2.0.jar;D:\Data\JavaProject\hbase\lib\commons-logging-1.2.jar;D:\Data\JavaProject\hbase\lib\commons-math3-3.1.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-client-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-common-2.5.1.jar;D:\Data\JavaProject\hbase\lib\jamon-runtime-2.4.1.jar;D:\Data\JavaProject\hbase\lib\java-xmlbuilder-0.4.jar;D:\Data\JavaProject\hbase\lib\protobuf-java-2.5.0.jar;D:\Data\JavaProject\hbase\lib\slf4j-log4j12-1.7.5.jar;D:\Data\JavaProject\hbase\lib\snappy-java-1.0.4.1.jar;D:\Data\JavaProject\hbase\lib\spymemcached-2.11.6.jar;D:\Data\JavaProject\hbase\lib\commons-digester-1.8.jar;D:\Data\JavaProject\hbase\lib\hbase-examples-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-it-1.2.6-tests.jar;D:\Data\JavaProject\hbase\lib\hbase-protocol-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jackson-jaxrs-1.9.13.jar;D:\Data\JavaProject\hbase\lib\jruby-complete-1.6.8.jar;D:\Data\JavaProject\hbase\lib\commons-daemon-1.0.13.jar;D:\Data\JavaProject\hbase\lib\hadoop-yarn-api-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hbase-procedure-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jasper-runtime-5.5.23.jar;D:\Data\JavaProject\hbase\lib\api-asn1-api-1.0.0-M20.jar;D:\Data\JavaProject\hbase\lib\commons-compress-1.4.1.jar;D:\Data\JavaProject\hbase\lib\commons-httpclient-3.1.jar;D:\Data\JavaProject\hbase\lib\jasper-compiler-5.5.23.jar;D:\Data\JavaProject\hbase\lib\jetty-sslengine-6.1.26.jar;D:\Data\JavaProject\hbase\lib\netty-all-4.0.23.Final.jar;D:\Data\JavaProject\hbase\lib\servlet-api-2.5-6.1.14.jar;D:\Data\JavaProject\hbase\lib\apacheds-i18n-2.0.0-M15.jar;D:\Data\JavaProject\hbase\lib\commons-beanutils-1.7.0.jar;D:\Data\JavaProject\hbase\lib\hbase-annotations-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-prefix-tree-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jackson-core-asl-1.9.13.jar;D:\Data\JavaProject\hbase\lib\hadoop-annotations-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-yarn-client-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-yarn-common-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hbase-common-1.2.6-tests.jar;D:\Data\JavaProject\hbase\lib\hbase-server-1.2.6-tests.jar;D:\Data\JavaProject\hbase\lib\commons-collections-3.2.2.jar;D:\Data\JavaProject\hbase\lib\commons-configuration-1.6.jar;D:\Data\JavaProject\hbase\lib\hbase-hadoop-compat-1.2.6.jar;D:\Data\JavaProject\hbase\lib\jackson-mapper-asl-1.9.13.jar;D:\Data\JavaProject\hbase\lib\hbase-hadoop2-compat-1.2.6.jar;D:\Data\JavaProject\hbase\lib\hbase-resource-bundle-1.2.6.jar;D:\Data\JavaProject\hbase\lib\commons-beanutils-core-1.8.0.jar;D:\Data\JavaProject\hbase\lib\findbugs-annotations-1.3.9-1.jar;D:\Data\JavaProject\hbase\lib\htrace-core-3.1.0-incubating.jar;D:\Data\JavaProject\hbase\lib\hbase-annotations-1.2.6-tests.jar;D:\Data\JavaProject\hbase\lib\hadoop-yarn-server-common-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hbase-external-blockcache-1.2.6.jar;D:\Data\JavaProject\hbase\lib\apacheds-kerberos-codec-2.0.0-M15.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-app-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-core-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-common-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-shuffle-2.5.1.jar;D:\Data\JavaProject\hbase\lib\hadoop-mapreduce-client-jobclient-2.5.1.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\xz-1.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\asm-3.2.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\guice-3.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\avro-1.7.4.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\junit-4.11.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\log4j-1.2.17.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\paranamer-2.3.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\commons-io-2.4.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\javax.inject-1.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\aopalliance-1.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jersey-core-1.9.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jersey-guice-1.9.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\guice-servlet-3.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\hamcrest-core-1.3.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jersey-server-1.9.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\netty-3.6.2.Final.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\leveldbjni-all-1.8.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\protobuf-java-2.5.0.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\snappy-java-1.0.4.1.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\commons-compress-1.4.1.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jackson-core-asl-1.9.13.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\hadoop-annotations-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\lib\jackson-mapper-asl-1.9.13.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-app-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-common-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-core-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-hs-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-hs-plugins-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-jobclient-2.7.6-tests.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-jobclient-2.7.6.jar;D:\programs\hadoop-2.7.6\share\hadoop\mapreduce\hadoop-mapreduce-client-shuffle-2.7.6.jar;D:\programs\JetBrains\IntelliJ IDEA 2017.3.4\lib\idea_rt.jar
2018-09-19 14:47:15,353 INFO  [main] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:java.library.path=D:\progra