1. 程式人生 > >hadoop HA場景下 java客戶端遠端訪問hdfs配置

hadoop HA場景下 java客戶端遠端訪問hdfs配置

當hadoop namenode是HA叢集時,客戶端遠端訪問hdfs有兩種實現方法:
方法1:將所有關於namenode的引數寫入Configuration物件中。
程式碼:

package com.lx;

import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache
.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; public class Demo { private static void uploadToHdfs() throws Exception { //本地檔案地址 String localSrc = "d:/PowerDesigner15_Evaluation.rar"; //存放在hdfs的目的地址 String dest = "/user/PowerDesigner15_Evaluation.rar"
; InputStream in = new BufferedInputStream(new FileInputStream(localSrc)); //得到配置物件 Configuration conf = new Configuration(); conf.set("fs.defaultFS", "hdfs://ns1"); conf.set("dfs.nameservices", "ns1"); conf.set("dfs.ha.namenodes.ns1", "nn1,nn2"); conf.set
("dfs.namenode.rpc-address.ns1.nn1", "hadoop1:9000"); conf.set("dfs.namenode.rpc-address.ns1.nn2", "hadoop2:9000"); conf.set("dfs.client.failover.proxy.provider.ns1", "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"); //檔案系統 FileSystem fs = FileSystem.get(new URI("hdfs://ns1"), conf, "hadoop"); //輸出流 OutputStream out = fs.create(new Path(dest)); //連線兩個流,形成通道,使輸入流向輸出流傳輸資料 IOUtils.copyBytes(in, out, 4096, true); } public static void main(String[] args) throws Exception{ // TODO Auto-generated method stub uploadToHdfs(); } }

方法2:將hadoop叢集配置檔案core-site.xml和hdfs-site.xml檔案拷貝到專案的src目錄下。
這裡寫圖片描述
程式碼:

package com.lx;

import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

public class Demo2 {

    public static void main(String[] args) throws Exception{
        // TODO Auto-generated method stub
        String src="d:/RoseHA.rar";
        String dst="/user/RoseHA.rar";

        InputStream in=new BufferedInputStream(new FileInputStream(src));

        Configuration conf=new Configuration();

        FileSystem fs=FileSystem.get(new URI("hdfs://ns1"), conf, "hadoop");

        OutputStream out=fs.create(new Path(dst));

        IOUtils.copyBytes(in, out, 4096, true);
    }

}