1 客戶端環境準備

根據自己電腦的作業系統拷貝對應的編譯後的hadoop jar包到非中文路徑

配置HADOOP_HOME的環境變數,並且在path中配置hadoop的bin

重啟電腦

2. HdfsClientDemo

建立一個Maven專案,在pom.xml中匯入相應的依賴,匯入失敗的話,試一試Reimport

<dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>
 
junit</artifactId>
            <version>RELEASE</version>
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-core</artifactId>
            <version>2.8.2</version>
 

        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.7.2</version>
        </dependency>
        <dependency>
            <groupId
 
>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.7.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>2.7.2</version>
        </dependency>
        <dependency>
            <groupId>jdk.tools</groupId>
            <artifactId>jdk.tools</artifactId>
            <version>1.8</version>
            <scope>system</scope>
            <systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
        </dependency>
</dependencies>

在src/main/resources目錄下建立log4j.properties檔案

log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=target/spring.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n

建立包和HdfsClientDemo類

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.net.URI;


public class HdfsClient {

    private static final String HADOOP_URI = "hdfs://hadoop100:9000";

    private Configuration configuration;
    private FileSystem fileSystem;


    @Before
    public void before() throws Exception {
        //configuration 物件相對應的就是 hadoop的那些配置檔案,比如修改當前客戶端上傳檔案的備份數量為1
        //configuration.set("replication","1");
        configuration = new Configuration();
        fileSystem = FileSystem.get(new URI(HADOOP_URI),configuration,"nty");
    }

    @After
    public void after() throws Exception {
        fileSystem.close();
    }

    /**
     * 建立目錄
     */
    @Test
    public void mkdir() throws Exception {
        fileSystem.mkdirs(new Path("/client_test"));
    }

    /**
     * 上傳檔案
     */
    @Test
    public void upload() throws Exception {
        fileSystem.copyFromLocalFile(new Path("d:\\Hadoop_test\\test1.txt"), new Path("/client_test"));
    }

    /**
     * 下載檔案
     */
    @Test
    public void download() throws Exception {
        fileSystem.copyToLocalFile(new Path("/client_test/test1.txt"), new Path("d:\\Hadoop_test\\test1_1.txt"));
    }

    /**
     * 刪除檔案
     *
     */
    @Test
    public void delete() throws Exception {
        fileSystem.delete(new Path("/output"),true);
    }

    /**
     * 重新命名
     */
    @Test
    public void rename() throws Exception {
        fileSystem.rename(new Path("/input"), new Path("/input_rename"));
    }

    /**
     * 檔案遍歷
     */
    @Test
    public void liststatus() throws Exception {
        FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
        for(FileStatus fs : fileStatuses){
            System.out.println(fs.isDirectory() ? (fs.getPath().getName() + " is directory") : (fs.getPath().getName() + " is file"));
        }

    }
    

}