1. 程式人生 > >Java 對HDFS介面呼叫

Java 對HDFS介面呼叫

使用HDFS提供的API來對分散式檔案系統中的檔案進行基本的操作,在linux下是不需要連線的,在windows下連線hdfs有三種方式:

Configuration  conf = new Configuration();
//第一種方式
conf.set("fs.defaultFS", "hdfs://10.49.85.152:9000");
//第二種設定configuration方式
conf.addResource(new Path("D:\\hadoop\\1.xml"));    
FileSystem hdfs = FileSystem.get(conf);
//第三種方式
FileSystem hdfs = FileSystem.get
(new URI("hdfs://10.49.85.152:9000"), conf);

下面是對基本的API的呼叫,以供參考:

import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import
org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; public class HDFSDemo2 { static Configuration conf; static FileSystem fs; static{ conf = new Configuration(); try { fs = FileSystem.get(new URI("hdfs://10.49.85.152:9000"
), conf); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (URISyntaxException e) { // TODO Auto-generated catch block e.printStackTrace(); } } // public HDFSDemo2(){ // conf = new Configuration(); // try { // fs = FileSystem.get(new URI("hdfs://10.49.85.152:9000"), conf); // } catch (IOException e) { // // TODO Auto-generated catch block // e.printStackTrace(); // } catch (URISyntaxException e) { // // TODO Auto-generated catch block // e.printStackTrace(); // } // } //建立資料夾 public static void mkdir(String name){ boolean flag = false; try { if (!fs.exists(new Path(name+"1"))) { flag = fs.mkdirs(new Path(name)); } System.out.println(flag); if(flag){ System.out.println("資料夾建立成功"); }else{ System.out.println("資料夾建立失敗"); } } catch (IllegalArgumentException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); }finally{ close(); } } //建立檔案 public static void createFile(String file, String text){ FSDataOutputStream outputStream = null; byte[] arg0 = text.getBytes(); try { outputStream = fs.create(new Path(file)); outputStream.write(arg0, 0, arg0.length); outputStream.close(); System.out.println("檔案建立成功"); } catch (IllegalArgumentException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); }finally{ close(); } } //對檔案重新命名 public static void rename(String oldName, String newName){ boolean flag; try { flag = fs.rename(new Path(oldName), new Path(newName)); if(flag){ System.out.println("檔案重新命名成功"); } } catch (IllegalArgumentException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); }finally{ close(); } } //判斷檔案是否存在 public static void existFile(String file){ boolean flag; try { flag = fs.exists(new Path(file)); if(flag){ System.out.println("存在"); }else { System.out.println("不存在"); } } catch (IllegalArgumentException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); }finally{ close(); } } //刪除檔案 public static void deleteFile(String file){ boolean flag; try { flag = fs.deleteOnExit(new Path(file)); if(flag){ System.out.println("成功刪除"); } } catch (IllegalArgumentException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); }finally{ close(); } } public static void close(){ try { fs.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } //上傳檔案 public static void uploadFile(String src, String dst){ try { fs.copyFromLocalFile(false, new Path(src), new Path(dst)); System.out.println("檔案上傳成功"); } catch (IllegalArgumentException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); }finally{ close(); } } //下載檔案 public static void downFile(String src, String dst){ try { fs.copyToLocalFile(new Path(src), new Path(dst)); System.out.println("檔案下載成功"); } catch (IllegalArgumentException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); }finally{ close(); } } //讀取檔案 public static String readFile(String file){ String str = ""; FSDataInputStream inputStream = null; byte[] arg0 = new byte[1024]; try { inputStream = fs.open(new Path(file)); inputStream.read(arg0); str = new String(arg0); } catch (IllegalArgumentException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); }finally{ close(); } return str; } //返回指定資料夾的檔案列表 public static void listName(String dir){ try { FileStatus files[] = fs.listStatus(new Path(dir)); for(FileStatus file : files){ System.out.println(file.getPath().getName() + "\t" + file.getLen()); } } catch (FileNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IllegalArgumentException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); }finally{ close(); } } public static void main(String[] args) throws IOException { // TODO Auto-generated method stub // HDFSDemo2 hdfs = new HDFSDemo2(); HDFSDemo2.mkdir("/test/t2"); // HDFSDemo2.createFile("/test/3.txt", "fdsf23432432違法所得"); // HDFSDemo2.createFile(args[0], args[1]); // HDFSDemo2.rename("/test/test2", "/test/test1"); // HDFSDemo2.existFile("/test/3.txt"); // HDFSDemo2.deleteFile("/test/5.txt"); // HDFSDemo2.downFile(args[0], args[1]); // String str = HDFSDemo2.readFile("/test/3.txt"); // System.out.println(str); // HDFSDemo2.listName("/test"); } }