Java 對HDFS介面呼叫
阿新 • • 發佈:2018-11-01
使用HDFS提供的API來對分散式檔案系統中的檔案進行基本的操作,在linux下是不需要連線的,在windows下連線hdfs有三種方式:
Configuration conf = new Configuration();
//第一種方式
conf.set("fs.defaultFS", "hdfs://10.49.85.152:9000");
//第二種設定configuration方式
conf.addResource(new Path("D:\\hadoop\\1.xml"));
FileSystem hdfs = FileSystem.get(conf);
//第三種方式
FileSystem hdfs = FileSystem.get (new URI("hdfs://10.49.85.152:9000"), conf);
下面是對基本的API的呼叫,以供參考:
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class HDFSDemo2 {
static Configuration conf;
static FileSystem fs;
static{
conf = new Configuration();
try {
fs = FileSystem.get(new URI("hdfs://10.49.85.152:9000" ), conf);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (URISyntaxException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
// public HDFSDemo2(){
// conf = new Configuration();
// try {
// fs = FileSystem.get(new URI("hdfs://10.49.85.152:9000"), conf);
// } catch (IOException e) {
// // TODO Auto-generated catch block
// e.printStackTrace();
// } catch (URISyntaxException e) {
// // TODO Auto-generated catch block
// e.printStackTrace();
// }
// }
//建立資料夾
public static void mkdir(String name){
boolean flag = false;
try {
if (!fs.exists(new Path(name+"1"))) {
flag = fs.mkdirs(new Path(name));
}
System.out.println(flag);
if(flag){
System.out.println("資料夾建立成功");
}else{
System.out.println("資料夾建立失敗");
}
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
close();
}
}
//建立檔案
public static void createFile(String file, String text){
FSDataOutputStream outputStream = null;
byte[] arg0 = text.getBytes();
try {
outputStream = fs.create(new Path(file));
outputStream.write(arg0, 0, arg0.length);
outputStream.close();
System.out.println("檔案建立成功");
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
close();
}
}
//對檔案重新命名
public static void rename(String oldName, String newName){
boolean flag;
try {
flag = fs.rename(new Path(oldName), new Path(newName));
if(flag){
System.out.println("檔案重新命名成功");
}
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
close();
}
}
//判斷檔案是否存在
public static void existFile(String file){
boolean flag;
try {
flag = fs.exists(new Path(file));
if(flag){
System.out.println("存在");
}else
{
System.out.println("不存在");
}
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
close();
}
}
//刪除檔案
public static void deleteFile(String file){
boolean flag;
try {
flag = fs.deleteOnExit(new Path(file));
if(flag){
System.out.println("成功刪除");
}
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
close();
}
}
public static void close(){
try {
fs.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
//上傳檔案
public static void uploadFile(String src, String dst){
try {
fs.copyFromLocalFile(false, new Path(src), new Path(dst));
System.out.println("檔案上傳成功");
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
close();
}
}
//下載檔案
public static void downFile(String src, String dst){
try {
fs.copyToLocalFile(new Path(src), new Path(dst));
System.out.println("檔案下載成功");
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
close();
}
}
//讀取檔案
public static String readFile(String file){
String str = "";
FSDataInputStream inputStream = null;
byte[] arg0 = new byte[1024];
try {
inputStream = fs.open(new Path(file));
inputStream.read(arg0);
str = new String(arg0);
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
close();
}
return str;
}
//返回指定資料夾的檔案列表
public static void listName(String dir){
try {
FileStatus files[] = fs.listStatus(new Path(dir));
for(FileStatus file : files){
System.out.println(file.getPath().getName() + "\t" + file.getLen());
}
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
close();
}
}
public static void main(String[] args) throws IOException {
// TODO Auto-generated method stub
// HDFSDemo2 hdfs = new HDFSDemo2();
HDFSDemo2.mkdir("/test/t2");
// HDFSDemo2.createFile("/test/3.txt", "fdsf23432432違法所得");
// HDFSDemo2.createFile(args[0], args[1]);
// HDFSDemo2.rename("/test/test2", "/test/test1");
// HDFSDemo2.existFile("/test/3.txt");
// HDFSDemo2.deleteFile("/test/5.txt");
// HDFSDemo2.downFile(args[0], args[1]);
// String str = HDFSDemo2.readFile("/test/3.txt");
// System.out.println(str);
// HDFSDemo2.listName("/test");
}
}