1. 程式人生 > >(6)Java 讀寫 hdfs檔案或者目錄

(6)Java 讀寫 hdfs檔案或者目錄

1.讀取單個檔案

			Date date = DateUtil.getSpecifiedDayBefore();
			String yesterday = DateUtil.dateToStr(date, "yyyy-MM-dd");
			String path = "hdfs://ip:9000/output_log/output_log_click" + yesterday;
			
		  Configuration conf = new Configuration();
			FileSystem fs = FileSystem.get(URI.create(path), conf);
			FSDataInputStream hdfsInStream = fs.open(new Path(path));
			InputStreamReader isr = new InputStreamReader(hdfsInStream, "utf-8");
			BufferedReader br = new BufferedReader(isr);
			String line;
			// int k = 0;
			while ((line = br.readLine()) != null) {
				System.out.println(line);
			}
		  
	  


2.讀取資料夾

		Date date = DateUtil.getSpecifiedDayBefore();
		String yesterday = DateUtil.dateToStr(date, "yyyy-MM-dd");
		String path = "hdfs://ip:9000/output_log/output_log_click" + yesterday;

		Configuration conf = new Configuration();
		FileSystem fs = FileSystem.get(URI.create(path), conf);
		FileStatus[] status = fs.listStatus(new Path(path));
		for (FileStatus file : status) {
			if (!file.getPath().getName().startsWith("newsMap")) {
                continue;
            }
			FSDataInputStream hdfsInStream = fs.open(file.getPath());
			InputStreamReader isr = new InputStreamReader(hdfsInStream, "utf-8");
			BufferedReader br = new BufferedReader(isr);
			String line;
			// int k = 0;
			while ((line = br.readLine()) != null) {
				System.out.println(line);
			}
			
		}