關於ftp和http下載斷點續傳
阿新 • • 發佈：2019-01-08
FTP下載斷點續傳
import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.SocketTimeoutException; import java.text.DecimalFormat; import java.util.ArrayList; import java.util.List; import java.util.TimeZone; import java.util.regex.Pattern; import org.apache.commons.net.ftp.FTPClient; import org.apache.commons.net.ftp.FTPClientConfig; import org.apache.commons.net.ftp.FTPFile; import org.apache.commons.net.ftp.FTPFileFilter; import org.apache.commons.net.ftp.FTPReply; import org.apache.log4j.Logger; public class ApacheFtpUtil { private FTPClient ftpClient; private String strIp; private int intPort; private String user; private String password; private static Logger logger = Logger.getLogger(ApacheFtpUtil.class.getName()); /* * * Ftp建構函式 */ public ApacheFtpUtil(String strIp, int intPort, String user, String Password) { this.strIp = strIp; this.intPort = intPort; this.user = user; this.password = Password; this.ftpClient = new FTPClient(); } public ApacheFtpUtil(String strIp, int intPort) { this.strIp = strIp; this.intPort = intPort; this.user = "anonymous"; this.password = "anonymous"; this.ftpClient = new FTPClient(); } /** * @return 判斷是否登入成功 * */ public boolean connectServer() { boolean isLogin = false; FTPClientConfig ftpClientConfig = new FTPClientConfig(); ftpClientConfig.setServerTimeZoneId(TimeZone.getDefault().getID()); this.ftpClient.setControlEncoding("GBK"); this.ftpClient.configure(ftpClientConfig); try { if (this.intPort > 0) { this.ftpClient.connect(this.strIp, this.intPort); } else { this.ftpClient.connect(this.strIp); } // FTP伺服器連接回答 int reply = this.ftpClient.getReplyCode(); if (!FTPReply.isPositiveCompletion(reply)) { this.ftpClient.disconnect(); throw new IOException("Can't connect to server '" + this.strIp + "'"); } isLogin = this.ftpClient.login(this.user, this.password); if(!isLogin){ closeServer(); throw new IOException("Can't login to server '" + this.strIp + "'"); } // 設定傳輸協議 this.ftpClient.enterLocalPassiveMode(); this.ftpClient.setFileType(FTPClient.BINARY_FILE_TYPE); logger.info("恭喜" + this.user + "成功登陸FTP伺服器:"+ftpClient.printWorkingDirectory()); } catch (Exception e) { e.printStackTrace(); logger.error(this.user + "登入FTP服務失敗！" + e.getMessage()); closeServer(); } this.ftpClient.setBufferSize(1024 * 8); this.ftpClient.setConnectTimeout(30 * 1000); this.ftpClient.setDataTimeout(60 * 1000); return isLogin; } /** * @退出關閉伺服器連結 * */ public void closeServer() { if (null != this.ftpClient && this.ftpClient.isConnected()) { try { boolean reuslt = this.ftpClient.logout();// 退出FTP伺服器 if (reuslt) { logger.info("成功退出伺服器"); } } catch (IOException e) { e.printStackTrace(); logger.warn("退出FTP伺服器異常！" + e.getMessage()); } finally { try { this.ftpClient.disconnect();// 關閉FTP伺服器的連線 } catch (IOException e) { e.printStackTrace(); logger.warn("關閉FTP伺服器的連線異常！"); } } } } /** * 取得相對於當前連線目錄的某個目錄下所有檔案列表 * Lists the files in the given FTP directory. * @param path * @return */ public List<String> getFileList(String path) { ftpClient.enterLocalPassiveMode(); List<String> list = new ArrayList<String>(); try { FTPFile[] ftplist = ftpClient.listFiles(path); int size = (ftplist == null) ? 0 : ftplist.length; for (int i = 0; i < size; i++) { FTPFile ftpFile = ftplist[i]; if (ftpFile.isFile()) { list.add(ftpFile.getName()); } } } catch (IOException e) { e.printStackTrace(); } return list; } public FTPFile[] getDataFileList(String path,final String pattern) throws IOException{ ftpClient.enterLocalPassiveMode(); FTPFile[] list = ftpClient.listFiles(path, new FTPFileFilter() { @Override public boolean accept(FTPFile file) { Pattern p = Pattern.compile(pattern); if (file.isFile() && p.matcher(file.getName()).find()) return true ; return false ; }}) ; return list; } /** * 獲取ftp遠端檔案大小 * @param filename * @return */ public long getFileSize(String filename) { long fileSize = -1; filename = filename.replaceAll("\\\\", "/"); try { FTPFile[] list = ftpClient.listFiles(filename); if(list!=null&&list.length>0){ return list[0].getSize(); } } catch (IOException e) { e.printStackTrace(); } return fileSize; } /*** * 上傳Ftp檔案過載方法 * @param localFile 當地檔案路徑 * @param romotUpLoadePath上傳伺服器絕對路徑,路徑以 "/"分開 * */ public boolean uploadFile(String localFilepath, String romotUpLoadePath) { File localFile = new File(localFilepath); BufferedInputStream inStream = null; boolean success = false; try { String romoteparent = romotUpLoadePath.substring(0,romotUpLoadePath.lastIndexOf("/")); createDir(romoteparent); inStream = new BufferedInputStream(new FileInputStream(localFile)); logger.info(localFile.getName() + "開始上傳....."); success = this.ftpClient.storeFile(romotUpLoadePath, inStream); if (success == true) { logger.info(localFile.getName() + "上傳成功"); return success; } } catch (FileNotFoundException e) { e.printStackTrace(); logger.error(localFile + "未找到"); } catch (IOException e) { e.printStackTrace(); } finally { if (inStream != null) { try { inStream.close(); } catch (IOException e) { e.printStackTrace(); } } } return success; } public static String formatSize(Long fileSize) { String formatSize = ""; DecimalFormat df = new DecimalFormat("0.##"); if (fileSize / (1024 * 1024 * 1024) > 0) { formatSize = df.format(Float.parseFloat(fileSize.toString())/ (1024 * 1024 * 1024))+ " GB"; } else if (fileSize / (1024 * 1024) > 0) { formatSize = df.format(Float.parseFloat(fileSize.toString())/ (1024 * 1024))+ " MB"; } else if (fileSize / (1024) > 0) { formatSize = df.format(Float.parseFloat(fileSize.toString())/ (1024))+ " KB"; } else { formatSize = fileSize + " 位元組"; } return formatSize; } /** * 斷點續傳FTP資源 * @param remote * @param local * @return * @throws IOException */ public long downloadFTP(String remote, String local,long lRemoteSize) { File f = new File(local); InputStream is = null; OutputStream out = null; BufferedInputStream bis = null; long bytesum = 0; int byteread = 0; long fileSize = 0; try { ftpClient.enterLocalPassiveMode(); //檔名稱 String fileName = remote.substring(remote.lastIndexOf('/')+1); if (f.exists()) { fileSize = f.length(); } out = new FileOutputStream(f, true); String progressInfo = "("+fileName+","+formatSize(fileSize+bytesum)+"/"+formatSize(lRemoteSize)+")"; //如果檔案已經下載完畢，直接返回 if (f.length() >= lRemoteSize) { System.out.println("ftp download progress：已下載完畢 "+progressInfo); out.close(); return f.length(); } ftpClient.setRestartOffset(fileSize); ftpClient.setBufferSize(1204 * 8); byte[] buffer = new byte[1204 * 8]; is = ftpClient.retrieveFileStream(remote);//result = ftpClient.retrieveFile(remote, out); bis = new BufferedInputStream(is); long step = lRemoteSize /100; long process= (fileSize+bytesum) /step; System.out.println("ftp download progress： "+process +"%"+progressInfo); while ((byteread = bis.read(buffer)) != -1) { bytesum += byteread; out.write(buffer, 0, byteread); long nowProcess = (fileSize+bytesum) /step; if(nowProcess > process){ process = nowProcess; System.out.println("ftp download progress： "+process +"% ("+fileName+","+formatSize(fileSize+bytesum)+"/"+formatSize(lRemoteSize)+")"); } } out.flush(); }catch(SocketTimeoutException e){ return -2; } catch(Exception e) { e.printStackTrace(); return -1; } finally{ try{ if(is!=null){ is.close(); ftpClient.completePendingCommand();//is的close()後面呼叫，避免程式死掉 } if(bis!=null){ bis.close(); } if(out!=null){ out.close(); } }catch(IOException ie){ ie.printStackTrace(); } } return f.length(); } /*** * @上傳資料夾 * @param localDirectory * 當地資料夾 * @param remoteDirectoryPath * Ftp 伺服器路徑 以"/"分隔 (FTP上的資料夾) * */ public boolean uploadDirectory(String localDirectory, String remoteDirectoryPath) { File src = new File(localDirectory); createDir(remoteDirectoryPath); File[] allFile = src.listFiles(); for (int currentFile = 0; currentFile < allFile.length; currentFile++) { if (!allFile[currentFile].isDirectory()) { String srcName = allFile[currentFile].getPath().toString(); uploadFile(srcName, remoteDirectoryPath); } } for (int currentFile = 0; currentFile < allFile.length; currentFile++) { if (allFile[currentFile].isDirectory()) { // 遞迴 String remoteDirPath =remoteDirectoryPath +"/"+allFile[currentFile].getName(); uploadDirectory(allFile[currentFile].getPath().toString(),remoteDirPath); } } return true; } /** * 在當前目錄建立目錄 */ private boolean createDir(String remoteDirectoryPath) { try { if(!this.ftpClient.changeWorkingDirectory(remoteDirectoryPath)){ String[] pathdir = remoteDirectoryPath.split("/"); String tempRemote = ""; for(int i=1;i<pathdir.length;i++){ tempRemote += ("/"+pathdir[i]); if(!this.ftpClient.changeWorkingDirectory(tempRemote)){ this.ftpClient.makeDirectory(tempRemote); } } } return true; } catch (IOException e) { e.printStackTrace(); logger.info(remoteDirectoryPath + "目錄建立失敗"); } return false; } /*** * @下載資料夾 * @param localDirectoryPath本地地址 * @param remoteDirectory 遠端資料夾 * */ public boolean downLoadDirectory(String localDirectoryPath,String remoteDirectory) { try { String fileName = new File(remoteDirectory).getName(); localDirectoryPath = localDirectoryPath + fileName + "//"; new File(localDirectoryPath).mkdirs(); FTPFile[] allFile = this.ftpClient.listFiles(remoteDirectory); for (int currentFile = 0; currentFile < allFile.length; currentFile++) { if (!allFile[currentFile].isDirectory()) { downloadFTP(remoteDirectory+allFile[currentFile].getName(),localDirectoryPath+allFile[currentFile].getName(),allFile[currentFile].getSize()); } } for (int currentFile = 0; currentFile < allFile.length; currentFile++) { if (allFile[currentFile].isDirectory()) { String strremoteDirectoryPath = remoteDirectory + "/"+ allFile[currentFile].getName(); downLoadDirectory(localDirectoryPath,strremoteDirectoryPath); } } } catch (IOException e) { e.printStackTrace(); logger.info("下載資料夾失敗"); return false; } return true; } // FtpClient的Set 和 Get 函式 public FTPClient getFtpClient() { return ftpClient; } public void setFtpClient(FTPClient ftpClient) { this.ftpClient = ftpClient; } public static void main(String[] args) throws IOException, Exception { ApacheFtpUtil ftp =new ApacheFtpUtil("ladsweb.nascom.nasa.gov",21,"anonymous","anonymous"); ftp.connectServer();//".*(h25v05|h25v06).*hdf$" FTPFile[] fileList=ftp.getDataFileList("allData/6/MOD13Q1/2016/145/",".*hdf$"); for(int i=0;i<fileList.length;i++){ FTPFile df = fileList[i]; System.out.println("開始下載檔案："+df.getName()); ftp.downloadFTP("allData/6/MOD13Q1/2016/145/"+df.getName(), "D:\\temp\\"+df.getName(), df.getSize()); } ftp.closeServer(); } }
HTTP斷點續傳
import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.RandomAccessFile; import java.net.HttpURLConnection; import java.net.URL; import java.text.DecimalFormat; public class DownloadHttp { public static String formatSize(Long fileSize) { String formatSize = ""; DecimalFormat df = new DecimalFormat("0.##"); float fsize = Float.parseFloat(fileSize.toString()); if (fileSize / (1024 * 1024 * 1024) > 0) { formatSize = df.format(fsize/ (1024 * 1024 * 1024))+ " GB"; } else if (fileSize / (1024 * 1024) > 0) { formatSize = df.format(fsize/ (1024 * 1024))+ " MB"; } else if (fileSize / (1024) > 0) { formatSize = df.format(fsize/ (1024))+ " KB"; } else { formatSize = fileSize + " 位元組"; } return formatSize; } /** * 斷點續傳HTTP模式資源 * @param srcHttpFile * @param destFile * @return */ public static long httpDownload(String srcHttpFile, String destFile) { srcHttpFile = srcHttpFile.replaceAll("\\\\", "/"); destFile = destFile.replaceAll("\\\\", "/"); // 下載網路檔案 long bytesum = 0; int byteread = 0; InputStream is = null; RandomAccessFile raf = null; HttpURLConnection httpConnection = null; long fileSize = 0; try { String fileName = srcHttpFile.substring(srcHttpFile.lastIndexOf('/')+1); URL url = new URL(srcHttpFile); httpConnection = (HttpURLConnection) url.openConnection(); File nfile = new File(destFile); if(nfile.exists()){ fileSize = nfile.length(); } byte[] buffer = new byte[1204 * 10]; httpConnection.setRequestProperty("User-Agent", "NetFox"); httpConnection.setRequestProperty("RANGE", "bytes=" + fileSize + "-"); if(httpConnection.getContentLength() == 0 ){ System.out.println("http download progress： 100% ("+fileName+","+formatSize(fileSize+bytesum)+")"); return fileSize; } is = httpConnection.getInputStream(); raf = new RandomAccessFile(destFile, "rw"); raf.seek(fileSize); long step = (fileSize + httpConnection.getContentLength())/100; long process= (fileSize + bytesum) /step; System.out.println("http download progress： "+process +"% ("+fileName+","+formatSize(fileSize+bytesum)+")"); while ((byteread = is.read(buffer)) != -1) { bytesum += byteread; raf.write(buffer, 0, byteread); long nowProcess = (fileSize+bytesum) /step; if(nowProcess > process){ process = nowProcess; System.out.println("http download progress： "+process +"% ("+fileName+","+formatSize(fileSize+bytesum)+")"); } } } catch (FileNotFoundException e) { e.printStackTrace(); return -1; } catch (IOException e) { e.printStackTrace(); return -2; } finally { try { if (raf != null) { raf.close(); } if (is != null) { is.close(); } if (httpConnection != null) { httpConnection.disconnect(); } } catch (IOException e) { e.printStackTrace(); } } return bytesum; } public static void main(String[] args) { String src="http://repo.spring.io/libs-release-local/org/springframework/spring/4.1.0.RELEASE/spring-framework-4.1.0.RELEASE-dist.zip"; httpDownload(src,"D:/temp/spring-framework-4.1.0.RELEASE-dist.zip"); } }