阿里雲OSS 分塊上傳的程式碼整理
阿新 • • 發佈：2018-12-23
廢話不多說，把程式碼貼出來。
檔案上傳程式碼：package com.changba.erp.utils; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.web.multipart.MultipartFile; import com.aliyun.oss.OSSClient; import com.aliyun.oss.model.CompleteMultipartUploadRequest; import com.aliyun.oss.model.InitiateMultipartUploadRequest; import com.aliyun.oss.model.InitiateMultipartUploadResult; import com.aliyun.oss.model.ListPartsRequest; import com.aliyun.oss.model.PartETag; import com.aliyun.oss.model.PartListing; import com.aliyun.oss.model.PartSummary; import com.aliyun.oss.model.UploadPartRequest; import com.aliyun.oss.model.UploadPartResult; public class AliyunOSSUpload implements Runnable { private MultipartFile localFile; private long startPos; private long partSize; private int partNumber; private String uploadId; private static String key; private static String bucketName; // 新建一個List儲存每個分塊上傳後的ETag和PartNumber protected static List<PartETag> partETags = Collections.synchronizedList(new ArrayList<PartETag>()); private static Logger logger = LoggerFactory.getLogger(FileUploader.class); protected static OSSClient client = null; /** * 建立構造方法 * * @param localFile * 要上傳的檔案 * @param startPos * 每個檔案塊的開始 * @param partSize * @param partNumber * @param uploadId * 作為塊的標識 * @param key * 上傳到OSS後的檔名 */ public AliyunOSSUpload(MultipartFile localFile, long startPos, long partSize, int partNumber, String uploadId, String key , String bucketName) { this.localFile = localFile; this.startPos = startPos; this.partSize = partSize; this.partNumber = partNumber; this.uploadId = uploadId; AliyunOSSUpload.key = key; AliyunOSSUpload.bucketName = bucketName; } /** * 分塊上傳核心方法(將檔案分成按照每個5M分成N個塊，並加入到一個list集合中) */ @Override public void run() { InputStream instream = null; try { // 獲取檔案流 instream = localFile.getInputStream(); // 跳到每個分塊的開頭 instream.skip(this.startPos); // 建立UploadPartRequest，上傳分塊 UploadPartRequest uploadPartRequest = new UploadPartRequest(); uploadPartRequest.setBucketName(bucketName); uploadPartRequest.setKey(key); uploadPartRequest.setUploadId(this.uploadId); uploadPartRequest.setInputStream(instream); uploadPartRequest.setPartSize(this.partSize); uploadPartRequest.setPartNumber(this.partNumber); UploadPartResult uploadPartResult = FileUploader.client.uploadPart(uploadPartRequest); logger.info("Part#" + this.partNumber + " done\n"); synchronized (partETags) { // 將返回的PartETag儲存到List中。 partETags.add(uploadPartResult.getPartETag()); } } catch (Exception e) { e.printStackTrace(); } finally { if (instream != null) { try { // 關閉檔案流 instream.close(); } catch (IOException e) { e.printStackTrace(); } } } } /** * 初始化分塊上傳事件並生成uploadID，用來作為區分分塊上傳事件的唯一標識 * * @return */ protected static String claimUploadId(String bucketName, String key) { InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, key); InitiateMultipartUploadResult result = FileUploader.client.initiateMultipartUpload(request); logger.info(result.getUploadId()); return result.getUploadId(); } /** * 將檔案分塊進行升序排序並執行檔案上傳。 * * @param uploadId */ protected static void completeMultipartUpload(String uploadId) { // 將檔案分塊按照升序排序 Collections.sort(partETags, new Comparator<PartETag>() { @Override public int compare(PartETag p1, PartETag p2) { return p1.getPartNumber() - p2.getPartNumber(); } }); CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETags); // 完成分塊上傳 FileUploader.client.completeMultipartUpload(completeMultipartUploadRequest); } /** * 列出檔案所有分塊的清單 * * @param uploadId */ protected static void listAllParts(String uploadId) { ListPartsRequest listPartsRequest = new ListPartsRequest(bucketName, key, uploadId); // 獲取上傳的所有分塊資訊 PartListing partListing = FileUploader.client.listParts(listPartsRequest); // 獲取分塊的大小 int partCount = partListing.getParts().size(); // 遍歷所有分塊 for (int i = 0; i < partCount; i++) { PartSummary partSummary = partListing.getParts().get(i); logger.info("分塊編號 " + partSummary.getPartNumber() + ", ETag=" + partSummary.getETag()); } } }
package com.changba.erp.utils; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.web.multipart.MultipartFile; import com.aliyun.oss.OSSClient; public class FileUploader { private static String endpoint = "http://oss-cn-hangzhou.aliyuncs.com"; private static String accessKeyId = "FAsc5DqUOpCMZqv8"; private static String accessKeySecret = "ewbsDctAbvNQbBl4d66mcoYgNC9N9T"; private static String bucketName = "mysong"; protected static OSSClient client = null; private static Logger logger = LoggerFactory.getLogger(FileUploader.class); public static String fileUpload(MultipartFile file) { // 建立一個可重用固定執行緒數的執行緒池。若同一時間執行緒數大於10，則多餘執行緒會放入佇列中依次執行 ExecutorService executorService = Executors.newFixedThreadPool(10); String key = file.getOriginalFilename(); // 獲取上傳檔案的名稱，作為在OSS上的檔名 // 建立OSSClient例項 client = new OSSClient(endpoint, accessKeyId, accessKeySecret); try { String uploadId = AliyunOSSUpload.claimUploadId(bucketName, key); // 設定每塊為 5M(除最後一個分塊以外，其他的分塊大小都要大於5MB) final long partSize = 5 * 1024 * 1024L; // 計算分塊數目 long fileLength = file.getSize(); int partCount = (int) (fileLength / partSize); if (fileLength % partSize != 0) { partCount++; } // 分塊 號碼的範圍是1~10000。如果超出這個範圍，OSS將返回InvalidArgument的錯誤碼。 if (partCount > 10000) { throw new RuntimeException("檔案過大(分塊大小不能超過10000)"); } else { logger.info("一共分了 " + partCount + " 塊"); } /** * 將分好的檔案塊加入到list集合中 */ for (int i = 0; i < partCount; i++) { long startPos = i * partSize; long curPartSize = (i + 1 == partCount) ? (fileLength - startPos) : partSize; // 執行緒執行。將分好的檔案塊加入到list集合中 executorService.execute(new AliyunOSSUpload(file, startPos, curPartSize, i + 1, uploadId, key, bucketName)); } /** * 等待所有分片完畢 */ // 關閉執行緒池（執行緒池不馬上關閉），執行以前提交的任務，但不接受新任務。 executorService.shutdown(); // 如果關閉後所有任務都已完成，則返回 true。 while (!executorService.isTerminated()) { try { // 用於等待子執行緒結束，再繼續執行下面的程式碼 executorService.awaitTermination(5, TimeUnit.SECONDS); } catch (InterruptedException e) { e.printStackTrace(); } } /** * partETags(上傳塊的ETag與塊編號（PartNumber）的組合) 如果校驗與之前計算的分塊大小不同，則丟擲異常 */ System.out.println(AliyunOSSUpload.partETags.size() +" ----- "+partCount); if (AliyunOSSUpload.partETags.size() != partCount) { throw new IllegalStateException("OSS分塊大小與檔案所計算的分塊大小不一致"); } else { logger.info("將要上傳的檔名 " + key + "\n"); } /* * 列出檔案所有的分塊清單並列印到日誌中，該方法僅僅作為輸出使用 */ AliyunOSSUpload.listAllParts(uploadId); /* * 完成分塊上傳 */ AliyunOSSUpload.completeMultipartUpload(uploadId); // 返回上傳檔案的URL地址 return endpoint + "/" + bucketName + "/" + client.getObject(bucketName, key).getKey(); } catch (Exception e) { logger.error("上傳失敗！", e); return "上傳失敗！"; } finally { AliyunOSSUpload.partETags.clear(); if (client != null) { client.shutdown(); } } } }