在瀏覽器進行大檔案分片上傳（java服務端實現）
阿新 • • 發佈：2018-11-09
最近在做web網盤的系統，網盤最基本的功能便是檔案上傳，但是檔案上傳當遇到大檔案的時候，在web端按傳統方式上傳簡直是災難，所以大檔案上傳可以採用分片上傳的辦法。其主要思路是：1.大檔案上傳時進行分片；2.分片上傳；3.對分片檔案進行合併。
思路比較清晰簡單，但一些問題在於：1.大檔案如何進行分片？2.分片如何進行記錄和儲存？3.如何校驗每個分片檔案的唯一性和順序性？4.如何合併檔案？
對於大檔案如何分片，這個主要是在前端進行解決，在這裡推薦大家用百度的WebUploader來實現前端所需。
對於對分片之後的檔案進行儲存的問題，我採用了臨時檔案儲存的辦法，臨時檔案儲存著每個分塊對應位元組位的狀態。
對於分片檔案的區分，這裡可以採用MD5碼的方式（不清楚MD5碼的可以先查一下），MD5碼簡單理解就像每個檔案的身份證一樣，每個不同的檔案都有自己唯一的MD5碼。
對於合併檔案的時候，前端在對檔案分片之後，在請求服務端合併的時候，請求中要帶上分片序號和大小，伺服器按照請求資料中給的分片序號和每片分塊大小算出開始位置，與讀取到的檔案片段資料，寫入檔案即可。這裡合併後的檔案會儲存倆個路徑，一個是當前網盤目錄下的路徑，一個是真實的永久路徑（目的是為了實現秒傳的功能）。
前端分片的程式碼就不貼了，主要用的百度的WebUploader。
這裡主要貼一些服務端的主要的程式碼
檔案上傳
/** * 上傳檔案 * * @param file 檔案 * @param wholeMd5 檔案整體md5碼 * @param name 檔名 * @param type 檔案型別 * @param lastModifiedDate 上傳時間 * @param size 檔案大小 * @param chunks 檔案分塊數 * @param chunk 正在執行的塊 */ @ApiOperation(value = "檔案上傳", hidden = true) @IgnoreUserToken @ApiResponses({ @ApiResponse(code = 500, response = RestError.class, message = "錯誤") }) @PostMapping(value = "upload") public ResponseEntity<Integer> fileUpload(@ApiParam(name = "檔案") @RequestPart MultipartFile file, @ApiParam(name = "md5") @RequestParam String wholeMd5, @ApiParam(name = "名稱") @RequestParam String name, @ApiParam(name = "型別") @RequestParam String type, @ApiParam(name = "日期") @RequestParam Date lastModifiedDate, @ApiParam(name = "大小") @RequestParam long size, @ApiParam(name = "開始位置") @RequestParam long start, @ApiParam(name = "結束位置") @RequestParam long end, @ApiParam(name = "總分塊數") @RequestParam(name = "chunks", defaultValue = "1") int chunks, @ApiParam(name = "第幾個分塊,從0開始") @RequestParam(name = "chunk", defaultValue = "0") int chunk) { try { log.info("檔案開始上傳"); this.fileServiceImpl.fileUpload(file.getInputStream(), wholeMd5, name, type, lastModifiedDate, size, chunks, chunk, start, end); return ResponseEntity.ok(1); } catch (Exception e) { return new ResponseEntity(RestError.IO_ERROR.setReason(e.getMessage()).toString(), HttpStatus.INTERNAL_SERVER_ERROR); } }
@Override public boolean fileUpload(InputStream fileIS, String wholeMd5, String name, String type, Date lastModifiedDate, long size, int chunks, int chunk, long start, long end) throws Exception { boolean result = false; try { File tempDirFile = new File(fileDir, TEMP_DIR); if (!tempDirFile.exists()) { tempDirFile.mkdirs(); } // 塊目錄資料夾 File wholeMd5FileDirectory = new File(tempDirFile.getAbsolutePath(), wholeMd5); if (!wholeMd5FileDirectory.exists()) { wholeMd5FileDirectory.mkdirs(); } // 塊檔案 File chunkFile = new File(wholeMd5FileDirectory.getAbsolutePath(), chunk + FILE_SEPARATOR + chunks + FILE_EXT); long chunkSize = end - start; if (!chunkFile.exists() || chunkFile.length() != chunkSize) { // 建立新的塊檔案 long startTime = System.currentTimeMillis(); log.info("建立建分片{} - {} ", start, end); int length = StreamUtils.copy(fileIS, new FileOutputStream(chunkFile)); long endTime = System.currentTimeMillis(); log.info("分片上傳耗時{}毫秒", (endTime - startTime)); if (length == (end - start)) { result = true; } } } catch (Exception e) { log.error("檔案上傳出錯{}", e.getCause()); e.printStackTrace(); throw e; } return result; }
檢查檔案的MD5
/**
* 檢查檔案的md5
*
* @param md5 檔案md5
* @param fileSize 檔案大小
* @return
*/
@ApiOperation(value = "檢查檔案的md5")
@GetMapping(value = "checkFileMd5/{md5}/{fileSize}/{md5CheckLength}")
@ApiResponses({
@ApiResponse(code = 500, response = RestError.class, message = "錯誤")
})
public ResponseEntity<Integer> checkFileMd5(@ApiParam("檔案md5碼") @PathVariable String md5,
@ApiParam("檔案大小") @PathVariable long fileSize,
@ApiParam("檔案用來檢查md5的長度") @PathVariable long md5CheckLength) {
try {
log.info("開始檢驗md5[{}],是否存在", md5);
return ResponseEntity.ok(this.fileServiceImpl.checkFileMd5(md5, fileSize, md5CheckLength) ? 1 : 0);
} catch (Exception e) {
return new ResponseEntity(RestError.DATABASE_ERROR.setReason(e.getMessage()).toString(), HttpStatus.INTERNAL_SERVER_ERROR);
}
} @Override
public boolean checkFileMd5(String md5, long fileSize, long md5CheckLength) {
Optional<UploadFileInfo> uploadFileInfo = this.uploadFileDao.findByMd5AndSize(md5, fileSize);
boolean isExist = false;
if (uploadFileInfo.isPresent()) {
File wholeFile = new File(this.fileDir, uploadFileInfo.get().getDfsPath());
if (wholeFile.exists() && wholeFile.length() == fileSize && md5.equals(FileUtils.md5(wholeFile, 0, md5CheckLength))) {
isExist = true;
}
}
log.info("{}的檔案{}存在", md5, isExist ? "" : "不");
return isExist;
}檢查分片是否存在
/**
* 檢查分片是否存在
*
* @param md5
* @param chunk
* @param chunks
* @param chunkStart
* @param chunkEnd
* @return
*/
@ApiOperation(value = "檢查分片是否存在")
@ApiResponses({
@ApiResponse(code = 500, response = RestError.class, message = "錯誤")
})
@GetMapping(value = "checkChunk/{md5}/{blockMd5}/{md5CheckLength}/{chunk}/{chunks}/{chunkStart}/{chunkEnd}")
public ResponseEntity<Integer> checkChunk(@ApiParam("檔案md5碼") @PathVariable String md5,
@ApiParam("分塊檔案md5碼") @PathVariable String blockMd5,
@ApiParam("用來檢測分塊檔案md5碼的長度") @PathVariable long md5CheckLength,
@ApiParam("第幾個分塊,從0開始") @PathVariable int chunk,
@ApiParam("總分塊數") @PathVariable int chunks,
@ApiParam("分塊開始位於的檔案位置") @PathVariable long chunkStart,
@ApiParam("分塊結束位於的檔案位置") @PathVariable long chunkEnd) {
try {
log.info("開始檢驗分片[{}]-[{}]的md5[{}],是否存在", chunk, chunks, blockMd5);
return ResponseEntity.ok(this.fileServiceImpl.checkChunk(md5, blockMd5, md5CheckLength, chunk, chunks, chunkStart, chunkEnd) ? 1 : 0);
} catch (Exception e) {
return new ResponseEntity(RestError.DATABASE_ERROR.setReason(e.getMessage()).toString(), HttpStatus.INTERNAL_SERVER_ERROR);
}
} @Override
public boolean checkChunk(String md5, String blockMd5, long md5CheckLength, int chunk, int chunks, long chunkStart, long chunkEnd) {
boolean isExist = false;
File chunkFile = new File(fileDir, TEMP_DIR + File.separator + md5 + File.separator + chunk + FILE_SEPARATOR + chunks + FILE_EXT);
if (chunkFile.exists() && chunkFile.length() == (chunkEnd - chunkStart)) {
String calBlockMd5 = FileUtils.md5(chunkFile, 0, md5CheckLength);
if (blockMd5.equals(calBlockMd5)) {
isExist = true;
}
}
log.info("{}的{}-{}分塊{}存在", md5, chunk, chunks, isExist ? "" : "不");
return isExist;
}合併檔案
/**
* 合併檔案
*
* @param fileInfo
* @return
*/
@ApiOperation(value = "合併檔案", notes = "把分片上傳的資料合併到一個檔案")
@ApiResponses({
@ApiResponse(code = 500, response = RestError.class, message = "錯誤")
})
@PostMapping(value = "mergeChunks")
public ResponseEntity<Integer> mergeChunks(@Validated @RequestBody FileInfo fileInfo, BindingResult bindingResult) {
log.info("開始合併檔案");
if (bindingResult.hasErrors()) {
log.error("錯誤的引數請求");
return new ResponseEntity("錯誤的引數請求", HttpStatus.BAD_REQUEST);
} else {
try {
DataEntity dataEntity = this.fileServiceImpl.mergeChunks(fileInfo);
log.info("合併檔案完成, 儲存的dataEntityId為:{}", dataEntity != null ? dataEntity.getId() : null);
return ResponseEntity.ok(dataEntity != null ? 1 : 0);
} catch (FileMargeException e) {
log.error(e.getMessage(), e);
return new ResponseEntity(RestError.FILE_MARGE_ERROR.setReason(e.getMessage()).toString(), HttpStatus.INTERNAL_SERVER_ERROR);
} catch (FileNotAllException e) {
log.error(e.getMessage(), e);
return new ResponseEntity(RestError.FILE_NOTALL_ERROR.setReason(e.getMessage()).toString(), HttpStatus.INTERNAL_SERVER_ERROR);
} catch (IOException e) {
log.error(e.getMessage(), e);
return new ResponseEntity(RestError.IO_ERROR.setReason(e.getMessage()).toString(), HttpStatus.INTERNAL_SERVER_ERROR);
}
}
} /**
* 合併檔案
*
* @param fileInfo
* @return {DataEntity}
* @throws FileNotAllException
* @throws IOException
*/
@Override
public DataEntity mergeChunks(FileInfo fileInfo) throws IOException, FileNotAllException, FileMargeException {
// 先檢查庫裡是否有檔案的存記錄
Optional<UploadFileInfo> uploadFileInfoOptional = this.uploadFileDao.findByMd5AndSize(fileInfo.getMd5(), fileInfo.getSize());
log.info("檢查檔案資訊是否在資料庫中存在");
UploadFileInfo uploadFileInfo = null;
if (uploadFileInfoOptional.isPresent()) {
log.info("檔案資訊:{}", fileInfo);
uploadFileInfo = uploadFileInfoOptional.get();
}
if (uploadFileInfo == null) {
uploadFileInfo = new UploadFileInfo();
}
//再檢查檔案是否存在
log.info("檢查真實檔案");
File wholeFile = new File(getRealFileRoot(), fileInfo.getMd5() + FILE_SEPARATOR + fileInfo.getName());
if (!wholeFile.exists() || wholeFile.length() != fileInfo.getSize()) {
log.info("檔案不存在或者檔案長度不符合! }");
if (wholeFile.exists()) {
log.info("長度為{}!={},", wholeFile.length(), fileInfo.getSize());
}
File tempDirFile = new File(fileDir, TEMP_DIR + File.separator + fileInfo.getMd5());
try {
if (tempDirFile.exists()) {
log.info("檔案分片目錄存在");
// 獲取該目錄下所有的碎片檔案
File[] partFiles = tempDirFile.listFiles((f, name) -> name.endsWith(FILE_EXT));
log.info("檔案分片個數為:", partFiles.length);
if (partFiles.length > 0) {
Arrays.sort(partFiles, (File f1, File f2) -> {
String name1 = f1.getName();
String name2 = f2.getName();
if (name1.length() < name2.length()) {
return -1;
} else if (name1.length() > name2.length()) {
return 1;
} else {
return name1.compareTo(name2);
}
});
long size = 0;
FileChannel resultFileChannel = new FileOutputStream(wholeFile, true).getChannel();
for (int i = 0; i < partFiles.length; i++) {
size += partFiles[i].length();
if (size > wholeFile.length()) {
log.info("合併第{}塊的檔案{}", i, partFiles[i].getName());
// FileUtils.copy(partFiles[i], wholeFile, size);
FileChannel inChannel = new FileInputStream(partFiles[i]).getChannel();
resultFileChannel.transferFrom(inChannel, resultFileChannel.size(), inChannel.size());
inChannel.close();
}
}
if (size < wholeFile.length()) {
log.info("分片檔案不完整");
throw new FileNotAllException();
}
}
log.info("刪除分片資料資訊");
this.threadPoolUtil.getExecutor().execute(() -> {
tempDirFile.listFiles(child -> child.delete());
tempDirFile.delete();
});
}
} catch (Exception e) {
throw new FileMargeException();
}
}
if (uploadFileInfo.getId() == null) {
log.info("儲存上傳的檔案資訊");
uploadFileInfo.setCreateTime(fileInfo.getCreateTime());
uploadFileInfo.setMd5(fileInfo.getMd5());
uploadFileInfo.setType(fileInfo.getType());
uploadFileInfo.setSize(wholeFile.length());
uploadFileInfo.setDfsPath(wholeFile.getAbsolutePath().substring(this.fileDir.length()+1));
this.uploadFileDao.save(uploadFileInfo);
}
// 檔案大小, 應該在合併完成的時候更新
log.info("獲取父目錄資訊");
DataEntity parent = this.getDataEntityById(fileInfo.getParentId());
// 如果檔案資訊裡包含檔案的相對路徑, 就應該建立檔案上傳的真實目錄
String path = fileInfo.getPath();
if (StringUtils.hasText(path)) {
log.info("包含相對目錄,進行相對目錄的建立");
path = FilenameUtils.getFullPathNoEndSeparator(path);
String[] paths = path.split("/");
for (String tempPath : paths) {
if (StringUtils.hasText(tempPath)) {
DataEntity dataEntity = this.dataEntityDao.findByNameAndParentAndUserId(tempPath, parent, UserUtil.getUserId());
if (dataEntity == null) {
dataEntity = new DataEntity();
dataEntity.setName(tempPath);
dataEntity.setDir(true);
dataEntity.setParent(parent);
parent = this.dataEntityDao.save(dataEntity);
} else {
parent = dataEntity;
}
}
}
}
log.info("建立目錄資訊");
DataEntity dataEntity = new DataEntity();
dataEntity.setName(fileInfo.getName());
dataEntity.setExt(fileInfo.getExt());
dataEntity.setDataType(fileInfo.getFileType());
dataEntity.setFileInfo(uploadFileInfo);
dataEntity.setParent(parent);
dataEntity.setSize(uploadFileInfo.getSize());
dataEntity = this.saveAndRenameFile(dataEntity);
this.saveAndCreateFile(dataEntity);
//判斷上傳檔案的型別，選擇呼叫解析介面
String fileType = fileInfo.getFileType();
if ("images".equals(fileType)||"vector".equals(fileType)||"terrain".equals(fileType)||"original".equals(fileType)) {
String resultInfo = analysis(dataEntity,fileInfo);
log.info("解析結果："+resultInfo);
}
return dataEntity;
}
關於秒傳功能，其實原理就是檢驗檔案MD5，在一個檔案上傳前先獲取檔案內容MD5值或者部分取值MD5，然後在查詢自己的記錄是否已存在相同的MD5，如果存在就直接從伺服器真實路徑取，而不需要重新進行分片上傳了，從而達到秒傳的效果。