Java POI大資料量的Excel匯入匯出
1. 大資料量的匯入
當Excel中的資料量超過10萬行時，在用POI讀取檔案流時很容易引起失敗，需要引入xlsx-streamer來進行資源的開啟，剩下的處理同POI處理上百行資料量類似：filePath=>FileInputStream=>Workbook=>Sheet=>Row=>Cell=>Java反射獲取值。
<dependency> <groupId>com.monitorjbl</groupId> <artifactId>xlsx-streamer</artifactId> <version>1.2.1</version> </dependency>
/** * 主要針對超過10萬行要解析的Excel的Workbook獲取； * * 快取到記憶體中的行數，預設是10行； * * 讀取資源時快取到記憶體的位元組大小，預設是1024； * * 資源必須開啟，InputStream或者File都可以，但是隻能開啟XLSX格式的檔案 * * @param filePath * @return * @throws Exception */ public static Workbook obtainWorkbookByStreamm(String filePath) throws Exception { // StreamingReader用於讀取Excel的內容，不能寫入，不能隨機讀取Excel的內容 FileInputStream in = new FileInputStream(filePath); Workbook workbook = StreamingReader.builder().rowCacheSize(100).bufferSize(4096).open(in); return workbook; }
2. 大資料量的匯出
分而治之的思想，將大資料量分批次的匯出到多個Excel檔案或者單個Excel檔案的多個sheet，也可匯出到多個Excel檔案後合併從單獨的檔案。
/** * 可選擇是分批次Excel匯出和多Sheet匯出; * * @author LEric */ public abstract class HugeDataExcelExportUtil {
public void handleMulti(HugeDataExcelOutputDomain domain) throws IOException { int totalRowNums = domain.getTotalRowNums(); int filterRowNums = domain.getFilterRowNums(); String fileName = new String(domain.getFileName().getBytes("ISO-8859-1"), Charset.defaultCharset()); String filePath = domain.getFilePath();
File file = new File(filePath); if (!file.exists()) { file.mkdirs(); } if (filterRowNums >= totalRowNums) { filterRowNums = totalRowNums; } int tempsize = (totalRowNums % filterRowNums) == 0 ? totalRowNums / filterRowNums : totalRowNums / filterRowNums + 1;
String tempExcelFile = filePath + fileName + ".xlsx"; FileOutputStream fos = new FileOutputStream(tempExcelFile); // 在記憶體當中保持 100 行 , 超過的資料放到硬碟中 SXSSFWorkbook workbook = new SXSSFWorkbook(100);
for (int i = 0; i < tempsize; i++) { Map<String, Object> params = new HashMap<>(); params.put("limit", i * filterRowNums); if (i == (totalRowNums / filterRowNums)) { params.put("offset", totalRowNums); } else { params.put("offset", (i + 1) * filterRowNums); } List<Map<String, Object>> dataList = getData(params); workbook = exportDataToExcel(workbook, dataList, i); }
try { workbook.write(fos); fos.flush(); } catch (Exception e) { throw new ExcelHandleException(e); } finally { if (Objects.nonNull(fos)) { fos.close(); } if (Objects.nonNull(workbook)) { workbook.dispose(); } } }
@SuppressWarnings("resource") public void handle(HugeDataExcelOutputDomain domain) throws IOException { int totalRowNums = domain.getTotalRowNums(); int filterRowNums = domain.getFilterRowNums(); String fileName = new String(domain.getFileName().getBytes("ISO-8859-1"), Charset.defaultCharset()); String filePath = domain.getFilePath();
File file = new File(filePath); if (!file.exists()) { file.mkdirs(); } if (filterRowNums >= totalRowNums) { filterRowNums = totalRowNums; } int tempsize = (totalRowNums % filterRowNums) == 0 ? totalRowNums / filterRowNums : totalRowNums / filterRowNums + 1;
for (int i = 0; i < tempsize; i++) { Map<String, Object> params = new HashMap<>(); params.put("limit", i * filterRowNums); if (i == (totalRowNums / filterRowNums)) { params.put("offset", totalRowNums); } else { params.put("offset", (i + 1) * filterRowNums); } List<Map<String, Object>> dataList = getData(params); String tempExcelFile = filePath + fileName + "[" + (i + 1) + "].xlsx"; FileOutputStream fos = new FileOutputStream(tempExcelFile);
// 在記憶體當中保持 100 行 , 超過的資料放到硬碟中 SXSSFWorkbook workbook = new SXSSFWorkbook(100); try { workbook = exportDataToExcel(workbook, dataList, i); workbook.write(fos); fos.flush(); } catch (Exception e) { throw new ExcelHandleException(e); } finally { if (Objects.nonNull(fos)) { fos.close(); } if (Objects.nonNull(workbook)) { workbook.dispose(); } } } }
public abstract List<Map<String, Object>> getData(Map<String, Object> params);
public abstract Map<String, Object> getHeaderMetadata();
/** * 可利用反射動態匹配對映實現 * * @param workbook * @param dataList * @param size * @return */ public SXSSFWorkbook exportDataToExcel(SXSSFWorkbook workbook, List<Map<String, Object>> dataList, int size) { Map<String, Object> objects = getHeaderMetadata(); String[] headNames = (String[]) objects.get("headNames"); String[] headEngNames = (String[]) objects.get("headEngNames"); String sheetName = (String) objects.get("sheetName");
Cell cell = null; Sheet sheet = workbook.createSheet(sheetName); Row row = sheet.createRow(0); for (int i = 0; i < headNames.length; i++) { cell = row.createCell(i); cell.setCellValue(headNames[i]); }
if (Objects.nonNull(dataList) && dataList.size() > 0 && !dataList.isEmpty()) { int rowIndex = 1; for (Map<String, Object> map : dataList) { row = sheet.createRow(rowIndex++); int index = 0; for (int i = 0; i < headEngNames.length; i++) { cell = row.createCell(index++); cell.setCellType(CellType.STRING); cell.setCellValue(("" + map.get(headEngNames[i])).replace("null", "")); } } } return workbook; } }