專案案例 || 將Excel資料批量匯入到資料庫
阿新 • • 發佈:2019-01-08
你在工作中是否遇到這樣的問題?資料一般存放在Excel表中,逐條遷移到資料庫中太麻煩,而且很多時候企業的資料量都是以萬起步,單條匯入顯然不現實。那麼該如何解決呢?
我們今天就給大家介紹一個用途非常廣泛的功能:批量匯入,在很多系統中,這也是必須實現的功能。而且當Excel表結構越複雜時,實現的難度就越高。
不管專案如何複雜,原理卻基本相同,一般是前臺頁面選擇Excel檔案,後臺獲取後進行資料轉化,然後迴圈執行Sql語句即可,瞭解這些原理後,一切也就變得簡單。
下面為了重點演示,專案進行簡化,只有批量匯入功能,採用Struts2框架,將Excel資料匯入到Mysql資料庫中。
專案結構如下圖:
Mysql資料庫user表結構
Excel中資料
如果Excel中必填項資料為空,提示匯入失敗,報告錯誤資訊及位置,專案演示圖:
如果資料正確,提示匯入資料庫成功:
具體實現過程
首先是JSP頁面,name的名稱是重點:
<form action="${pageContext.request.contextPath }/excelsave.action" method="post" enctype="multipart/form-data"> <input type="file" name="fileinput" multiple="multiple"/> <button type="submit" name="subimit">上傳</button> </form>
前臺點選上傳後跳轉到action處理,action中首先定義:
//這裡特別注意獲取fileinput要和頁面的匯入時name屬性一致。
private File fileinput;
//檔名用得到就獲取,一般用不到。
private String fileinputFileName;
//下面是get和set方法,省略
然後在方法體中直接呼叫:
//引數為獲取到的檔案
ExcelUtil(fileinput);
ExcelUtil是處理Excel工具類,直接使用,程式碼如下:
/**解析excel 工具類*/ @SuppressWarnings("rawtypes") public class ExcelUtil { public FileInputStream fis ; public HSSFWorkbook workBook; public HSSFSheet sheet; public XMLUtil parseXmlUtil; public StringBuffer errorString; /**當前實體類的code**/ public String curEntityCode; /**表頭map物件:key:entityCode, value:headMap(index,headTitle)**/ public Map curEntityHeadMap ; /**欄位的必填:key:entityCode+headTitle, value:true(必填),false(不必填)**/ public Map curEntityColRequired; /**存放每一行的資料**/ public List listDatas ; public ExcelUtil(File excelFile){ try { if(excelFile == null){ throw new FileNotFoundException(); } fis = new FileInputStream(excelFile); workBook = new HSSFWorkbook(fis); parseXmlUtil = new XMLUtil(); errorString = new StringBuffer(); readExcelData(); } catch (FileNotFoundException e) { e.printStackTrace(); }catch (IOException e) { e.printStackTrace(); } } /**開始從excel讀取資料**/ public void readExcelData(){ int sheetSize = workBook.getNumberOfSheets(); for(int i=0;i<sheetSize;i++){ sheet = workBook.getSheetAt(i); String entityName = workBook.getSheetName(i); readSheetData(sheet,entityName); } } /**讀每個sheet頁的資料**/ public void readSheetData(HSSFSheet sheet,String entityName){ int rowNumbers = sheet.getPhysicalNumberOfRows(); Map ent = (Map) parseXmlUtil.getEntityMap().get(entityName); this.setCurEntityCode((String) ent.get("code")); if(rowNumbers == 0){ System.out.println("excel中資料為空!"); errorString.append(Constans.ERROR_EXCEL_NULL); } List colList = (List) parseXmlUtil.getColumnListMap().get(entityName); int xmlRowNum = colList.size(); HSSFRow excelRow = sheet.getRow(0); int excelFirstRow = excelRow.getFirstCellNum(); int excelLastRow = excelRow.getLastCellNum(); if(xmlRowNum != (excelLastRow-excelFirstRow)){ System.out.println("xml列數與excel列數不相符,請檢查"); errorString.append(Constans.ERROR_EXCEL_COLUMN_NOT_EQUAL); } readSheetHeadData(sheet); readSheetColumnData(sheet,entityName); } /**讀取sheet頁中的表頭資訊**/ @SuppressWarnings({ "unchecked", "static-access"}) public void readSheetHeadData(HSSFSheet sheet){ Map headMap = new HashMap(); curEntityHeadMap = new HashMap(); curEntityColRequired = new HashMap(); HSSFRow excelheadRow = sheet.getRow(0); int excelLastRow = excelheadRow.getLastCellNum(); String headTitle = ""; for(int i=0;i<excelLastRow;i++){ HSSFCell cell = excelheadRow.getCell(i); headTitle = this.getStringCellValue(cell).trim(); if(headTitle.endsWith("*")){ curEntityColRequired.put(this.getCurEntityCode()+"_"+headTitle,true); }else{ curEntityColRequired.put(this.getCurEntityCode()+"_"+headTitle,false); } headMap.put(i, headTitle); } curEntityHeadMap.put(this.getCurEntityCode(), headMap); } /**讀取sheet頁裡面的資料**/ @SuppressWarnings({ "unchecked", "static-access" }) public void readSheetColumnData(HSSFSheet sheet,String entityName){ HSSFRow excelheadRow = sheet.getRow(0); int excelLastcell = excelheadRow.getLastCellNum(); //excel總列數 int excelRowNum = sheet.getLastRowNum(); //excel總行數 Map headMap = (Map) this.getCurEntityHeadMap().get(this.getCurEntityCode()); Map colMap = parseXmlUtil.getColumnMap(); listDatas =new ArrayList(); for(int i=1;i<excelRowNum+1;i++){//行迴圈 HSSFRow columnRow = sheet.getRow(i); if(columnRow != null){ Map curRowCellMap = new HashMap(); for(int j =0; j<excelLastcell;j++){ //列迴圈 int cout = headMap.get(j).toString().indexOf("*"); String headTitle =""; if(cout == -1){ headTitle = headMap.get(j).toString(); }else{ headTitle = headMap.get(j).toString().substring(0, cout); } Map curColMap = (Map) colMap.get(entityName+"_"+headTitle); String curColCode = (String) curColMap.get("code"); String curColType = (String) curColMap.get("type"); HSSFCell colCell = columnRow.getCell(j); String value =this.getStringCellValue(colCell); if(value != null){ value = value.trim(); } String xmlColType = (String) curColMap.get("type"); int intVal = 0; if(xmlColType.equals("int")){ if(value != null) { intVal = Integer.valueOf(value); } curRowCellMap.put(curColCode, intVal); //將這一行的資料以code-value的形式存入map }else{ curRowCellMap.put(curColCode, value); } /**驗證cell資料**/ validateCellData(i+1,j+1,colCell,entityName,headTitle,curColType,listDatas); } listDatas.add(curRowCellMap); } } if(this.getErrorString().length() ==0){//如果沒有任何錯誤,就儲存 saveExcelData(entityName); System.out.println("匯入資料庫成功"); }else{ //清理所有的快取clearMap();現在暫時未清理 String[] strArr = errorString.toString().split("<br>"); for(String s: strArr){ System.out.println(s); } } } /**驗證單元格資料**/ @SuppressWarnings("static-access") public void validateCellData(int curRow,int curCol,HSSFCell colCell,String entityName,String headName,String curColType,List listDatas){ List rulList = (List) parseXmlUtil.getColumnRulesMap().get(entityName+"_"+headName); if(rulList != null && rulList.size()>0){ for(int i=0 ; i<rulList.size() ; i++){ Map rulM = (Map) rulList.get(i); String rulName = (String) rulM.get("name"); String rulMsg = (String) rulM.get("message"); String cellValue = ""; if(this.getStringCellValue(colCell)==null) { //System.out.println("第"+curRow+"行,第"+curCol+"列:"+rulMsg); }else { cellValue = this.getStringCellValue(colCell).trim(); } if(rulName.equals(Constans.RULE_NAME_NULLABLE)){ if(cellValue.equals("")||cellValue == null){ errorString.append("匯入失敗,錯誤資訊:第"+curRow+"行,第"+curCol+"列:"+rulMsg+"<br>"); } } //////這裡寫其他的驗證規則。。。 } } } /**儲存excel裡面的資料**/ @SuppressWarnings("unchecked") public void saveExcelData(String entityName){ List<User> users= new ArrayList(); for(int i = 0 ; i<this.getListDatas().size();i++){ Map excelCol = (Map) this.getListDatas().get(i); //得到第 i 行的資料 User user = new User(); try { User obj = (User) BeanToMapUtil.convertMap(user.getClass(), excelCol); users.add(obj); } catch (IntrospectionException e) { e.printStackTrace(); } catch (IllegalAccessException e) { e.printStackTrace(); } catch (InstantiationException e) { e.printStackTrace(); } catch (InvocationTargetException e) { e.printStackTrace(); } } /**批量儲存資料**/ Dao dao = new Dao(); for(int i = 0;i<users.size();i++){ try{ dao.saveUser(users.get(i)); }catch(Exception e){ e.printStackTrace(); } } } /** * 獲得單元格字串 * @throws UnSupportedCellTypeException */ public static String getStringCellValue(HSSFCell cell) { if (cell == null){ return null; } String result = ""; switch (cell.getCellType()) { case HSSFCell.CELL_TYPE_BOOLEAN: result = String.valueOf(cell.getBooleanCellValue()); break; case HSSFCell.CELL_TYPE_NUMERIC: if (HSSFDateUtil.isCellDateFormatted(cell)) { java.text.SimpleDateFormat TIME_FORMATTER = new java.text.SimpleDateFormat( "yyyy-MM-dd"); result = TIME_FORMATTER.format(cell.getDateCellValue()); } else{ double doubleValue = cell.getNumericCellValue(); result = "" + doubleValue; } break; case HSSFCell.CELL_TYPE_STRING: if (cell.getRichStringCellValue() == null){ result = null; } else{ result = cell.getRichStringCellValue().getString(); } break; case HSSFCell.CELL_TYPE_BLANK: result = null; break; case HSSFCell.CELL_TYPE_FORMULA: try{ result = String.valueOf(cell.getNumericCellValue()); }catch(Exception e){ result = cell.getRichStringCellValue().getString(); } break; default: result = ""; } return result; } public String getCurEntityCode() { return curEntityCode; } public void setCurEntityCode(String curEntityCode) { this.curEntityCode = curEntityCode; } public Map getCurEntityHeadMap() { return curEntityHeadMap; } public void setCurEntityHeadMap(Map curEntityHeadMap) { this.curEntityHeadMap = curEntityHeadMap; } public XMLUtil getParseXmlUtil() { return parseXmlUtil; } public void setParseXmlUtil(XMLUtil parseXmlUtil) { this.parseXmlUtil = parseXmlUtil; } public Map getCurEntityColRequired() { return curEntityColRequired; } public void setCurEntityColRequired(Map curEntityColRequired) { this.curEntityColRequired = curEntityColRequired; } public List getListDatas() { return listDatas; } public void setListDatas(List listDatas) { this.listDatas = listDatas; } public StringBuffer getErrorString() { return errorString; } public void setErrorString(StringBuffer errorString) { this.errorString = errorString; } }
專案中定義一個XML檔案,主要做一些條件限制,比如使用者名稱不能為空,email不能重複等,所以就有一個XML解析類:
/**解析xml工具類*/
@SuppressWarnings("rawtypes")
public class XMLUtil {
/**entity map物件,key:name ,value:entity的屬性map集**/
public Map entityMap ;
/**column map 物件,key:entityName_colName , value:column的屬性map集 **/
public Map columnMap;
/**rule map 物件,key:entityName_colName_ruleName, value: rule 的map集:找到一行rule**/
public Map ruleMap ;
/**rules map 物件, key:entityName_colName, value: rules 的map集:找到該column下所有的rule**/
public Map columnRulesMap ;
/**entity--column map: key:entityName, value: column list:根據實體類名得到所有的列**/
public Map columnListMap ;
/**column list**/
public List columnList ;
/**開始解析xml檔案**/
public XMLUtil(){
SAXReader reader = new SAXReader();
InputStream in = getClass().getClassLoader().getResourceAsStream("user.xml");//讀取檔案流,Url為controller.xml檔案
try {
Document doc = reader.read(in);//獲得檔案例項
Element root = doc.getRootElement();
Iterator itEntity = root.elements("entity").iterator();
while(itEntity.hasNext()){
Element entity = (Element) itEntity.next();
parseEntity(entity);
}
/**測試entityMap 是否正確**/
Map enMap = (Map) this.getEntityMap().get("使用者表");
Set<?> set = enMap.keySet();
Iterator it = set.iterator();
while(it.hasNext()){
String uu = (String) it.next();
}
}catch(Exception e){
e.printStackTrace();
}
}
/**開始解析entity**/
@SuppressWarnings("unchecked")
public void parseEntity(Element entity){
if(entity != null){
/**對資料進行初始化設定**/
columnListMap = new HashMap();
columnMap = new HashMap();
entityMap = new HashMap();
ruleMap = new HashMap();
columnRulesMap = new HashMap();
columnList = new ArrayList();
setEntityMap(entity);
String entityName = entity.attributeValue("name");
Iterator itColumn = entity.elements("column").iterator();
while(itColumn.hasNext()){
Element column = (Element) itColumn.next();
setColumnMap(entityName,column);
}
columnListMap.put(entityName, columnList);
}
}
/**將entity放入entityMap中**/
@SuppressWarnings("unchecked")
public void setEntityMap(Element entity){
Map ent = new HashMap();
String name = entity.attributeValue("name");
String code = entity.attributeValue("code");
ent.put("name", name);
ent.put("code", code);
entityMap.put(name, ent);
}
/**將column放入columnMap中**/
@SuppressWarnings("unchecked")
public void setColumnMap(String entityName,Element column){
if(column != null){
Map col = new HashMap();
String name = column.attributeValue("name");
String code = column.attributeValue("code");
String type = column.attributeValue("type");
col.put("name", name);
col.put("code", code);
col.put("type", type);
String columnMapKey = entityName+"_"+name; //eg: 使用者表_使用者名稱
columnMap.put(columnMapKey, col);
columnList.add(col);
Iterator ruleIt = column.elements("rules").iterator(); //獲得rules
while(ruleIt.hasNext()){
Element rules = (Element)ruleIt.next();
Iterator rule = rules.elements("rule").iterator(); //獲得 rule
while(rule.hasNext()){
Element ruleValid = (Element) rule.next(); //獲得每一行rule
setRuleMap(entityName,name,ruleValid);
}
}
}
}
/**將 rule 驗證規則放入ruleMap中**/
@SuppressWarnings("unchecked")
public void setRuleMap(String entityName,String columnName,Element ruleValid){
if(ruleValid != null){
String ruleName = ruleValid.attributeValue("name");
String ruleMsg = ruleValid.attributeValue("message");
Map ruleValidMap = new HashMap();
ruleValidMap.put("name", ruleName);
ruleValidMap.put("message", ruleMsg);
String ruleStrKey = entityName+"_"+columnName+"_"+ruleName;
String colStrKey = entityName+"_"+columnName;
if(this.getColumnRulesMap().containsKey(colStrKey)){
List valids = (List) this.getColumnRulesMap().get(colStrKey);
valids.add(ruleValidMap);
}else{
List valids = new ArrayList();
valids.add(ruleValidMap);
this.columnRulesMap.put(colStrKey, valids); //將每個column下的所有rules存入該map中
}
ruleMap.put(ruleStrKey, ruleValidMap); //將每個column下的一條rule存入該map中
}
}
/**所有的get set 方法**/
public Map getEntityMap() {
return entityMap;
}
public void setEntityMap(Map entityMap) {
this.entityMap = entityMap;
}
public Map getColumnMap() {
return columnMap;
}
public void setColumnMap(Map columnMap) {
this.columnMap = columnMap;
}
public Map getRuleMap() {
return ruleMap;
}
public void setRuleMap(Map ruleMap) {
this.ruleMap = ruleMap;
}
public Map getColumnRulesMap() {
return columnRulesMap;
}
public void setColumnRulesMap(Map columnRulesMap) {
this.columnRulesMap = columnRulesMap;
}
public Map getColumnListMap() {
return columnListMap;
}
public void setColumnListMap(Map columnListMap) {
this.columnListMap = columnListMap;
}
}
XML解析類定義完成後,ExcelUtil會呼叫處理,處理完成後返回,匯入結束。
至於資料庫連線,任何方式都行,這裡不做要求,另外需要定義一個實體User,屬性根據自己的業務設定。
以上就是具體實現過程,如果你有任何問題,歡迎留言,我們共同交流討論。
還可以微信關注和置頂我的公眾號“SL社群”(slshequ),獲取原始碼。