1. 程式人生 > >專案案例 || 將Excel資料批量匯入到資料庫

專案案例 || 將Excel資料批量匯入到資料庫

你在工作中是否遇到這樣的問題?資料一般存放在Excel表中,逐條遷移到資料庫中太麻煩,而且很多時候企業的資料量都是以萬起步,單條匯入顯然不現實。那麼該如何解決呢?

我們今天就給大家介紹一個用途非常廣泛的功能:批量匯入,在很多系統中,這也是必須實現的功能。而且當Excel表結構越複雜時,實現的難度就越高。

不管專案如何複雜,原理卻基本相同,一般是前臺頁面選擇Excel檔案,後臺獲取後進行資料轉化,然後迴圈執行Sql語句即可,瞭解這些原理後,一切也就變得簡單。

下面為了重點演示,專案進行簡化,只有批量匯入功能,採用Struts2框架,將Excel資料匯入到Mysql資料庫中。

專案結構如下圖:

Mysql資料庫user表結構

Excel中資料

如果Excel中必填項資料為空,提示匯入失敗,報告錯誤資訊及位置,專案演示圖:

如果資料正確,提示匯入資料庫成功:


具體實現過程


首先是JSP頁面,name的名稱是重點:

<form action="${pageContext.request.contextPath }/excelsave.action" method="post" enctype="multipart/form-data">
    <input type="file" name="fileinput" multiple="multiple"/>
    <button type="submit" name="subimit">上傳</button>
</form>

前臺點選上傳後跳轉到action處理,action中首先定義:

//這裡特別注意獲取fileinput要和頁面的匯入時name屬性一致。
private File fileinput;
//檔名用得到就獲取,一般用不到。
private String fileinputFileName;
//下面是get和set方法,省略

然後在方法體中直接呼叫:

//引數為獲取到的檔案
ExcelUtil(fileinput);

ExcelUtil是處理Excel工具類,直接使用,程式碼如下:

/**解析excel 工具類*/
@SuppressWarnings("rawtypes")
public class ExcelUtil {
	
	public FileInputStream fis ;
	public HSSFWorkbook workBook;
	public HSSFSheet sheet;
	public XMLUtil parseXmlUtil;
	public StringBuffer errorString;
	
	/**當前實體類的code**/
	public String curEntityCode;
	
	/**表頭map物件:key:entityCode, value:headMap(index,headTitle)**/
	public Map curEntityHeadMap ;
	
	/**欄位的必填:key:entityCode+headTitle, value:true(必填),false(不必填)**/
	public Map curEntityColRequired;
	
	/**存放每一行的資料**/
	public  List listDatas ;
	
	
	public ExcelUtil(File excelFile){
		try {
			if(excelFile == null){
				throw new FileNotFoundException();
			}
		   fis = new FileInputStream(excelFile);		   
		   workBook = new HSSFWorkbook(fis);
		   parseXmlUtil = new XMLUtil();
		   errorString = new StringBuffer();
		   readExcelData();
		   	   				
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		}catch (IOException e) {
			e.printStackTrace();
		}	
	}
	
	
	/**開始從excel讀取資料**/	
	public void readExcelData(){
		int sheetSize = workBook.getNumberOfSheets();
		for(int i=0;i<sheetSize;i++){
			sheet = workBook.getSheetAt(i);
			String entityName = workBook.getSheetName(i);
			readSheetData(sheet,entityName);
		}
		
	}
	
	/**讀每個sheet頁的資料**/
	public void readSheetData(HSSFSheet sheet,String entityName){
		
		   int rowNumbers = sheet.getPhysicalNumberOfRows();
		   Map ent = (Map) parseXmlUtil.getEntityMap().get(entityName);
		   this.setCurEntityCode((String) ent.get("code"));
		   if(rowNumbers == 0){
			   System.out.println("excel中資料為空!");
			   errorString.append(Constans.ERROR_EXCEL_NULL);
		   }		  
		   List colList = (List) parseXmlUtil.getColumnListMap().get(entityName);
		   int xmlRowNum = colList.size();
		   HSSFRow excelRow = sheet.getRow(0);
		   int excelFirstRow = excelRow.getFirstCellNum();
		   int excelLastRow = excelRow.getLastCellNum();
		   if(xmlRowNum  != (excelLastRow-excelFirstRow)){
			     System.out.println("xml列數與excel列數不相符,請檢查");
			     errorString.append(Constans.ERROR_EXCEL_COLUMN_NOT_EQUAL);
		   }
		   readSheetHeadData(sheet);
			   
		   readSheetColumnData(sheet,entityName);
		   
		 
		 	   
	   }
	
   /**讀取sheet頁中的表頭資訊**/
   @SuppressWarnings({ "unchecked", "static-access"})
	public void readSheetHeadData(HSSFSheet sheet){
	   
		   Map headMap = new HashMap();
		   curEntityHeadMap = new HashMap();
		   curEntityColRequired = new HashMap();
		   HSSFRow excelheadRow = sheet.getRow(0);
		   int excelLastRow = excelheadRow.getLastCellNum();
		   String headTitle = "";
		   for(int i=0;i<excelLastRow;i++){
			   HSSFCell cell = excelheadRow.getCell(i);
			   headTitle = this.getStringCellValue(cell).trim();
			   if(headTitle.endsWith("*")){
				   curEntityColRequired.put(this.getCurEntityCode()+"_"+headTitle,true);
			   }else{
				   curEntityColRequired.put(this.getCurEntityCode()+"_"+headTitle,false);
			   }
			   headMap.put(i, headTitle);
		   }
		   curEntityHeadMap.put(this.getCurEntityCode(), headMap);
	   }
   
   /**讀取sheet頁裡面的資料**/
   @SuppressWarnings({ "unchecked", "static-access" })
   public void readSheetColumnData(HSSFSheet sheet,String entityName){
	   
	   HSSFRow excelheadRow = sheet.getRow(0);
	   int excelLastcell = excelheadRow.getLastCellNum();   //excel總列數
	   int excelRowNum = sheet.getLastRowNum();  //excel總行數
	   Map headMap = (Map) this.getCurEntityHeadMap().get(this.getCurEntityCode());	   
	   Map colMap = parseXmlUtil.getColumnMap();
	   listDatas =new ArrayList();
	   
	   for(int i=1;i<excelRowNum+1;i++){//行迴圈		  		   
		   HSSFRow columnRow = sheet.getRow(i);	
		   if(columnRow != null){
			   Map curRowCellMap = new HashMap();
			   for(int j =0; j<excelLastcell;j++){ //列迴圈
				   int cout =  headMap.get(j).toString().indexOf("*");
				   String headTitle ="";
				   if(cout == -1){
					  headTitle = headMap.get(j).toString();
				   }else{
					   headTitle =  headMap.get(j).toString().substring(0, cout);
				   }			   		    
				   Map curColMap =  (Map) colMap.get(entityName+"_"+headTitle);
				   String curColCode = (String) curColMap.get("code");
				   String curColType = (String) curColMap.get("type");
				   HSSFCell colCell = columnRow.getCell(j);
				   String value =this.getStringCellValue(colCell);
				   if(value != null){
					   value = value.trim();
				   }			  
				   String xmlColType = (String) curColMap.get("type");
				   int intVal = 0;
				   if(xmlColType.equals("int")){
					   if(value != null) {
						   intVal = Integer.valueOf(value);
					   }
					   curRowCellMap.put(curColCode, intVal);  //將這一行的資料以code-value的形式存入map
				   }else{
				      curRowCellMap.put(curColCode, value); 
				   }
				   /**驗證cell資料**/
				   validateCellData(i+1,j+1,colCell,entityName,headTitle,curColType,listDatas);
			   }
			   listDatas.add(curRowCellMap);
		   } 
	   }
	   
	   if(this.getErrorString().length() ==0){//如果沒有任何錯誤,就儲存
		   saveExcelData(entityName);
		   System.out.println("匯入資料庫成功");
	   }else{
		   //清理所有的快取clearMap();現在暫時未清理
		   String[] strArr = errorString.toString().split("<br>");
		   for(String s: strArr){
			   System.out.println(s);
		   }
		   
	   }
	   
	   
   }
   /**驗證單元格資料**/
   @SuppressWarnings("static-access")
   public void validateCellData(int curRow,int curCol,HSSFCell colCell,String entityName,String headName,String curColType,List listDatas){
	   
	   List rulList = (List) parseXmlUtil.getColumnRulesMap().get(entityName+"_"+headName);
	   if(rulList != null && rulList.size()>0){
		   for(int i=0 ; i<rulList.size() ; i++){
			   Map rulM = (Map) rulList.get(i);
			   String rulName = (String) rulM.get("name");
			   String rulMsg = (String) rulM.get("message");
			   String cellValue = "";
			   if(this.getStringCellValue(colCell)==null) {
				   //System.out.println("第"+curRow+"行,第"+curCol+"列:"+rulMsg);
			   }else {
				   cellValue = this.getStringCellValue(colCell).trim();
			   }
			   if(rulName.equals(Constans.RULE_NAME_NULLABLE)){		   
				   if(cellValue.equals("")||cellValue == null){
					   errorString.append("匯入失敗,錯誤資訊:第"+curRow+"行,第"+curCol+"列:"+rulMsg+"<br>");
				   }
			   }
			//////這裡寫其他的驗證規則。。。
			 
			}
		   
	   }
   }
   
   /**儲存excel裡面的資料**/
   @SuppressWarnings("unchecked")
   public void saveExcelData(String entityName){
	   
       List<User> users= new ArrayList();
	   for(int i = 0 ; i<this.getListDatas().size();i++){
		   Map excelCol = (Map) this.getListDatas().get(i);  //得到第 i 行的資料	   
		   User user = new User();
		   try {
			User obj = (User) BeanToMapUtil.convertMap(user.getClass(), excelCol);
			users.add(obj);	 
		} catch (IntrospectionException e) {			
			e.printStackTrace();
		} catch (IllegalAccessException e) {		
			e.printStackTrace();
		} catch (InstantiationException e) {		
			e.printStackTrace();
		} catch (InvocationTargetException e) {
			e.printStackTrace();
		}
			  
	   }
	   /**批量儲存資料**/
	   Dao dao = new Dao();
	   for(int i = 0;i<users.size();i++){
		   try{
		   dao.saveUser(users.get(i));
		   
		   }catch(Exception e){
			   e.printStackTrace();
		   }
	   }
	   
   }
   
   /**
	 * 獲得單元格字串
	 * @throws UnSupportedCellTypeException 
	 */
	public static String getStringCellValue(HSSFCell cell) {
		if (cell == null){
			return null;
		}

		String result = "";
		switch (cell.getCellType()) {
			case HSSFCell.CELL_TYPE_BOOLEAN:
				result = String.valueOf(cell.getBooleanCellValue());
				break;
			case HSSFCell.CELL_TYPE_NUMERIC:
				if (HSSFDateUtil.isCellDateFormatted(cell)) {
					java.text.SimpleDateFormat TIME_FORMATTER = new java.text.SimpleDateFormat(
							"yyyy-MM-dd");
					result = TIME_FORMATTER.format(cell.getDateCellValue());
				}
				else{
					double doubleValue = cell.getNumericCellValue();
					result = "" + doubleValue;
				}
				break;
			case HSSFCell.CELL_TYPE_STRING:
				if (cell.getRichStringCellValue() == null){
					result = null;
				}
				else{
					result = cell.getRichStringCellValue().getString();
				}
				break;
			case HSSFCell.CELL_TYPE_BLANK:
				result = null;
				break;
			case HSSFCell.CELL_TYPE_FORMULA:
				try{
					result = String.valueOf(cell.getNumericCellValue());   
		        }catch(Exception e){
		        	result = cell.getRichStringCellValue().getString();
		        }
				break;
			default:
				result = "";
		}
		
		return result;
	}

	
	public String getCurEntityCode() {
		return curEntityCode;
	}
	public void setCurEntityCode(String curEntityCode) {
		this.curEntityCode = curEntityCode;
	}
	public Map getCurEntityHeadMap() {
		return curEntityHeadMap;
	}
	public void setCurEntityHeadMap(Map curEntityHeadMap) {
		this.curEntityHeadMap = curEntityHeadMap;
	}
	public XMLUtil getParseXmlUtil() {
		return parseXmlUtil;
	}
	public void setParseXmlUtil(XMLUtil parseXmlUtil) {
		this.parseXmlUtil = parseXmlUtil;
	}
	public Map getCurEntityColRequired() {
		return curEntityColRequired;
	}
	public void setCurEntityColRequired(Map curEntityColRequired) {
		this.curEntityColRequired = curEntityColRequired;
	}
	public List getListDatas() {
		return listDatas;
	}
	public void setListDatas(List listDatas) {
		this.listDatas = listDatas;
	}
	public StringBuffer getErrorString() {
		return errorString;
	}
	public void setErrorString(StringBuffer errorString) {
		this.errorString = errorString;
	}

}

專案中定義一個XML檔案,主要做一些條件限制,比如使用者名稱不能為空,email不能重複等,所以就有一個XML解析類:

/**解析xml工具類*/
@SuppressWarnings("rawtypes")
public class XMLUtil {

	 /**entity map物件,key:name ,value:entity的屬性map集**/
	public Map entityMap ;
	
	/**column map 物件,key:entityName_colName , value:column的屬性map集 **/
	public Map columnMap;
	
	/**rule map 物件,key:entityName_colName_ruleName, value: rule 的map集:找到一行rule**/
	public Map ruleMap ;
	
	/**rules map 物件, key:entityName_colName, value: rules 的map集:找到該column下所有的rule**/
	public Map  columnRulesMap ;
	
	/**entity--column map: key:entityName, value: column list:根據實體類名得到所有的列**/
	public Map columnListMap ;
	
   /**column list**/
	public List columnList ;
	
	 
    /**開始解析xml檔案**/
	public XMLUtil(){
		SAXReader reader = new SAXReader();
		InputStream in = getClass().getClassLoader().getResourceAsStream("user.xml");//讀取檔案流,Url為controller.xml檔案
		try {
			Document doc = reader.read(in);//獲得檔案例項
			Element root = doc.getRootElement();	
			Iterator itEntity = root.elements("entity").iterator();
			while(itEntity.hasNext()){
				Element entity = (Element) itEntity.next();
				parseEntity(entity);
			}
			
			/**測試entityMap 是否正確**/
			Map enMap = (Map) this.getEntityMap().get("使用者表");
			Set<?> set = enMap.keySet();
			Iterator it = set.iterator();
			while(it.hasNext()){
				String uu = (String) it.next();
			}
		}catch(Exception e){
			e.printStackTrace();
		}
		
	}
	
	 /**開始解析entity**/
	@SuppressWarnings("unchecked")
	public void parseEntity(Element entity){
		if(entity != null){
			
			/**對資料進行初始化設定**/
			columnListMap = new HashMap();
			columnMap = new HashMap();
			entityMap = new HashMap();
			ruleMap = new HashMap();
			columnRulesMap = new HashMap();
			columnList = new ArrayList();
			
			setEntityMap(entity);			
			String entityName = entity.attributeValue("name");
			Iterator itColumn = entity.elements("column").iterator();
			while(itColumn.hasNext()){
				Element column = (Element) itColumn.next();
				setColumnMap(entityName,column);
			}
			columnListMap.put(entityName, columnList);
		}
	}
	 
	
	
	/**將entity放入entityMap中**/
	@SuppressWarnings("unchecked")
	public void setEntityMap(Element entity){		
		Map ent = new HashMap();
		String name = entity.attributeValue("name");
		String code = entity.attributeValue("code");
		ent.put("name", name);
		ent.put("code", code);
		entityMap.put(name, ent);			
	}
	
	/**將column放入columnMap中**/
	@SuppressWarnings("unchecked")
	public void setColumnMap(String entityName,Element column){
		if(column != null){		
			Map col = new HashMap();
			String name = column.attributeValue("name");
			String code = column.attributeValue("code");
			String type = column.attributeValue("type");
			col.put("name", name);
			col.put("code", code);
			col.put("type", type);
			String columnMapKey = entityName+"_"+name;    //eg:  使用者表_使用者名稱
			columnMap.put(columnMapKey, col);		
			columnList.add(col);
			Iterator ruleIt = column.elements("rules").iterator();  //獲得rules
			while(ruleIt.hasNext()){
				Element rules = (Element)ruleIt.next(); 
    			Iterator rule  = rules.elements("rule").iterator();   //獲得 rule
    			while(rule.hasNext()){
    				Element ruleValid = (Element) rule.next();     //獲得每一行rule
    				setRuleMap(entityName,name,ruleValid);    				
    			}
			}
		}
	}
		
    /**將 rule 驗證規則放入ruleMap中**/
	@SuppressWarnings("unchecked")
	public void setRuleMap(String entityName,String columnName,Element ruleValid){
		if(ruleValid != null){			
			String ruleName = ruleValid.attributeValue("name");
			String ruleMsg = ruleValid.attributeValue("message");
			Map ruleValidMap = new HashMap();
			ruleValidMap.put("name", ruleName);
			ruleValidMap.put("message", ruleMsg);
			String ruleStrKey = entityName+"_"+columnName+"_"+ruleName;
			String colStrKey = entityName+"_"+columnName;
			if(this.getColumnRulesMap().containsKey(colStrKey)){
    			List valids = (List) this.getColumnRulesMap().get(colStrKey);
    			valids.add(ruleValidMap);
    		}else{
    			List valids = new ArrayList();
    			valids.add(ruleValidMap);
    			this.columnRulesMap.put(colStrKey, valids);  //將每個column下的所有rules存入該map中
    		}
			ruleMap.put(ruleStrKey, ruleValidMap); //將每個column下的一條rule存入該map中
		}
	}
	


	/**所有的get set 方法**/
	public Map getEntityMap() {
		return entityMap;
	}

	public void setEntityMap(Map entityMap) {
		this.entityMap = entityMap;
	}

	public Map getColumnMap() {
		return columnMap;
	}

	public void setColumnMap(Map columnMap) {
		this.columnMap = columnMap;
	}

	public Map getRuleMap() {
		return ruleMap;
	}

	public void setRuleMap(Map ruleMap) {
		this.ruleMap = ruleMap;
	}

	public Map getColumnRulesMap() {
		return columnRulesMap;
	}

	public void setColumnRulesMap(Map columnRulesMap) {
		this.columnRulesMap = columnRulesMap;
	}

	public Map getColumnListMap() {
		return columnListMap;
	}

	public void setColumnListMap(Map columnListMap) {
		this.columnListMap = columnListMap;
	}
}

XML解析類定義完成後,ExcelUtil會呼叫處理,處理完成後返回,匯入結束。

至於資料庫連線,任何方式都行,這裡不做要求,另外需要定義一個實體User,屬性根據自己的業務設定。

以上就是具體實現過程,如果你有任何問題,歡迎留言,我們共同交流討論。

還可以微信關注和置頂我的公眾號“SL社群”(slshequ),獲取原始碼。