java excel匯入並多執行緒批量插入資料庫
阿新 • • 發佈：2019-02-06
最近寫了個excel匯入並多執行緒持久化到資料庫的功能，搗鼓了一天才弄好，先記錄下來防止自己忘了。
（1）先controller類中方法。
@AccessLog @ApiOperation(value = "匯入excel", httpMethod = "POST", notes = "匯入excel") @RequestMapping(value = "/importExcel",method = RequestMethod.POST) @ApiImplicitParams({ @ApiImplicitParam(name="postionId",value="崗位ID",dataType="long", paramType = "query"), @ApiImplicitParam(name="typeId",value="型別ID(1：崗位 2：人員)",dataType="int", paramType = "query"), @ApiImplicitParam(name="agencyId",value="部門ID",dataType="long", paramType = "query") }) public ResponseResult importExcel(@RequestParam(value="file") MultipartFile file, Long postionId, Integer typeId, Long agencyId) { SelAgencyAndPostionVO selAgencyAndPostionVO = new SelAgencyAndPostionVO(agencyId,postionId,typeId); if (null == selAgencyAndPostionVO) { return new ResponseResult(ExceptionCode.PARAM_IS_NULL); } //型別標識（1：崗位 2：人員） typeId = selAgencyAndPostionVO.getTypeId(); if (null == typeId) { log.info("1", "typeId is null"); return new ResponseResult(ExceptionCode.PARAM_IS_NULL); } //獲取上傳的檔名稱； String name = file.getOriginalFilename(); //判斷是否為excel型別檔案 if(!name.endsWith(".xls") && !name.endsWith(".xlsx")){ log.info("匯入的檔案不是excel型別"); return new ResponseResult<>("匯入的檔案不是excel型別"); } try { HttpServletRequest request = ((ServletRequestAttributes) RequestContextHolder. getRequestAttributes()).getRequest(); //上傳至絕對路徑 String path = request.getSession().getServletContext().getRealPath(File.separator); String uploadDir = path+"upload"+File.separator; log.info(this.getClass().getName()+"臨時儲存圖片路徑saveImgUrl："+uploadDir); File f = new File(uploadDir); //如果不存在該路徑就建立 if (!f.exists()) { f.mkdir(); } //獲取檔名 String uuid= new Date().getTime()+"_"+UUID.randomUUID().toString(). replace("-","").substring(0,6); //檔案儲存絕對路徑 String newName = uploadDir+ uuid + "_"+name; //上傳檔案位置 File dir = new File(uploadDir); if (!dir.exists()) { dir.mkdirs(); } File imgFile = new File(newName); //存入臨時記憶體 FileUtils.writeByteArrayToFile(imgFile, file.getBytes()); //獲取excel中的資料資訊 List<Map<String, Object>> maps = ImportExcelFileUtil.getDataFromExcel(newName,typeId == 1 ? new ElPositionDTO() : typeId == 2 ? new ElUserInfoDTO(): null); //刪除臨時儲存的圖片 if(imgFile.exists() && imgFile.isFile()) { imgFile.delete(); } if (CollectionUtils.isEmpty(maps)) { log.error("ElAttachmentController的importExcel方法獲取匯入的excel資料為空"); return new ResponseResult<>(ExceptionCode.METHOD_FAILURE); } //獲取的是成功插入的次數 int row = elAgencyPositionUserService.importBatchData(maps,selAgencyAndPostionVO); String result = ""; if ((maps.size() - row) == 0 ) { result = "全部匯入成功"+row+"條"; } else if ((maps.size() - row) > 0) { result ="匯入成功"+row+"條，匯入失敗" + (maps.size() - row) + "條（錯誤或重複）"; } return new ResponseResult(result); }catch(BusinessException e){ log.error("ElAttachmentController的importExcel方法error"+e.getMessage(),e); return new ResponseResult<>(e); }catch (Exception e) { log.error("ElAttachmentController的importExcel異常"+e.getMessage(), e); return new ResponseResult(ExceptionCode.INTERFACE_USE_FAILURE); } }
（2）InportExcelFileUtil類處理excel檔案的資訊。此excel方法是通用的方法
package com.xxx.utils; import com.xxx.dto.ElUserInfoDTO; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.poi.hssf.usermodel.HSSFCell; import org.apache.poi.hssf.usermodel.HSSFWorkbook; import org.apache.poi.ss.formula.functions.T; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.FileInputStream; import java.io.InputStream; import java.lang.reflect.Field; import java.text.DecimalFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; /** * Author xz * Date 2018/6/11、9:18 * Version 1.0 **/ public class ImportExcelFileUtil { private static final Logger log = LoggerFactory.getLogger(ImportExcelFileUtil.class); private final static String excel2003L =".xls"; //2003- 版本的excel private final static String excel2007U =".xlsx"; //2007+ 版本的excel /** * 拼裝單個obj 通用 * * @param obj * @param row * @return * @throws Exception */ private static Map<String, Object> dataObj(Object obj, Row row) throws Exception { Class<?> rowClazz= obj.getClass(); Field[] fields = FieldUtils.getAllFields(rowClazz); if (fields == null || fields.length < 1) { return null; } //容器 Map<String, Object> map = new HashMap<String, Object>(); //注意excel表格欄位順序要和obj欄位順序對齊 （如果有多餘欄位請另作特殊下標對應處理） for (int j = 0; j < fields.length; j++) { map.put(fields[j].getName(), getVal(row.getCell(j))); } return map; } /** * 處理val * * @param cell * @return */ public static String getVal(Cell cell) { Object value = null; DecimalFormat df = new DecimalFormat("0"); //格式化字元型別的數字 SimpleDateFormat sdf = new SimpleDateFormat("yyy-MM-dd"); //日期格式化 DecimalFormat df2 = new DecimalFormat("0.00"); //格式化數字 switch (cell.getCellType()) { case Cell.CELL_TYPE_STRING: value = cell.getRichStringCellValue().getString(); break; case Cell.CELL_TYPE_NUMERIC: if("General".equals(cell.getCellStyle().getDataFormatString())){ value = df.format(cell.getNumericCellValue()); }else if("m/d/yy".equals(cell.getCellStyle().getDataFormatString())){ value = sdf.format(cell.getDateCellValue()); }else{ value = df2.format(cell.getNumericCellValue()); } break; case Cell.CELL_TYPE_BOOLEAN: value = cell.getBooleanCellValue(); break; case Cell.CELL_TYPE_BLANK: value = ""; break; default: break; } return value.toString(); } /** * * 讀取出filePath中的所有資料資訊 * * @param filePath excel檔案的絕對路徑 * @param obj * @return */ public static List<Map<String, Object>> getDataFromExcel(String filePath, Object obj){ if (null == obj) { return null; } List<Map<String, Object>> ret = null; FileInputStream fis =null; Workbook wookbook = null; int lineNum = 0; Sheet sheet = null; try{ //獲取一個絕對地址的流 fis = new FileInputStream(filePath); wookbook = getWorkbook(fis,filePath); //得到一個工作表 sheet = wookbook.getSheetAt(0); //獲得表頭 Row rowHead = sheet.getRow(0); //列數 int rows = rowHead.getPhysicalNumberOfCells(); //行數 lineNum = sheet.getLastRowNum(); if(0 == lineNum){ log.info("ImportExcelFileUtil中的getDataFromExcel方法匯入的Excel內沒有資料！"); } ret = getData(sheet, lineNum, rows, obj); } catch (Exception e){ e.printStackTrace(); } return ret; } * @param obj * @return */ public static List<Map<String, Object>> getData(Sheet sheet, int lineNum, int rowNum, Object obj){ List<Map<String, Object>> ret = null; try { //容器 ret = new ArrayList<Map<String, Object>>(); //獲得所有資料 for(int i = 1; i <= lineNum; i++){ //獲得第i行物件 Row row = sheet.getRow(i); if(row!=null){ //裝載obj ret.add(dataObj(obj,row)); } } } catch (Exception e) { e.printStackTrace(); } return ret; } /** * 描述：根據檔案字尾，自適應上傳檔案的版本 * * @param inStr,fileName * @return * @throws Exception */ public static Workbook getWorkbook(InputStream inStr, String fileName) throws Exception{ Workbook wb = null; String fileType = fileName.substring(fileName.lastIndexOf(".")); if(excel2003L.equals(fileType)){ wb = new HSSFWorkbook(inStr); //2003- }else if(excel2007U.equals(fileType)){ wb = new XSSFWorkbook(inStr); //2007+ }else{ throw new Exception("解析的檔案格式有誤！"); } return wb; } public static void main(String[] args) throws Exception{ ElUserInfoDTO dto = new ElUserInfoDTO(); List<Map<String, Object>> dataFromExcel = getDataFromExcel("D:\\img\\測試4.xls", dto); for (int i = 0; i < dataFromExcel.size(); i++) { for (Map.Entry<String, Object> entry : dataFromExcel.get(i).entrySet()) { System.out.println("Key = " + entry.getKey() + ", Value = " + entry.getValue()); } } System.out.println(dataFromExcel); } }
（3）建立多執行緒，並計算執行緒數，此實現的執行緒是Call,為了可以返回成功的結果
public int importBatchData(List<Map<String, Object>> list,SelAgencyAndPostionVO selAgencyAndPostionVO) { //部門主鍵ID Long agencyId = selAgencyAndPostionVO.getAgencyId(); //型別ID（1：崗位 2：人員 ） Integer typeId = selAgencyAndPostionVO.getTypeId(); //崗位主鍵ID Long postionId = selAgencyAndPostionVO.getPostionId(); int row = 0; try { if (typeId == 1) { row = savePositionInfoList(list,agencyId); } else if (typeId == 2) { Long orgId = elAppInfoService.getOrg().getOrgId(); //在匯入之前，把同orgId下的使用者全部放進快取，防止重複匯入 ElUserInfo elUserInfo = new ElUserInfo(); elUserInfo.setOrgId(orgId); List<ElUserInfo> users = userInfoMapper.getUsersByOrgId(elUserInfo); //Redis的key值 String userCodeKey = AppConstants.Flag.USERKEY + orgId; //存入Redis之前，把之前的清空 redisCacheService.deleteRedisCacheByKey(userCodeKey); //規則：key==>"userCode"+orgId,value==>users 存入Redis redisCacheService.setRedisCacheByKey(userCodeKey,JSON.toJSONString(users),3L,TimeUnit.MINUTES); row = saveUserInfoList(list,agencyId,postionId); } } catch (Exception e) { e.printStackTrace(); } return row;
public int saveUserInfoList(List<Map<String, Object>> list, Long agencyId, Long postionId) {
Integer row = 1;
Integer successCount = 0;
int count = 50;// 一個執行緒處理50條資料
int listSize = list.size();// 資料集合大小
int runThreadSize = (listSize / count) + 1; // 開啟的執行緒數
List<Map<String, Object>> newlist = null;// 存放每個執行緒的執行資料
ExecutorService executor = Executors.newFixedThreadPool(runThreadSize);// 建立一個執行緒池，數量和開啟執行緒的數量一樣
// 建立兩個個計數器
CountDownLatch begin = new CountDownLatch(1);
CountDownLatch end = new CountDownLatch(runThreadSize);
// 迴圈建立執行緒
for (int i = 0; i < runThreadSize; i++) {
if ((i + 1) == runThreadSize) {
int startIndex;
startIndex = (i * count);
int endIndex = list.size();
newlist = list.subList(startIndex, endIndex);
} else {
int startIndex = (i * count);
int endIndex = (i + 1) * count;
newlist = list.subList(startIndex, endIndex);
}
//執行緒類，處理資料持久化
UserInfoThread userInfoThread = new UserInfoThread(newlist,begin,end,agencyId,postionId);
//executor.execute(userInfoThread);
Future<Integer> submit = executor.submit(userInfoThread);
try {
//提交成功的次數
row = submit.get();
successCount += row;
} catch (InterruptedException e1) {
log.error("ElAgencyPositionUserServiceImpl的saveUserInfoList方法error"+e1.getMessage(),e1);
} catch (ExecutionException e2) {
log.error("ElAgencyPositionUserServiceImpl的saveUserInfoList方法error"+e2.getMessage(),e2);
}
}
try{
begin.countDown();
end.await();
//執行完關閉執行緒池
executor.shutdown();
}catch (Exception e) {
log.error("ElAgencyPositionUserServiceImpl的saveUserInfoList方法error"+e.getMessage(),e);
}
return successCount;
}（4）UserInfoThread具體實現業務
package com.xxx.service;
import com.alibaba.fastjson.JSON;
import com.xxx.utils.redis.RedisCacheService;
import com.xxx.bean.ElUserInfo;
import com.xxx.common.AppConstants;
import com.xxx.dao.UserInfoMapper;
import com.xxx.utils.SpringUtil;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.StringUtils;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import static com.xxx.utils.MD5Utils.MD5;
/**
* Author xz
* Date 2018/6/11、17:24
* Version 1.0
**/
public class UserInfoThread implements Callable<Integer> {
private static final Logger log = LoggerFactory.getLogger(PostionThread.class);
private List<Map<String, Object>> list;
private CountDownLatch begin;
private CountDownLatch end;
private Long agencyId;
private Long postionId;
private UserInfoMapper userInfoMapper;
private OrgEmployeeService orgEmployeeService;
private RedisCacheService redisCacheService;
private ElAppInfoService elAppInfoService;
//建立個建構函式初始化 list,和其他用到的引數
public UserInfoThread(List<Map<String, Object>> list,CountDownLatch begin,CountDownLatch end, Long agencyId, Long postionId){
this.list = list;
this.begin = begin;
this.end = end;
this.agencyId = agencyId;
this.postionId = postionId;
userInfoMapper = (UserInfoMapper)SpringUtil.getBean("userInfoMapper");
orgEmployeeService = (OrgEmployeeService)SpringUtil.getBean(OrgEmployeeService.class);
redisCacheService = (RedisCacheService)SpringUtil.getBean(RedisCacheService.class);
elAppInfoService = (ElAppInfoService)SpringUtil.getBean(ElAppInfoService.class);
}
@Override
public Integer call(){
int row = 0;
try {
List<ElUserInfo> userList = new ArrayList<ElUserInfo>();
if (CollectionUtils.isNotEmpty(list)) {
//組織id
Long orgId = elAppInfoService.getOrg().getOrgId();
A:for (int i = 0; i < list.size(); i++) {
Map<String, Object> map = list.get(i);
String userSex = map.get("userSex").toString().trim();
String userName = map.get("userName").toString().trim();
String userTel = map.get("userTel").toString().trim();
String passWord = map.get("passWord").toString().trim();
String key = AppConstants.Flag.USERKEY+orgId;
//匯入的人員資訊欄位有一個是為空的，就不持久化資料庫。
if (StringUtils.isEmpty(userSex)) {
continue;
}
if (StringUtils.isEmpty(userName)) {
continue;
}
if (StringUtils.isEmpty(userTel)) {
continue;
}
if (StringUtils.isEmpty(passWord)) {
continue;
}
//獲取的是資料庫存在的同orgId下使用者資訊，以json字串形式
String userListValue = redisCacheService.getRedisCacheByKey(key);
//把json字串轉ElUserInfo使用者物件
List<ElUserInfo> elUserInfos = JSON.parseArray(userListValue, ElUserInfo.class);
//去重，若有重複的就結束此次迴圈
for (ElUserInfo userInfo: elUserInfos) {
if (userTel.equals(userInfo.getUserTel())) {
continue A;
}
}
if ("男".equals(userSex)) {
userSex = "0";
} else if ("女".equals(userSex)){
userSex = "1";
}
ElUserInfo user = new ElUserInfo();
user.setUserName(userName);
user.setUserTel(userTel);
user.setPassWord(MD5(passWord));
user.setUserSex(userSex);
user.setPositionId(postionId);
user.setAgencyId(agencyId);
user.setCreateDate(new Date());
user.setUpdateDate(new Date());
user.setDelMark(0);
user.setRoleId(1L);
user.setEmployeeId(0L);
user.setOrgId(orgId);
userList.add(user);
}
if (CollectionUtils.isNotEmpty(userList)) {
//先持久化本地
row = userInfoMapper.createBatchUserInfoList(userList);
if (row > 0) {
//持久化成功後同步組織平臺
String add = orgEmployeeService.addOrganRoleUserToPlatform(userList, "add");
if (!StringUtils.isEmpty(add) && !"-1".equals(add) && !"null".equals(add)) {
//同步成功後,修改OrgId和EmployeeId
for (ElUserInfo user : userList) {
user.setEmployeeId(1L);
}
//以使用者手機號碼為唯一標示，批量修改OrgId和EmployeeId
userInfoMapper.updateBatchOrgId(userList);
}
log.info(this.getClass().getName()+"的UserInfoThread"+add.toString());
}
}
}
//....
//執行完讓執行緒直接進入等待
// begin.await();
} catch (Exception e) {
log.error("elPositionInfoServiceImpl的UserInfoThread方法error"+e.getMessage(),e);
}finally{
//這裡要主要了，當一個執行緒執行完了計數要減一不要這個執行緒會被一直掛起
//，end.countDown()，這個方法就是直接把計數器減一的
end.countDown();
}
return row;
}
}
@Component
public class SpringUtil implements ApplicationContextAware {
private static ApplicationContext applicationContext;
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
if(SpringUtil.applicationContext == null) {
SpringUtil.applicationContext = applicationContext;
}
}
//獲取applicationContext
public static ApplicationContext getApplicationContext() {
return applicationContext;
}
//通過name獲取 Bean.
public static Object getBean(String name){
return getApplicationContext().getBean(name);
}
//通過class獲取Bean.
public static <T> T getBean(Class<T> clazz){
return getApplicationContext().getBean(clazz);
}
//通過name,以及Clazz返回指定的Bean
public static <T> T getBean(String name,Class<T> clazz){
return getApplicationContext().getBean(name, clazz);
}
}（5）最後，歡迎大家關注【碼農新銳】公眾號，加入我們一起來進階Java。
