thinkphp 3.2.2 匯入 以及 5萬條以上大檔案Excel表格匯入
阿新 • • 發佈：2018-12-12
一、首先說一下自己以前用的方式：以前是採用PHPoffice類+import.class.php檔案匯入。程式碼如下：
1.html程式碼：
<a href="#" style="margin-right: 10px" id="leading_in" data-toggle="modal" data-target="#myModal_leading_in"> <span class="glyphicon glyphicon-log-in" style="margin-right: 5px;color: #00BD8B"></span>匯入 </a>
<!-- 匯入 模態框 --> <div class="modal fade" id="myModal_leading_in" tabindex="-1" role="dialog" aria-labelledby="myModalLabel-add" aria-hidden="true"> <div class="modal-dialog"> <div class="modal-content"> <div class="modal-header"> <button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button> <h4 class="modal-title" id="myModalLabel-add">匯入</h4> </div> <form id="form_upload" enctype="multipart/form-data" action="__URL__/upload_iplib" method="post"> <div class="modal-body" style="margin-bottom:20px;"> <div class="form-group" > <div class="col-sm-9" style="margin-bottom:10px;"> <a href="__ROOT__/DownLoad/iplib.xlsx" style="color:#22a7f0;" >下載模板</a> </div> </div><br/> <div class="form-group"> <div class="col-sm-9"> <div class="widget-main"> <input type="file" id="id-input-file-2" name="file"/> </div> </div> </div> <div class="space-4"></div> </div> <div class="modal-footer"> <input type="submit" class="btn btn-info" value="確定"> <button type="button" class="btn btn-default" data-dismiss="modal">關閉</button></div> </form> </div><!-- /.modal-content --> </div><!-- /.modal --> </div>
2.PHP程式碼：
//IP地址庫匯入 public function upload_iplib(){ $upload = new \Think\Upload();// 例項化上傳類 $upload->maxSize = 3145728 ;// 設定附件上傳大小 $upload->exts = array('xls', 'xlsx');// 設定附件上傳型別 $upload->rootPath = './Uploads/'; // 設定附件上傳根目錄 $upload->savePath = ''; // 設定附件上傳（子）目錄 $upload->saveName = 'time'; $upload->autoSub = false; // 上傳檔案 $info = $upload->upload(); if(!$info) {// 上傳錯誤提示錯誤資訊 $this->error($upload->getError()); } $result = Import::excel($info['file']['savename']); $config = Policy::iplib_col(); $error_msg = ""; $status = 'success'; if(!empty($result)){ foreach ($result as $key => $value) { $data = array(); foreach ($value as $k => $v) { if(is_null($v)){ continue; } if (isset($config[$k])) { if ($v!='') { $data[$config[$k]] = Policy::col2value($k, $v); } } } if($result['status'] === false) { $status = 'failed'; $error_msg .= $res['message']."(excel:".($key + 1).") "; }else{ M('ip_address_library')->add($data); } } $this->ajaxReturn(['status' => $status]); }else{ $status='error'; $error_msg ='沒有匯入任何資訊！'; $this->ajaxReturn(['status' => $status,'message' => $error_msg]); } }
3.import.class.php
<?php
/**
* Created by zhou.
* User: zhou
* Date: 2015/9/17
* Time: 15:16
*/
namespace Home\Common;
use PHPExcel;
use PHPExcel_IOFactory;
Vendor('Classes.PHPExcel');
class Import
{
public static function excel($filePath = "")
{
$filePath = "./Uploads/".$filePath; // 要讀取的檔案的路徑
$PHPExcel = new PHPExcel(); // 拿到例項，待會兒用
$PHPReader = new \PHPExcel_Reader_Excel2007(); // Reader很關鍵，用來讀excel檔案
if (!$PHPReader->canRead($filePath)) { // 這裡是用Reader嘗試去讀檔案，07不行用05，05不行就報錯。注意，這裡的return是Yii框架的方式。
$PHPReader = new \PHPExcel_Reader_Excel5();
if (!$PHPReader->canRead($filePath)) {
$errorMessage = "Can not read file.";
return $this->render('error', ['errorMessage' => $errorMessage]);
}
}
$PHPExcel = $PHPReader->load($filePath); // Reader讀出來後，載入給Excel例項
$currentSheet = $PHPExcel->getSheet(0); // 拿到第一個sheet（工作簿？）
$result = [];
foreach ($currentSheet->getRowIterator() as $row) { // 行迭代器
$cellIterator = $row->getCellIterator(); // 拿到行中的cell迭代器
$cellIterator->setIterateOnlyExistingCells(false); // 設定cell迭代器，遍歷所有cell，哪怕cell沒有值
$lineVal = [];
foreach ($cellIterator as $cell) {
if ($cell->getDataType() == \PHPExcel_Cell_DataType::TYPE_NUMERIC) { // 這裡是比較cell中資料型別是不是number
$cellStyleFormat = $cell->getStyle( $cell->getCoordinate() )->getNumberFormat(); // 接下來這兩句是拿到這個number的格式
$formatCode = $cellStyleFormat->getFormatCode(); // 如果是普通的數字，formatCode將為General, 如果是如6/12/91 12:00的時間格式，formatCode將為/d/yy h:mm（反正就是時間格式了）
if ($formatCode == 'yyyy/m/d\ h:mm;@') {
$value = \PHPExcel_Shared_Date::ExcelToPHP($cell->getValue()) - 28800;
} else {
$value = $cell->getValue();
}
} else {
$value = $cell->getValue();
}
array_push($lineVal, RemoveXSS($value));
}
array_push($result, $lineVal);
}
$config = $result[0];
$result_info = [];
foreach ($result as $key => $value) {
if ($key == 0) {
unset($result[$key]);
continue;
}
foreach ($value as $k => $v) {
$result_info[$key][$config[$k]] = $v;
}
}
return $result_info;
}
}二、由於工作中碰到 要一下子匯入5萬條資料，頁面會超時，導致NGINX掛死。所以修改方案，按照匯入CSV格式的檔案，先切割為1萬行一個的小檔案，再匯入。程式碼如下：
1.html程式碼：
<a href="#" style="margin-left: 15px;cursor: pointer;" id="leading_in" data-toggle="modal" data-target="#myModal_leading_in">
<span class="glyphicon glyphicon-log-in" style="margin-right: 5px;color: #00BD8B"></span>匯入 </a><!-- 匯入 模態框 -->
<div class="modal fade" id="myModal_leading_in" tabindex="-1" role="dialog"
aria-labelledby="myModalLabel-add" aria-hidden="true">
<div class="modal-dialog">
<div class="modal-content">
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal"
aria-hidden="true">×
</button>
<h4 class="modal-title" id="myModalLabel-add">匯入</h4>
</div>
<form id="form_upload" enctype="multipart/form-data" action="__URL__/before_upload" method="post">
<div class="modal-body" style="margin-bottom:20px;">
<div class="form-group" >
<div class="col-sm-9" style="margin-bottom:10px;">
<a href="__PUBLIC__/Download/policy_isms.xlsx" style="color:#22a7f0;" >下載模板</a>
</div>
</div><br/>
<div class="form-group">
<div class="col-sm-9">
<div class="widget-main">
<input type="file" id="id-input-file-2" name="file"/>
<input type='hidden' name="count" value="0">
</div>
</div>
</div>
<div class="space-4"></div>
</div>
<div class="modal-footer">
<input type="submit" class="btn btn-info" value="確定">
<button type="button" class="btn btn-default" data-dismiss="modal">關閉</button>
</div>
</form>
</div><!-- /.modal-content -->
</div><!-- /.modal -->
</div>
2.jQuery程式碼，注意這裡是有個遞迴過程，先走before_upload將檔案進行上傳和拆分，再傳引數給前端，如果傳過來的引數表明還有待匯入的檔案 則再走一遍import_result()函式，如果沒有則提示匯入成功！
需要要引入一個jquery 外掛jquery.form.js
<script type="text/javascript" src="__PUBLIC__/js/jquery.form.js"></script>var import_condition = {};
$(document).ready(function(){
$("#form_upload").ajaxForm({
success:function(data){
if(data.status === 1) {
confirm('沒有資料', {
btn: ['確定'] //按鈕
}, function () {
closeAll();
});
} else if (data.status === '3') {
import_condition.count = data.count;
import_result();
}else{
alert(data.info);
}
}
});
})
function import_result(){
$.post("__URL__/info_upload",import_condition , function(data){
if(data.status === 'success') {
alert('匯入成功！');
} else if (data.status === '3') {
import_condition.count = data.count;
import_result();
}else{
alert(data.message)
}
});
}3.PHP程式碼（上傳並切割檔案的方法）：
//上傳並分割csv檔案
public function before_upload(){
$upload = new \Think\Upload();// 例項化上傳類
$upload->maxSize = 0 ;// 設定附件上傳大小
$upload->exts = array('csv');// 設定附件上傳型別
$upload->rootPath = './Uploads/excel/'; // 設定附件上傳根目錄
$upload->savePath = ''; // 設定附件上傳（子）目錄
$upload->saveName = 'time';
$upload->autoSub = false;
// 上傳檔案
$info = $upload->upload();
if(!$info) {// 上傳錯誤提示錯誤資訊
$this->error($upload->getError());
}
$count = I('count');
$rootPath = C('DOWN_PATH');
$filePath = 'excel/';
if($info['file']['savename']){
$cmd = "cd ". $rootPath . $filePath . "\n";
$cmd .= "split -a 1 -l 10000 " . $info['file']['savename']." -d zp_";
//dump($cmd);die;
$result = system($cmd);
if ($result)
{
$status = true;
}
$cmd = "cd ".$rootPath.$filePath."\n";
$cmd .= "rm -rf ".$info['file']['savename'];
$res = system($cmd);
}
//dump($rootPath . $filePath.'zp_'.$count);die;
if(file_exists($rootPath . $filePath.'zp_'.$count)){
$this->ajaxReturn(array('status'=>'3','count'=>$count));
}else{
$this->ajaxReturn(array('status'=>'1'));
}
}4.匯入的方法：
public function info_upload(){
$count = I('count');
$rootPath = C('DOWN_PATH');
$cmd = "cd ".$rootPath."excel/"."\n";
$cmd.= 'mv zp_'.$count." zp_".$count.".csv"."\n";
$cmd .= "chmod -R 777 zp_".$count.".csv";
system($cmd);
$filename = "zp_".$count.".csv";
$file = fopen($rootPath."excel/".$filename,'r');
while(!feof($file)){
$dat[] = fgetcsv($file);
}
$dat = eval('return ' . iconv('EUC-CN', 'utf-8', var_export($dat, true)) . ';');
foreach ($dat as $key => $value) {
if (!$value) {
unset($dat[$key]);
}
}
fclose($file);
//dump($data);die;
$config = Policy::col();
$error_msg = "";
$status = 'success';
$top_data = array(
'0' => "指令ID",
'1' => "指令型別",
'2' => "機房名稱",
'3' => "生效時間",
'4' => "過期時間",
'5' => "操作型別",
'6' => "域名",
'7' => "URL",
'8' => "關鍵字",
'9' => "源IP地址",
'10' => "目的IP地址",
'11' => "源埠",
'12' => "目的埠",
'13' => "傳輸層協議",
);
if($count == 0){
//檔案被切割後只有第一個存在標題，需要去掉
unset($dat[0]);
}
//轉化成需要的格式
foreach ($dat as $key => $value) {
foreach ($value as $k => $val) {
$result[$key-1][$top_data[$k]] = $val;
}
}
if(!empty($result)){
foreach ($result as $key => $value) {
$data = array();
foreach ($value as $k => $v) {
if(is_null($v)){
continue;
}
if (isset($config[$k])) {
if ($v!='') {
//轉換成 欄位名 =>value的形式
$data[$config[$k]] = Policy::col2value($k, $v);
//dump([$config[$k]]);die;
}
}
}
$data['effect_time'] = strtotime($data['effect_time']);
$data['expired_time'] = strtotime($data['expired_time']);
if($result['status'] === false) {
$status = 'failed';
$error_msg .= $res['message']."(excel:".($key + 1).") ";
}else{
//dump($data);die;
$rule = $data;
//去掉不是規則的data陣列中的元素
unset($rule['type']);
unset($rule['house_id']);
unset($rule['effect_time']);
unset($rule['expired_time']);
unset($rule['operationtype']);
unset($rule['commandid']);
foreach ($rule as $key => $value) {
if($value !=''){
if($key == '關鍵字'){
$data['rule'][] = array('subtype'=>$key,'valueStart'=>str_replace('，', ',', $value),'keywordRange'=>'0,1,2');
}else{
//將規則統一歸到$data['rule']裡面
$data['rule'][] = array('subtype'=>$key,'valueStart'=>$value);
}
}
}
$data['level'] = $this->judge_level($data['rule'],$data['type']);
//去掉規則漢字的元素 因為規則已被歸到$data['rule']裡面
unset($data['域名']);
unset($data['URL']);
unset($data['關鍵字']);
unset($data['源IP地址']);
unset($data['目的IP地址']);
unset($data['源埠']);
unset($data['目的埠']);
unset($data['傳輸層協議']);
$data['idc_id'] = M('basic_idc')->select()[0]['idc_id'];
$data['owner'] = $_SESSION['username'];
$this->info_build_xml($data);
//停頓一毫秒，太快反應不過來會丟檔案
usleep(100);
}
}
//迴圈完了該檔案裡的內容就刪掉，防止與下次匯入發生重合
$cmd = "cd ".$rootPath."excel/"."\n";
$cmd .= "rm zp_".$count.".csv";
system($cmd);
if(file_exists($rootPath."excel/".'zp_'.($count+1))){
$this->ajaxReturn(['status' => '3','count'=>$count+1]);
}else{
$this->ajaxReturn(['status' => $status]);
}
}else{
$status='error';
$error_msg ='沒有匯入任何資訊！';
$this->ajaxReturn(['status' => $status,'message' => $error_msg]);
}
}