Swoole一鍵操作基於阿里雲的RDS資料庫遷移+OSS檔案搬遷
阿新 • • 發佈：2018-12-14
傳統的資料庫搬遷思路是把資料庫表的結構及資料都查詢出來，然後通過迴圈進行資料結構重組拼接。然後匯出！資料量少的話，這樣當然是沒毛病。當資料量太大的時候，伺服器的記憶體開銷就吃不住了，很容易炸掉，導致伺服器癱掉。當然我之前也這麼幹的 ，所以也一直想辦法解決這個問題 ，當你體會到大資料搬遷的那酸爽、提心吊膽你就明白了
一天我看到一條sql
SELECT COUNT( * ) as cnt FROM information_schema.TABLES WHERE TABLE_SCHEMA = 舊資料庫名
CREATE TABLE IF NOT EXISTS 新資料庫名.表名 LIKE 舊資料庫名.表名
INSERT 新資料庫名.表名 SELECT * FROM 舊資料庫名.表名
相信看到這幾條sql 就應該有思路了，沒錯就是這三條sql!
廢話不說了 上程式碼
<?php namespace action\swoole; /** * 換域名 */ class changesiteAction extends \Action{ private $bucket = 'oss'; private $oldPrefix = ''; private $newPrefix = ''; private $maxkeys = 1000; private $initDB = array('aaa',bbb',ccc','ccc'); //多分支資料庫 private $upgrading = array( 'tag' => 'copying', 'room' => '1'); private $config = array( 'demo_action_admin_certificateconfig', 'demo_action_admin_dealerentranceconfig', 'demo_action_cash_config', 'demo_action_customer_config', 'demo_action_dealer_config', 'demo_action_delivery_config', 'demo_action_itemcomm_config', 'demo_action_poster_config', 'demo_action_seckill_config', 'demo_action_shop_config', 'demo_action_user_config', 'demo_action_admin_skinconfig', 'demo_action_wechat_config', 'demo_action_xiaoi_config', 'demo_action_yunpay_config', 'demo_model_item_config', 'demo_sysapi_ultraconfig', 'demo_sysapi_payconfig' ); //重新連線 public function reload(){ if($this->linker->server->reload()) $message = array('msgcode'=>'RELOAD'); else $message = array('msgcode'=>'FAILURE'); PUSH($this->linker,$message); } //心跳連線 public function heartbeat() { $res = $this->linker->messaging; $roomid = $res['roomid']; PUSH($this->linker,array('errcode'=>'heartbeat'),'adventure_'.$roomid); return false; } public function init(){ // 此時已經可以進行通訊！ PUSH($this->linker,$this->upgrading); return $this->upgrading; } //第一步 public function copyStart(){ $arg = $this->linker->messaging; if(empty($arg['newsite'])) return PUSH($this->linker,array('errcode'=>'ERR_POST_NEWSITE','errmsg'=>'新域名為空')); if(empty($arg['oldsite'])) return PUSH($this->linker,array('errcode'=>'ERR_POST_OLDSITE','errmsg'=>'舊域名為空')); $newsite=strtolower(trim($arg['newsite'])); $oldsite=strtolower(trim($arg['oldsite'])); $m = $this->m(DBMATRIX); //檢測舊域名是否存在 $site = $m->sel()->from('customer')->where(array("domain = ",$oldsite))->exe(); if(empty($site)){ PUSH($this->linker,array('errcode'=>'ERR_EXIST_DOMAIN','errmsg'=>'舊域名不存在')); return false; } // 檢查新域名是否被使用 $mysql = "select id from demo_customer where domain = '{$newsite}'"; $myres = $m->exe(false,$mysql); if(!empty($myres)){ PUSH($this->linker,array('errcode'=>'DOMAIN_HAS_USE','errmsg'=>'新域名已使用過')); return false; } //查詢資料庫表的數量 $oldDB = preg_replace("/[\W]/", "_", $oldsite); $sql = "SELECT COUNT( * ) as cnt FROM information_schema.TABLES WHERE TABLE_SCHEMA = '{$oldDB}'"; $tables_num = $this->m($this->initDB[$site['branch']])->exe(false,$sql)['cnt']; //新增域名修改記錄表 $mysql="select * from demo_changesite_log where newsite='{$newsite}' and oldsite = '{$oldsite}'"; $logres=$this->m(DBMATRIX)->exe(false,$mysql); if(empty($logres)){ $data['sql_sum'] = $tables_num; $data['siteid']=$site['id']; $data['newsite']=$newsite; $data['oldsite']=$oldsite; $data['step'] = 1; //$data['admid'] = $_SESSION['admid']; $res=$this->m(DBMATRIX)->ins('changesite_log')->values($data)->exe(); if(empty($res)){ PUSH($this->linker,array('errcode'=>'ERR_INS_LOG','errmsg'=>'修改記錄新增失敗')); return false; } return PUSH($this->linker,array('errcode'=>'FINISHREADY','id'=>$res,'numsql'=>$tables_num,'step'=>1)); } return PUSH($this->linker,array('errcode'=>'FINISHREADY','id'=>$logres['id'],'numsql'=>$tables_num,'step'=>$logres['step'])); } //第二步 public function copyData(){ $arg = $this->linker->messaging; if(empty($arg['id'])) return PUSH($this->linker,array('errcode'=>'ERR_POST_ID','errmsg'=>'ID為空')); $log = $this->m(DBMATRIX)->sel('oldsite,newsite')->from('changesite_log')->where(array('id =',$arg['id']))->exe();dump($log); if(empty($log)) return PUSH($this->linker,array('errcode'=>'ERR_LOG','errmsg'=>'記錄不存在','id'=>$arg['id'])); $newsite=strtolower(trim($log['newsite'])); $oldsite=strtolower(trim($log['oldsite'])); $oldDB = preg_replace("/[\W]/", "_", $oldsite); $newDB = preg_replace("/[\W]/", "_", $newsite); $site=$this->m(DBMATRIX)->sel('id,branch')->from('customer')->where(array('domain=',$oldsite))->exe();dump($site); //切換資料庫 $sql='use `'.$newDB.'`'; $res=$this->m($this->initDB[$site['branch']])->exe(false,$sql); if(!$res){ //建立新資料庫 $sql='create database `'.$newDB.'`'; $this->m($this->initDB[$site['branch']])->exe(false,$sql); $sql='use `'.$newDB.'`'; $res = $this->m($this->initDB[$site['branch']])->exe(false,$sql); if(!empty($res)) return PUSH($this->linker,array('errcode'=>'ERR_DATABASW','errmsg'=>'資料庫建立失敗')); } /** * 這裡直接進行表結構及資料的複製，直接省去由於害怕資料丟失的備份 * CREATE TABLE admin_jingzhunfenxiao_com.admin2 LIKE 3n4w_jingzhunfenxiao_com.demo_admin; 複製表結構 * INSERT admin_jingzhunfenxiao_com.admin2 SELECT * FROM 3n4w_jingzhunfenxiao_com.demo_admin; 複製表資料 * array(286) { [0] => array(1) { ["Tables_in_zzz_zzz_com"] => string(15) "demo_agent_item" } [1] => array(1) { ["Tables_in_zzz_zzz_com"] => string(17) "demo_agent_record" } [2] => array(1) { ["Tables_in_zzz_zzz_com"] => string(15) "demo_agent_spec" } } **/ $sql = "show tables"; $tables = $this->m($oldsite)->exe(true,$sql); $numsql = 0;$errsql = ''; if(!empty($tables)){ $m = $this->m($oldsite); foreach ($tables as $k => $va) { $numsql = $k; $tablename = $va['Tables_in_'.$oldDB]; $ssql = "show tables {$tablename}"; $tab = $m->exe(false,$ssql); if(empty($tab)){ $tsql = "CREATE TABLE IF NOT EXISTS {$newDB}.{$tablename} LIKE {$oldDB}.{$tablename}"; $tres = $m->exe(false,$tsql); if(empty($tres)){ $errsql = $k.'-1';break; } $dsql = "INSERT {$newDB}.{$tablename} SELECT * FROM {$oldDB}.{$tablename}"; $dres = $m->exe(false,$dsql); } PUSH($this->linker,array('errcode'=>'SUCCESS_CREATE','num'=>$numsql+1)); } //有錯誤導致建立沒有繼續執行，記錄錯誤位置 if(!empty($errsql)){ $mysql="update demo_changesite_log set step=2 , sqlid={$errsql} where id={$arg['id']}"; $myres=$this->m(DBMATRIX)->exe(false,$mysql);echo $mysql;die; return PUSH($this->linker,array('errcode'=>'ERR_CREATE','errmsg'=>'資料斷開連線')); } } $mysql="update demo_changesite_log set step=2 , sqlid={$numsql} where id={$arg['id']}"; $myres=$this->m(DBMATRIX)->exe(false,$mysql); return PUSH($this->linker,array('errcode'=>'FINISHCOPYDATA','errmsg'=>'複製成功')); } //第三步 複製oss檔案 // SOCKET 模式搬遷 public function startSocketCopy(){ $arg = $this->linker->messaging; if(empty($arg['id'])) return PUSH($this->linker,array('errcode'=>'ERR_POST_ID','errmsg'=>'ID為空')); $log = $this->m(DBMATRIX)->sel('oldsite,newsite')->from('changesite_log')->where(array('id =',$arg['id']))->exe(); if(empty($log)) return PUSH($this->linker,array('errcode'=>'ERR_LOG','errmsg'=>'記錄不存在')); $newsite=strtolower(trim($log['newsite'])); $oldsite=strtolower(trim($log['oldsite'])); $this->oldPrefix = $oldsite.'/'; $this->newPrefix = $newsite.'/'; $this->copyInSocket($this->fullTheList($this->oldPrefix)); $mysql="update demo_changesite_log set step=3 where id={$arg['id']}"; $myres=$this->m(DBMATRIX)->exe(false,$mysql); return PUSH($this->linker,array('errcode'=>'FINISHCOPYOSS','errmsg'=>'OSS檔案搬遷完成')); } //第四步 修改總站配置及刪除舊庫 public function alterConfig(){ $arg = $this->linker->messaging; if(empty($arg['id'])) return PUSH($this->linker,array('errcode'=>'ERR_POST_ID','errmsg'=>'ID為空')); $log = $this->m(DBMATRIX)->sel('oldsite,newsite')->from('changesite_log')->where(array('id =',$arg['id']))->exe(); if(empty($log)) return PUSH($this->linker,array('errcode'=>'ERR_LOG','errmsg'=>'記錄不存在')); $newsite=strtolower(trim($log['newsite'])); $oldsite=strtolower(trim($log['oldsite'])); $m=$this->m(DBMATRIX); //搬總站配置 for($i=0;!empty($this->config[$i]);$i++){ // 找出新域名名配置 $mysql = "select id,http_host from `".$this->config[$i]."` where `http_host` = '{$newsite}'"; $config_from = $m->exe(false,$mysql); if($config_from) continue; // 找出原域名配置 $mysql = "select id,http_host from `".$this->config[$i]."` where `http_host` = '{$oldsite}'"; $config_from = $m->exe(false,$mysql); if(!$config_from){ $mysql = "insert into `".$this->config[$i]."`(http_Host) values('{$newsite}') "; $myre = $m->exe(false,$mysql); if(!$myre) return PUSH($this->linker,array('errcode'=>'ERR_CONFIG','errmsg'=>'資料斷開連線')); }else{ // 新域名配置插入表 $mysql = "update `".$this->config[$i]."` set http_host='{$newsite}' where http_host = '{$oldsite}'"; $myre = $this->m(DBMATRIX)->exe(false,$mysql); if(!$myre) return PUSH($this->linker,array('errcode'=>'ERR_CONFIG','errmsg'=>'資料斷開連線')); } } //修改customer $site = $m->sel('id,version,branch')->from('customer')->where(array('domain=',$newsite))->exe(); if(!$site){ $site = $m->sel('id,version,branch')->from('customer')->where(array('domain=',$oldsite))->exe(); $res = $m->upd('customer')->set(array('domain'=>$newsite))->where(array('id=',$site['id']))->exe(); if(!$res) return PUSH($this->linker,array('errcode'=>'ERR_CUSTOMER','errmsg'=>'資料斷開連線')); } //把新域名寫入域名修改表（customer_update） $list=$m->sel()->from('customer_update')->where(array('siteid=',$site['id']," and domain=",$newsite))->exe(); if(!$list){ $data = date("Y-m-d H:i:s"); $sql="insert into demo_customer_update (siteid,domain,time) values({$site['id']},'{$newsite}','{$data}')"; $log=$m->exe(false,$sql); if(!$log) return PUSH($this->linker,array('errcode'=>'ERR_INSERT','errmsg'=>'資料斷開連線')); } //刪除舊資料庫 $database = preg_replace("/[\W]/", "_", $oldsite); $sql='DROP DATABASE `'.$database.'`'; //$res= $this->m($this->initDB[$site['branch']])->exe(false,$sql); //if(!$res) return PUSH($this->linker,array('errcode'=>'ERR_DROP_DATABASE','errmsg'=>'資料斷開連線')); $mysql="update demo_changesite_log set step=6 ,state=1 where id={$arg['id']}"; $res=$m->exe(false,$mysql); return PUSH($this->linker,array('errcode'=>'FINISHTASK','errmsg'=>'任務完成')); } private function fullTheList($Prefix){ $ObjectListInfo = A($this->linker,'oss/object/listObjects',array($this->bucket,array('max-keys'=>$this->maxkeys,'prefix'=>$Prefix))); $ObjectList = $subObjectList = $ObjectListInfo->getObjectList(); $PrefixList = $subPrefixList = $ObjectListInfo->getPrefixList(); /* 每次只能獲取1000條 迴圈拼接第1000條後面的檔案或資料夾到陣列中 */ while(true){ $numObject = count($subObjectList); $numPrefix = count($subPrefixList); if($numObject+$numPrefix == 0) break; if($numObject+$numPrefix < $this->maxkeys) break; /* 只有檔案時，取檔案 只有資料夾時，取資料夾 都有時，取較大值 */ while(true){ if(!$numObject) {$lastObject = $subPrefixList[$numPrefix-1]->getPrefix();break;} if(!$numPrefix) {$lastObject = $subObjectList[$numObject-1]->getKey();break;} $Key = $subObjectList[$numPrefix-1]->getKey(); $Prefix = $subPrefixList[$numObject-1]->getPrefix(); $lastObject = strcmp($Key,$Prefix)>0?$Key:$Prefix; break; } $subObjectListInfo = A($this->linker,'oss/object/listObjects',array($this->bucket,array('max-keys'=>$this->maxkeys,'prefix'=>$Prefix,'marker'=>$lastObject))); $subObjectList = $subObjectListInfo->getObjectList(); $subPrefixList = $subObjectListInfo->getPrefixList(); $ObjectList = array_merge($ObjectList,$subObjectList); $PrefixList = array_merge($PrefixList,$subPrefixList); } return array('ObjectList'=>$ObjectList,'PrefixList'=>$PrefixList); } /* 遞迴函式：傳入完整的檔案與資料夾列表 引數：array( 'ObjectList'=>$ObjectList, 'PrefixList'=>$PrefixList ); */ private function copyInSocket($the){ foreach($the['ObjectList'] as $obj){ $key = $obj->getKey(); if(preg_match('/\+/',$key)) continue; $newKey = $this->newKey($this->oldPrefix,$this->newPrefix,$key); $res = A($this->linker,'oss/object/copyObject',array($this->bucket,$key,$this->bucket,$newKey)); PUSH($this->linker,array('errcode'=>'COPYOBJECT')); } foreach($the['PrefixList'] as $pre) $this->copyInSocket($this->fullTheList($pre->getPrefix())); } private function newKey($oldPrefix,$newPrefix,$obj){ return preg_replace('/^'.str_replace('/','\/',$oldPrefix).'/',$newPrefix,$obj); } }
前端程式碼
html
<!DOCTYPE html> <html> <head> <meta charset="utf-8"> <title>換域名</title> <meta name="renderer" content="webkit"> <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"> <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"> <link rel="stylesheet" href="http://3n4w.oss-cn-shenzhen.aliyuncs.com/public/client-resource/layui-master/dist/css/layui.css"> <link rel="stylesheet" type="text/css" href="/admin/public/common.css"> <link rel="stylesheet" type="text/css" href="/admin/public/reset.css"/> <link rel="stylesheet" type="text/css" href="domain.css"/> <!-- 注意：如果你直接複製所有程式碼到本地，上述css路徑需要改成你本地的 --> </head> <body > <blockquote class="layui-elem-quote"> <span class="title">域名更換操作</span> </blockquote> <div class="content"> <div class="layui-inline"> <div class="layui-form-item"> <label class="layui-form-label">舊域名</label> <div class="layui-input-inline"> <input type="text" name="username" lay-verify="required" placeholder="請輸入" autocomplete="off" class="layui-input" id="oldsite"> </div> </div> <div class="layui-form-item"> <label class="layui-form-label">新域名</label> <div class="layui-input-inline"> <input type="text" name="username" lay-verify="required" placeholder="請輸入" autocomplete="off" class="layui-input" id="newsite"> </div> </div> </div> <div class="layui-inline"> <button class="layui-btn save-btn fl" id="save-btn">儲存</button> <button class="layui-btn reload fl" id="reload">重啟</button> </div> </div> <div class="progress" id="progress"> <fieldset class="layui-elem-field layui-field-title" style="margin-top: 30px;"> <legend>操作進度</legend> </fieldset> <ul class="layui-timeline"> <li class="layui-timeline-item"> <i class="layui-icon layui-timeline-axis"></i> <div class="layui-timeline-content layui-text"> <div class="layui-timeline-title"><label class="step1">換域名準備階段</label><label class="state st1"></label></div> </div> </li> <li class="layui-timeline-item"> <i class="layui-icon layui-timeline-axis"> </i> <div class="layui-timeline-content layui-text"> <div class="layui-timeline-title"><label class=" step2">資料庫表資料複製</label><label class="state st2"></label></div> </div> </li> <li class="layui-timeline-item"> <i class="layui-icon layui-timeline-axis"></i> <div class="layui-timeline-content layui-text"> <div class="layui-timeline-title"><label class="step3">OSS儲存檔案複製</label><label class="state st3"></label></div> </div> </li> <li class="layui-timeline-item"> <i class="layui-icon layui-timeline-axis"></i> <div class="layui-timeline-content layui-text"> <div class="layui-timeline-title"><label class="step4">配置檔案修改</label><label class="state st4"></label></div> </div> </li> <li class="layui-timeline-item"> <i class="layui-icon layui-timeline-axis"></i> <div class="layui-timeline-content layui-text"> <div class="layui-timeline-title"><label class="step5">操作已完成</label></div> </div> </li> </ul> </div> <script type="text/javascript" src="http://3n4w.oss-cn-shenzhen.aliyuncs.com/public/client-resource/layui-master/dist/layui.js"></script> <!-- 注意：如果你直接複製所有程式碼到本地，上述js路徑需要改成你本地的 --> <script type="text/javascript" src="domain.js"></script> <script> </script> </body> </html>
js
layui.use(['form'], function() {
var form = layui.form,
$ = layui.jquery;
var websocket = new WebSocket('ws://192.168.1.1:9997?swoole=dev&mca=swoole/changesite/init&http_host=www.baidu.com');
var save_btn = document.getElementById("save-btn");
var oldsite = document.getElementById("oldsite");
var newsite = document.getElementById("newsite");
var progress = document.getElementById("progress");
var finishcopy = document.getElementById("finishcopy");
var reload_btn = document.getElementById('reload');
reload_btn.onclick = function(){
var copyStart = {
mca : 'swoole/changesite/reload'};
websocket.send(JSON.stringify(copyStart));
}
var reload = {
mca : 'swoole/changesite/reload',
arg : 'arg1',};
var beating = {
mca : 'swoole/changesite/beating',
arg : 'arg1',};
var copyNum = 0;var id = 0;var step =0;
var copyReady = function(num){console.log(num);
$('.step1').addClass('has-end').parents('.layui-timeline-content').siblings('.layui-icon').addClass('has-over');
$('.st1').html('已完成');
$('.st2').html("正在複製<b class='has-copy'>0</b> /<b class='tabnum'>"+num+" </b>");
if(step==1){
copyData();
}
if(step==2){
startSocketCopy();
}
if(step==3){
$('.step2').addClass('has-end').parents('.layui-timeline-content').siblings('.layui-icon').addClass('has-over');
$('.st2').html("已完成");
alterConfig();
}
}
var copyData = function(){
var copyStart = {
mca : 'swoole/changesite/copyData',
id : id};
websocket.send(JSON.stringify(copyStart));
}
var startSocketCopy = function(){
$('.step2').addClass('has-end').parents('.layui-timeline-content').siblings('.layui-icon').addClass('has-over');
$('.st2').html("已完成");
$('.st3').html("正在複製<b class='has-file'>0</b>");
var copyStart = {
mca : 'swoole/changesite/startSocketCopy',
id : id};console.log(copyStart);
websocket.send(JSON.stringify(copyStart));
}
var alterConfig = function(){
$('.step3').addClass('has-end').parents('.layui-timeline-content').siblings('.layui-icon').addClass('has-over');
$('.st3').html("已完成");
$('.st4').html("正在修改配置檔案");
var copyStart = {
mca : 'swoole/changesite/alterConfig',
id : id};
websocket.send(JSON.stringify(copyStart));
}
var taskEnd = function(){
$('.step4').addClass('has-end').parents('.layui-timeline-content').siblings('.layui-icon').addClass('has-over');
$('.st4').html("已完成");
$('.step5').addClass('has-end').parents('.layui-timeline-content').siblings('.layui-icon').addClass('has-over');
}
save_btn.onclick = function(){
progress.style.display = 'block';
var copyStart = {
mca : 'swoole/changesite/copyStart',
oldsite : oldsite.value,
newsite : newsite.value};
websocket.send(JSON.stringify(copyStart));
}
websocket.onopen = function (evt) {
console.log("Connected to WebSocket server.");
websocket.send(JSON.stringify(reload));
window.setInterval(function(){websocket.send(JSON.stringify(beating));}, 10000);
};
websocket.onmessage = function (evt) {
var that = this;
var data = eval('('+evt.data+')');
console.log(data);
switch(data.message.errcode){
case 'FINISHREADY':
id = data.message.id;
step = data.message.step;
copyReady(data.message.numsql);
break;
case 'FINISHCOPYDATA':
startSocketCopy();
break;
case 'FINISHCOPYOSS':
alterConfig();
break;
case 'FINISHTASK':
taskEnd();
break;
case 'SUCCESS_CREATE':
$('.has-copy').html(data.message.num);console.log(data.message.num);
break;
case 'COPYOBJECT':
$('.has-file').html(++copyNum);
break;
//default:console.log(data.message.errmsg);break;
}
};
websocket.onclose = function (evt) {console.log("Disconnected");};
websocket.onerror = function (evt, e) {console.log('Error occured: ');console.log(evt.data);};
});css
.content {
padding: 20px 80px;
}
button.layui-btn.save-btn.fl {
margin-left: 100px;
}
.layui-timeline {
padding-left: 100px;
}
.finish {
background-color: #a53333;
}
.progress{
display: none;
}
.state{
color: #c74949;
padding-left: 75px;
}
.has-over{
background-color: #c74949;
}
.has-end{
color: #c74949;
}