OceanBase三節點部署&&擴容
環境資訊搭建三節點(1-1-1)建立資源池和租戶檢視資料分佈
環境資訊
| IP | OB目錄 | 埠 |
|---|---|---|
| 192.168.43.89 | /data/observer | [2881,2882] |
| 192.168.43.233 | /data/observer | [2881,2882] |
| 192.168.43.223 | /data/observer | [2881,2882] |
| IP | OBProxy目錄 | 埠 |
|---|---|---|
| 192.168.43.89 | /data/obproxy | [2883,2884] |
搭建三節點(1-1-1)
本次使用筆記本上的虛擬機器方式部署，配置如下：
注：記憶體最少8G
使用離線方式安裝：
注：本文側重使用離線方式安裝，在3.4注意中敘述了線上安裝方式
1 .準備工作 [root使用者]
1.1 selinux關閉
#臨時關閉
setenforce 0
getenforce
#開機不啟動selinux，需重啟生效。已臨時關閉，本次不需要重啟生效。
sed -i 's/=enforcing/=disabled/g' /etc/selinux/config
#檢視配置已生效
cat /etc/selinux/config
1.2 firewalld關閉
#關閉防火牆
systemctl stop firewalld
#開機不啟動防火牆
systemctl disable firewalld
1.3 修改主機名
#修改主機名為oceanbase，當前已生效，退出會話，再登入後顯示新主機名
hostnamectl set-hostname oceanbase1
hostnamectl set-hostname oceanbase2
hostnamectl set-hostname oceanbase3
1.4 配置hosts
#主機名解析新增主機資訊
cat >> /etc/hosts << EOF
192.168.43.89 oceanbase1
192.168.43.233 oceanbase2
192.168.43.223 oceanbase3
EOF
#檢視主機名資訊
cat /etc/hosts
1.5 建立互信 [可選]
#配置互信 免密 為了方便還是建立互信比較好
oceanbase1：
rm -rf /root/.ssh
[root@oceanbase1 ~]# ssh-keygen
cd /root/.ssh
mv id_rsa.pub authorized_keys
scp -r /root/.ssh oceanbase2:/root
scp -r /root/.ssh oceanbase3:/root
#為了方便，我配置了互信
1.6 安裝目錄準備
以root使用者安裝時，預設安裝在/root/observer目錄下，本文以 root使用者安裝在/data/observer目錄下。建立過程。
#檢視安裝目錄大小
[root@oceanbase1 ~]# df -h
檔案系統 容量 已用 可用 已用% 掛載點
/dev/sdb 107G 22G 80G 21% /data
1.7 sysctl.conf配置
#新增內容
echo "fs.aio-max-nr=1048576" >> /etc/sysctl.conf
#配置生效
sysctl -p
1.8 開啟檔案數限制
#新增內容
cat >> /etc/security/limits.conf << EOF
* soft nofile 655350
* hard nofile 655350
EOF
#退出當前會話，重新登入，使配置生效。
#檢查open files當前值，應為655350，否則後續啟動叢集會報錯
ulimit -n
1.9 時間同步服務 [可選]
#以ob1為主時鐘
[root@oceanbase1 ~]# yum install -y ntp
[root@oceanbase1 ~]# vi /etc/ntp.conf
server 127.127.1.0 iburst
systemctl restart ntpd.service
[root@oceanbase2 ~]# vi /etc/ntp.conf
server 192.168.43.89
restrict 192.168.43.89 mask 255.255.240.0 nomodify notrap
systemctl restart ntpd.service
ntpdate -u 192.168.43.89
[root@oceanbase3 ~]# vi /etc/ntp.conf
server 192.168.52.183
restrict 192.168.52.183 mask 255.255.240.0 nomodify notrap
systemctl restart ntpd.service
ntpdate -u 192.168.52.183
#執行以下命令驗證配置是否成功[官方文件中也有時鐘源配置流程]：
[root@oceanbase2 ~]# ntpdate -u 192.168.43.89
6 Aug 20:40:00 ntpdate[5211]: adjust time server 192.168.43.89 offset -0.003421 sec
[root@oceanbase2 ~]# ntpstat
unsynchronised
polling server every 8 s
[root@oceanbase2 ~]# timedatectl
Local time: 五 2021-08-06 20:40:27 CST
Universal time: 五 2021-08-06 12:40:27 UTC
RTC time: 五 2021-08-06 12:40:24
Time zone: Asia/Shanghai (CST, +0800)
NTP enabled: yes
NTP synchronized: no
RTC in local TZ: no
DST active: n/a
#說明 NTP 服務生效。
同樣使用OBD快速部署，配置檔案mini-distributed-with-obproxy-example.yaml:
根據實際情況在官網下載：
https://gitee.com/oceanbase/obdeploy/blob/master/example/mini-distributed-with-obproxy-example.yaml#
## Only need to configure when remote login is required
# user:
# username: root
# password: 111111
# key_file: your ssh-key file path if need
# port: your ssh port, default 22
# timeout: ssh connection timeout (second), default 30
oceanbase-ce:
servers:
- name: z1
# Please don't use hostname, only IP can be supported
ip: 192.168.43.89
- name: z2
ip: 192.168.43.233
- name: z3
ip: 192.168.43.223
global:
# Please set devname as the network adaptor's name whose ip is in the setting of severs.
# if set severs as "127.0.0.1", please set devname as "lo"
# if current ip is 192.168.1.10, and the ip's network adaptor's name is "eth0", please use "eth0"
devname: ens33
#建立分散式架構時需要根據自己的網絡卡名字更改
cluster_id: 1
# please set memory limit to a suitable value which is matching resource.
memory_limit: 8G
#記憶體設定 最小為8G，不需要做變動
system_memory: 4G
stack_size: 512K
cpu_count: 10
#CPU匯流排程數，不確定的就用lscpu查詢一下，
cache_wash_threshold: 1G
__min_full_resource_pool_memory: 268435456
workers_per_cpu_quota: 8
#CPU工作執行緒數，根據實際情況設定值，言外之意就是cpu_count中你想拿出來多少給他工作用
schema_history_expire_time: 1d
# The value of net_thread_count had better be same as cpu's core number.
net_thread_count: 4
major_freeze_duty_time: Disable
minor_freeze_times: 10
enable_separate_sys_clog: 0
enable_merge_by_turn: FALSE
datafile_disk_percentage: 20
syslog_level: INFO
enable_syslog_recycle: true
max_syslog_file_count: 4
# observer cluster name, consistent with obproxy's cluster_name
appname: ob_cluster
root_password:
proxyro_password:
z1:
mysql_port: 2881
rpc_port: 2882
home_path: /data/observer
zone: zone1
z2:
mysql_port: 2881
rpc_port: 2882
home_path: /data/observer
zone: zone2
z3:
mysql_port: 2881
rpc_port: 2882
home_path: /data/observer
zone: zone3
obproxy:
servers:
- 192.168.43.89
global:
listen_port: 2883
prometheus_listen_port: 2884
home_path: /data/obproxy
# oceanbase root server list
# format: ip:mysql_port,ip:mysql_port
rs_list: 192.168.43.89:2881;192.168.43.233:2881;192.168.43.223:2881
enable_cluster_checkout: false
# observer cluster name, consistent with oceanbase-ce's appname
cluster_name: ob_cluster
obproxy_sys_password:
observer_sys_password:
#如上引數大家可在官網查詢具體解釋，在這找了兩個比較重要的敘述了一下，若解釋錯誤請大家以官網為準
2.離線安裝OBD
2.1檢查OBD離線RPM安裝包
通過官網去下載離線包上傳到自己伺服器
[root@oceanbase1 ~]# cd /opt
[root@oceanbase1 opt]# ll
ob-deploy-1.0.2-2.el7.x86_64.rpm
2.2本機安裝OBD
[root@oceanbase1 opt]# rpm -ivh ob-deploy-1.0.2-2.el7.x86_64.rpm
警告：ob-deploy-1.0.2-2.el7.x86_64.rpm: 頭V4 RSA/SHA1 Signature, 金鑰 ID e9b4a7aa: NOKEY
準備中... ################################# [100%]
正在升級/安裝...
1:ob-deploy-1.0.2-2.el7 ################################# [100%]
Installation of obd finished successfully
Please source /etc/profile.d/obd.sh to enable it
#安裝過程確實依賴可根據實際情況安裝，可使用本yum源或網上下載RPM包依賴，可以使用阿里雲映象根據自己版本去下載
https://mirrors.aliyun.com/centos/7.9.2009/os/x86_64/Packages/
3 本地OceanBase映象
當前主機不能連線公網，將之前OceanBase資料庫的rpm離線軟體包映象到本地。
3.1 檢查離線包
[root@oceanbase1 opt]# ll
-rw-r--r--. 1 root root 8074612 5月 30 21:59 obproxy-3.1.0-1.el7.x86_64.rpm
-rw-r--r-- 1 root root 42035940 8月 3 15:29 oceanbase-ce-3.1.0-2.el7.x86_64.rpm
-rw-r--r-- 1 root root 158972 8月 3 15:34 oceanbase-ce-libs-3.1.0-2.el7.x86_64.rpm
[root@oceanbase1 opt2]# ll
-rw-r--r-- 1 root root 658620 8月 6 20:50 libobclient-2.0.0-2.el7.x86_64.rpm
-rw-r--r-- 1 root root 41916564 8月 6 20:50 obclient-2.0.0-2.el7.x86_64.rpm
#obclient客戶端也一併下載，並且安裝上，便於一會使用obclient
3.2 將OceanBase數離線軟體包加入本地映象
在中控機器(192.168.43.89)執行以下命令，將OceanBase資料庫的離線軟體包加入本地映象：
[root@oceanbase1 opt]# pwd
/opt
[root@oceanbase1 opt]# obd mirror clone *.rpm
name: obproxy
version: 3.1.0
release:1.el7
arch: x86_64
md5: 0b17cf0459a3b53c5a2febb6572894d183154c64
add obproxy-3.1.0-1.el7.x86_64.rpm to local mirror
name: oceanbase-ce
version: 3.1.0
release:2.el7
arch: x86_64
md5: afd11d52f83eef4b456d77969fde620c4bfba85e
add oceanbase-ce-3.1.0-2.el7.x86_64.rpm to local mirror
name: oceanbase-ce-libs
version: 3.1.0
release:2.el7
arch: x86_64
md5: 47300ca1ac4c62493caf3e9235b105e242e533b5
add oceanbase-ce-libs-3.1.0-2.el7.x86_64.rpm to local mirror
3.3 將obd遠端獲取安裝的remote檔案改個名，讓obd執行時候不走外網的映象，走本地的映象
mv /root/.obd/mirror/remote /root/.obd/mirror/remotebak
3.4 注：若您的機器可以訪問公網，並能夠新增三方yum軟體源，可以使用 OB的官方軟體源安裝 OBD，並且忽略掉2.2-3.3步驟：
#線上安裝方法
[root@oceanbase1 ~]# yum install -y yum-utils
[root@oceanbase1 ~]# yum-config-manager --add-repo https://mirrors.aliyun.com/oceanbase/OceanBase.repo
[root@oceanbase1 ~]# yum install -y ob-deploy
4.obd 部署:
[root@oceanbase1 data]# obd cluster deploy ob_cluster -c mini-distributed-with-obproxy-example.yaml
Found a higher version
name: oceanbase-ce
version: 3.1.0
release:2.el7
arch: x86_64
md5: afd11d52f83eef4b456d77969fde620c4bfba85e
Do you want to use it? [y/n]: y
Package oceanbase-ce-3.1.0 is available.
obproxy-3.1.0 already installed.
install oceanbase-ce-3.1.0 for local ok
+-----------------------------------------------------------------------------+
| Packages |
+--------------+---------+---------+------------------------------------------+
| Repository | Version | Release | Md5 |
+--------------+---------+---------+------------------------------------------+
| obproxy | 3.1.0 | 1.el7 | 0b17cf0459a3b53c5a2febb6572894d183154c64 |
| oceanbase-ce | 3.1.0 | 2.el7 | afd11d52f83eef4b456d77969fde620c4bfba85e |
+--------------+---------+---------+------------------------------------------+
Open ssh connection ok
Remote obproxy-3.1.0-0b17cf0459a3b53c5a2febb6572894d183154c64 repository install ok
Remote obproxy-3.1.0-0b17cf0459a3b53c5a2febb6572894d183154c64 repository lib check ok
Remote oceanbase-ce-3.1.0-afd11d52f83eef4b456d77969fde620c4bfba85e repository install ok
Remote oceanbase-ce-3.1.0-afd11d52f83eef4b456d77969fde620c4bfba85e repository lib check !!
[WARN] z1(192.168.43.89) oceanbase-ce-3.1.0-afd11d52f83eef4b456d77969fde620c4bfba85e require: libmariadb.so.3
[WARN] z2(192.168.43.233) oceanbase-ce-3.1.0-afd11d52f83eef4b456d77969fde620c4bfba85e require: libmariadb.so.3
[WARN] z3(192.168.43.223) oceanbase-ce-3.1.0-afd11d52f83eef4b456d77969fde620c4bfba85e require: libmariadb.so.3
Try to get lib-repository
Found a higher version
name: oceanbase-ce-libs
version: 3.1.0
release:2.el7
arch: x86_64
md5: 47300ca1ac4c62493caf3e9235b105e242e533b5
Do you want to use it? [y/n]: y
Package oceanbase-ce-libs-3.1.0 is available.
install oceanbase-ce-libs-3.1.0 for local ok
Use oceanbase-ce-libs-3.1.0-47300ca1ac4c62493caf3e9235b105e242e533b5 for oceanbase-ce-3.1.0-afd11d52f83eef4b456d77969fde620c4bfba85e
Remote oceanbase-ce-libs-3.1.0-47300ca1ac4c62493caf3e9235b105e242e533b5 repository install ok
Remote oceanbase-ce-3.1.0-afd11d52f83eef4b456d77969fde620c4bfba85e repository lib check ok
Cluster status check ok
192.168.43.89 init cluster work home
z1(192.168.43.89) initializes cluster work home
z2(192.168.43.233) initializes cluster work home
z3(192.168.43.223) initializes cluster work home
ob_cluster deployed
#看到ob_cluster deployed即證明叢集初始化成功
4.1啟動叢集:
[root@oceanbase1 data]# obd cluster start ob_cluster
Get local repositories and plugins ok
Open ssh connection ok
Cluster param config check ok
Check before start observer ok
Check before start obproxy ok
Start observer ok
observer program health check ok
Connect to observer ok
Initialize cluster
Cluster bootstrap ok
Wait for observer init ok
+--------------------------------------------------+
| observer |
+----------------+---------+------+-------+--------+
| ip | version | port | zone | status |
+----------------+---------+------+-------+--------+
| 192.168.43.223 | 3.1.0 | 2881 | zone3 | active |
| 192.168.43.233 | 3.1.0 | 2881 | zone2 | active |
| 192.168.43.89 | 3.1.0 | 2881 | zone1 | active |
+----------------+---------+------+-------+--------+
Start obproxy ok
obproxy program health check ok
Connect to obproxy ok
Initialize cluster
+-------------------------------------------------+
| obproxy |
+---------------+------+-----------------+--------+
| ip | port | prometheus_port | status |
+---------------+------+-----------------+--------+
| 192.168.43.89 | 2883 | 2884 | active |
+---------------+------+-----------------+--------+
ob_cluster running
#叢集啟動時候可能會在Cluster bootstrap步驟失敗，失敗之後我重新啟動了一次，第二次成功，可根據失敗常見原因進行排查，當status全都為active時，cluster和proxy已經正常啟動
初始化OceanBase叢集失敗的常見原因有【來自官方問答榜】：
1、機器間的時鐘誤差過大，可以利用ntpq、clockdiff等檢查機器間的始終誤差。
2、資訊指定有無，比如zone名稱有誤，或者網絡卡名和ip地址沒對上等。
3、其他問題，如硬體問題。
具體原因可以檢視日誌：
observer.log observer執行時的日誌
rootserver.log observer上rootserver的日誌
#猜測我的虛擬機器可能因為網路原因時鐘誤差比較大，可以配置上ntp服務進行時間同步
小插曲1：
obd安裝OceanBase和obproxy，安裝後通過obproxy 2883連線報錯，連線2881埠正常：
解決流程：
通過觀察obproxy日誌，發現proxyro使用者密碼被拒絕；
[2021-08-03 15:55:22.075669] WARN [PROXY.SM] print_mysql_complete_log (ob_mysql_sm.cpp:3899) [4117][Y0-7FF1ABC7B3A0] [lt=39] [dc=0] finishing mysql tunnel((sm_id=7, cs_id=2147483655, proxy_sessid=3, ss_id=3, server_sessid=3221497608, client_ip={*Not IP address [0]*:0}, server_ip={192.168.43.89:2881}, server_trace_id=, proxy_user_name=proxyro@sys#ob_cluster, database_name=, is_flow_controlled=false, cpu_flow_control_count=0, memory_flow_control_count=0, sql=, sql_cmd=NULL, result={is_trans_completed:true, is_resp_completed:true, ending_type:2, is_partition_hit:true, has_new_sys_var:false, has_proxy_idc_name_user_var:false, is_server_db_reset:false, reserved_len:0, connection_id:0, scramble_buf:"", is_resultset_resp:false, server_capabilities_lower_.capability:0, ok_packet_action_type:2, last_ok_pkt_len:12, rewritten_last_ok_pkt_len:0, error_pkt:field_count:255, errcode:1045, sqlstate:"42000", message:
"Access denied for user 'proxyro'@'xxx.xxx.xxx.xxx' (using password: NO)"})
解決方案：通過2881埠登入observer：將proxyro密碼設定為空。安裝時在yaml檔案沒有設定proxyro的密碼，此處不解，略過！
設定空之後通過2883可正常登入
此時：2881和2883埠登入時root密碼都為空，為了方便沒有設定密碼。可根據需要alter一下
小插曲2：
解釋：當status有狀態為inactive的請檢查程序是否正常啟動，或網路是否暢通，我的是因為程序沒有啟動
5.建立資源池和租戶
5.1通過obproxy 連線到observer， 建立資源及租戶
[root@oceanbase1 data]# obclient -u root -p -h 127.0.0.1 -P 2883 -A
Enter password:
Welcome to the OceanBase. Commands end with ; or \g.
Your MySQL connection id is 9
Server version: 5.6.25 OceanBase 3.1.0 (r-00672c3c730c3df6eef3b359eae548d8c2db5ea2) (Built Jun 22 2021 12:46:28)
Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
#加上-A引數
MySQL [(none)]> use oceanbase;
Database changed
#建立資源池，unit數量為1，資源根據自己伺服器配置給，我給了最小
MySQL [oceanbase]> create resource unit testunit1 max_cpu=1, min_cpu=1, max_memory='1G', min_memory='1G', max_iops=10000, min_iops=1000, max_session_num=1000000, max_disk_size='10G';
Query OK, 0 rows affected (0.048 sec)
MySQL [oceanbase]> create resource pool testpool_1 unit='testunit1', unit_num=1;
Query OK, 0 rows affected (0.075 sec)
MySQL [oceanbase]> CREATE TENANT IF NOT EXISTS testtenant_1
-> charset='utf8mb4',
-> replica_num=3,
-> zone_list=('zone1','zone2','zone3'),
-> primary_zone='RANDOM',
-> resource_pool_list=('testpool_1');
Query OK, 0 rows affected (11.333 sec)
#根據配置高低速度也會不同，等待。。。。。
5.2通過自己建立的租戶testtenant_1登入建立表:
[root@oceanbase1 data]# obclient -u root@testtenant_1 -p -h 127.0.0.1 -P 2881 -A
Enter password:
Welcome to the OceanBase. Commands end with ; or \g.
Your MySQL connection id is 3221501050
Server version: 5.7.25 OceanBase 3.1.0 (r-00672c3c730c3df6eef3b359eae548d8c2db5ea2) (Built Jun 22 2021 12:46:28)
Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
MySQL [(none)]> show databases;
+--------------------+
| Database |
+--------------------+
| oceanbase |
| information_schema |
| mysql |
| test |
+--------------------+
4 rows in set (0.014 sec)
MySQL [(none)]> create database t1;
Query OK, 1 row affected (0.260 sec)
MySQL [(none)]> create tablegroup t1 partition by hash partitions 3;
Query OK, 0 rows affected (0.319 sec)
MySQL [(none)]> use t1;
Database changed
MySQL [t1]> create table tt1(id int , name char(20));
Query OK, 0 rows affected (0.779 sec)
MySQL [t1]> create table tt2(id int , name char(20)) tablegroup=t1 partition by hash (id) partitions 3 ;
Query OK, 0 rows affected (0.639 sec)
MySQL [t1]> create table tt3(id int , name char(20)) tablegroup=t1 partition by hash(id) partitions 3 ;
Query OK, 0 rows affected (0.697 sec)
#簡單演示一下建庫建表
5.3檢視資料分佈，使用sys租戶：
[root@oceanbase1 data]# obclient -u root -p -h 127.0.0.1 -P 2883 -A
MySQL [(none)]> SELECT t.tenant_id, a.tenant_name, t.table_name, d.database_name, t1.tablegroup_name , t.part_num , t2.partition_id, t2.role , t2.ZONE, t2.svr_ip, concat(t2.svr_ip, ':', t2.svr_port) observer , t2.data_size , a.primary_zone , IF(t.locality = '' OR t.locality IS NULL, a.locality, t.locality) AS locality FROM oceanbase.__all_tenant AS a JOIN oceanbase.__all_virtual_database AS d ON ( a.tenant_id = d.tenant_id ) JOIN oceanbase.__all_virtual_table AS t ON (t.tenant_id = d.tenant_id AND t.database_id = d.database_id) JOIN oceanbase.__all_virtual_meta_table t2 ON (t.tenant_id = t2.tenant_id AND (t.table_id=t2.table_id OR t.tablegroup_id=t2.table_id) AND t2.ROLE IN (1) ) LEFT JOIN oceanbase.__all_virtual_tablegroup AS t1 ON (t.tenant_id = t1.tenant_id and t.tablegroup_id = t1.tablegroup_id) WHERE a.tenant_id IN (1001,1002 ) AND t.table_type IN (3) and table_name in ('tt1','tt2','tt3') ORDER BY t.tenant_id, t1.tablegroup_name, d.database_name, t.table_name, t2.partition_id;
+-----------+--------------+------------+---------------+-----------------+----------+--------------+------+-------+----------------+---------------------+-----------+--------------+---------------------------------------------+
| tenant_id | tenant_name | table_name | database_name | tablegroup_name | part_num | partition_id | role | ZONE | svr_ip | observer | data_size | primary_zone | locality |
+-----------+--------------+------------+---------------+-----------------+----------+--------------+------+-------+----------------+---------------------+-----------+--------------+---------------------------------------------+
| 1001 | testtenant_1 | tt1 | t1 | NULL | 1 | 0 | 1 | zone3 | 192.168.43.223 | 192.168.43.223:2882 | 0 | RANDOM | FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3 |
| 1001 | testtenant_1 | tt2 | t1 | t1 | 3 | 0 | 1 | zone2 | 192.168.43.233 | 192.168.43.233:2882 | 0 | RANDOM | FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3 |
| 1001 | testtenant_1 | tt2 | t1 | t1 | 3 | 1 | 1 | zone3 | 192.168.43.223 | 192.168.43.223:2882 | 0 | RANDOM | FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3 |
| 1001 | testtenant_1 | tt2 | t1 | t1 | 3 | 2 | 1 | zone1 | 192.168.43.89 | 192.168.43.89:2882 | 0 | RANDOM | FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3 |
| 1001 | testtenant_1 | tt3 | t1 | t1 | 3 | 0 | 1 | zone2 | 192.168.43.233 | 192.168.43.233:2882 | 0 | RANDOM | FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3 |
| 1001 | testtenant_1 | tt3 | t1 | t1 | 3 | 1 | 1 | zone3 | 192.168.43.223 | 192.168.43.223:2882 | 0 | RANDOM | FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3 |
| 1001 | testtenant_1 | tt3 | t1 | t1 | 3 | 2 | 1 | zone1 | 192.168.43.89 | 192.168.43.89:2882 | 0 | RANDOM | FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3 |
+-----------+--------------+------------+---------------+-----------------+----------+--------------+------+-------+----------------+---------------------+-----------+--------------+---------------------------------------------+
7 rows in set (0.077 sec)
#t1為上面建的tablegroup name，tt1、tt2、tt3是上面建的table name
附圖：
5.4sys租戶檢視目前資源分佈：
[root@oceanbase1 data]# obclient -u root -p -h 127.0.0.1 -P 2883 -A
Enter password:
Welcome to the OceanBase. Commands end with ; or \g.
Your MySQL connection id is 11
Server version: 5.6.25 OceanBase 3.1.0 (r-00672c3c730c3df6eef3b359eae548d8c2db5ea2) (Built Jun 22 2021 12:46:28)
Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
MySQL [(none)]> use oceanbase;
Database changed
MySQL [oceanbase]> select t1.name resource_pool_name, t2.`name` unit_config_name, t2.max_cpu, t2.min_cpu, round(t2.max_memory / 1024 / 1024 / 1024) max_mem_gb, round(t2.min_memory / 1024 / 1024 / 1024) min_mem_gb, t3.unit_id, t3.zone, concat(t3.svr_ip, ':', t3.`svr_port`) observer, t4.tenant_id, t4.tenant_name from __all_resource_pool t1 join __all_unit_config t2 on (t1.unit_config_id = t2.unit_config_id) join __all_unit t3 on (t1.`resource_pool_id` = t3.`resource_pool_id`) left join __all_tenant t4 on (t1.tenant_id = t4.tenant_id) order by t1.`resource_pool_id`, t2.`unit_config_id`, t3.unit_id;
+--------------------+------------------+---------+---------+------------+------------+---------+-------+---------------------+-----------+--------------+
| resource_pool_name | unit_config_name | max_cpu | min_cpu | max_mem_gb | min_mem_gb | unit_id | zone | observer | tenant_id | tenant_name |
+--------------------+------------------+---------+---------+------------+------------+---------+-------+---------------------+-----------+--------------+
| sys_pool | sys_unit_config | 5 | 2.5 | 1 | 1 | 1 | zone1 | 192.168.43.89:2882 | 1 | sys |
| sys_pool | sys_unit_config | 5 | 2.5 | 1 | 1 | 2 | zone2 | 192.168.43.233:2882 | 1 | sys |
| sys_pool | sys_unit_config | 5 | 2.5 | 1 | 1 | 3 | zone3 | 192.168.43.223:2882 | 1 | sys |
| testpool_1 | testunit1 | 1 | 1 | 1 | 1 | 1001 | zone1 | 192.168.43.89:2882 | 1001 | testtenant_1 |
| testpool_1 | testunit1 | 1 | 1 | 1 | 1 | 1002 | zone2 | 192.168.43.233:2882 | 1001 | testtenant_1 |
| testpool_1 | testunit1 | 1 | 1 | 1 | 1 | 1003 | zone3 | 192.168.43.223:2882 | 1001 | testtenant_1 |
+--------------------+------------------+---------+---------+------------+------------+---------+-------+---------------------+-----------+--------------+
6 rows in set (0.011 sec)
附圖：
6.伺服器擴容：[ 1-1-1 ---> 2 -2-2 ]
| IP | OB目錄 | 埠 |
|---|---|---|
| 192.168.43.89 | /data/observer2 | [3881,3882] |
| 192.168.43.233 | /data/observer2 | [3881,3882] |
| 192.168.43.223 | /data/observer2 | [3881,3882] |
擴容埠配置
6.1設定環境變數，建立observer目錄(三臺機器均執行)
[root@oceanbase1 47300ca1ac4c62493caf3e9235b105e242e533b5]# pwd
/root/.obd/repository/oceanbase-ce-libs/3.1.0/47300ca1ac4c62493caf3e9235b105e242e533b5
[root@oceanbase1 47300ca1ac4c62493caf3e9235b105e242e533b5]# ll
總用量 460
lrwxrwxrwx 1 root root 15 8月 3 15:46 libaio.so -> libaio.so.1.0.1
lrwxrwxrwx 1 root root 15 8月 3 15:46 libaio.so.1 -> libaio.so.1.0.1
-rwxr-xr-x 1 root root 7848 8月 3 15:46 libaio.so.1.0.1
lrwxrwxrwx 1 root root 15 8月 3 15:46 libmariadb.so -> libmariadb.so.3
-rwxr-xr-x 1 root root 460648 8月 3 15:46 libmariadb.so.3
export LD_LIBRARY_PATH=/root/.obd/repository/oceanbase-ce-libs/3.1.0/47300ca1ac4c62493caf3e9235b105e242e533b5/
mkdir -p /data/observer2/store/{sort_dir,sstable,clog,ilog,slog}
6.2手工啟動observe程序
#官方地址：https://www.oceanbase.com/docs/knowledge-base/knowledge-base/V1.0.0/how-to-scale-out-the-oceanbase-cluster
192.168.43.89:
cd /data/observer2 &&/root/.obd/repository/oceanbase-ce/3.1.0/56f57e9843e719d830ec03c206d914f4b3adc82b/bin/observer -r "192.168.43.89:2882:2881;192.168.43.233:2882:2881;192.168.43.223:2882:2881" -o __min_full_resource_pool_memory=268435456,memory_limit=8G,system_memory=4G,stack_size=512K,cpu_count=10,cache_wash_threshold=1G,workers_per_cpu_quota=8,schema_history_expire_time=1d,net_thread_count=4,major_freeze_duty_time=Disable,minor_freeze_times=10,enable_separate_sys_clog=0,enable_merge_by_turn=False,datafile_disk_percentage=20,enable_syslog_recycle=True,max_syslog_file_count=4 -z zone1 -p 3881 -P 3882 -n ob_cluster -c 1 -d /data/observer2/store -i ens33 -l INFO
192.168.43.223:
cd /data/observer2 && /root/.obd/repository/oceanbase-ce/3.1.0/56f57e9843e719d830ec03c206d914f4b3adc82b/bin/observer -r "192.168.43.89:2882:2881;192.168.43.233:2882:2881;192.168.43.223:2882:2881" -o __min_full_resource_pool_memory=268435456,memory_limit=8G,system_memory=4G,stack_size=512K,cpu_count=10,cache_wash_threshold=1G,workers_per_cpu_quota=8,schema_history_expire_time=1d,net_thread_count=4,major_freeze_duty_time=Disable,minor_freeze_times=10,enable_separate_sys_clog=0,enable_merge_by_turn=False,datafile_disk_percentage=20,enable_syslog_recycle=True,max_syslog_file_count=4 -z zone2 -p 3881 -P 3882 -n ob_cluster -c 1 -d /data/observer2/store -i ens33 -l INFO
192.168.43.233:
cd /data/observer2 &&/root/.obd/repository/oceanbase-ce/3.1.0/56f57e9843e719d830ec03c206d914f4b3adc82b/bin/observer -r "192.168.43.89:2882:2881;192.168.43.233:2882:2881;192.168.43.223:2882:2881" -o __min_full_resource_pool_memory=268435456,memory_limit=8G,system_memory=4G,stack_size=512K,cpu_count=10,cache_wash_threshold=1G,workers_per_cpu_quota=8,schema_history_expire_time=1d,net_thread_count=4,major_freeze_duty_time=Disable,minor_freeze_times=10,enable_separate_sys_clog=0,enable_merge_by_turn=False,datafile_disk_percentage=20,enable_syslog_recycle=True,max_syslog_file_count=4 -z zone3 -p 3881 -P 3882 -n ob_cluster -c 1 -d /data/observer2/store -i ens33 -l INFO
6.3目前有兩個ob程序，說明手動啟動已經成功
註解：如下：
/root/.obd/repository/oceanbase-ce/3.1.0/56f57e9843e719d830ec03c206d914f4b3adc82b/bin/observer -r "192.168.43.89:2882:2881;192.168.43.233:2882:2881;192.168.43.223:2882:2881" -o __min_full_resource_pool_memory=268435456,memory_limit=8G,system_memory=4G,stack_size=512K,cpu_count=10,cache_wash_threshold=1G,workers_per_cpu_quota=8,schema_history_expire_time=1d,net_thread_count=4,major_freeze_duty_time=Disable,minor_freeze_times=10,enable_separate_sys_clog=0,enable_merge_by_turn=False,datafile_disk_percentage=20,enable_syslog_recycle=True,max_syslog_file_count=4 -z zone3 -p 3881 -P 3882 -n ob_cluster -c 1 -d /data/observer2/store -i ens33 -l INFO
其中：
3881：表示 MySQL 的訪問埠。
3882：表示遠端訪問埠。
zone_1：表示 Zone 名稱。
/data/observer2/store：表示資料盤目錄。
datafile_disk_percentage=20：表示資料盤的佔用比率為 20%。
192.168.43.89和192.168.43.223 和 192.168.43.233：表示 rs_list。
memory_limit=8GB：表示程序啟動記憶體上限為 8 GB。
WARN：表示 log_level 為 WARNING 級別。
使用 -o 引數時，需滿足以下條件：
不分大小寫，但是推薦按照 observer.config.bin 中的變數名稱來寫。
引數不能包含以下特殊字元：
空格、\r、\n、\t
引數名和引數值中間必須有等號（=）。
引數之間使用逗號（,）進行分割。
#和上面yaml檔案中配置的一樣，只不過此處使用的是手動啟動ob程序
新增server
MySQL [oceanbase]> alter system add server '192.168.43.89:3882' zone 'zone1';
MySQL [oceanbase]> alter system add server '192.168.43.233:3882' zone 'zone2';
MySQL [oceanbase]> alter system add server '192.168.43.223:3882' zone 'zone3';
6.4檢視資源情況，看每個zone內是否已經有兩個observer
select zone,concat(svr_ip,':',svr_port) observer, cpu_capacity,cpu_total,cpu_assigned,cpu_assigned_percent, mem_capacity,mem_total,mem_assigned,mem_assigned_percent, unit_Num,round(`load`,2) `load`, round(cpu_weight,2) cpu_weight, round(memory_weight,2) mem_weight, leader_count from __all_virtual_server_stat order by zone,svr_ip;
注：如圖所示，因資源限制問題，233上observer2無法啟動程序，接下來的擴容未親測。
7.租戶擴容
7.1升級資源的unit數量
MySQL [oceanbase]> alter resource pool testpool_1 unit_num=2;
7.2擴容後檢視一下資料分佈：
MySQL [oceanbase]> SELECT t.tenant_id, a.tenant_name, t.table_name, d.database_name, t1.tablegroup_name , t.part_num , t2.partition_id, t2.role , t2.ZONE, t2.svr_ip, concat(t2.svr_ip, ':', t2.svr_port) observer , t2.data_size , a.primary_zone , IF(t.locality = '' OR t.locality IS NULL, a.locality, t.locality) AS locality FROM oceanbase.__all_tenant AS a JOIN oceanbase.__all_virtual_database AS d ON ( a.tenant_id = d.tenant_id ) JOIN oceanbase.__all_virtual_table AS t ON (t.tenant_id = d.tenant_id AND t.database_id = d.database_id) JOIN oceanbase.__all_virtual_meta_table t2 ON (t.tenant_id = t2.tenant_id AND (t.table_id=t2.table_id OR t.tablegroup_id=t2.table_id) AND t2.ROLE IN (1) ) LEFT JOIN oceanbase.__all_virtual_tablegroup AS t1 ON (t.tenant_id = t1.tenant_id and t.tablegroup_id = t1.tablegroup_id) WHERE a.tenant_id IN (1001,1002 ) AND t.table_type IN (3) and table_name in ('tt1','tt2','tt3') ORDER BY t.tenant_id, t1.tablegroup_name, d.database_name, t.table_name, t2.partition_id;
8.租戶縮容
MySQL [oceanbase]> alter resource pool testpool_1 unit_num=1;
MySQL [oceanbase]> select gmt_create,gmt_modified,job_id,job_type,job_status,return_code,progress,tenant_id from __all_rootservice_job;
檢視__all_rootservice_job可檢視進度，等待狀態變成SUCCESS
刪除server
MySQL [oceanbase]> ALTER SYSTEM DELETE SERVER 'ip:port' [,'ip:port'…] [ZONE [=] 'zone']
總結：搭建過程一定要注意細節，細節決定成敗，遇到錯誤可以多看看observer.log、必要時也可以檢視obproxy.log檔案，OB的日誌數量比較多，可以通過篩選找出對你有用的日誌，通過錯誤去官網檢視說明，最後說一句，有時間一定要多搭建幾遍ob叢集，熟能生巧，次數多了才能深入瞭解OB。
注：本文不探討效能，只講過程。