corosync+pacemaker+crm實現drbd高可用
系統：Centos 7.2 內核3.10.0-327.el7.x86_64 # 內核需要高於2.6.32
Drbd : 192.168.8.111：node1/dev/drdb0 /mydeta
192.168.8.112 : node2
Mysql_vip: 192.168.8.200 #下章實現
# 需要的軟件包：
mariadb-5.5.53-linux-i686.tar.gz
drbd84-utils-8.9.1-1.el7.elrepo.x86_64.rpm
kmod-drbd84-8.4.6-1.el7.elrepo.x86_64.rpm
yum 需要安裝的軟件 pcs, ansible, crmsh, libxslt
# drbd是基於主機名通信的在/etc/hosts中增加
# 以本地ip地址如192.168.8.111 node1這樣的方式添加一個地址一行、 127.0.0.1 不要填uname -n的名稱
# 安裝crmsh yum源如下 [[email protected] ~]# vim /etc/yum.repos.d/ha.repo
[network_ha-clustering_Stable]
name=Stable High Availability/Clustering packages (CentOS_CentOS-7)
type=rpm-md
baseurl=http://download.opensuse.org/repositories/network:/ha-clustering:/Stable/CentOS_CentOS-7/
gpgcheck=1
gpgkey=http://download.opensuse.org/repositories/network:/ha-clustering:/Stable/CentOS_CentOS-7//repodata/repomd.xml.key
enabled=1
# 安裝crmsh 單臺主機安裝就行
yum -y install crmsh libxslt ansible pcs
# 主機互信
[[email protected] ~]# ssh-keygen
[[email protected] ~]# cat .ssh/id_rsa.pub > .ssh/authorized_keys
[[email protected] ~]# chmod go= .ssh/authorized_keys
[[email protected] ~]# scp -p .ssh/id_rsa.pub .ssh/authorized_keys 192.168.8.112:/root/.ssh/
# 出現以下錯誤
scp: /root/.ssh/: No such file or directory
# 解決方案
連接node2創建.ssh mkdir /root/.ssh
# drbd配置段
# 安裝drbd前先準備兩塊硬盤 node1 跟node2 同樣配置
rpm -ivh http://elrepo.reloumirrors.net/elrepo/el7/x86_64/RPMS/drbd84-utils-8.9.1-1.el7.elrepo.x86_64.rpm
yum -y install http://elrepo.reloumirrors.net/elrepo/el7/x86_64/RPMS/kmod-drbd84-8.4.6-1.el7.elrepo.x86_64.rpm
[[email protected] drbd-8.4.8-1]# modprobe drbd
[[email protected] drbd-8.4.8-1]# lsmod | grep drbd
drbd 405537 0
libcrc32c 12644 2 xfs,drbd
# 將新建的磁盤分區
[[email protected] sbin]# fdisk /dev/sdb# 自己新建就成、兩邊大小保持一致跟raid1一樣
# 配置drbd
[[email protected] sbin]#
# drbd配置: http://502245466.blog.51cto.com/7559397/1298945
# drbd常見錯誤: http://www.j3j5.com/post-128.html
[[email protected] drbd.d]# vim global_common.conf
global {
usage-count no;
}
common {
protocol C;
handlers {
pri-on-incon-degr "/usr/lib/drbd/notify-pri-on-incon-degr.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
pri-lost-after-sb "/usr/lib/drbd/notify-pri-lost-after-sb.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
local-io-error "/usr/lib/drbd/notify-io-error.sh; /usr/lib/drbd/notify-emergency-shutdown.sh; echo o > /proc/sysrq-trigger ; halt -f";
}
startup {
}
options {
}
disk {
on-io-error detach;
}
net {
cram-hmac-alg "sha1";
shared-secret "xiong123";
}
syncer {
rate 1024M;
}
}
[[email protected] drbd.d]# vim mydrbd.res
resource mydrbd {
device /dev/drbd0;
disk /dev/sdb1;
meta-disk internal;
on node1{on為開啟，node1為節點名稱
address 192.168.8.111:7789;
}
on node2{
address 192.168.8.112:7789;
}
}
[[email protected] drbd.d]# scp mydrbd.res global_common.conf node2:/usr/local/drbd/etc/drbd.d/
############測試drbd是否成功######
[[email protected] drbd.d]# drbdadm create-md mydrbd
initializing activity log
NOT initializing bitmap
Writing meta data...
New drbd meta data block successfully created
# 配置啟動否則corosync ocf無法直接定義drbd啟動
[[email protected] sbin]# systemctl enable drbd.service
# 此處為測試drbd是否完整 兩邊都得同時啟動
[[email protected] sbin]# systemctl start drbd.service
[[email protected] drbd.d]# drbdadm primary --force mydrbd # 將主機強制成為主節點
[[email protected] drbd.d]# drbd-overview
0:mydrbd/0 Connected Primary/Secondary UpToDate/UpToDate# Primary/Secondary 主從，另一端是Secondary/Primary
#格式磁盤
[[email protected] drbd.d]# mkfs.ext4 /dev/drbd0
# 掛載 兩個節點都需要創建 /mydata/
[[email protected] drbd.d]# mount /dev/drbd0 /mydata/
[[email protected] drbd.d]# df -Th
/dev/drbd0 ext4 4.8G 20M 4.6G 1% /mydata
復制一個文件至/mydata目錄中，並將主drbd設置為從
[[email protected] ~]# umount /mydata/
[[email protected] ~]# drbdadm secondary mydrbd
[[email protected] ~]# drbd-overview
0:mydrbd/0 Connected Secondary/Secondary UpToDate/Diskless
# 節點2操作配置 無需格式化/dev/drbd0
[[email protected] drbd.d]# drbdadm primary mydrbd
[[email protected] drbd.d]# mkdir /mydata
[[email protected] drbd.d]# mount /dev/drbd0 /mydata/
[[email protected] drbd.d]# cd /mydata/
[[email protected] mydata]# ls
issue lost+found
# 最後將drbd服務都停止
[[email protected] /]# umount /mydata/
[[email protected] /]# systemctl stop drbd
[[email protected] /]# systemctl stop drbd
############################測試結束########################################
# pcs配置drbd高可用
# 安裝
# 安裝pcs軟件如果報/sbin/xxx命令的名字將crmsh.repo先改成crmsh.repo.bak 再安裝
[[email protected] sbin]# ansible ha -m yum -a "name=pcs state=installed"
# pcs認證需要
[[email protected] sbin]# ansible ha -m shell -a ‘echo "xiong" | passwd --stdin hacluster‘
# 啟動pcs服務
[[email protected] sbin]# ansible ha -m service -a "name=pcsd state=started enabled=yes"
# 認證集群服務
[[email protected] sbin]# pcs cluster auth node1 node2
Username: hacluster
Password:
node1: Authorized
node2: Authorized
# 添加一個集群名為myha 並設置組播地址為235.35.32.24
[[email protected] sbin]# pcs cluster setup --name myha node1 node2 --mcast0 235.35.32.24
# 檢查集群服務是否正常
[[email protected] sbin]# corosync-cmapctl | grep mem
runtime.totem.pg.mrp.srp.memb_commit_token_rx (u64) = 4
runtime.totem.pg.mrp.srp.memb_commit_token_tx (u64) = 4
runtime.totem.pg.mrp.srp.memb_join_rx (u64) = 5
runtime.totem.pg.mrp.srp.memb_join_tx (u64) = 3
runtime.totem.pg.mrp.srp.memb_merge_detect_rx (u64) = 332
runtime.totem.pg.mrp.srp.memb_merge_detect_tx (u64) = 332
runtime.totem.pg.mrp.srp.members.1.config_version (u64) = 0
runtime.totem.pg.mrp.srp.members.1.ip (str) = r(0) ip(192.168.8.111)
runtime.totem.pg.mrp.srp.members.1.join_count (u32) = 1
runtime.totem.pg.mrp.srp.members.1.status (str) = joined
runtime.totem.pg.mrp.srp.members.2.config_version (u64) = 0
runtime.totem.pg.mrp.srp.members.2.ip (str) = r(0) ip(192.168.8.112)
runtime.totem.pg.mrp.srp.members.2.join_count (u32) = 1
runtime.totem.pg.mrp.srp.members.2.status (str) = joined
# 檢查狀態
[[email protected] sbin]# pcs status
Cluster name: myha
WARNING: no stonith devices and stonith-enabled is not false
Stack: corosync
Current DC: node1 (version 1.1.13-10.el7-44eb2dd) - partition with quorum
2 nodes and 0 resources configured
Online: [ node1 node2 ]
# 配置高可用
crm(live)configure# primitive mydrbd ocf:linbit:drbd params drbd_resource="mystore" op monitor role="Master" interval=30s timeout=30s op monitor role="Slave" timeout=30s op start timeout=240s op stop timeout=100s
####################正確模板##########################################
primitive mydatas ocf:linbit:drbd \
params drbd_resource=mystore \
op monitor role=Master interval=10s timeout=20s \
op monitor role=Slave interval=20s timeout=20s \
op start timeout=240s interval=0 \
op stop timeout=100s interval=0
######################################################################
crm(live)configure# ms mydrbd_master mydrbd meta clone-max=2 clone-node-max=1 master-max=1 master-node-max=1
crm(live)configure# verify #檢查是否成功
crm(live)# status
Online: [ node73 node74 ]
Master/Slave Set: ms_mydatas [mydatas]
Masters: [ node73 ]
Slaves: [ node74 ]
# 配置文件掛載目錄
crm(live)configure# primitive mount ocf:heartbeat:Filesystem params device="/dev/drbd0" directory="/mydata" fstype="ext4" op start timeout=60s op stop timeout=60s
# 定義排序約束 讓mount一直跟著ms_mydatas的主
crm(live)configure# colocation mount_with_ms_mydatas inf: mount ms_mydatas:Master
# 檢查是否成功
[[email protected] ~]# df -Th
/dev/drbd0 ext4 16G 45M 15G 1% /mydata
[[email protected] ~]# drbd-overview
0:mystore/0 Connected Primary/Secondary UpToDate/UpToDate /mydata ext4 16G 45M 15G 1%
####################### drbd主從配置文件###############################
primitive mount Filesystem \
params device="/dev/drbd0" directory="/mydata" fstype=ext4 \
op start timeout=60s interval=0 \
op stop timeout=60s interval=0
primitive mydatas ocf:linbit:drbd \
params drbd_resource=mystore \
op monitor role=Master interval=10s timeout=20s \
op monitor role=Slave interval=20s timeout=20s \
op start timeout=240s interval=0 \
op stop timeout=100s interval=0
ms ms_mydatas mydatas \
meta clone-max=2 clone-node-max=1 master-max=1 master-node-max=1 notify=true
colocation mount_with_ms_mydatas inf: mount ms_mydatas:Master
#########################################################################
# 錯誤整理
1、ERROR: error: unpack_resources:Resource start-up disabled since no STONITH resources have been defined
error: unpack_resources:Either configure some or disable STONITH with the stonith-enabled option
error: unpack_resources:NOTE: Clusters with shared data need STONITH to ensure data integrity
解決：crm(live)configure# property stonith-enabled=false
# 錯誤
[[email protected] drbd.d]# drbdadm primary mystore
0: State change failed: (-2) Need access to UpToDate data
Command ‘drbdsetup-84 primary 0‘ terminated with exit code 17
# 解決方案
[[email protected] drbd.d]# drbdadm primary mystore --force
# 錯誤
[[email protected] drbd.d]# pcs status
Error: cluster is not currently running on this node
# 解決方案
[[email protected] drbd.d]# pcs cluster start --all
# 啟動主從失敗
Failed Actions:
* mydrbd_stop_0 on node74 ‘not configured‘ (6): call=8, status=complete, exitreason=‘none‘,
last-rc-change=‘Wed May 24 15:27:42 2017‘, queued=0ms, exec=138ms
* mydrbd_stop_0 on node73 ‘not configured‘ (6): call=8, status=complete, exitreason=‘none‘,
last-rc-change=‘Wed May 24 15:27:41 2017‘, queued=0ms, exec=134ms
# 解決方案
設置各種超時時長都加上"s" primitive mydrbd ocf:linbit:drbd params drbd_resource="mystore" op monitor role="Master" interval=30s timeout=30s op monitor role="Slave" timeout=30s op start timeout=240s op stop timeout=100s
drbd yum包下載地址：http://elrepo.reloumirrors.net/elrepo/el7/x86_64/RPMS/
配置參閱鏈接： http://blog.csdn.net/qq_21127313/article/details/68210977
本文出自 “xiong” 博客，請務必保留此出處http://xiong51.blog.51cto.com/5239058/1929054
corosync+pacemaker+crm實現drbd高可用