NTP服務配置

NTP客戶端配置

# vim /etc/ntp.conf
server 92.0.0.250
### 手動同步下時間
# ntpdate -u 92.0.0.250
### 啟動服務
# systemctl start ntpd
# systemctl enable ntpd
### 檢查同步
# ntpq -p

監控節點配置（方式一）

以下指令在anode節點執行

為CEPH集群生成UUID

# uuidgen
cb9321ef-c7b4-48f7-a1bf-5c75deede6ee
# tee /etc/ceph/ceph.conf << EOF
[global]
fsid = cb9321ef-c7b4-48f7-a1bf-5c75deede6ee

[mon.anode]
host = anode
mon addr = 92.0.0.11:6789
EOF
 

創建集群密鑰環以及監視器密鑰

# ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon ‘allow *‘

創建管理員密鑰環，生成client.admin用戶，然後添加用戶到環上

# ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon ‘allow *‘ --cap osd ‘allow *‘ --cap mds ‘allow *‘ --cap mgr ‘allow *‘
 

創建一個bootstrap-osd密鑰環，生成client.bootstrap-osd用戶，然後添加用戶到環上

# ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon ‘profile bootstrap-osd‘

將生成的密鑰添加到ceph.mon.keyring

# ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
# ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
 

創建監視器映射

# monmaptool --create --add anode 92.0.0.11 --fsid cb9321ef-c7b4-48f7-a1bf-5c75deede6ee /tmp/monmap

創建默認數據目錄

# mkdir -p /var/lib/ceph/mon/ceph-anode

使用監視器映射和密鑰環填充監視器服務

# ceph-mon --mkfs -i anode --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring

更新配置文件

# tee /etc/ceph/ceph.conf << EOF
[global]
fsid = cb9321ef-c7b4-48f7-a1bf-5c75deede6ee
auth cluster required = cephx
auth service required = cephx
auth client required = cephx
osd journal size = 1024
osd pool default size = 2
osd pool default min size = 1
osd pool default pg num = 128
osd pool default pgp num = 128
osd crush chooseleaf type = 1

[mon.anode]
host = anode
mon addr = 92.0.0.11:6789
EOF

標記監視器服務就緒

# touch /var/lib/ceph/mon/ceph-anode/done

啟動監視器服務

### 有兩種方式啟動服務

### 方式一：修改數據目錄權限
# chown -R ceph:ceph /var/lib/ceph

### 方式二：修改服務啟動腳本
# vim /usr/lib/systemd/system/[email protected]
ExecStart=/usr/bin/ceph-mon -f --cluster ${CLUSTER} --id %i --setuser root --setgroup root
# systemctl daemon-reload

### 啟動服務
# systemctl start ceph-mon@anode
# systemctl enable ceph-mon@anode

### 檢測服務
# ceph -s
  services:
    mon: 1 daemons, quorum anode
    mgr: no daemons active
    osd: 0 osds: 0 up, 0 in

添加監控節點

以下指令在bnode節點執行

拷貝配置文件和客戶端密鑰

# scp [email protected]:/etc/ceph/ceph.conf /etc/ceph/
# scp [email protected]:/etc/ceph/ceph.client.admin.keyring /etc/ceph/

獲取監控集群密鑰環

# ceph auth get mon. -o /tmp/ceph.mon.keyring

獲取監控集群映射文件

# ceph mon getmap -o /tmp/monmap

使用監視器映射和密鑰環填充新的監視器服務

# ceph-mon --mkfs -i bnode --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring

修改配置文件

### 配置文件的mon.anode是讓你在沒有配置bnode的時候讓ceph mon命令知道怎麽去連接ceph monitor集群（刪除後會發現命令報錯），在配置好bnode後，可以刪除anode的配置也可以不刪除，後面配置cnode的時候可以使用bnode的配置也可以使用anode的配置
# vim /etc/ceph/ceph.conf
[global]
fsid = cb9321ef-c7b4-48f7-a1bf-5c75deede6ee
auth cluster required = cephx
auth service required = cephx
auth client required = cephx
osd journal size = 1024
osd pool default size = 2
osd pool default min size = 1
osd pool default pg num = 128
osd pool default pgp num = 128
osd crush chooseleaf type = 1

[mon.bnode]
host = bnode
mon addr = 92.0.0.12:6789

標記監視器服務就緒

# touch /var/lib/ceph/mon/ceph-bnode/done

啟動監視器服務

# chown -R ceph:ceph /var/lib/ceph

### 啟動服務
# systemctl start ceph-mon@bnode
# systemctl enable ceph-mon@bnode

### 檢測服務
# ceph -s
  services:
    mon: 2 daemons, quorum anode,bnode
    mgr: no daemons active
    osd: 0 osds: 0 up, 0 in

同理添加cnode節點（配置文件可以拷貝anode節點的也可以拷貝bnode節點的）

# ceph -s
  services:
    mon: 3 daemons, quorum anode,bnode,cnode
    mgr: no daemons active
    osd: 0 osds: 0 up, 0 in

配置管理服務

以下指令在anode節點執行

創建密鑰文件

# name="kolla"
# mkdir -p /var/lib/ceph/mgr/ceph-$name
# ceph auth get-or-create mgr.$name mon ‘allow profile mgr‘ osd ‘allow *‘ mds ‘allow *‘ -o /var/lib/ceph/mgr/ceph-$name/keyring

啟動服務

# chown -R ceph:ceph /var/lib/ceph
# systemctl start ceph-mgr@$name
# systemctl status ceph-mgr@$name
# systemctl enable ceph-mgr@$name

查詢服務

# ceph -s
  services:
    mon: 3 daemons, quorum anode,bnode,cnode
    mgr: kolla(active)
    osd: 2 osds: 2 up, 2 in

同理配置好bnode和cnode節點

# ceph -s
  services:
    mon: 3 daemons, quorum anode,bnode,cnode
    mgr: kolla(active, starting)
    osd: 2 osds: 2 up, 2 in

查詢命令幫助

# ceph tell mgr help

監控節點配置（方式二）

以下指令在anode節點執行

為CEPH集群生成UUID

# uuidgen
cb9321ef-c7b4-48f7-a1bf-5c75deede6ee
# tee /etc/ceph/ceph.conf << EOF
[global]
fsid = cb9321ef-c7b4-48f7-a1bf-5c75deede6ee
auth cluster required = cephx
auth service required = cephx
auth client required = cephx
osd journal size = 1024
osd pool default size = 2
osd pool default min size = 1
osd pool default pg num = 128
osd pool default pgp num = 128
osd crush chooseleaf type = 1
mon initial members = anode, bnode, cnode
mon host = 92.0.0.11, 92.0.0.12, 92.0.0.13
EOF

創建集群密鑰環以及監視器密鑰

# ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon ‘allow *‘

創建管理員密鑰環，生成client.admin用戶，然後添加用戶到環上

# ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon ‘allow *‘ --cap osd ‘allow *‘ --cap mds ‘allow *‘ --cap mgr ‘allow *‘

創建一個bootstrap-osd密鑰環，生成client.bootstrap-osd用戶，然後添加用戶到環上

# ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon ‘profile bootstrap-osd‘

將生成的密鑰添加到ceph.mon.keyring

# ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
# ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring

創建監視器映射

# monmaptool --create --add anode 92.0.0.11 --add bnode 92.0.0.12 --add cnode 92.0.0.13 --fsid cb9321ef-c7b4-48f7-a1bf-5c75deede6ee --clobber /tmp/monmap
# monmaptool --print /tmp/monmap

創建默認數據目錄

# mkdir -p /var/lib/ceph/mon/ceph-anode

使用監視器映射和密鑰環填充監視器服務

# ceph-mon --mkfs -i anode --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring

標記監視器服務就緒

# touch /var/lib/ceph/mon/ceph-anode/done

啟動監視器服務

### 啟動服務
# chown -R ceph:ceph /var/lib/ceph
# systemctl start ceph-mon@anode
# systemctl enable ceph-mon@anode

### 檢測服務
# ceph daemon mon.anode mon_status

添加監控節點

以下指令在bnode節點執行

拷貝配置文件、密鑰和監視器映射

# scp [email protected]:/etc/ceph/ceph.conf /etc/ceph/
# scp [email protected]:/tmp/ceph.mon.keyring /tmp
# scp [email protected]:/tmp/monmap /tmp

創建默認數據目錄

# mkdir -p /var/lib/ceph/mon/ceph-bnode

使用監視器映射和密鑰環填充監視器服務

# ceph-mon --mkfs -i bnode --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring

標記監視器服務就緒

# touch /var/lib/ceph/mon/ceph-bnode/done

啟動監視器服務

### 啟動服務
# chown -R ceph:ceph /var/lib/ceph
# systemctl start ceph-mon@banode
# systemctl enable ceph-mon@bnode

### 檢測服務
# ceph daemon mon.bnode mon_status

同理添加cnode節點

添加OSDS

方式一：腳本（dnode）

以下指令在dnode節點執行

拷貝配置文件和密鑰環

# scp [email protected]:/etc/ceph/ceph.conf /etc/ceph/
# scp [email protected]:/etc/ceph/ceph.client.admin.keyring /etc/ceph/
# scp [email protected]:/var/lib/ceph/bootstrap-osd/ceph.keyring /var/lib/ceph/bootstrap-osd/

創建OSD

# ceph-volume lvm create --data /dev/vdb
Running command: ceph-authtool --gen-print-key
Running command: ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 2d6a968f-9c10-4c57-adb6-be0a1341581b
Running command: vgcreate --force --yes ceph-cb9321ef-c7b4-48f7-a1bf-5c75deede6ee /dev/vdb
 stdout: Physical volume "/dev/vdb" successfully created
 stdout: Volume group "ceph-cb9321ef-c7b4-48f7-a1bf-5c75deede6ee" successfully created
Running command: lvcreate --yes -l 100%FREE -n osd-block-2d6a968f-9c10-4c57-adb6-be0a1341581b ceph-cb9321ef-c7b4-48f7-a1bf-5c75deede6ee
 stdout: Logical volume "osd-block-2d6a968f-9c10-4c57-adb6-be0a1341581b" created.
Running command: ceph-authtool --gen-print-key
Running command: mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-0
Running command: chown -R ceph:ceph /dev/dm-4
Running command: ln -s /dev/ceph-cb9321ef-c7b4-48f7-a1bf-5c75deede6ee/osd-block-2d6a968f-9c10-4c57-adb6-be0a1341581b /var/lib/ceph/osd/ceph-0/block
Running command: ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-0/activate.monmap
 stderr: got monmap epoch 5
Running command: ceph-authtool /var/lib/ceph/osd/ceph-0/keyring --create-keyring --name osd.0 --add-key AQDR0sFaI6fMAxAAhoZKkXR29nUPbWCeAAibkg==
 stdout: creating /var/lib/ceph/osd/ceph-0/keyring
 stdout: added entity osd.0 auth auth(auid = 18446744073709551615 key=AQDR0sFaI6fMAxAAhoZKkXR29nUPbWCeAAibkg== with 0 caps)
Running command: chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/keyring
Running command: chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/
Running command: ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 0 --monmap /var/lib/ceph/osd/ceph-0/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-0/ --osd-uuid 2d6a968f-9c10-4c57-adb6-be0a1341581b --setuser ceph --setgroup ceph
--> ceph-volume lvm prepare successful for: /dev/vdb
Running command: ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-cb9321ef-c7b4-48f7-a1bf-5c75deede6ee/osd-block-2d6a968f-9c10-4c57-adb6-be0a1341581b --path /var/lib/ceph/osd/ceph-0
Running command: ln -snf /dev/ceph-cb9321ef-c7b4-48f7-a1bf-5c75deede6ee/osd-block-2d6a968f-9c10-4c57-adb6-be0a1341581b /var/lib/ceph/osd/ceph-0/block
Running command: chown -R ceph:ceph /dev/dm-4
Running command: chown -R ceph:ceph /var/lib/ceph/osd/ceph-0
Running command: systemctl enable ceph-volume@lvm-0-2d6a968f-9c10-4c57-adb6-be0a1341581b
 stderr: Created symlink from /etc/systemd/system/multi-user.target.wants/[email protected] to /usr/lib/systemd/system/[email protected].
Running command: systemctl start ceph-osd@0
--> ceph-volume lvm activate successful for osd ID: 0
--> ceph-volume lvm activate successful for osd ID: None
--> ceph-volume lvm create successful for: /dev/vdb

查詢集群狀態

# ceph -s
  services:
    mon: 3 daemons, quorum anode,bnode,cnode
    mgr: no daemons active
    osd: 1 osds: 1 up, 1 in

方式二：手動（enode）

以下指令在enode節點執行

拷貝配置文件和密鑰環

# scp [email protected]:/etc/ceph/ceph.conf /etc/ceph/
# scp [email protected]:/etc/ceph/ceph.client.admin.keyring /etc/ceph/
# scp [email protected]:/var/lib/ceph/bootstrap-osd/ceph.keyring /var/lib/ceph/bootstrap-osd/

為OSD生成UUID

# UUID=$(uuidgen)

為OSD創建cephx密鑰

# OSD_SECRET=$(ceph-authtool --gen-print-key)

創建OSD

### ceph命令-i原本是要指定一個json文件，這邊使用管道操作中的"-"將前面echo的輸出結果作為文件
# ID=$(echo "{\"cephx_secret\": \"$OSD_SECRET\"}" | ceph osd new $UUID -i - -n client.bootstrap-osd -k /var/lib/ceph/bootstrap-osd/ceph.keyring)

為OSD創建數據目錄

# mkdir /var/lib/ceph/osd/ceph-$ID

掛載磁盤到數據目錄上

# mkfs.xfs /dev/vdb
# mount /dev/vdb /var/lib/ceph/osd/ceph-$ID

修改fstab配置

### 在配置文件末尾添加
# vim /etc/fstab
/dev/vdb /var/lib/ceph/osd/ceph-1               xfs     defaults        0 0

創建OSD密鑰文件

# ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-$ID/keyring --name osd.$ID --add-key $OSD_SECRET

初始化OSD數據目錄

# ceph-osd -i $ID --mkfs --osd-uuid $UUID

啟動服務

# chown -R ceph:ceph /var/lib/ceph/osd/ceph-$ID
# systemctl enable ceph-osd@$ID
# systemctl start ceph-osd@$ID

查詢集群狀態

# ceph -s
  services:
    mon: 3 daemons, quorum anode,bnode,cnode
    mgr: no daemons active
    osd: 2 osds: 2 up, 2 in

添加MDS節點

以下指令在anode節點執行

創建 MDS 數據目錄

# mkdir -p /var/lib/ceph/mds/ceph-anode

創建密鑰環

# ceph-authtool --create-keyring /var/lib/ceph/mds/ceph-anode/keyring --gen-key -n mds.anode

導入密鑰環並設置權限

# ceph auth add mds.anode osd "allow rwx" mds "allow" mon "allow profile mds" -i /var/lib/ceph/mds/ceph-anode/keyring

修改配置文件

# vim /etc/ceph/ceph.conf
[global]
fsid = cb9321ef-c7b4-48f7-a1bf-5c75deede6ee
auth cluster required = cephx
auth service required = cephx
auth client required = cephx
osd journal size = 1024
osd pool default size = 2
osd pool default min size = 1
osd pool default pg num = 128
osd pool default pgp num = 128
osd crush chooseleaf type = 1

[mon.anode]
host = anode
mon addr = 92.0.0.11:6789

[mds.anode]
host = anode

啟動服務

# chown -R ceph:ceph /var/lib/ceph
# systemctl start ceph-mds@anode
# systemctl status ceph-mds@anode
# systemctl enable ceph-mds@anode

參考文檔

ceph集群的部署
ceph集群之-添加刪除Monitor

CEPH安裝教程（中）