多es 叢集資料遷移方案

前言

加入新公司的第二個星期的星期二 遇到另一個專案需要技術性支援：驗證es多叢集的資料備份方案，需要我參與驗證，在這個專案中需要關注到兩個叢集的互通性。es叢集是部署在不同的k8s環境中，K8s環境未必可以互相通訊。在這個過程中也遇到很多問題，關關難過關關過。

因為是新入職的公司的第一個完成的任務，特記錄如下

環境

• k8s版本：1.18.3

• es版本：7.8

• logstash版本：7.8

• ceph： s3

驗證方案

-方案1 ： logstash

• 方案2： ceph s3
• s3 操作：https://www.cnblogs.com/sunhongleibibi/p/11661123.html
• 部署資源清單: 連結：https://pan.baidu.com/s/13wxwywdu_9g-Szxy_ugC7w
  
  提取碼：anch
• PS：連結為部署資源清單和驗證方案可編輯圖

es 雙叢集環境

[root@localhost a]# ctl get po -n a

NAME                                    READY   STATUS    RESTARTS   AGE

elasticsearch-client-7bf748d697-gnrjz   1/1     Running   0          23h

elasticsearch-data-0                    1/1     Running   0          23h

elasticsearch-master-7569856dd7-z7q4z   1/1     Running   0          23h

logstash-69f7b75f5-vl4k2                1/1     Running   0          23h

-------------------------------------------------------------------------------

[root@localhost a]# ctl get po -n b

NAME                                    READY   STATUS    RESTARTS   AGE

elasticsearch-client-84f954845b-4tj2s   1/1     Running   0          2d1h

elasticsearch-data-0                    1/1     Running   0          2d1h

elasticsearch-master-58b5bf9d-62nfg     1/1     Running   0          2d1h

資料節點儲存為nfs

go向es插入資料

/**

 * @Author: anchnet

 * @Description:

 * @File:  main.go

 * @Version: 1.0.0

 * @Date: 2021/7/2 13:07

 */

package main

import (

	"bytes"

	"encoding/json"

	"flag"

	"fmt"

	"github.com/elastic/go-elasticsearch/v8"

	"io/ioutil"

	"strconv"

	"time"

)

func main() {

	start := time.Now() // 獲取當前時間

	//支援引數

	var (

		count int    // 起始數

		total int    // 截至數

		index string // index

		id    string // id

		title string // title

	)

	flag.IntVar(&count, "c", 1, "起始數")

	flag.IntVar(&total, "e", 1, "截至數")

	flag.StringVar(&index, "i", "", "index")

	flag.StringVar(&id, "d", "", "id")

	flag.StringVar(&title, "t", "", "title")

	// 解析引數

	flag.Parse()

	if index == "" {

		index = "demo"

	}

	if id == "" {

		id = "id_1"

	}

	if title == "" {

		title = "安暢"

	}

	fmt.Println("count：", count)

	fmt.Println("total：", total)

	fmt.Println("index：", index)

	fmt.Println("id：", id)

	fmt.Println("title：", title)

	addresses := []string{"http://192.168.201.113:31001"}

	config := elasticsearch.Config{

		Addresses: addresses,

		Username:  "elastic",

		Password:  "l605eslS0mYOYEB4grNU",

		CloudID:   "",

		APIKey:    "",

	}

	// new client

	es, err := elasticsearch.NewClient(config)

	if err != nil {

		fmt.Println(err, "Error creating the client")

	}

	//Get(*es, index, id)

	//Update(*es, index, id)

	//Get(*es, index, id)

	create(*es, index, count, total)

	//Search(*es, index, title)

	elapsed := time.Since(start)

	fmt.Println("該函式執行完成耗時：", elapsed)

}

func create(es elasticsearch.Client, index string, count int, total int) bool {

	//var wg sync.WaitGroup

	// Create creates a new document in the index.

	// Returns a 409 response when a document with a same ID already exists in the index.

	for i := count; i < total; i++ {

		//wg.Add(1)

		k := strconv.Itoa(i)

		var buf bytes.Buffer

		content, err := ioutil.ReadFile("./test.log") // just pass the file name

		if err != nil {

			fmt.Print(err)

		}

		doc := map[string]interface{}{

			"title":   "安暢是一家怎麼樣的公司呢？" + k,

			"content": content,

			"time":    time.Now().Unix(),

			"date":    time.Now(),

		}

		if err := json.NewEncoder(&buf).Encode(doc); err != nil {

			fmt.Println(err, "Error encoding doc")

			return false

		}

		func() {

			time.Sleep(1 * time.Millisecond)

			res, err := es.Create(index, "idx_"+k, &buf)

			if err != nil {

				fmt.Println(err, "Error create response")

			}

			//wg.Done()

			defer res.Body.Close()

			fmt.Println(res.String())

		}()

	}

	//wg.Wait()

	return true

}

//go run main.go -c 3 -e 10000

方案一驗證：

驗證向a叢集寫入資料

# kubectl exec -it elasticsearch-client-7bf748d697-gnrjz  -n a -- curl -u elastic:l605eslS0mYOYEB4grNU http://elasticsearch-client.a:9200/_cat/indices

yellow open demo                        sOGM8_9UTnuEUyCpLRx6Mg 1 1  10000     0 74.3mb 74.3mb

logstash-cm.yaml

apiVersion: v1

kind: ConfigMap

metadata:

  name: logstash-config

  namespace: a

data:

  containers.conf: |

    input {

      elasticsearch {

         hosts => ["http://elasticsearch-client.a:9200"]

         user => "elastic"

         password => "l605eslS0mYOYEB4grNU"

         index => "demo"

         #size => 10000

       #  scroll => "1m"

        # codec => "json"

         #docinfo => true

     }

    }

     #filter {

       #sleep {

         #time => "1"

         #every => 10

       #}

    #}

    output {

      elasticsearch {

         hosts => ["elasticsearch-client.b:9200"]

         user => "elastic"

         password => "bcsvxqPxIKIJ5p5e20zZ"

         index => "demo"

         document_type => "%{[@metadata][_type]}"

         document_id => "%{[@metadata][_id]}"

      }

    # stdout { codec => rubydebug { metadata => true } }

    }

驗證b叢集

PS：在反覆驗證下 在現有叢集環境下，在資料量為74.3M 時，由a es叢集向b叢集 資料傳輸的時間大約為3S.

方案二驗證：

• 推薦此方案
• 已解決的問題：1、es官方映象與s3外掛的整合 2、bucket的許可權問題 3、es配置過載 4、a es叢集向s3快照 到b es叢集恢復

Dockerfile

es映象增加s3外掛

FROM elasticsearch:7.8.0

ADD https://artifacts.elastic.co/downloads/elasticsearch-plugins/repository-s3/repository-s3-7.8.0.zip /usr/share/elasticsearch/

RUN yes | elasticsearch-plugin install file:///usr/share/elasticsearch/repository-s3-7.8.0.zip

建立s3 bucket

# s3cmd ls | grep zisefeizhu

2021-07-06 09:24  s3://zisefeizhu

# s3cmd info s3://zisefeizhu

s3://zisefeizhu/ (bucket):

   Location:  default

   Payer:     BucketOwner

   Expiration Rule: none

   Policy:    none

   CORS:      none

   ACL:       *anon*: READ

   ACL:       anchnet: FULL_CONTROL

   URL:       http://49.235.65.51:7480/zisefeizhu/

  # s3cmd setacl s3://zisefeizhu/ --acl-public --recursive

es 結合s3

以下操作在a es 叢集的每個Pod 上執行

# elasticsearch-keystore add s3.client.default.access_key

# elasticsearch-keystore add s3.client.default.secret_key

# elasticsearch-keystore list

keystore.seed

s3.client.default.access_key

s3.client.default.secret_key

建立一個S3型別的倉庫

# ctl exec -it elasticsearch-client-7bf748d697-gnrjz -n a -- curl -u elastic:l605eslS0mYOYEB4grNU -H 'Content-Type: application/json' -XPUT http://elasticsearch-client.a:9200/_snapshot/s3_obs_repository  -d '

{

  "type": "s3",

  "settings": {

    "endpoint": "49.235.65.51:7480",

    "bucket": "s3://zisefeizhu",

    "base_path": "/zisefeizhu",

    "max_snapshot_bytes_per_sec": "200mb",

     "max_restore_bytes_per_sec": "200mb"

   }

}

'

！報錯：

{"error":{"root_cause":[{"type":"repository_verification_exception","reason":"[s3_obs_repository] path [/zisefeizhu] is not accessible on master node"}],"type":"repository_verification_exception","reason":"[s3_obs_repository] path [/zisefeizhu] is not accessible on master node","caused_by":{"type":"i_o_exception","reason":"Unable to upload object [/zisefeizhu/tests-FLt0bsA2QcCTc_l9JeO78w/master.dat] using a single upload","caused_by":{"type":"sdk_client_exception","reason":"sdk_client_exception: Failed to connect to service endpoint: ","caused_by":{"type":"i_o_exception","reason":"Connect timed out"}}}},"status":500}

PS：master node 無法訪問base_path ？？？？

master node 也已經添加了s3的訪問許可權、且s3的bucket 也已經公開了許可權。

es 與ceph s3 關聯問題解決

1、es 叢集的master節點和data節點 增加 s3的key

2、es配叢集過載配置

# kubectl exec -it elasticsearch-client-7bf748d697-gnrjz  -n a -- curl -u elastic:l605eslS0mYOYEB4grNU -XPOST http://elasticsearch-client.a:9200/_nodes/reload_secure_settings

a叢集建立S3型別的快照倉庫

# ctl exec -it elasticsearch-client-7bf748d697-gnrjz -n a -- curl -u elastic:l605eslS0mYOYEB4grNU -H 'Content-Type: application/json' -XPUT http://elasticsearch-client.a:9200/_snapshot/s3_obs_repository  -d '

  {

    "type": "s3",

    "settings": {

       "endpoint": "http://49.235.65.51:7480",

       "bucket": "zisefeizhu",

       "base_path": "/zisefeizhu",

       "max_snapshot_bytes_per_sec": "200mb",

       "max_restore_bytes_per_sec": "200mb"

     }

}

'

## 成功會返回：

 {"acknowledged":true}

## 檢視

# ctl exec -it elasticsearch-client-7bf748d697-gnrjz -n a -- curl -u elastic:l605eslS0mYOYEB4grNU  -XGET http://elasticsearch-client.a:9200/_snapshot/s3_obs_repository/_all

{"snapshots":[]}

## 刪除

# ctl exec -it elasticsearch-client-7bf748d697-gnrjz -n a -- curl -u elastic:l605eslS0mYOYEB4grNU  -XDELETE http://elasticsearch-client.a:9200/_snapshot/s3_obs_repository

{"acknowledged":true}

a叢集建立快照

# ctl exec -it elasticsearch-client-7bf748d697-gnrjz -n a -- curl -u elastic:l605eslS0mYOYEB4grNU -H 'Content-Type: application/json' -XPUT http://elasticsearch-client.a:9200/_snapshot/s3_obs_repository/snapshot_zisefeizhu  -d '

{

  "indices": "demo",

  "ignore_unavailable": true,

  "include_global_state": false

}

'

## s3 檢視快照大小

# s3cmd du -H s3://zisefeizhu

  74M      41 objects s3://zisefeizhu/   #和demo 索引大小一致

## 檢視倉庫下的所有快照

# ctl exec -it elasticsearch-client-7bf748d697-gnrjz -n a -- curl -u elastic:l605eslS0mYOYEB4grNU  -XGET http://elasticsearch-client.a:9200/_snapshot/s3_obs_repository/_all

{"snapshots":[{"snapshot":"snapshot_zisefeizhu","uuid":"OhTUdg63QuGEtiynLQUh2A","version_id":7080099,"version":"7.8.0","indices":["demo"],"include_global_state":false,"state":"SUCCESS","start_time":"2021-07-13T05:23:53.378Z","start_time_in_millis":1626153833378,"end_time":"2021-07-13T05:24:09.226Z","end_time_in_millis":1626153849226,"duration_in_millis":15848,"failures":[],"shards":{"total":1,"failed":0,"successful":1}}]

## 檢視 snapshot_zisefeizhu 快照的概要狀態

# ctl exec -it elasticsearch-client-7bf748d697-gnrjz -n a -- curl -u elastic:l605eslS0mYOYEB4grNU  -XGET http://elasticsearch-client.a:9200/_snapshot/s3_obs_repository/snapshot_zisefeizhu

{"snapshots":[{"snapshot":"snapshot_zisefeizhu","uuid":"OhTUdg63QuGEtiynLQUh2A","version_id":7080099,"version":"7.8.0","indices":["demo"],"include_global_state":false,"state":"SUCCESS","start_time":"2021-07-13T05:23:53.378Z","start_time_in_millis":1626153833378,"end_time":"2021-07-13T05:24:09.226Z","end_time_in_millis":1626153849226,"duration_in_millis":15848,"failures":[],"shards":{"total":1,"failed":0,"successful":1}}]}

## 檢視 snapshot_zisefeizhu 快照的詳細狀態

# ctl exec -it elasticsearch-client-7bf748d697-gnrjz -n a -- curl -u elastic:l605eslS0mYOYEB4grNU  -XGET http://elasticsearch-client.a:9200/_snapshot/s3_obs_repository/snapshot_zisefeizhu/_status

{"snapshots":[{"snapshot":"snapshot_zisefeizhu","repository":"s3_obs_repository","uuid":"OhTUdg63QuGEtiynLQUh2A","state":"SUCCESS","include_global_state":false,"shards_stats":{"initializing":0,"started":0,"finalizing":0,"done":1,"failed":0,"total":1},"stats":{"incremental":{"file_count":34,"size_in_bytes":77950953},"total":{"file_count":34,"size_in_bytes":77950953},"start_time_in_millis":1626153833378,"time_in_millis":15848},"indices":{"demo":{"shards_stats":{"initializing":0,"started":0,"finalizing":0,"done":1,"failed":0,"total":1},"stats":{"incremental":{"file_count":34,"size_in_bytes":77950953},"total":{"file_count":34,"size_in_bytes":77950953},"start_time_in_millis":1626153833627,"time_in_millis":14222},"shards":{"0":{"stage":"DONE","stats":{"incremental":{"file_count":34,"size_in_bytes":77950953},"total":{"file_count":34,"size_in_bytes":77950953},"start_time_in_millis":1626153833627,"time_in_millis":14222}}}}}}]}

## 刪除快照

# ctl exec -it elasticsearch-client-7bf748d697-gnrjz -n a -- curl -u elastic:l605eslS0mYOYEB4grNU  -XDELETE http://elasticsearch-client.a:9200/_snapshot/s3_obs_repository/snapshot_zisefeizhu

• ignore_unavailable true 在建立快照的過程中會忽略快照請求中不存在的索引。
• ignore_unavailable 的預設值為false，即如果要做快照的索引不存在，快照請求就會失敗。
• include_global_state false 能夠防止叢集的全域性狀態被作為快照的一部分儲存起來。
• include_global_state的預設值為true
• partial預設值是false，如果快照中的一個或多個主分片不是可用狀態，叢集會等待直到主分片全部可用才開始做快照。 通過設定 partial 為 true 可以忽略主分片全部可用檢查直接做快照。

b叢集增加s3 key

# ctl get po -n b

NAME                                    READY   STATUS    RESTARTS   AGE

elasticsearch-client-7bf748d697-tfh26   1/1     Running   0          4m7s

elasticsearch-data-0                    1/1     Running   0          4m12s

elasticsearch-master-7569856dd7-828hp   1/1     Running   0          4m13s

# ctl exec -it elasticsearch-client-7bf748d697-tfh26 -n b -- bash

[root@elasticsearch-client-7bf748d697-tfh26 elasticsearch]# elasticsearch-plugin list

repository-s3

[root@elasticsearch-client-7bf748d697-tfh26 elasticsearch]# elasticsearch-keystore list

keystore.seed

[root@elasticsearch-client-7bf748d697-tfh26 elasticsearch]# elasticsearch-keystore add s3.client.default.access_key

Enter value for s3.client.default.access_key:

[root@elasticsearch-client-7bf748d697-tfh26 elasticsearch]# elasticsearch-keystore add s3.client.default.secret_key

Enter value for s3.client.default.secret_key:

[root@elasticsearch-client-7bf748d697-tfh26 elasticsearch]# exit

# ctl exec -it elasticsearch-data-0 -n b -- bash

[root@elasticsearch-data-0 elasticsearch]# elasticsearch-keystore add s3.client.default.access_key

Enter value for s3.client.default.access_key:

[root@elasticsearch-data-0 elasticsearch]# elasticsearch-keystore add s3.client.default.secret_key

Enter value for s3.client.default.secret_key:

[root@elasticsearch-data-0 elasticsearch]# exit

# ctl exec -it elasticsearch-master-7569856dd7-828hp -n b -- bash

[root@elasticsearch-master-7569856dd7-828hp elasticsearch]# elasticsearch-keystore add s3.client.default.access_key

Enter value for s3.client.default.access_key:

[root@elasticsearch-master-7569856dd7-828hp elasticsearch]# elasticsearch-keystore add s3.client.default.secret_key

Enter value for s3.client.default.secret_key:

[root@elasticsearch-master-7569856dd7-828hp elasticsearch]# elasticsearch-keystore add s3.client.default.secret_key

Setting s3.client.default.secret_key already exists. Overwrite? [y/N]y

Enter value for s3.client.default.secret_key:

[root@elasticsearch-master-7569856dd7-828hp elasticsearch]# exit

## 過載配置

# ctl exec -it elasticsearch-client-7bf748d697-tfh26  -n b -- curl -u elastic:ygYv2Zj7kCiIr3rTpSpY -XPOST http://elasticsearch-client.b:9200/_nodes/reload_secure_settings

{"_nodes":{"total":3,"successful":3,"failed":0},"cluster_name":"elasticsearch","nodes":{"ECOASZjXQlqgeGAEEh93cg":{"name":"elasticsearch-client"},"hdJqqU95RqWwDu9IRUiL1g":{"name":"elasticsearch-master"},"WvafQLYwTyWimASYT8aK6A":{"name":"elas

b叢集建立s3型別的快照倉庫

# ctl exec -it  elasticsearch-client-7bf748d697-tfh26 -n b -- curl -u elastic:ygYv2Zj7kCiIr3rTpSpY -H 'Content-Type: application/json' -XPUT http://elasticsearch-client.b:9200/_snapshot/s3_obs_repository  -d '

   {

     "type": "s3",

     "settings": {

        "endpoint": "http://49.235.65.51:7480",

        "bucket": "zisefeizhu",

        "base_path": "/zisefeizhu",

        "max_snapshot_bytes_per_sec": "200mb",

        "max_restore_bytes_per_sec": "200mb"

      }

 }

'

{"acknowledged":true}

b叢集恢復快照

### b叢集已有索引

# ctl exec -it elasticsearch-client-7bf748d697-tfh26  -n b -- curl -u elastic:ygYv2Zj7kCiIr3rTpSpY http://elasticsearch-client.B:9200/_cat/indices

green open .monitoring-es-7-2021.07.13 iPnlqDtXSja5sUiGRC9xqg 1 0 32 2  262kb  262kb

green open .security-7                 lB-855UKS_CKJrefolaN8g 1 0  7 0 23.7kb 23.7kb

### 在b叢集檢視a叢集的快照

# ctl exec -it  elasticsearch-client-7bf748d697-tfh26 -n b -- curl -u elastic:ygYv2Zj7kCiIr3rTpSpY  -XGET http://elasticsearch-client.b:9200/_snapshot/s3_obs_repository/_all?pretty=true

{

  "snapshots" : [

    {

      "snapshot" : "snapshot_zisefeizhu",

      "uuid" : "eG6ndGcGRX61CnnTJzI50w",

      "version_id" : 7080099,

      "version" : "7.8.0",

      "indices" : [

        "demo"

      ],

      "include_global_state" : false,

      "state" : "SUCCESS",

      "start_time" : "2021-07-13T05:30:40.333Z",

      "start_time_in_millis" : 1626154240333,

      "end_time" : "2021-07-13T05:30:53.365Z",

      "end_time_in_millis" : 1626154253365,

      "duration_in_millis" : 13032,

      "failures" : [ ],

      "shards" : {

        "total" : 1,

        "failed" : 0,

        "successful" : 1

      }

    }

  ]

}

### 在b叢集恢復快照

# ctl exec -it  elasticsearch-client-7bf748d697-tfh26 -n b -- curl -u elastic:ygYv2Zj7kCiIr3rTpSpY -H 'Content-Type: application/json' -XPOST http://elasticsearch-client.b:9200/_snapshot/s3_obs_repository/snapshot_zisefeizhu/_restore  -d '

{

"indices": "demo",

"index_settings": {

  "index.number_of_replicas": 1  //副本數

},

"rename_pattern": "demo",

"rename_replacement": "demo_s3"

}     

{"accepted":true}

## 檢視b叢集索引

# ctl exec -it elasticsearch-client-7bf748d697-tfh26  -n b -- curl -u elastic:ygYv2Zj7kCiIr3rTpSpY http://elasticsearch-client.b:9200/_cat/indices | grep demo

yellow open demo                        Avsk9cr9RvSQTEIHxGUHHg 1 1 10000 0   74.3mb   74.3mb

• 方案二 完美結束

es異地雙活：https://www.cnblogs.com/yjmyzz/p/14586637.html