手動搭建Kubernetes1.8高可用叢集（6）calico
阿新 • • 發佈：2018-12-30
一、準備
3、Node
3、建立目錄
/etc/cni/net.d/ /etc/calico/certs /opt/cni/bin/ 許可權0755 屬主kube
二、配置所有節點
1、連結etcd證書
ln /etc/ssl/etcd/ssl/ca.pem /etc/calico/certs/ca_cert.crt ln /etc/ssl/etcd/ssl/node-node1.pem /etc/calico/certs/cert.crt ln /etc/ssl/etcd/ssl/node-node1-key.pem /etc/calico/certs/key.pem
2、/etc/cni/net.d/10-calico.conflist
{
"name": "cni0",
"cniVersion":"0.3.1",
"plugins":[
{
"nodename": "node1",
"type": "calico",
"etcd_endpoints": "https://192.168.1.121:2379,https://192.168.1.122:2379,https://192.168.1.126:2379",
"etcd_cert_file": "/etc/ssl/etcd/ssl/node-node1.pem",
"etcd_key_file": "/etc/ssl/etcd/ssl/node-node1-key.pem",
"etcd_ca_cert_file": "/etc/ssl/etcd/ssl/ca.pem",
"log_level": "info",
"ipam": {
"type": "calico-ipam"
},
"kubernetes": {
"kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml"
}
},
{
"type":"portmap",
"capabilities":{
"portMappings":true
}
}
]
}
3、/usr/local/bin/calicoctl
#!/bin/bash /usr/bin/docker run -i --privileged --rm \ --net=host --pid=host \ -e ETCD_ENDPOINTS=https://192.168.1.121:2379,https://192.168.1.122:2379,https://192.168.1.126:2379 \ -e ETCD_CA_CERT_FILE=/etc/calico/certs/ca_cert.crt \ -e ETCD_CERT_FILE=/etc/calico/certs/cert.crt \ -e ETCD_KEY_FILE=/etc/calico/certs/key.pem \ -v /usr/bin/docker:/usr/bin/docker \ -v /var/run/docker.sock:/var/run/docker.sock \ -v /var/run/calico:/var/run/calico \ -v /etc/calico/certs:/etc/calico/certs:ro \ --memory=170M --cpu-shares=100 \ quay.io/calico/ctl:v1.5.0 \ "[email protected]"
4、複製外掛， 許可權0755 屬主kube
docker run --rm -v /opt/cni/bin:/cnibindir quay.io/coreos/hyperkube:v1.8.3_coreos.0 /bin/cp -r /opt/cni/bin/. /cnibindir/ docker run --rm -v /opt/cni/bin:/cnibindir quay.io/calico/cni:v1.10.0 sh -c 'cp /opt/cni/bin/* /cnibindir/'
此時Node應該已經Ready
三、設定ipPool
1、設定
echo '{
"kind": "ipPool",
"spec": {"disabled": false, "ipip": {"enabled": true, "mode": "always"},
"nat-outgoing": true},
"apiVersion": "v1",
"metadata": {"cidr": "10.233.64.0/18"}
}' | calicoctl create -f -
2、驗證
[[email protected] ~]# calicoctl get ipPool CIDR 10.233.64.0/18
[[email protected] ~]# curl --cacert /etc/ssl/etcd/ssl/ca.pem --cert /etc/ssl/etcd/ssl/admin-node1.pem --key /etc/ssl/etcd/ssl/admin-node1-key.pem https://localhost:2379/v2/keys/calico/v1/ipam/v4/pool {"action":"get","node":{"key":"/calico/v1/ipam/v4/pool","dir":true,"nodes":[{"key":"/calico/v1/ipam/v4/pool/10.233.64.0-18","value":"{\"cidr\":\"10.233.64.0/18\",\"ipip\":\"tunl0\",\"ipip_mode\":\"always\",\"masquerade\":true,\"ipam\":true,\"disabled\":false}","modifiedIndex":200,"createdIndex":200}],"modifiedIndex":23,"createdIndex":23}}
四、建立calico manifest
1、/etc/kubernetes/calico-config.yml
kind: ConfigMap apiVersion: v1 metadata: name: calico-config namespace: kube-system data: etcd_endpoints: "https://192.168.1.121:2379,https://192.168.1.122:2379,https://192.168.1.126:2379" etcd_ca: "/calico-secrets/ca_cert.crt" etcd_cert: "/calico-secrets/cert.crt" etcd_key: "/calico-secrets/key.pem" cluster_type: "kubespray,bgp" calico_backend: "bird"
2、/etc/kubernetes/calico-node.yml
--- # This manifest installs the calico/node container, as well # as the Calico CNI plugins and network config on # each master and worker node in a Kubernetes cluster. kind: DaemonSet apiVersion: extensions/v1beta1 metadata: name: calico-node namespace: kube-system labels: k8s-app: calico-node spec: selector: matchLabels: k8s-app: calico-node template: metadata: labels: k8s-app: calico-node annotations: scheduler.alpha.kubernetes.io/critical-pod: '' kubespray.etcd-cert/serial: E0C25EE5CFA19DC6 spec: hostNetwork: true serviceAccountName: calico-node tolerations: - effect: NoSchedule operator: Exists containers: # Runs calico/node container on each Kubernetes node. This # container programs network policy and routes on each # host. - name: calico-node image: quay.io/calico/node:v2.5.0 env: # The location of the Calico etcd cluster. - name: ETCD_ENDPOINTS valueFrom: configMapKeyRef: name: calico-config key: etcd_endpoints # Choose the backend to use. - name: CALICO_NETWORKING_BACKEND valueFrom: configMapKeyRef: name: calico-config key: calico_backend # Cluster type to identify the deployment type - name: CLUSTER_TYPE valueFrom: configMapKeyRef: name: calico-config key: cluster_type # Disable file logging so `kubectl logs` works. - name: CALICO_DISABLE_FILE_LOGGING value: "true" # Set Felix endpoint to host default action to ACCEPT. - name: FELIX_DEFAULTENDPOINTTOHOSTACTION value: "RETURN" # should be set in etcd before deployment # # Configure the IP Pool from which Pod IPs will be chosen. # - name: CALICO_IPV4POOL_CIDR # value: "192.168.0.0/16" # - name: CALICO_IPV4POOL_IPIP # value: "always" # Disable IPv6 on Kubernetes. - name: FELIX_IPV6SUPPORT value: "false" # Set Felix logging to "info" - name: FELIX_LOGSEVERITYSCREEN value: "info" # Disable autocreation of pools - name: CALICO_NO_DEFAULT_POOLS value: "true" # Enable libnetwork - name: CALICO_LIBNETWORK_ENABLED value: "true" # Set MTU for tunnel device used if ipip is enabled - name: FELIX_PROMETHEUSMETRICSENABLED value: "false" - name: FELIX_PROMETHEUSMETRICSPORT value: "9091" - name: FELIX_PROMETHEUSGOMETRICSENABLED value: "true" - name: FELIX_PROMETHEUSPROCESSMETRICSENABLED value: "true" # Location of the CA certificate for etcd. - name: ETCD_CA_CERT_FILE valueFrom: configMapKeyRef: name: calico-config key: etcd_ca # Location of the client key for etcd. - name: ETCD_KEY_FILE valueFrom: configMapKeyRef: name: calico-config key: etcd_key # Location of the client certificate for etcd. - name: ETCD_CERT_FILE valueFrom: configMapKeyRef: name: calico-config key: etcd_cert - name: IP valueFrom: fieldRef: fieldPath: status.hostIP - name: NODENAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: FELIX_HEALTHENABLED value: "true" - name: FELIX_IGNORELOOSERPF value: "False" securityContext: privileged: true resources: limits: cpu: 300m memory: 500M requests: cpu: 150m memory: 64M livenessProbe: httpGet: path: /liveness port: 9099 periodSeconds: 10 initialDelaySeconds: 10 failureThreshold: 6 readinessProbe: httpGet: path: /readiness port: 9099 periodSeconds: 10 volumeMounts: - mountPath: /lib/modules name: lib-modules readOnly: true - mountPath: /var/run/calico name: var-run-calico readOnly: false - mountPath: /calico-secrets name: etcd-certs volumes: # Used by calico/node. - name: lib-modules hostPath: path: /lib/modules - name: var-run-calico hostPath: path: /var/run/calico # Used to install CNI. - name: cni-bin-dir hostPath: path: /opt/cni/bin - name: cni-net-dir hostPath: path: /etc/cni/net.d # Mount in the etcd TLS secrets. - name: etcd-certs hostPath: path: "/etc/calico/certs" updateStrategy: rollingUpdate: maxUnavailable: 20% type: RollingUpdate
3、/etc/kubernetes/calico-node-sa.yml
--- apiVersion: v1 kind: ServiceAccount metadata: name: calico-node namespace: kube-system labels: kubernetes.io/cluster-service: "true"
4、/etc/kubernetes/calico-cr.yml
--- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: calico-node namespace: kube-system rules: - apiGroups: [""] resources: - pods - nodes verbs: - get
5、/etc/kubernetes/calico-crb.yml
--- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: name: calico-node roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: calico-node subjects: - kind: ServiceAccount name: calico-node namespace: kube-system
6、建立 （可以把上面5個檔案合成一個）
kubectl create -f calico-config.yml kubectl create -f calico-node.yml kubectl create -f calico-node-sa.yml kubectl create -f calico-cr.yml kubectl create -f calico-crb.yml
四、驗證
1、kubectl get node
[[email protected] kubernetes]# kubectl get node NAME STATUS ROLES AGE VERSION node1 Ready master 3h v1.8.3+coreos.0 node2 Ready master,node 3h v1.8.3+coreos.0 node3 Ready node 2h v1.8.3+coreos.0
2、calicoctl get nodes -o wide
[[email protected] kubernetes]# calicoctl get nodes -o wide NAME ASN IPV4 IPV6 node1 (64512) 192.168.1.123/32 node2 (64512) 192.168.1.124/32 node3 (64512) 192.168.1.125/32
3、calicoctl node status
[[email protected] kubernetes]# calicoctl node status Calico process is running. IPv4 BGP status +---------------+-------------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +---------------+-------------------+-------+----------+-------------+ | 192.168.1.124 | node-to-node mesh | up | 09:57:30 | Established | | 192.168.1.125 | node-to-node mesh | up | 09:57:30 | Established | +---------------+-------------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found.