6.1
創建kube-proxy證書

創建證書簽名請求

[root@k8s-master1 kube-proxy]# cat kube-proxy-csr.json
{
"CN": "system:kube-proxy",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "SZ",
"L": "SZ",
"O": "k8s",
"OU": "4Paradigm"
}
]
}
[root@k8s-master1 kube-proxy]#

?CN：指定該證書的 User 為 system:kube-proxy ；

生成證書和私鑰

[root@k8s-master1 kube-proxy]# cfssl gencert -ca=/etc/kubernetes/cert/ca.pem -ca-key=/etc/kubernetes/cert/ca-key.pem -config=/etc/kubernetes/cert/ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
2018/08/30 21:58:31 [INFO] generate received request
2018/08/30 21:58:31 [INFO] received CSR
2018/08/30 21:58:31 [INFO] generating key: rsa-2048
2018/08/30 21:58:31 [INFO] encoded CSR
2018/08/30 21:58:31 [INFO] signed certificate with serial number 62542245638277052495817543993296923487092361674
2018/08/30 21:58:31 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@k8s-master1 kube-proxy]#
 

6.2
創建和分發kubeconfig文件

[root@k8s-master1 kube-proxy]# source /opt/k8s/bin/environment.sh
[root@k8s-master1 kube-proxy]# echo ${KUBE_APISERVER}
https://192.168.211.127:8443
[root@k8s-master1 kube-proxy]# kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/cert/ca.pem --embed-certs=true --server=https://192.168.211.127:8443 --kubeconfig=kube-proxy.kubeconfig
Cluster "kubernetes" set.
[root@k8s-master1 kube-proxy]# kubectl config set-credentials kube-proxy --client-certificate=kube-proxy.pem --client-key=kube-proxy-key.pem --embed-certs=true --kubeconfig=kube-proxy.kubeconfig
User "kube-proxy" set.
[root@k8s-master1 kube-proxy]# kubectl config set-context default --cluster=kubernetes --user=kube-proxy --kubeconfig=kube-proxy.kubeconfig
Context "default" created.
[root@k8s-master1 kube-proxy]# kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
Switched to context "default".
[root@k8s-master1 kube-proxy]# ls
kube-proxy.csr? kube-proxy-csr.json? kube-proxy-key.pem? kube-proxy.kubeconfig? kube-proxy.pem
[root@k8s-master1 kube-proxy]#
 

分發

[root@k8s-master1 kube-proxy]# cp kube-proxy.kubeconfig /etc/kubernetes/
[root@k8s-master1 kube-proxy]# scp kube-proxy.kubeconfig root@k8s-master2:/etc/kubernetes/
kube-proxy.kubeconfig? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?? 100% 6219? ?? 6.1KB/s?? 00:00? ?
[root@k8s-master1 kube-proxy]# scp kube-proxy.kubeconfig root@k8s-master3:/etc/kubernetes/
kube-proxy.kubeconfig? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?? 100% 6219? ?? 6.1KB/s?? 00:00? ?
[root@k8s-master1 kube-proxy]# scp kube-proxy.kubeconfig root@k8s-node3:/etc/kubernetes/
root@k8s-node3‘s password:
kube-proxy.kubeconfig? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?? 100% 6219? ?? 6.1KB/s?? 00:00? ?
[root@k8s-master1 kube-proxy]#

6.3
創建kube-proxy配置文件

創建 kube-proxy config 文件模

[root@k8s-master1 kube-proxy]# echo ${CLUSTER_CIDR}
172.30.0.0/16
[root@k8s-master1 kube-proxy]# cat kube-proxy.config.yaml.template
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: ##NODE_IP##
clientConnection:
kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig
clusterCIDR: 172.30.0.0/16
healthzBindAddress: ##NODE_IP##:10256
hostnameOverride: ##NODE_NAME##
kind: KubeProxyConfiguration
metricsBindAddress: ##NODE_IP##:10249
mode: "ipvs"
[root@k8s-master1 kube-proxy]#

bindAddress : 監聽地址；
clientConnection.kubeconfig : 連接 apiserver 的 kubeconfig 文件；
clusterCIDR : kube-proxy 根據 --cluster-cidr 判斷集群內部和外部流量，
指定 --cluster-cidr 或 --masquerade-all 選項後 kube-proxy 才會對訪問
Service IP 的請求做 SNAT；
hostnameOverride : 參數值必須與 kubelet 的值一致，否則 kube-proxy 啟動後會
找不到該 Node，從而不會創建任何 ipvs 規則；
mode : 使用 ipvs 模式；

分發

[root@k8s-master1 kube-proxy]# cp kube-proxy.config.yaml.template /etc/kubernetes/kube-proxy.config.yaml
[root@k8s-master1 kube-proxy]# scp kube-proxy.config.yaml.template root@k8s-master2:/etc/kubernetes/kube-proxy.config.yaml
kube-proxy.config.yaml.template? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?? 100%? 315? ?? 0.3KB/s?? 00:00? ?
[root@k8s-master1 kube-proxy]# scp kube-proxy.config.yaml.template root@k8s-master3:/etc/kubernetes/kube-proxy.config.yaml
kube-proxy.config.yaml.template? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?? 100%? 315? ?? 0.3KB/s?? 00:00? ?
[root@k8s-master1 kube-proxy]# scp kube-proxy.config.yaml.template root@k8s-node3:/etc/kubernetes/kube-proxy.config.yaml
root@k8s-node3‘s password:
kube-proxy.config.yaml.template ? ? ??

修改NODE_IP和NODE_NAME
所有節點的都根據節點的ip和hostname修改
參考下面的

[root@k8s-master1 kube-proxy]# cat /etc/kubernetes/kube-proxy.config.yaml
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 192.168.211.128
clientConnection:
kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig
clusterCIDR: 172.30.0.0/16
healthzBindAddress: 192.168.211.128:10256
hostnameOverride: k8s-master1
kind: KubeProxyConfiguration
metricsBindAddress: 192.168.211.128:10249
mode: "ipvs"
[root@k8s-master1 kube-proxy]#

6.4
創建和分發kube-proxy systemd unit 文件

[root@k8s-master1 kube-proxy]# cat kube-proxy.service
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart=/opt/k8s/bin/kube-proxy ? --config=/etc/kubernetes/kube-proxy.config.yaml ? --alsologtostderr=true ? --logtostderr=false ? --log-dir=/var/log/kubernetes ? --v=2
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
[root@k8s-master1 kube-proxy]#

註意
WorkingDirectory=/var/lib/kube-proxy
這個目錄手動去創建

分發到所有節點

[root@k8s-master1 kube-proxy]# mkdir -p /var/lib/kube-proxy
[root@k8s-master1 kube-proxy]# ls
kube-proxy.config.yaml.template? kube-proxy-csr.json? kube-proxy.kubeconfig? kube-proxy.service
kube-proxy.csr? ? ? ? ? ? ? ? ?? kube-proxy-key.pem?? kube-proxy.pem
[root@k8s-master1 kube-proxy]# scp kube-proxy.service root@k8s-master1:/etc/systemd/system
kube-proxy.service? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? 100%? 450? ?? 0.4KB/s?? 00:00? ?
[root@k8s-master1 kube-proxy]# scp kube-proxy.service root@k8s-master2:/etc/systemd/system
kube-proxy.service? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? 100%? 450? ?? 0.4KB/s?? 00:00? ?
[root@k8s-master1 kube-proxy]# scp kube-proxy.service root@k8s-master3:/etc/systemd/system
kube-proxy.service? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? 100%? 450? ?? 0.4KB/s?? 00:00? ?
[root@k8s-master1 kube-proxy]# scp kube-proxy.service root@k8s-node3:/etc/systemd/system
root@k8s-node3‘s password:
kube-proxy.service? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? 100%? 450? ?? 0.4KB/s?? 00:00? ?
[root@k8s-master1 kube-proxy]#

6.5
啟動服務

systemctl daemon-reload && systemctl enable kube-proxy && systemctl restart kube-proxy

啟動失敗報錯:

[root@k8s-master1 kubernetes]# cat kube-proxy.ERROR
Log file created at: 2018/08/30 22:26:09
Running on machine: k8s-master1
Binary: Built with gc go1.9.3 for linux/amd64
Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg
F0830 22:26:09.387614? ? 4255 helpers.go:119] error: unable to load in-cluster configuration, KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT must be defined
goroutine 1 [running]:

文件格式問題，註意參考格式見下

[root@k8s-master1 kubernetes]# cat /etc/kubernetes/kube-proxy.config.yaml
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 192.168.211.128
clientConnection:
? kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig ?
clusterCIDR: 172.30.0.0/16
healthzBindAddress: 192.168.211.128:10256
hostnameOverride: k8s-master1
kind: KubeProxyConfiguration
metricsBindAddress: 192.168.211.128:10249
mode: "ipvs"
[root@k8s-master1 kubernetes]#

kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig ? ? ## 註意這個前面的空格，沒有就會報上面的錯誤

檢查端口

[root@k8s-master1 kubernetes]# netstat -lnpt|grep kube-prox
tcp? ? ? ? 0? ? ? 0 192.168.211.128:10256?? 0.0.0.0:*? ? ? ? ? ? ?? LISTEN? ? ? 5349/kube-proxy? ??
tcp? ? ? ? 0? ? ? 0 192.168.211.128:10249?? 0.0.0.0:*? ? ? ? ? ? ?? LISTEN? ? ? 5349/kube-proxy? ??
[root@k8s-master1 kubernetes]#

查看ip路由規則

[root@k8s-master1 kubernetes]# /usr/sbin/ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
? -> RemoteAddress:Port? ? ? ? ?? Forward Weight ActiveConn InActConn
TCP? 10.254.0.1:443 rr persistent 10800
? -> 192.168.211.128:6443? ? ? ?? Masq? ? 1? ? ? 0? ? ? ? ? 0? ? ? ??
? -> 192.168.211.129:6443? ? ? ?? Masq? ? 1? ? ? 0? ? ? ? ? 0? ? ? ??
? -> 192.168.211.130:6443? ? ? ?? Masq? ? 1? ? ? 0? ? ? ? ? 0? ? ? ??
[root@k8s-master1 kubernetes]#

部署k8s ssl集群實踐14:work節點部署kube-proxy