1. 程式人生 > >Linux lvs介紹

Linux lvs介紹

linux lvs

lvs:Linux Virtual Server,4層路由器,相比7層代理,4層代理過程中不受端口數量限制,因為沒到應用層,ipvs在input連上強轉報文到路由 ,有網站測試能達到400w並發。跟iptables配置類似,lvs的框架是ipvs,規則工具ipvsadm。

調度

  • 靜態方法:僅根據算法本身進行調度;

    1. RR:roundrobin,輪詢;
    2. WRR:Weighted RR,加權輪詢;
    3. SH:Source Hashing,實現session sticky,源IP地址hash;將來自於同一個IP地址的請求始終發往第一次挑中的RS,從而實現會話綁定;
    4. DH:Destination Hashing;目標地址哈希,將發往同一個目標地址的請求始終轉發至第一次挑中的RS,典型使用場景是正向代理緩存場景中的負載均衡;
  • 動態常用方法:主要根據每RS當前的負載狀態及調度算法進行調度;
    1. LC:least connections :Overhead=activeconns*256+inactiveconns 算法:一個活動進程開銷 是 非活動進程開銷的 256倍 (這只是一個大概和理值)
    2. WLC:Weighted LC #默認算法,Overhead=(activeconns*256+inactiveconns)/weight
    3. SED:Shortest Expection Delay #不考慮非活動資源 算術理解,Overhead=(activeconns+1)*256/weight
    4. NQ:Never Queue #SED升級,權重1,10兩臺 , 前10個會走權重10的那臺,而另外一臺一個都不用處理,NQ就是處理這個情況。

nat模式

工作流程圖
技術分享圖片
準備條件:

vs:node1
rs:node2, node3, node4
node1:      
172.16.86.249   #作為私網
192.168.1.200   # 作為公網
node2:
172.16.86.250    網關 172.16.86.249
node3:
172.16.86.248    網關 172.16.86.249
node4 
172.16.86.251    網關 172.16.86.249

1、vs安裝ipvsadm

[root@node1 ~]# yum install ipvsadm

2、服務管理

#添加
#ipvsadm -A|E -t|u|f service-address [-s scheduler] [-p [timeout]]
 #-t: TCP協議的端口,VIP:TCP_PORT
 #-u: UDP協議的端口,VIP:UDP_PORT
 #-f:firewall MARK,是一個數字;
#______________________________________________________________________________
[root@node1 ~]# ipvsadm -A -t 192.168.1.200:80 -s rr
#修改
[root@node1 ~]# ipvsadm -E -t 192.168.1.200:80 -s wrr
#刪除
[root@node1 ~]# ipvsadm -D -t 192.168.1.200:80

3、節點管理

#增改
#ipvsadm -a|e -t|u|f service-address -r server-address [-g|i|m] [-w weight]
#lvs類型:
    #-g: gateway, dr類型
    #-i: ipip, tun類型
    #-m: masquerade, nat類型
# -w weight:權重;

#______________________________________________________________________________
[root@node1 ~]# ipvsadm -a -t 192.168.1.200:80 -r 172.16.86.250 -m     #可以在rs的ip後面加端口  默認是把前面的端口映射到後面的端口
[root@node1 ~]# ipvsadm -a -t 192.168.1.200:80 -r 172.16.86.248 -m
[root@node1 ~]# ipvsadm -a -t 192.168.1.200:80 -r 172.16.86.251 -m
#刪:
#ipvsadm -d -t|u|f service-address -r server-address
[root@node1 ~]# ipvsadm -d -t 172.16.86.249:80 -r 172.16.86.251

3、查看

[root@node1 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight權重 ActiveConn正在連接的 InActConn 非活動連接數量
TCP  192.168.1.200:80 wlc   #默認是wlc調度
  -> 172.16.86.248:80             Masq    1                      0          0
  -> 172.16.86.250:80             Masq    1                      0          0
  -> 172.16.86.251:80             Masq    1                      0          0

[root@node1 ~]# ipvsadm -ln --stats
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port               Conns連接數   InPkts入棧報文數  OutPkts出棧報文數  InBytes入棧字節 OutBytes出棧字節
  -> RemoteAddress:Port
TCP  192.168.1.200:80               282989   1708574  1419365  115244K  141336K
  -> 172.16.86.248:80               169787  1026019   852945 69195245 84705145
  -> 172.16.86.250:80                56599   341318   281848 23003003 28178841
  -> 172.16.86.251:80                56603   341237   284572 23046538 28452109

[root@node1 ~]# watch -n.1 ‘ipvsadm -Ln --rate‘
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port                 CPS每秒建立連接    InPPS每秒入棧報文數   OutPPS    InBPS每秒入棧字節數   OutBPS
  -> RemoteAddress:Port
TCP  192.168.1.200:80                 1699    10176     8482   686755   850122
  -> 172.16.86.248:80                 1019     6106     5089   412064   508979
  -> 172.16.86.250:80                  340     2035     1696   137359   170579
  -> 172.16.86.251:80                  340     2035     1696   137332   170564

4、先來測試rr輪詢算法

[root@node1 ~]# ipvsadm -E -t 192.168.1.200:80 -s rr

[root@node1 ~]# curl http://192.168.1.200/
node4
[root@node1 ~]# curl http://192.168.1.200/
node3
[root@node1 ~]# curl http://192.168.1.200/
node2

wrr

[root@node1 ~]# ipvsadm -E -t 192.168.1.200:80 -s wrr
[root@node1 ~]# ipvsadm -e -t 192.168.1.200:80 -r 172.16.86.248 -m -w 3
[root@node1 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight  ActiveConn InActConn
TCP  192.168.1.200:80 wrr
  -> 172.16.86.248:80             Masq    3      0          1           #node3
  -> 172.16.86.250:80             Masq    1      0          1           #node2
  -> 172.16.86.251:80             Masq    1      0          3           #node4

[root@node1 ~]# curl http://192.168.1.200/
node4
[root@node1 ~]# curl http://192.168.1.200/
node3
[root@node1 ~]# curl http://192.168.1.200/
node2
[root@node1 ~]# curl http://192.168.1.200/
node3
[root@node1 ~]# curl http://192.168.1.200/
node3

5、保存規則

#查看rpm包中的腳本
[root@node1 ~]# cat /usr/lib/systemd/system/ipvsadm.service
[Unit]
Description=Initialise the Linux Virtual Server
After=syslog.target network.target

[Service]
Type=oneshot
ExecStart=/bin/bash -c "exec /sbin/ipvsadm-restore < /etc/sysconfig/ipvsadm"
ExecStop=/bin/bash -c "exec /sbin/ipvsadm-save -n > /etc/sysconfig/ipvsadm"
ExecStop=/sbin/ipvsadm -C
RemainAfterExit=yes

[Install]
WantedBy=multi-user.target

#把規則保存到配置文件中
[root@node1 ~]# ipvsadm -S -n > /etc/sysconfig/ipvsadm

清空
[root@node1 ~]# ipvsadm -C
[root@node1 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn

#重載
[root@node1 ~]# ipvsadm -R < /etc/sysconfig/ipvsadm
[root@node1 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.1.200:80 wrr
  -> 172.16.86.248:80             Masq    3      0          0
  -> 172.16.86.250:80             Masq    1      0          0
  -> 172.16.86.251:80             Masq    1      0          0

dr模型

工作流程圖
技術分享圖片

限制響應級別:arp_ignore
0:默認值,表示可使用本地任意接口上配置的任意地址進行響應;
1: 僅在請求的目標IP配置在本地主機的接收到請求報文接口上時,才給予響應;
限制通告級別:arp_announce
0:默認值,把本機上的所有接口的所有信息向每個接口上的網絡進行通告;
1:盡量避免向非本地連接網絡進行通告;
2:必須避免向非本網絡通告;

這兩個參數一個是通告設置,防止外部連接,一個是響應設置,防止向外發送

實驗準備:

node1: dip192.168.1.200   vip:192.168.1.205  vs
node2: 192.168.1.201  vip:192.168.1.205   rs
node3: 192.168.1.202   vip:192.168.1.205  rs

node1 配置 vip ? 如果vip等於dip,下面廣播域不要是自己

[root@node1 ~]# ip addr add 192.168.1.205/32 broadcast 192.168.1.205 dev ens34:0
[root@node1 ~]# ip addr delete 192.168.1.205/32 broadcast 192.168.1.205 dev ens34:0

rs: node2 node3 配置

#!/bin/bash
#
vip=192.168.1.205
mask=‘255.255.255.255‘

case $1 in
start)
    echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
    echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce

    ip addr add  $vip/32  broadcast $vip  dev lo:0
    #發往vip的報文,必須要從lo:0 出去   所以這個lo:0網卡是用來發報文的,相當於 修改 源ip
    ip route add  $vip dev lo:0
    ;;
stop)
    ip addr del $vip/32 dev lo:0
    ip route delete $vip dev lo:0

    echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
    echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce

    ;;
*) 
    echo "Usage $(basename $0) start|stop"
    exit 1
    ;;
esac 

vs:

[root@node1 ~]# ipvsadm -A -t 192.168.1.205:80 -s rr
#這裏添加節點還有另外一個意義:就是通知本機 rs節點會有vip,你只要把包發送回去就行
[root@node1 ~]# ipvsadm -a -t 192.168.1.205:80 -r 192.168.1.201 -g
[root@node1 ~]# ipvsadm -a -t 192.168.1.205:80 -r 192.168.1.202 -g

查看

[root@node1 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.1.205:80 rr
  -> 192.168.1.201:80             Route  路由類型   1      0          0
  -> 192.168.1.202:80             Route   1      0          0

測試

marvindeMacBook-Pro:~ marvin$ curl http://192.168.1.205/
node3
marvindeMacBook-Pro:~ marvin$ curl http://192.168.1.205/
node2
marvindeMacBook-Pro:~ marvin$ curl http://192.168.1.205/
node3
marvindeMacBook-Pro:~ marvin$ curl http://192.168.1.205/
node2

抓包分析

[root@node1 ~]#  tcpdump -i any -nn    port 80

#響應在rs節點  
[root@node2 ~]#  tcpdump -i any -nn    port 80
10:18:54.367851 IP 192.168.1.205.80 > 192.168.1.104.55631: Flags [P.], seq 1:235, ack 78, win 227, options [nop,nop,TS val 352696164 ecr 945230025], length 234: HTTP: HTTP/1.1 200 OK

vs配置腳本

#!/bin/bash

vip=‘192.168.1.205‘
iface=‘ens34:0‘
mask=‘255.255.255.255‘
port=‘80‘
rs1=‘192.168.1.201‘
rs2=‘192.168.1.202‘
scheduler=‘wrr‘
type=‘-g‘

case $1 in
start)
    ip addr add $vip/32 broadcast $vip dev $iface
    iptables -F

    ipvsadm -A -t ${vip}:${port} -s $scheduler
    ipvsadm -a -t ${vip}:${port} -r ${rs1} $type -w 1
    ipvsadm -a -t ${vip}:${port} -r ${rs2} $type -w 1
    ;;
stop)
    ipvsadm -C
    ip addr delete $vip/32  broadcast $vip dev $iface
    ;;
*)
    echo "Usage $(basename $0) start|stop"
    exit 1
    ;;
esac

tun模型

工作流程圖
技術分享圖片

fullnat模型

工作流程圖
技術分享圖片

把多個服務標記為一個服務

iptables -t mangle -A PREROUTING -d 192.168.1.200 -p tcp -m multoport --dports 80,443 -j MARK --set-mark 3   #標記3 隨意
iptables -t mangle -vnL
ipvsadm -A -f 3 s sh   #對標記3的報文定義一個服務
ipvsadm -a -f 3 -r 192.168.1.201 -g
ipvsadm -a -f 3 -r 192.168.1.202 -g

持久連接

-p 實現持久連接
每端口持久:每個端口對應定義為一個集群服務,每集群服務單獨調度;
ipvsadm -A 192.168.1.200:80 -s rr -p
每防火墻標記持久:基於防火墻標記定義集群服務;可實現將多個端口上的應用統一調度,即所謂的port Affinity;
ipvsadm -A -f 3 -s rr -p
每客戶端持久:基於0端口定義集群服務,即將客戶端對所有應用的請求統統調度至後端主機,必須定義為持久模式;
ipvsadm -A -t 192.168.1.200:0 -s rr -p
ipvsadm -a -t 192.168.1.200:0 -r 192.168.1.201 -g
ipvsadm -a -t 192.168.1.200:0 -r 192.168.1.202 -g

監測(啟動)ldirectord

ldirectord功能:檢查rs服務器是否有壞掉,如果壞掉,ipvsadm做delete,如果壞了修復會做ipvsadm add
配置腳本說明

checktimeout=3                  超時時常
checkinterval=1                 每個一秒檢查一次,可以設置長點,減少壓力
fallback=127.0.0.1:80           如果real server全跪了,本機提供
autoreload=yes                   配置文件發生修改,自動加載
logfile="/var/log/ldirectord.log"
quiescent=no
virtual=5                       #防火墻標記  ip:port 的話是正常標記
    real=172.16.0.7:80 gate 2
    real=172.16.0.8:80 gate 1
    fallback=127.0.0.1:80 gate    #如果rs都壞了,啟用這項
    service=http           #用http協議方式監測,關閉這個選項,會用四層方式探測
    scheduler=wrr
    checktype=negotiate  #談判方式,不是一次擊斃
    checkport=80
    request="index.html"    
    receive="CentOS"    #index.html中帶有CentOS

安裝配置

[root@node1 packages]# wget ftp://ftp.pbone.net/mirror/ftp5.gwdg.de/pub/opensuse/repositories/network:/ha-clustering:/Stable/CentOS_CentOS-6/x86_64/ldirectord-3.9.5-3.1.x86_64.rpm
[root@node1 packages]# yum install ldirectord-3.9.5-3.1.x86_64.rpm

[root@node1 packages]# rpm -ql ldirectord
/etc/ha.d
/etc/ha.d/resource.d
/etc/ha.d/resource.d/ldirectord
/etc/init.d/ldirectord
/etc/logrotate.d/ldirectord
/usr/lib/ocf/resource.d/heartbeat/ldirectord
/usr/sbin/ldirectord
/usr/share/doc/ldirectord-3.9.5
/usr/share/doc/ldirectord-3.9.5/COPYING
/usr/share/doc/ldirectord-3.9.5/ldirectord.cf
/usr/share/man/man8/ldirectord.8.gz

[root@node1 packages]# cp /usr/share/doc/ldirectord-3.9.5/ldirectord.cf /etc/ha.d/

[root@node1 packages]# vim /etc/ha.d/ldirectord.cf
# Global Directives
checktimeout=3
checkinterval=1
#fallback=127.0.0.1:80
#fallback6=[::1]:80
autoreload=yes
logfile="/var/log/ldirectord.log"
#logfile="local0"
#emailalert="[email protected]"
#emailalertfreq=3600
#emailalertstatus=all
quiescent=no

virtual=192.168.1.205:80
        real=192.168.1.201:80 gate
        real=192.168.1.202:80 gate
        fallback=127.0.0.1:80 gate
#       service=http
        scheduler=rr
        #persistent=600
        #netmask=255.255.255.255
        protocol=tcp
        checktype=negotiate
        checkport=80
#       request="index.html"
#       receive="Test Page"
#       virtualhost=www.x.y.z

啟動服務

[root@node1 packages]# /etc/init.d/ldirectord start
Starting ldirectord (via systemctl):                       [  OK  ]
[root@node1 packages]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.1.205:80 rr
  -> 192.168.1.201:80             Route   1      0          0
  -> 192.168.1.202:80             Route   1      0          0
marvindeMacBook-Pro:~ marvin$ curl http://192.168.1.205/
node3
marvindeMacBook-Pro:~ marvin$ curl http://192.168.1.205/
node2

停掉一個節點

[root@node3 ~]# systemctl stop mynginx

[root@node1 packages]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.1.205:80 rr
  -> 192.168.1.201:80             Route   1      0          6

marvindeMacBook-Pro:~ marvin$ curl http://192.168.1.205/
node2
marvindeMacBook-Pro:~ marvin$ curl http://192.168.1.205/
node2

全部停掉 ,會自動啟動本地服務

[root@node1 packages]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.1.205:80 rr
  -> 127.0.0.1:80                 Route   1      0          0

marvindeMacBook-Pro:~ marvin$ curl http://192.168.1.205/
sorry
marvindeMacBook-Pro:~ marvin$ curl http://192.168.1.205/
sorry

啟用一臺,恢復正常

[root@node2 ~]# systemctl start mynginx
[root@node1 packages]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.1.205:80 rr
  -> 192.168.1.201:80             Route   1      0          0

Linux lvs介紹