Keepalived

Keepalived介绍

官网

#官网
https://keepalived.org/
#文档
https://keepalived.org/doc/
http://keepalived.org/documentation.html
#下载地址
https://keepalived.org/download.html

工作模式

抢占模式（实时、延时抢占）、非抢占模式

配置介绍

专业术语	注释
RS	真实服务器
SIP	源IP，从那台来的
DIP	目的IP，要去哪台
VIP	虚拟IP
vrrp_instance	虚拟路由器
MASTER	主节点
BACKUP	备用节点
priority	优先级（越大越高）
virtual_router_id	虚拟路由ID，一主一从的vrrp_instance是一套，套内是一样的，每套的ID都不同。一台主服务器一台从服务器可以配置多套

配置文件

global_defs {
   router_id nginx-lvs1  #尽量每套VIP的都不同
}

vrrp_script chk_lvs {
    script "/home/yanfaapp/rcs/app/keepalived/sbin/lvs_check.sh" #检查脚本存放位置
    interval 2  #脚本检查间隔时间 2s
    weight -20  #权重降低20
    fall 3      #失败3次后降权
    rise 1      #成功1伺候恢复
}

vrrp_instance VI_1 {
    state BACKUP  		#角色，在抢占式中主节点为MASTER，备用节点为BACKUP；非抢占式中都配置为BACKUP
    interface ens18 	#网卡名称
    virtual_router_id 11
    priority 100  		#优先级配置
    advert_int 1  		#每1发送一次VRRP通告
    nopreempt     		#非抢占式中配置， 注意：配置后除非主节点的keepalived关闭，否则VIP不会漂移到备用节点
    preempt_delay 60 	# 抢占延迟模式,延迟60s,默认的是300s
    unicast_src_ip 192.168.31.217	# 配置单播，单播源IP
    unicast_peer {
        192.168.31.218 				# 单播目的IP
    }
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.31.198/24   dev ens18 label ens18:1 #配置VIP
    }
    track_script {
        chk_lvs		#引用上边配置的检查脚本
    }
}
virtual_server 192.168.31.198 9000 { #真实服务器，需要转发给后台的服务器
    delay_loop 6
    lb_algo rr		# 调度算法配置
    lb_kind DR      # 和LVS同台部署时需要配置LVS运行模式
  #  #persistence_timeout 5  #长链接配置
    protocol TCP

    real_server 192.168.31.219 9000 {
        weight 1
        TCP_CHECK {
            connect_port 9000 	#检查9000端口
            connect_timeout 3 	#3s内未建立链接视为失败
            retry 3				#失败3次后将不再转发流量过去
            delay_before_retry 3#失败后等待3s再次尝试
        }
    }
    real_server 192.168.31.220 9000 {
        weight 1
        TCP_CHECK {
            connect_port 9000
            connect_timeout 3
            retry 3
            delay_before_retry 3
        }
    }
    real_server 192.168.31.221 9000 {
        weight 1
        TCP_CHECK {
            connect_port 9000
            connect_timeout 3
            retry 3
            delay_before_retry 3
        }
    }
}

检查脚本

抢占式

#!/bin/bash

ip=$(hostname -I | awk '{print $1}')
dt=$(date +'%Y%m%d %H:%M:%S')
echo "$0--${ip}--${dt}" >> /tmp/kp.log

# 检查 LVS 的状态
if [ -e /proc/net/ip_vs ]; then
	if [[ $(cat /proc/net/ip_vs | grep TCP) != '' ]];then 
		echo '状态正常'
		exit 0		# 正常状态
	else 
		echo "${ip}--${dt}: LVS is not running. Take necessary actions" >> /tmp/kp.log
		echo '异常状态'
		exit 1		# 异常状态
	fi
else 
	echo "${ip}--${dt}: LVS is not running. Take necessary actions" >> /tmp/kp.log
	exit 1
fi

非抢占式

#!/bin/bash

ip=$(hostname -I | awk '{print $1}')
dt=$(date +'%Y%m%d %H:%M:%S')
echo "$0--${ip}--${dt}" >> /tmp/kp.log

# 检查 LVS 的状态
if [ -e /proc/net/ip_vs ]; then
	if [[ $(cat /proc/net/ip_vs | grep TCP) != '' ]];then 
		echo '状态正常'
		sudo systemctl stop keepalived  #由于备用节点不会抢占，只能先关了主节点，其他用户记得给sudo权限
		exit 0		# 正常状态
	else 
		echo "${ip}--${dt}: LVS is not running. Take necessary actions" >> /tmp/kp.log
		echo '异常状态'
		exit 1		# 异常状态
	fi
else 
	echo "${ip}--${dt}: LVS is not running. Take necessary actions" >> /tmp/kp.log
	exit 1
fi

给sudo权限

#root用户操作
visudo -f /etc/sudoers.d/yanfaapp_keepalived
#添加
username ALL=(ALL) NOPASSWD: /bin/systemctl restart keepalived
username ALL=(ALL) NOPASSWD: /bin/systemctl stop keepalived
username ALL=(ALL) NOPASSWD: /bin/systemctl status keepalived
#检查是否配置正确
visudo -c
#测试是否生效，切换到username用户运行
sudo systemctl start keepalived

安装、配置

安装

centos 部署 lvs+keepalived

#!/bin/bash
######install  keepalived  server########
version="2.0.16"
UNPATH="/home/yanfaapp"
INPATH="/home/yanfaapp/rcs/app"
DPATH="/home/yanfaapp/rcs/data"
router_id="lvs-1"
NET="ens18"
role="BACKUP"
virtual_router_id=11
priority=100

SIP="192.168.31.217"
DIP="192.168.31.218"
VIP="192.168.31.198"

RS_IP1="192.168.31.219"
RS_IP2="192.168.31.220"
RS_IP3="192.168.31.221"
RS_POST="9000"
# 检查是否有keepalived安装包
if [ ! -f ${UNPATH}/keepalived-${version}.tar.gz ];then
    echo "keepalived-${version}.tar.gz not found in ${UNPATH}"
    exit 1
fi
if [ ! -f  ${INPATH} ];then
mkdir -p ${INPATH}
fi

if [ ! -f  ${DPATH} ];then
mkdir -p ${DPATH}
fi

cd ${UNPATH}
tar -zxvf ${UNPATH}/keepalived-${version}.tar.gz
yum -y install openssl openssl-devel
cd ${UNPATH}/keepalived-${version}/ 
./configure   --prefix=${INPATH}/keepalived-${version}
mkdir ${INPATH}/keepalived-${version}
make && make install
mkdir /etc/keepalived
cp ${INPATH}/keepalived-${version}/etc/keepalived/keepalived.conf /etc/keepalived/
cp ${UNPATH}/keepalived-${version}/keepalived/etc/init.d/keepalived /etc/rc.d/init.d/
cp ${INPATH}/keepalived-${version}/etc/sysconfig/keepalived  /etc/sysconfig/
cp ${INPATH}/keepalived-${version}/sbin/keepalived /usr/sbin/
#配置文件
cat > /etc/keepalived/keepalived.conf << EOF
global_defs {
   router_id ${router_id}
}

vrrp_script chk_lvs {
    script "${INPATH}/keepalived/sbin/lvs_check.sh"
    interval 2
    weight -20
    fall 3
    rise 1
}

vrrp_instance VI_1 {
    state ${role}
    interface ${NET}
    virtual_router_id ${virtual_router_id}
    priority ${priority}
    advert_int 1
    nopreempt
    unicast_src_ip ${SIP}
    unicast_peer {
        ${DIP}
    }
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        ${VIP}/24   dev ${NET} label ${NET}:1
    }
    track_script {
        chk_lvs
    }
}
virtual_server ${VIP} ${RS_POST} {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    # persistence_timeout 5
    protocol TCP

    real_server ${RS_IP1} ${RS_POST} {
        weight 1
        TCP_CHECK {
            connect_port ${RS_POST}
            connect_timeout 3
            retry 3
            delay_before_retry 3
        }
    }
    real_server ${RS_IP2} ${RS_POST} {
        weight 1
        TCP_CHECK {
            connect_port ${RS_POST}
            connect_timeout 3
            retry 3
            delay_before_retry 3
        }
    }
    real_server ${RS_IP3} ${RS_POST} {
        weight 1
        TCP_CHECK {
            connect_port ${RS_POST}
            connect_timeout 3
            retry 3
            delay_before_retry 3
        }
    }
}

EOF

cat > ${INPATH}/keepalived/sbin/lvs_check.sh << EOF
#!/bin/bash

ip=$(hostname -I | awk '{print $1}')
dt=$(date +'%Y%m%d %H:%M:%S')
echo "$0--${ip}--${dt}" >> /tmp/kp.log

# 检查 LVS 的状态
if [ -e /proc/net/ip_vs ]; then
	if [[ $(cat /proc/net/ip_vs | grep TCP) != '' ]];then 
		echo '状态正常'
		systemctl start keepalived
		exit 0		# 正常状态
	else 
		echo "${ip}--${dt}: LVS is not running. Take necessary actions" >> /tmp/kp.log
		echo '异常状态'
		systemctl stop keepalived
		exit 1		# 异常状态
	fi
else 
	echo "${ip}--${dt}: LVS is not running. Take necessary actions" >> /tmp/kp.log
	exit 1
fi

EOF

ubuntu 部署keepalived

#!/bin/bash
######install  keepalived  server########
version="2.0.16"
UNPATH="/home/yanfaapp"
INPATH="/home/yanfaapp/rcs/app"
DPATH="/home/yanfaapp/rcs/data"
router_id="lvs-1"
NET="ens18"
role="BACKUP"
virtual_router_id=11
priority=100

SIP="192.168.31.217"
DIP="192.168.31.218"
VIP="192.168.31.198"

RS_IP1="192.168.31.219"
RS_IP2="192.168.31.220"
RS_IP3="192.168.31.221"
RS_POST="9000"
# 检查是否有keepalived安装包
if [ ! -f ${UNPATH}/keepalived-${version}.tar.gz ];then
    echo "在路径 ${UNPATH} 下找不到 keepalived-${version}.tar.gz"
    exit 1
fi
if [ ! -f  ${INPATH} ];then
mkdir -p ${INPATH}
fi

if [ ! -f  ${DPATH} ];then
mkdir -p ${DPATH}
fi

cd ${UNPATH}
tar -zxvf ${UNPATH}/keepalived-${version}.tar.gz

apt -y install make gcc ipvsadm build-essential pkg-config automake autoconf \
libipset-dev libnl-3-dev libnl-genl-3-dev \
libssl-dev libxtables-dev libip4tc-dev libip6tc-dev libmagic-dev libsnmp-dev \
libglib2.0-dev libpcre2-dev libnftnl-dev libmnl-dev libsystemd-dev

cd ${UNPATH}/keepalived-${version}/ 
./configure   --prefix=${INPATH}/keepalived-${version}
mkdir ${INPATH}/keepalived-${version}
make && make install
mkdir /etc/keepalived
cp ${INPATH}/keepalived-${version}/etc/keepalived/keepalived.conf /etc/keepalived/
#配置文件
cat > /etc/keepalived/keepalived.conf << EOF
global_defs {
   router_id ${router_id}
}

vrrp_script chk_lvs {
    script "${INPATH}/keepalived/sbin/lvs_check.sh"
    interval 2
    weight -20
    fall 3
    rise 1
}

vrrp_instance VI_1 {
    state ${role}
    interface ${NET}
    virtual_router_id ${virtual_router_id}
    priority ${priority}
    advert_int 1
    nopreempt
    unicast_src_ip ${SIP}
    unicast_peer {
        ${DIP}
    }
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        ${VIP}/24   dev ${NET} label ${NET}:1
    }
    track_script {
        chk_lvs
    }
}
virtual_server ${VIP} ${RS_POST} {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    # persistence_timeout 5
    protocol TCP

    real_server ${RS_IP1} ${RS_POST} {
        weight 1
        TCP_CHECK {
            connect_port ${RS_POST}
            connect_timeout 3
            retry 3
            delay_before_retry 3
        }
    }
    real_server ${RS_IP2} ${RS_POST} {
        weight 1
        TCP_CHECK {
            connect_port ${RS_POST}
            connect_timeout 3
            retry 3
            delay_before_retry 3
        }
    }
    real_server ${RS_IP3} ${RS_POST} {
        weight 1
        TCP_CHECK {
            connect_port ${RS_POST}
            connect_timeout 3
            retry 3
            delay_before_retry 3
        }
    }
}

EOF

cat > ${INPATH}/keepalived/sbin/lvs_check.sh << EOF
#!/bin/bash

ip=$(hostname -I | awk '{print $1}')
dt=$(date +'%Y%m%d %H:%M:%S')
echo "$0--${ip}--${dt}" >> /tmp/kp.log

# 检查 LVS 的状态
if [ -e /proc/net/ip_vs ]; then
	if [[ $(cat /proc/net/ip_vs | grep TCP) != '' ]];then 
		echo '状态正常'
		systemctl start keepalived
		exit 0		# 正常状态
	else 
		echo "${ip}--${dt}: LVS is not running. Take necessary actions" >> /tmp/kp.log
		echo '异常状态'
		systemctl stop keepalived
		exit 1		# 异常状态
	fi
else 
	echo "${ip}--${dt}: LVS is not running. Take necessary actions" >> /tmp/kp.log
	exit 1
fi
EOF

RS服务器配置

ubuntu

# DR 模式
cat >> /etc/sysctl.conf << EOF
net.ipv4.conf.all.arp_ignore=1
net.ipv4.conf.all.arp_announce=2
#net.ipv4.conf.lo.arp_ignore=1
#net.ipv4.conf.lo.arp_announce=2
EOF

ip addr add 192.168.31.198/32 dev lo label lo:1
#测试通过后，添加到卡机自启

centos

# DR 模式
cat >> /etc/sysctl.conf << EOF
net.ipv4.conf.all.arp_ignore=1
net.ipv4.conf.all.arp_announce=2
#net.ipv4.conf.lo.arp_ignore=1
#net.ipv4.conf.lo.arp_announce=2
EOF

vim /etc/sysconfig/network-scripts/ifcfg-lo:1
# 添加
DEVICE=lo:1
IPADDR=192.168.31.198
NETMASK=255.255.255.255
NETWORK=127.0.0.0
# 重启网卡
systemctl restart network

检查

#启动LVS
systemctl start ipvsamd
#启动keepalived
systemctl start keepalived
#检查lvs是否生效
ipvsamd -ln
#检查keepalived
ip a 
#其他服务器（非LVS、RS服务器） 
telnet VIP 端口
#查看转发到那台RS服务器，在VIP所在的服务器运行
ipvsamd -lnc 

LVS

介绍

运行模式

模式	特性	适用场景
NAT	调度器做 SNAT/DNAT，Real Server 响应需经调度器返回	网络拓扑复杂，Real Server 分布较广时使用
TUN	使用 IP 隧道技术，Real Server 可跨网段	Real Server 跨子网部署时使用
DR	Real Server 直接响应客户端，性能最优	大流量、高并发场景，Real Server 和调度器在同一局域网内

特性/模式	TUN 模式	DR 模式	NAT 模式
跨网段支持	✅	❌	✅
性能	中等	最高	较低
配置复杂度	中等	简单	简单
是否需要VIP	是	是	是
响应路径	RS -> 客户端	RS-> 客户端	RS -> Director -> 客户端

算法

算法名称	类型	是否支持权重	适用场景	特点
RR	静态	❌	均匀负载	简单高效
WRR	静态	✅	性能差异大	控制灵活
LC	动态	❌	长连接服务	自动适应负载
WLC	动态	✅	异构环境	综合性能佳
LBLC	动态	❌	客户端亲和	保持连接一致性
LBLCR	动态	❌	缓存集群	支持缓存复制
DH	静态	❌	固定目标	固定转发策略
SH	静态	❌	会话保持	同一用户固定节点

部署

内核配置

kernel.sysrq=0
net.ipv4.ip_forward=0
net.ipv4.conf.all.send_redirects=0
net.ipv4.conf.default.send_redirects=0
net.ipv4.conf.all.accept_source_route=0
net.ipv4.conf.default.accept_source_route=0
net.ipv4.conf.all.accept_redirects=0
net.ipv4.conf.default.accept_redirects=0
net.ipv4.conf.all.secure_redirects=0
net.ipv4.conf.default.secure_redirects=0
net.ipv4.icmp_echo_ignore_broadcasts=1
net.ipv4.icmp_ignore_bogus_error_responses=1
net.ipv4.conf.all.rp_filter=1
net.ipv4.conf.default.rp_filter=1
net.ipv4.tcp_syncookies=1
kernel.dmesg_restrict=1
net.ipv6.conf.all.accept_redirects=0
net.ipv6.conf.default.accept_redirects=0