使用kubeadm部署高可用IPV4/IPV6集群-v1.32

原创

小陈运维

发布于 2025-03-23 17:38:46

2290

文章被收录于专栏：小陈运维小陈运维

使用kubeadm部署高可用IPV4/IPV6集群

https://github.com/cby-chen/Kubernetes 开源不易，帮忙点个star，谢谢了

k8s基础系统环境配置

配置IP

# 注意！
# 若虚拟机是进行克隆的那么网卡的UUID和MachineID会重复
# 需要重新生成新的UUIDUUID和MachineID
# UUID和MachineID重复无法DHCP获取到IPV6地址
ssh root@192.168.1.189 "rm -rf /etc/machine-id; systemd-machine-id-setup;reboot"
ssh root@192.168.1.190 "rm -rf /etc/machine-id; systemd-machine-id-setup;reboot"
ssh root@192.168.1.191 "rm -rf /etc/machine-id; systemd-machine-id-setup;reboot"
ssh root@192.168.1.192 "rm -rf /etc/machine-id; systemd-machine-id-setup;reboot"
ssh root@192.168.1.193 "rm -rf /etc/machine-id; systemd-machine-id-setup;reboot"
# 
# 查看当前的网卡列表和 UUID：
# nmcli con show
# 删除要更改 UUID 的网络连接：
# nmcli con delete uuid <原 UUID>
# 重新生成 UUID：
# nmcli con add type ethernet ifname <接口名称> con-name <新名称>
# 重新启用网络连接：
# nmcli con up <新名称>

# 更改网卡的UUID
# 先配置静态IP之后使用ssh方式配置不断连
ssh root@192.168.1.189 "nmcli con delete uuid d1141403-18c6-3149-907c-ed5f09663a7f;nmcli con add type ethernet ifname ens160 con-name ens160;nmcli con up ens160"
ssh root@192.168.1.190 "nmcli con delete uuid d1141403-18c6-3149-907c-ed5f09663a7f;nmcli con add type ethernet ifname ens160 con-name ens160;nmcli con up ens160"
ssh root@192.168.1.191 "nmcli con delete uuid d1141403-18c6-3149-907c-ed5f09663a7f;nmcli con add type ethernet ifname ens160 con-name ens160;nmcli con up ens160"
ssh root@192.168.1.192 "nmcli con delete uuid d1141403-18c6-3149-907c-ed5f09663a7f;nmcli con add type ethernet ifname ens160 con-name ens160;nmcli con up ens160"
ssh root@192.168.1.193 "nmcli con delete uuid d1141403-18c6-3149-907c-ed5f09663a7f;nmcli con add type ethernet ifname ens160 con-name ens160;nmcli con up ens160"

# 参数解释
# 
# ssh ssh root@192.168.1.21
# 使用SSH登录到IP为192.168.1.21的主机，使用root用户身份。
# 
# nmcli con delete uuid 708a1497-2192-43a5-9f03-2ab936fb3c44
# 删除 UUID 为 708a1497-2192-43a5-9f03-2ab936fb3c44 的网络连接，这是 NetworkManager 中一种特定网络配置的唯一标识符。
# 
# nmcli con add type ethernet ifname ens160 con-name ens160
# 添加一种以太网连接类型，并指定接口名为 ens160，连接名称也为 ens160。
# 
# nmcli con up ens160
# 开启 ens160 这个网络连接。
# 
# 简单来说，这个命令的作用是删除一个特定的网络连接配置，并添加一个名为 ens160 的以太网连接，然后启用这个新的连接。

# 修改静态的IPv4地址
ssh root@192.168.1.189 "nmcli con mod ens160 ipv4.addresses 192.168.1.21/24; nmcli con mod ens160 ipv4.gateway  192.168.1.1; nmcli con mod ens160 ipv4.method manual; nmcli con mod ens160 ipv4.dns "8.8.8.8"; nmcli con up ens160"
ssh root@192.168.1.190 "nmcli con mod ens160 ipv4.addresses 192.168.1.22/24; nmcli con mod ens160 ipv4.gateway  192.168.1.1; nmcli con mod ens160 ipv4.method manual; nmcli con mod ens160 ipv4.dns "8.8.8.8"; nmcli con up ens160"
ssh root@192.168.1.191 "nmcli con mod ens160 ipv4.addresses 192.168.1.23/24; nmcli con mod ens160 ipv4.gateway  192.168.1.1; nmcli con mod ens160 ipv4.method manual; nmcli con mod ens160 ipv4.dns "8.8.8.8"; nmcli con up ens160"
ssh root@192.168.1.192 "nmcli con mod ens160 ipv4.addresses 192.168.1.24/24; nmcli con mod ens160 ipv4.gateway  192.168.1.1; nmcli con mod ens160 ipv4.method manual; nmcli con mod ens160 ipv4.dns "8.8.8.8"; nmcli con up ens160"
ssh root@192.168.1.193 "nmcli con mod ens160 ipv4.addresses 192.168.1.25/24; nmcli con mod ens160 ipv4.gateway  192.168.1.1; nmcli con mod ens160 ipv4.method manual; nmcli con mod ens160 ipv4.dns "8.8.8.8"; nmcli con up ens160"


# 参数解释
# 
# ssh root@192.168.1.189
# 使用SSH登录到IP为192.168.1.189的主机，使用root用户身份。
# 
# "nmcli con mod ens160 ipv4.addresses 192.168.1.21/24"
# 修改ens160网络连接的IPv4地址为192.168.1.21，子网掩码为 24。
# 
# "nmcli con mod ens160 ipv4.gateway 192.168.1.1"
# 修改ens160网络连接的IPv4网关为192.168.1.1。
# 
# "nmcli con mod ens160 ipv4.method manual"
# 将ens160网络连接的IPv4配置方法设置为手动。
# 
# "nmcli con mod ens160 ipv4.dns "8.8.8.8"
# 将ens160网络连接的IPv4 DNS服务器设置为 8.8.8.8。
# 
# "nmcli con up ens160"
# 启动ens160网络连接。
# 
# 总体来说，这条命令是通过SSH远程登录到指定的主机，并使用网络管理命令 (nmcli) 修改ens160网络连接的配置，包括IP地址、网关、配置方法和DNS服务器，并启动该网络连接。

# 我这里有公网的IPv6的地址，但是是DHCP动态的，无法固定，使用不方便
# 所以我配置了内网的IPv6地址，可以实现固定的访问地址

# 我使用的方式。只配置IPv6地址不配置网关DNS
ssh root@192.168.1.21 "nmcli con mod ens160 ipv6.addresses fc00::21/8; nmcli con up ens160"
ssh root@192.168.1.22 "nmcli con mod ens160 ipv6.addresses fc00::22/8; nmcli con up ens160"
ssh root@192.168.1.23 "nmcli con mod ens160 ipv6.addresses fc00::23/8; nmcli con up ens160"
ssh root@192.168.1.24 "nmcli con mod ens160 ipv6.addresses fc00::24/8; nmcli con up ens160"
ssh root@192.168.1.25 "nmcli con mod ens160 ipv6.addresses fc00::25/8; nmcli con up ens160"

# IPv6地址路由DNS，样例
ssh root@192.168.1.21 "nmcli con mod ens160 ipv6.addresses fc00:43f4:1eea:1::10; nmcli con mod ens160 ipv6.gateway fc00:43f4:1eea:1::1; nmcli con mod ens160 ipv6.method manual; nmcli con mod ens160 ipv6.dns "2400:3200::1"; nmcli con up ens160"
ssh root@192.168.1.22 "nmcli con mod ens160 ipv6.addresses fc00:43f4:1eea:1::20; nmcli con mod ens160 ipv6.gateway fc00:43f4:1eea:1::1; nmcli con mod ens160 ipv6.method manual; nmcli con mod ens160 ipv6.dns "2400:3200::1"; nmcli con up ens160"
ssh root@192.168.1.23 "nmcli con mod ens160 ipv6.addresses fc00:43f4:1eea:1::30; nmcli con mod ens160 ipv6.gateway fc00:43f4:1eea:1::1; nmcli con mod ens160 ipv6.method manual; nmcli con mod ens160 ipv6.dns "2400:3200::1"; nmcli con up ens160"
ssh root@192.168.1.24 "nmcli con mod ens160 ipv6.addresses fc00:43f4:1eea:1::40; nmcli con mod ens160 ipv6.gateway fc00:43f4:1eea:1::1; nmcli con mod ens160 ipv6.method manual; nmcli con mod ens160 ipv6.dns "2400:3200::1"; nmcli con up ens160"
ssh root@192.168.1.25 "nmcli con mod ens160 ipv6.addresses fc00:43f4:1eea:1::50; nmcli con mod ens160 ipv6.gateway fc00:43f4:1eea:1::1; nmcli con mod ens160 ipv6.method manual; nmcli con mod ens160 ipv6.dns "2400:3200::1"; nmcli con up ens160"

# 参数解释
# 
# ssh root@192.168.1.21
# 通过SSH连接到IP地址为192.168.1.21的远程主机，使用root用户进行登录。
# 
# "nmcli con mod ens160 ipv6.addresses fc00:43f4:1eea:1::10"
# 使用nmcli命令修改ens160接口的IPv6地址为fc00:43f4:1eea:1::10。
# 
# "nmcli con mod ens160 ipv6.gateway fc00:43f4:1eea:1::1"
# 使用nmcli命令修改ens160接口的IPv6网关为fc00:43f4:1eea:1::1。
# 
# "nmcli con mod ens160 ipv6.method manual"
# 使用nmcli命令将ens160接口的IPv6配置方法修改为手动配置。
# 
# "nmcli con mod ens160 ipv6.dns "2400:3200::1"
# 使用nmcli命令设置ens160接口的IPv6 DNS服务器为2400:3200::1。
# 
# "nmcli con up ens160"
# 使用nmcli命令启动ens160接口。
# 
# 这个命令的目的是在远程主机上配置ens160接口的IPv6地址、网关、配置方法和DNS服务器，并启动ens160接口。

# 查看网卡配置
# nmcli device show ens160
# nmcli con show ens160
[root@localhost ~]#  cat /etc/NetworkManager/system-connections/ens160.nmconnection 
[connection]
id=ens160
uuid=d199c6e0-4212-4bf8-9a7b-a2da247ca759
type=ethernet
interface-name=ens160
timestamp=1742703386

[ethernet]

[ipv4]
address1=192.168.1.21/24,192.168.1.1
dns=192.168.1.99;
method=manual

[ipv6]
addr-gen-mode=default
address1=fc00::21/8
method=auto

[proxy]

设置主机名

hostnamectl set-hostname k8s-master01
hostnamectl set-hostname k8s-master02
hostnamectl set-hostname k8s-master03
hostnamectl set-hostname k8s-node01
hostnamectl set-hostname k8s-node02

配置yum源

# 其他系统的源地址
# https://help.mirrors.cernet.edu.cn/

# 对于私有仓库
sed -e 's|^mirrorlist=|#mirrorlist=|g' -e 's|^#baseurl=http://mirror.centos.org/\$contentdir|baseurl=http://192.168.1.123/centos|g' -i.bak  /etc/yum.repos.d/CentOS-*.repo

# 对于 Ubuntu
sed -i 's/cn.archive.ubuntu.com/mirrors.ustc.edu.cn/g' /etc/apt/sources.list

# epel扩展源
sudo yum install -y epel-release
sudo sed -e 's!^metalink=!#metalink=!g' \
    -e 's!^#baseurl=!baseurl=!g' \
    -e 's!https\?://download\.fedoraproject\.org/pub/epel!https://mirror.nju.edu.cn/epel!g' \
    -e 's!https\?://download\.example/pub/epel!https://mirror.nju.edu.cn/epel!g' \
    -i /etc/yum.repos.d/epel{,-testing}.repo

# 对于 CentOS 7
sudo sed -e 's|^mirrorlist=|#mirrorlist=|g' \
         -e 's|^#baseurl=http://mirror.centos.org/centos|baseurl=https://mirror.nju.edu.cn/centos|g' \
         -i.bak \
         /etc/yum.repos.d/CentOS-*.repo

# 对于 CentOS 8
sudo sed -e 's|^mirrorlist=|#mirrorlist=|g' \
         -e 's|^#baseurl=http://mirror.centos.org/$contentdir|baseurl=https://mirror.nju.edu.cn/centos|g' \
         -i.bak \
         /etc/yum.repos.d/CentOS-*.repo

# 对于CentOS 9
-------

安装一些必备工具

# 对于 Ubuntu
apt update && apt upgrade -y && apt install -y wget psmisc vim net-tools nfs-kernel-server telnet lvm2 git tar curl

# 对于 CentOS 7
yum update -y && yum -y install  wget psmisc vim net-tools nfs-utils telnet yum-utils device-mapper-persistent-data lvm2 git tar curl

# 对于 CentOS 8
yum update -y && yum -y install wget psmisc vim net-tools nfs-utils telnet yum-utils device-mapper-persistent-data lvm2 git network-scripts tar curl

# 对于 CentOS 9
yum update -y && yum -y install wget psmisc vim net-tools nfs-utils telnet yum-utils device-mapper-persistent-data lvm2 git tar curl

关闭防火墙

# Ubuntu忽略，CentOS执行
systemctl disable --now firewalld

关闭SELinux

# Ubuntu忽略，CentOS执行
setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config

关闭交换分区

sed -ri 's/.*swap.*/#&/' /etc/fstab
swapoff -a && sysctl -w vm.swappiness=0

cat /etc/fstab
# /dev/mapper/centos-swap swap                    swap    defaults        0 0

网络配置（俩种方式二选一）

# Ubuntu忽略，CentOS执行，CentOS9不支持方式一

# 方式一
# systemctl disable --now NetworkManager
# systemctl start network && systemctl enable network

# 方式二
cat > /etc/NetworkManager/conf.d/calico.conf << EOF 
[keyfile]
unmanaged-devices=interface-name:cali*;interface-name:tunl*
EOF
systemctl restart NetworkManager

进行时间同步

# 服务端
# apt install chrony -y
yum install chrony -y
cat > /etc/chrony.conf << EOF 
pool ntp.aliyun.com iburst
driftfile /var/lib/chrony/drift
makestep 1.0 3
rtcsync
allow 192.168.1.0/24
local stratum 10
keyfile /etc/chrony.keys
leapsectz right/UTC
logdir /var/log/chrony
EOF

systemctl restart chronyd ; systemctl enable chronyd

# 客户端
# apt install chrony -y
yum install chrony -y
cat > /etc/chrony.conf << EOF 
pool 192.168.1.21 iburst
driftfile /var/lib/chrony/drift
makestep 1.0 3
rtcsync
keyfile /etc/chrony.keys
leapsectz right/UTC
logdir /var/log/chrony
EOF

systemctl restart chronyd ; systemctl enable chronyd

#使用客户端进行验证
chronyc sources -v

配置ulimit

ulimit -SHn 65535
cat >> /etc/security/limits.conf <<EOF
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* seft memlock unlimited
* hard memlock unlimitedd
EOF

配置免密登录

# apt install -y sshpass
yum install -y sshpass
ssh-keygen -f /root/.ssh/id_rsa -P ''
export IP="192.168.1.21 192.168.1.22 192.168.1.23 192.168.1.24 192.168.1.25"
export SSHPASS=123123
for HOST in $IP;do
     sshpass -e ssh-copy-id -o StrictHostKeyChecking=no $HOST
done

添加启用源

# Ubuntu忽略，CentOS执行

# 为 RHEL-9 SL-9 或 CentOS-9 安装 ELRepo 
yum install https://www.elrepo.org/elrepo-release-9.el9.elrepo.noarch.rpm -y 
sed -i "s@mirrorlist@#mirrorlist@g" /etc/yum.repos.d/elrepo.repo 
sed -i "s@elrepo.org/linux@mirrors.tuna.tsinghua.edu.cn/elrepo@g" /etc/yum.repos.d/elrepo.repo 

# 为 RHEL-8或 CentOS-8配置源
yum install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm -y 
sed -i "s@mirrorlist@#mirrorlist@g" /etc/yum.repos.d/elrepo.repo 
sed -i "s@elrepo.org/linux@mirrors.tuna.tsinghua.edu.cn/elrepo@g" /etc/yum.repos.d/elrepo.repo 

# 为 RHEL-7 SL-7 或 CentOS-7 安装 ELRepo 
yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm -y 
sed -i "s@mirrorlist@#mirrorlist@g" /etc/yum.repos.d/elrepo.repo 
sed -i "s@elrepo.org/linux@mirrors.tuna.tsinghua.edu.cn/elrepo@g" /etc/yum.repos.d/elrepo.repo 

# 查看可用安装包
yum  --disablerepo="*"  --enablerepo="elrepo-kernel"  list  available

升级内核至4.18版本以上

# Ubuntu忽略，CentOS执行

# 安装最新的内核
# 我这里选择的是稳定版kernel-ml   如需更新长期维护版本kernel-lt  
yum -y --enablerepo=elrepo-kernel  install  kernel-ml

# 查看已安装那些内核
rpm -qa | grep kernel

# 查看默认内核
grubby --default-kernel

# 若不是最新的使用命令设置
grubby --set-default $(ls /boot/vmlinuz-* | grep elrepo)

# 重启生效
reboot

# v8 整合命令为：
yum install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm -y ; sed -i "s@mirrorlist@#mirrorlist@g" /etc/yum.repos.d/elrepo.repo ; sed -i "s@elrepo.org/linux@mirrors.tuna.tsinghua.edu.cn/elrepo@g" /etc/yum.repos.d/elrepo.repo ; yum  --disablerepo="*"  --enablerepo="elrepo-kernel"  list  available -y ; yum  --enablerepo=elrepo-kernel  install kernel-lt -y ; grubby --default-kernel ; reboot 

# v7 整合命令为：
yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm -y ; sed -i "s@mirrorlist@#mirrorlist@g" /etc/yum.repos.d/elrepo.repo ; sed -i "s@elrepo.org/linux@mirrors.tuna.tsinghua.edu.cn/elrepo@g" /etc/yum.repos.d/elrepo.repo ; yum  --disablerepo="*"  --enablerepo="elrepo-kernel"  list  available -y ; yum  --enablerepo=elrepo-kernel  install  kernel-lt -y ; grubby --set-default $(ls /boot/vmlinuz-* | grep elrepo) ; grubby --default-kernel ; reboot 

# 离线版本 
yum install -y /root/cby/kernel-lt-*-1.el7.elrepo.x86_64.rpm ; grubby --set-default $(ls /boot/vmlinuz-* | grep elrepo) ; grubby --default-kernel ; reboot

安装ipvsadm

# 对于CentOS7离线安装
# yum install /root/centos7/ipset-*.el7.x86_64.rpm /root/centos7/lm_sensors-libs-*.el7.x86_64.rpm  /root/centos7/ipset-libs-*.el7.x86_64.rpm /root/centos7/sysstat-*.el7_9.x86_64.rpm  /root/centos7/ipvsadm-*.el7.x86_64.rpm  -y

# 对于 Ubuntu
# apt install ipvsadm ipset sysstat conntrack -y

# 对于 CentOS
yum install ipvsadm ipset sysstat conntrack libseccomp -y
cat >> /etc/modules-load.d/ipvs.conf <<EOF 
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF

systemctl restart systemd-modules-load.service

lsmod | grep -e ip_vs -e nf_conntrack
ip_vs_sh               16384  0
ip_vs_wrr              16384  0
ip_vs_rr               16384  0
ip_vs                 237568  6 ip_vs_rr,ip_vs_sh,ip_vs_wrr
nf_conntrack          217088  3 nf_nat,nft_ct,ip_vs
nf_defrag_ipv6         24576  2 nf_conntrack,ip_vs
nf_defrag_ipv4         16384  1 nf_conntrack
libcrc32c              16384  5 nf_conntrack,nf_nat,nf_tables,xfs,ip_vs

修改内核参数

cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720

net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384

net.ipv6.conf.all.disable_ipv6 = 0
net.ipv6.conf.default.disable_ipv6 = 0
net.ipv6.conf.lo.disable_ipv6 = 0
net.ipv6.conf.all.forwarding = 1
EOF

sysctl --system

所有节点配置hosts本地解析

cat > /etc/hosts <<EOF
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6

192.168.1.21 k8s-master01
192.168.1.22 k8s-master02
192.168.1.23 k8s-master03
192.168.1.24 k8s-node01
192.168.1.25 k8s-node02
192.168.1.36 lb-vip

fc00::21 k8s-master01
fc00::22 k8s-master02
fc00::23 k8s-master03
fc00::24 k8s-node01
fc00::25 k8s-node02
EOF

配置kubernetes安装源

Debian / Ubuntu

在配置中添加镜像（注意修改为自己需要的版本号）：

curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg

cat <<EOF | tee /etc/apt/sources.list.d/kubernetes.list
deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://mirrors.tuna.tsinghua.edu.cn/kubernetes/core:/stable:/v1.32/deb/ /
# deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://mirrors.tuna.tsinghua.edu.cn/kubernetes/addons:/cri-o:/stable:/v1.32/deb/ /
EOF

安装必要应用：

apt-get update
apt-get install -y kubelet kubeadm kubectl
systemctl enable kubelet && systemctl start kubelet

# 如安装指定版本
# apt install kubelet=1.28.2-00 kubeadm=1.28.2-00 kubectl=1.28.2-00

CentOS / RHEL / Fedora

执行如下命令（注意修改为自己需要的版本号）：

cat <<EOF | tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=kubernetes
baseurl=https://mirrors.tuna.tsinghua.edu.cn/kubernetes/yum/repos/kubernetes-el7-$basearch
name=Kubernetes
baseurl=https://mirrors.tuna.tsinghua.edu.cn/kubernetes/core:/stable:/v1.32/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v1.32/rpm/repodata/repomd.xml.key

[cri-o]
name=CRI-O
baseurl=https://mirrors.tuna.tsinghua.edu.cn/kubernetes/addons:/cri-o:/stable:/v1.32/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/addons:/cri-o:/prerelease:/main/rpm/repodata/repomd.xml.key
EOF

安装必要应用：

yum update
yum install -y kubelet kubeadm kubectl

# 如安装指定版本
# yum install kubelet-1.28.2-0 kubeadm-1.28.2-0 kubectl-1.28.2-0

systemctl enable kubelet && systemctl start kubelet

# 将 SELinux 设置为 禁用
setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config

ps: 由于官网未开放同步方式, 可能会有索引gpg检查失败的情况, 这时请用 yum install -y --nogpgcheck kubelet kubeadm kubectl 安装

k8s基本组件安装

注意：二选其一即可

安装Containerd作为Runtime

# https://github.com/containernetworking/plugins/releases/
wget https://mirrors.chenby.cn/https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-amd64-v1.6.2.tgz


#创建cni插件所需目录
mkdir -p /etc/cni/net.d /opt/cni/bin 
#解压cni二进制包
tar xf cni-plugins-linux-amd64-v*.tgz -C /opt/cni/bin/

# https://github.com/containerd/containerd/releases/
wget https://mirrors.chenby.cn/https://github.com/containerd/containerd/releases/download/v2.0.4/containerd-2.0.4-linux-amd64.tar.gz

#解压
tar -xzf containerd-*-linux-amd64.tar.gz -C /usr/local/

#创建服务启动文件
cat > /etc/systemd/system/containerd.service <<EOF
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target

[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd
Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
LimitNPROC=infinity
LimitCORE=infinity
LimitNOFILE=infinity
TasksMax=infinity
OOMScoreAdjust=-999

[Install]
WantedBy=multi-user.target
EOF

配置Containerd所需的模块

cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF

加载模块

systemctl restart systemd-modules-load.service

配置Containerd所需的内核

cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables  = 1
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF

# 加载内核
sysctl --system

创建Containerd的配置文件

# 创建默认配置文件
mkdir -p /etc/containerd
containerd config default | tee /etc/containerd/config.toml

# 修改Containerd的配置文件

# sed -i "s#SystemdCgroup\ \=\ false#SystemdCgroup\ \=\ true#g" /etc/containerd/config.toml
# cat /etc/containerd/config.toml | grep SystemdCgroup

# 沙箱pause镜像
sed -i "s#registry.k8s.io#registry.aliyuncs.com/chenby#g" /etc/containerd/config.toml
cat /etc/containerd/config.toml | grep sandbox

# 配置加速器
[root@k8s-master01 ~]# vim /etc/containerd/config.toml
[root@k8s-master01 ~]# cat /etc/containerd/config.toml | grep certs.d -C 5

    [plugins.'io.containerd.cri.v1.images'.pinned_images]
      sandbox = 'registry.aliyuncs.com/chenby/pause:3.10'

    [plugins.'io.containerd.cri.v1.images'.registry]
      config_path = '/etc/containerd/certs.d'

    [plugins.'io.containerd.cri.v1.images'.image_decryption]
      key_model = 'node'

  [plugins.'io.containerd.cri.v1.runtime']
[root@k8s-master01 ~]# 

mkdir /etc/containerd/certs.d/docker.io -pv
cat > /etc/containerd/certs.d/docker.io/hosts.toml << EOF
server = "https://docker.io"
[host."https://xxxxxxxxxxx.com"]
  capabilities = ["pull", "resolve"]
EOF

启动并设置为开机启动

systemctl daemon-reload
# 用于重新加载systemd管理的单位文件。当你新增或修改了某个单位文件（如.service文件、.socket文件等），需要运行该命令来刷新systemd对该文件的配置。

systemctl enable --now containerd.service
# 启用并立即启动docker.service单元。docker.service是Docker守护进程的systemd服务单元。

systemctl stop containerd.service
# 停止运行中的docker.service单元，即停止Docker守护进程。

systemctl start containerd.service
# 启动docker.service单元，即启动Docker守护进程。

systemctl restart containerd.service
# 重启docker.service单元，即重新启动Docker守护进程。

systemctl status containerd.service
# 显示docker.service单元的当前状态，包括运行状态、是否启用等信息。

配置crictl客户端连接的运行时位置

# https://github.com/kubernetes-sigs/cri-tools/releases/
wget https://mirrors.chenby.cn/https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.32.0/crictl-v1.32.0-linux-amd64.tar.gz

#解压
tar xf crictl-v*-linux-amd64.tar.gz -C /usr/bin/
#生成配置文件
cat > /etc/crictl.yaml <<EOF
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF

#测试
systemctl restart  containerd
crictl info

安装docker作为Runtime

解压docker程序

# 二进制包下载地址：https://download.docker.com/linux/static/stable/x86_64/
wget https://mirrors.tuna.tsinghua.edu.cn/docker-ce/linux/static/stable/x86_64/docker-28.0.2.tgz

#解压
tar xf docker-*.tgz 
#拷贝二进制文件
cp docker/* /usr/bin/

创建containerd的service文件

#创建containerd的service文件,并且启动
cat >/etc/systemd/system/containerd.service <<EOF
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target

[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/bin/containerd
Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
LimitNPROC=infinity
LimitCORE=infinity
LimitNOFILE=1048576
TasksMax=infinity
OOMScoreAdjust=-999

[Install]
WantedBy=multi-user.target
EOF

# 设置开机自启
systemctl enable --now containerd.service

准备docker的service文件

#准备docker的service文件
cat > /etc/systemd/system/docker.service <<EOF
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service cri-docker.service docker.socket containerd.service
Wants=network-online.target
Requires=docker.socket containerd.service

[Service]
Type=notify
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always
StartLimitBurst=3
StartLimitInterval=60s
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity
Delegate=yes
KillMode=process
OOMScoreAdjust=-500

[Install]
WantedBy=multi-user.target
EOF

准备docker的socket文件

#准备docker的socket文件
cat > /etc/systemd/system/docker.socket <<EOF
[Unit]
Description=Docker Socket for the API

[Socket]
ListenStream=/var/run/docker.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker

[Install]
WantedBy=sockets.target
EOF

配置加速器

# 配置加速器
mkdir /etc/docker/ -pv
cat >/etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "registry-mirrors": [
    "https://xxxxxxxxxxxx.com"
  ],
  "max-concurrent-downloads": 10,
  "log-driver": "json-file",
  "log-level": "warn",
  "log-opts": {
    "max-size": "10m",
    "max-file": "3"
    },
  "data-root": "/var/lib/docker"
}
EOF

启动docker

groupadd docker
#创建docker组

systemctl daemon-reload
# 用于重新加载systemd管理的单位文件。当你新增或修改了某个单位文件（如.service文件、.socket文件等），需要运行该命令来刷新systemd对该文件的配置。

systemctl enable --now docker.socket
# 启用并立即启动docker.socket单元。docker.socket是一个systemd的socket单元，用于接收来自网络的Docker API请求。

systemctl enable --now docker.service
# 启用并立即启动docker.service单元。docker.service是Docker守护进程的systemd服务单元。

systemctl stop docker.service
# 停止运行中的docker.service单元，即停止Docker守护进程。

systemctl start docker.service
# 启动docker.service单元，即启动Docker守护进程。

systemctl restart docker.service
# 重启docker.service单元，即重新启动Docker守护进程。

systemctl status docker.service
# 显示docker.service单元的当前状态，包括运行状态、是否启用等信息。

docker info
#验证

解压cri-docker

# 由于1.24以及更高版本不支持docker所以安装cri-docker
# 下载cri-docker 
# https://github.com/Mirantis/cri-dockerd/releases/
wget  https://mirrors.chenby.cn/https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.16/cri-dockerd-0.3.16.amd64.tgz

# 解压cri-docker
tar xvf cri-dockerd-*.amd64.tgz 
cp -r cri-dockerd/  /usr/bin/
chmod +x /usr/bin/cri-dockerd/cri-dockerd

写入启动cri-docker配置文件

# 写入启动配置文件
cat >  /usr/lib/systemd/system/cri-docker.service <<EOF
[Unit]
Description=CRI Interface for Docker Application Container Engine
Documentation=https://docs.mirantis.com
After=network-online.target firewalld.service docker.service
Wants=network-online.target
Requires=docker.service

[Service]
Type=notify
ExecStart=/usr/bin/cri-dockerd/cri-dockerd --network-plugin=cni --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.7
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always
StartLimitBurst=3
StartLimitInterval=60s
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity
Delegate=yes
KillMode=process

[Install]
WantedBy=multi-user.target
EOF

写入cri-docker的socket配置文件

# 写入socket配置文件
cat > /usr/lib/systemd/system/cri-docker.socket <<EOF
[Unit]
Description=CRI Docker Socket for the API
PartOf=cri-docker.service

[Socket]
ListenStream=%t/cri-dockerd.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker

[Install]
WantedBy=sockets.target
EOF

启动cri-docker

systemctl daemon-reload
# 用于重新加载systemd管理的单位文件。当你新增或修改了某个单位文件（如.service文件、.socket文件等），需要运行该命令来刷新systemd对该文件的配置。

systemctl enable --now cri-docker.service
# 启用并立即启动cri-docker.service单元。cri-docker.service是cri-docker守护进程的systemd服务单元。

systemctl restart cri-docker.service
# 重启cri-docker.service单元，即重新启动cri-docker守护进程。

systemctl status docker.service
# 显示docker.service单元的当前状态，包括运行状态、是否启用等信息。

高可用keepalived、haproxy

安装keepalived和haproxy服务

yum -y install keepalived haproxy

修改haproxy配置文件（配置文件一样）

# cp /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg.bak

cat >/etc/haproxy/haproxy.cfg<<"EOF"
global
 maxconn 2000
 ulimit-n 16384
 log 127.0.0.1 local0 err
 stats timeout 30s

defaults
 log global
 mode http
 option httplog
 timeout connect 5000
 timeout client 50000
 timeout server 50000
 timeout http-request 15s
 timeout http-keep-alive 15s


frontend monitor-in
 bind *:33305
 mode http
 option httplog
 monitor-uri /monitor

frontend k8s-master
 bind 0.0.0.0:9443
 bind 127.0.0.1:9443
 mode tcp
 option tcplog
 tcp-request inspect-delay 5s
 default_backend k8s-master


backend k8s-master
 mode tcp
 option tcplog
 option tcp-check
 balance roundrobin
 default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
 server  k8s-master01  192.168.1.21:6443 check
 server  k8s-master02  192.168.1.22:6443 check
 server  k8s-master03  192.168.1.23:6443 check
EOF

Master01配置keepalived master节点

#cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak

cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived

global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5 
    weight -5
    fall 2
    rise 1
}
vrrp_instance VI_1 {
    state MASTER
    # 注意网卡名
    interface ens160 
    mcast_src_ip 192.168.1.21
    virtual_router_id 51
    priority 100
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        192.168.1.36
    }
    track_script {
      chk_apiserver 
} }

EOF

Master02配置keepalived backup节点

# cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak

cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived

global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5 
    weight -5
    fall 2
    rise 1

}
vrrp_instance VI_1 {
    state BACKUP
    # 注意网卡名
    interface ens160
    mcast_src_ip 192.168.1.22
    virtual_router_id 51
    priority 80
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        192.168.1.36
    }
    track_script {
      chk_apiserver 
} }

EOF

Master03配置keepalived backup节点

# cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak

cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived

global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5 
    weight -5
    fall 2
    rise 1

}
vrrp_instance VI_1 {
    state BACKUP
    # 注意网卡名
    interface ens160
    mcast_src_ip 192.168.1.23
    virtual_router_id 51
    priority 50
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        192.168.1.36
    }
    track_script {
      chk_apiserver 
} }

EOF

健康检查脚本配置（lb主机）

cat >  /etc/keepalived/check_apiserver.sh << EOF
#!/bin/bash

err=0
for k in \$(seq 1 3)
do
    check_code=\$(pgrep haproxy)
    if [[ \$check_code == "" ]]; then
        err=\$(expr \$err + 1)
        sleep 1
        continue
    else
        err=0
        break
    fi
done

if [[ \$err != "0" ]]; then
    echo "systemctl stop keepalived"
    /usr/bin/systemctl stop keepalived
    exit 1
else
    exit 0
fi
EOF

# 给脚本授权

chmod +x /etc/keepalived/check_apiserver.sh

启动服务

systemctl daemon-reload
# 用于重新加载systemd管理的单位文件。当你新增或修改了某个单位文件（如.service文件、.socket文件等），需要运行该命令来刷新systemd对该文件的配置。
systemctl enable --now haproxy.service
# 启用并立即启动haproxy.service单元。haproxy.service是haproxy守护进程的systemd服务单元。
systemctl enable --now keepalived.service
# 启用并立即启动keepalived.service单元。keepalived.service是keepalived守护进程的systemd服务单元。
systemctl status haproxy.service
# haproxy.service单元的当前状态，包括运行状态、是否启用等信息。
systemctl status keepalived.service
# keepalived.service单元的当前状态，包括运行状态、是否启用等信息。

测试高可用

# 能ping同
[root@k8s-node02 ~]# ping 192.168.1.36

# 能telnet访问
[root@k8s-node02 ~]# telnet 192.168.1.36 9443

# 关闭主节点，看vip是否漂移到备节点

初始化安装

整改镜像

# 查看最新版本有那些镜像
[root@k8s-master01 ~]# kubeadm config images list --image-repository registry.aliyuncs.com/google_containers
registry.aliyuncs.com/google_containers/kube-apiserver:v1.32.3
registry.aliyuncs.com/google_containers/kube-controller-manager:v1.32.3
registry.aliyuncs.com/google_containers/kube-scheduler:v1.32.3
registry.aliyuncs.com/google_containers/kube-proxy:v1.32.3
registry.aliyuncs.com/google_containers/coredns:v1.11.3
registry.aliyuncs.com/google_containers/pause:3.10
registry.aliyuncs.com/google_containers/etcd:3.5.16-0
[root@k8s-master01 ~]# 

# 只有一个CRI的情况下
kubeadm config images pull --image-repository registry.aliyuncs.com/google_containers
# 指定CRI拉去镜像
kubeadm config images pull --image-repository registry.aliyuncs.com/google_containers --cri-socket unix:///var/run/cri-dockerd.sock
kubeadm config images pull --image-repository registry.aliyuncs.com/google_containers --cri-socket unix:///var/run/containerd/containerd.sock

修改初始化配置

# 创建默认配置
kubeadm config print init-defaults > kubeadm-init.yaml
# 这是我使用的配置文件
cat > kubeadm.yaml << EOF
apiVersion: kubeadm.k8s.io/v1beta4
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 192.168.1.21
  bindPort: 6443
nodeRegistration:
  # criSocket: unix:///run/containerd/containerd.sock
  criSocket: unix:///var/run/cri-dockerd.sock
  imagePullPolicy: IfNotPresent
  imagePullSerial: true
  kubeletExtraArgs:
  - name: "node-ip"
    value: "192.168.1.21,fc00::21"
  name: k8s-master01
  taints:
  - effect: PreferNoSchedule
    key: node-role.kubernetes.io/master
timeouts:
  controlPlaneComponentHealthCheck: 4m0s
  discovery: 5m0s
  etcdAPICall: 2m0s
  kubeletHealthCheck: 4m0s
  kubernetesAPICall: 1m0s
  tlsBootstrap: 5m0s
  upgradeManifests: 5m0s
---
apiServer:
  certSANs:
    - x.oiox.cn
    - z.oiox.cn
    - k8s-master01
    - k8s-master02
    - k8s-master03
    - 192.168.1.21
    - 192.168.1.22
    - 192.168.1.23
    - 192.168.1.24
    - 192.168.1.25
    - 192.168.1.26
    - 192.168.1.27
    - 192.168.1.28
    - 192.168.1.29
    - 127.0.0.1
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta4
caCertificateValidityPeriod: 87600h0m0s
certificateValidityPeriod: 8760h0m0s
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
encryptionAlgorithm: RSA-2048
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: 1.32.3
networking:
  dnsDomain: cluster.local
  podSubnet: 172.16.0.0/12,fc00:2222::/64
  serviceSubnet: 10.96.0.0/16,fd00:1111::/112
proxy: {}
scheduler: {}
controlPlaneEndpoint: "192.168.1.36:9443"
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
---
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 0s
    enabled: true
  x509:
    clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 0s
    cacheUnauthorizedTTL: 0s
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
cpuManagerReconcilePeriod: 0s
evictionPressureTransitionPeriod: 0s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageMinimumGCAge: 0s
kind: KubeletConfiguration
cgroupDriver: systemd
logging: {}
memorySwap: {}
nodeStatusReportFrequency: 0s
nodeStatusUpdateFrequency: 0s
rotateCertificates: true
runtimeRequestTimeout: 0s
shutdownGracePeriod: 0s
shutdownGracePeriodCriticalPods: 0s
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 0s
syncFrequency: 0s
volumeStatsAggPeriod: 0s
EOF

开始初始化

[root@k8s-master01 ~]# kubeadm init --config=kubeadm.yaml



************************

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of control-plane nodes by copying certificate authorities
and service account keys on each node and then running the following as root:

  kubeadm join 192.168.1.36:9443 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:1a6196cd63edf4e78f39d34d448d6333d25e1ad0ff650839260fc7df25ec8a92 \
	--control-plane 

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.1.36:9443 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:1a6196cd63edf4e78f39d34d448d6333d25e1ad0ff650839260fc7df25ec8a92 
[root@k8s-master01 ~]# 



# 重新初始化

# 只有一个CRI的情况下
kubeadm reset
# 指定CRI重置
kubeadm reset --cri-socket unix:///var/run/cri-dockerd.sock
kubeadm reset --cri-socket unix:///var/run/containerd/containerd.sock

配置kubectl

# 配置kubectl
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

配置证书

# 使用脚本将这如果你睡拷贝到其他maser节点
USER=root
CONTROL_PLANE_IPS="192.168.1.22 192.168.1.23"
for host in ${CONTROL_PLANE_IPS}; do
    scp /etc/kubernetes/pki/ca.crt "${USER}"@$host:
    scp /etc/kubernetes/pki/ca.key "${USER}"@$host:
    scp /etc/kubernetes/pki/sa.key "${USER}"@$host:
    scp /etc/kubernetes/pki/sa.pub "${USER}"@$host:
    scp /etc/kubernetes/pki/front-proxy-ca.crt "${USER}"@$host:
    scp /etc/kubernetes/pki/front-proxy-ca.key "${USER}"@$host:
    scp /etc/kubernetes/pki/etcd/ca.crt "${USER}"@$host:etcd-ca.crt
    # 如果你正使用外部 etcd，忽略下一行
    scp /etc/kubernetes/pki/etcd/ca.key "${USER}"@$host:etcd-ca.key
done

# 在其他的maser上面执行 ，将证书文件放入所需目录
USER=root
mkdir -p /etc/kubernetes/pki/etcd
mv /${USER}/ca.crt /etc/kubernetes/pki/
mv /${USER}/ca.key /etc/kubernetes/pki/
mv /${USER}/sa.pub /etc/kubernetes/pki/
mv /${USER}/sa.key /etc/kubernetes/pki/
mv /${USER}/front-proxy-ca.crt /etc/kubernetes/pki/
mv /${USER}/front-proxy-ca.key /etc/kubernetes/pki/
mv /${USER}/etcd-ca.crt /etc/kubernetes/pki/etcd/ca.crt
# 如果你正使用外部 etcd，忽略下一行
mv /${USER}/etcd-ca.key /etc/kubernetes/pki/etcd/ca.key

初始化Master2

# 在maser02上执行操作，将加入控制节点
kubeadm config print join-defaults > kubeadm-join-master-02.yaml
cat > kubeadm-join-master-02.yaml << EOF
apiVersion: kubeadm.k8s.io/v1beta4
caCertPath: /etc/kubernetes/pki/ca.crt
discovery:
  bootstrapToken:
    apiServerEndpoint: 192.168.1.36:9443
    token: abcdef.0123456789abcdef
    caCertHashes:
    - "sha256:1a6196cd63edf4e78f39d34d448d6333d25e1ad0ff650839260fc7df25ec8a92"
    unsafeSkipCAVerification: true
  tlsBootstrapToken: abcdef.0123456789abcdef
kind: JoinConfiguration
controlPlane:
  localAPIEndpoint:
    advertiseAddress: "192.168.1.22"
    bindPort: 6443
nodeRegistration:
  # criSocket: unix:///run/containerd/containerd.sock
  criSocket: unix:///var/run/cri-dockerd.sock
  imagePullPolicy: IfNotPresent
  imagePullSerial: true
  name: k8s-master02
  taints:
  - effect: PreferNoSchedule
    key: node-role.kubernetes.io/master
  kubeletExtraArgs:
  - name: "node-ip"
    value: "192.168.1.22,fc00::22"
timeouts:
  controlPlaneComponentHealthCheck: 4m0s
  discovery: 5m0s
  etcdAPICall: 2m0s
  kubeletHealthCheck: 4m0s
  kubernetesAPICall: 1m0s
  tlsBootstrap: 5m0s
  upgradeManifests: 5m0s
EOF

kubeadm join --config=kubeadm-join-master-02.yaml

初始化Master3

# 在maser03上执行操作，将加入控制节点
kubeadm config print join-defaults > kubeadm-join-master-03.yaml
cat > kubeadm-join-master-03.yaml << EOF
apiVersion: kubeadm.k8s.io/v1beta4
caCertPath: /etc/kubernetes/pki/ca.crt
discovery:
  bootstrapToken:
    apiServerEndpoint: 192.168.1.36:9443
    token: abcdef.0123456789abcdef
    caCertHashes:
    - "sha256:1a6196cd63edf4e78f39d34d448d6333d25e1ad0ff650839260fc7df25ec8a92"
    unsafeSkipCAVerification: true
  tlsBootstrapToken: abcdef.0123456789abcdef
kind: JoinConfiguration
controlPlane:
  localAPIEndpoint:
    advertiseAddress: "192.168.1.23"
    bindPort: 6443
nodeRegistration:
  # criSocket: unix:///run/containerd/containerd.sock
  criSocket: unix:///var/run/cri-dockerd.sock
  imagePullPolicy: IfNotPresent
  imagePullSerial: true
  name: k8s-master03
  taints:
  - effect: PreferNoSchedule
    key: node-role.kubernetes.io/master
  kubeletExtraArgs:
  - name: "node-ip"
    value: "192.168.1.23,fc00::23"
timeouts:
  controlPlaneComponentHealthCheck: 4m0s
  discovery: 5m0s
  etcdAPICall: 2m0s
  kubeletHealthCheck: 4m0s
  kubernetesAPICall: 1m0s
  tlsBootstrap: 5m0s
  upgradeManifests: 5m0s
EOF

kubeadm join --config=kubeadm-join-master-03.yaml

初始化Node1

# 在node01上执行操作，将加入工作节点
kubeadm config print join-defaults > kubeadm-join-node-01.yaml
cat > kubeadm-join-node-01.yaml << EOF
apiVersion: kubeadm.k8s.io/v1beta4
caCertPath: /etc/kubernetes/pki/ca.crt
discovery:
  bootstrapToken:
    apiServerEndpoint: 192.168.1.36:9443
    token: abcdef.0123456789abcdef
    caCertHashes:
    - "sha256:1a6196cd63edf4e78f39d34d448d6333d25e1ad0ff650839260fc7df25ec8a92"
    unsafeSkipCAVerification: true
  tlsBootstrapToken: abcdef.0123456789abcdef
kind: JoinConfiguration
nodeRegistration:
  # criSocket: unix:///run/containerd/containerd.sock
  criSocket: unix:///var/run/cri-dockerd.sock
  imagePullPolicy: IfNotPresent
  imagePullSerial: true
  name: k8s-node01
  taints: null
  kubeletExtraArgs:
  - name: "node-ip"
    value: "192.168.1.24,fc00::24"
timeouts:
  controlPlaneComponentHealthCheck: 4m0s
  discovery: 5m0s
  etcdAPICall: 2m0s
  kubeletHealthCheck: 4m0s
  kubernetesAPICall: 1m0s
  tlsBootstrap: 5m0s
  upgradeManifests: 5m0s
EOF

kubeadm join --config=kubeadm-join-node-01.yaml

初始化Node2

# 在node02上执行操作，将加入工作节点
kubeadm config print join-defaults > kubeadm-join-node-02.yaml
cat > kubeadm-join-node-02.yaml << EOF
apiVersion: kubeadm.k8s.io/v1beta4
caCertPath: /etc/kubernetes/pki/ca.crt
discovery:
  bootstrapToken:
    apiServerEndpoint: 192.168.1.36:9443
    token: abcdef.0123456789abcdef
    caCertHashes:
    - "sha256:1a6196cd63edf4e78f39d34d448d6333d25e1ad0ff650839260fc7df25ec8a92"
    unsafeSkipCAVerification: true
  tlsBootstrapToken: abcdef.0123456789abcdef
kind: JoinConfiguration
nodeRegistration:
  # criSocket: unix:///run/containerd/containerd.sock
  criSocket: unix:///var/run/cri-dockerd.sock
  imagePullPolicy: IfNotPresent
  imagePullSerial: true
  name: k8s-node02
  taints: null
  kubeletExtraArgs:
  - name: "node-ip"
    value: "192.168.1.25,fc00::25"
timeouts:
  controlPlaneComponentHealthCheck: 4m0s
  discovery: 5m0s
  etcdAPICall: 2m0s
  kubeletHealthCheck: 4m0s
  kubernetesAPICall: 1m0s
  tlsBootstrap: 5m0s
  upgradeManifests: 5m0s
EOF

kubeadm join --config=kubeadm-join-node-02.yaml

查看集群状态

[root@k8s-master01 ~]# kubectl get nodes
NAME           STATUS     ROLES           AGE     VERSION
k8s-master01   NotReady   control-plane   3m56s   v1.32.3
k8s-master02   NotReady   control-plane   2m3s    v1.32.3
k8s-master03   NotReady   control-plane   40s     v1.32.3
k8s-node01     NotReady   <none>          8s      v1.32.3
k8s-node02     NotReady   <none>          5s      v1.32.3
[root@k8s-master01 ~]#

安装网络插件

注意二选其一即可，建议在此处创建好快照后在进行操作，后续出问题可以回滚

centos7 要升级libseccomp 不然无法安装网络插件

# https://github.com/opencontainers/runc/releases
# 升级runc
# wget https://mirrors.chenby.cn/https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64

install -m 755 runc.amd64 /usr/local/sbin/runc
cp -p /usr/local/sbin/runc  /usr/local/bin/runc
cp -p /usr/local/sbin/runc  /usr/bin/runc

#查看当前版本
[root@k8s-master-1 ~]# rpm -qa | grep libseccomp
libseccomp-2.5.2-2.el9.x86_64

#下载高于2.4以上的包
# yum -y install http://rpmfind.net/linux/centos/8-stream/BaseOS/x86_64/os/Packages/libseccomp-2.5.1-1.el8.x86_64.rpm
# 清华源
# yum -y install https://mirrors.tuna.tsinghua.edu.cn/centos/8-stream/BaseOS/x86_64/os/Packages/libseccomp-2.5.1-1.el8.x86_64.rpm

安装Calico

更改calico网段

# 安装operator
kubectl create -f https://mirrors.chenby.cn/https://raw.githubusercontent.com/projectcalico/calico/v3.29.2/manifests/tigera-operator.yaml


# 下载配置文件
curl https://mirrors.chenby.cn/https://raw.githubusercontent.com/projectcalico/calico/v3.29.2/manifests/custom-resources.yaml -O




# 修改地址池
vim custom-resources.yaml
apiVersion: operator.tigera.io/v1
kind: Installation
metadata:
  name: default
spec:
  calicoNetwork:
    ipPools:
    - name: default-ipv4-ippool
      blockSize: 26
      cidr: 172.16.0.0/12 
      encapsulation: VXLANCrossSubnet
      natOutgoing: Enabled
      nodeSelector: all()

---
apiVersion: operator.tigera.io/v1
kind: APIServer
metadata:
  name: default
spec: {}


# 执行安装
kubectl create -f custom-resources.yaml



# 安装客户端
curl -L https://mirrors.chenby.cn/https://github.com/projectcalico/calico/releases/download/v3.28.2/calicoctl-linux-amd64 -o calicoctl


# 给客户端添加执行权限
chmod +x ./calicoctl


# 查看集群节点
./calicoctl get nodes
# 查看集群节点状态
./calicoctl node status
#查看地址池
./calicoctl get ipPool
./calicoctl get ipPool -o yaml

查看容器状态

# calico 初始化会很慢 需要耐心等待一下，大约十分钟左右
[root@k8s-master01 ~]# kubectl get pod -A | grep calico
NAMESPACE          NAME                                       READY   STATUS              RESTARTS   AGE
calico-apiserver   calico-apiserver-6c6d4589d6-hfzpg          1/1     Running             0          6m3s
calico-apiserver   calico-apiserver-6c6d4589d6-rs27g          1/1     Running             0          6m3s
calico-system      calico-kube-controllers-7cdf8468d9-9jc22   1/1     Running             0          6m3s
calico-system      calico-node-2qk9k                          1/1     Running             0          6m3s
calico-system      calico-node-755hv                          1/1     Running             0          6m3s
calico-system      calico-node-rncvq                          1/1     Running             0          6m3s
calico-system      calico-node-t694l                          1/1     Running             0          6m3s
calico-system      calico-node-txwr6                          1/1     Running             0          6m3s
calico-system      calico-typha-58c46dd757-8sn77              1/1     Running             0          6m3s
calico-system      calico-typha-58c46dd757-lsnkh              1/1     Running             0          5m57s
calico-system      calico-typha-58c46dd757-wpz64              1/1     Running             0          5m57s
calico-system      csi-node-driver-84xbq                      2/2     Running             0          6m3s
calico-system      csi-node-driver-gl8m7                      2/2     Running             0          6m3s
calico-system      csi-node-driver-lf4xp                      2/2     Running             0          6m3s
calico-system      csi-node-driver-mlwnf                      2/2     Running             0          6m3s
calico-system      csi-node-driver-pqpkb                      2/2     Running             0          6m3s

安装cilium(推荐)

安装helm

# [root@k8s-master01 ~]# curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
# [root@k8s-master01 ~]# chmod 700 get_helm.sh
# [root@k8s-master01 ~]# ./get_helm.sh

wget https://mirrors.huaweicloud.com/helm/v3.17.2/helm-v3.17.2-linux-amd64.tar.gz
tar xvf helm-*-linux-amd64.tar.gz
cp linux-amd64/helm /usr/local/bin/

安装cilium

# 添加源
helm repo add cilium https://helm.cilium.io

# 修改为国内源
helm pull cilium/cilium
tar xvf cilium-*.tgz
cd cilium/

# sed -i "s#quay.io/#quay.m.daocloud.io/#g" values.yaml

# 默认参数安装
helm install  cilium ./cilium/ -n kube-system

# 启用ipv6
# helm install cilium ./cilium/ --namespace kube-system --set ipv6.enabled=true

# 启用路由信息和监控插件
# helm install cilium ./cilium/ --namespace kube-system --set ipv6.enabled=true --set hubble.relay.enabled=true --set hubble.ui.enabled=true --set prometheus.enabled=true --set operator.prometheus.enabled=true --set hubble.enabled=true --set hubble.metrics.enabled="{dns,drop,tcp,flow,port-distribution,icmp,http}"

查看

[root@k8s-master01 ~]# kubectl  get pod -A | grep cil
NAMESPACE     NAME                               READY   STATUS    RESTARTS   AGE
kube-system   cilium-2tnfb                       1/1     Running   0          60s
kube-system   cilium-5tgcb                       1/1     Running   0          60s
kube-system   cilium-6shf5                       1/1     Running   0          60s
kube-system   cilium-ccbcx                       1/1     Running   0          60s
kube-system   cilium-cppft                       1/1     Running   0          60s
kube-system   cilium-operator-675f685d59-7q27q   1/1     Running   0          60s
kube-system   cilium-operator-675f685d59-kwmqz   1/1     Running   0          60s
[root@k8s-master01 ~]#

-------字数限制

https://github.com/cby-chen/Kubernetes

关于

https://www.oiox.cn/

https://www.oiox.cn/index.php/start-page.html

CSDN、GitHub、知乎、开源中国、思否、掘金、简书、华为云、阿里云、腾讯云、哔哩哔哩、今日头条、新浪微博、个人博客

全网可搜《小陈运维》

原创声明：本文系作者授权腾讯云开发者社区发表，未经许可，不得转载。

如有侵权，请联系 cloudcommunity@tencent.com 删除。

linux

centos

容器

kubernetes

原创声明：本文系作者授权腾讯云开发者社区发表，未经许可，不得转载。

如有侵权，请联系 cloudcommunity@tencent.com 删除。

使用kubeadm部署高可用IPV4/IPV6集群-v1.32

使用kubeadm部署高可用IPV4/IPV6集群-v1.32

使用kubeadm部署高可用IPV4/IPV6集群

k8s基础系统环境配置

配置IP

设置主机名

配置yum源

安装一些必备工具

关闭防火墙

关闭SELinux

关闭交换分区

网络配置（俩种方式二选一）

进行时间同步

配置ulimit

配置免密登录

添加启用源

升级内核至4.18版本以上

安装ipvsadm

修改内核参数

所有节点配置hosts本地解析

配置kubernetes安装源

Debian / Ubuntu

CentOS / RHEL / Fedora

k8s基本组件安装

安装Containerd作为Runtime

配置Containerd所需的模块

加载模块

配置Containerd所需的内核

创建Containerd的配置文件

启动并设置为开机启动

配置crictl客户端连接的运行时位置

安装docker作为Runtime

解压docker程序

创建containerd的service文件

准备docker的service文件

准备docker的socket文件

配置加速器

启动docker

解压cri-docker

写入启动cri-docker配置文件

写入cri-docker的socket配置文件

启动cri-docker

高可用keepalived、haproxy

安装keepalived和haproxy服务

修改haproxy配置文件（配置文件一样）

Master01配置keepalived master节点

Master02配置keepalived backup节点

Master03配置keepalived backup节点

健康检查脚本配置（lb主机）

启动服务

测试高可用

初始化安装

整改镜像

修改初始化配置

开始初始化

配置kubectl

配置证书

初始化Master2

初始化Master3

初始化Node1

初始化Node2

查看集群状态

安装网络插件

安装Calico

更改calico网段

查看容器状态

安装cilium(推荐)

安装helm

安装cilium

查看

社区

活动

圈层

关于

腾讯云开发者

热门产品

热门推荐

更多推荐