新增俩台node节点加进k8s集群。
以下操作均在所有机器操作
yum install -y epel-release conntrack ntpdate ntp ipvsadm ipset jq iptables curl sysstat libseccomp wget unzip net-tools fuse-devel fuse fuse-libs
systemctl stop firewalld
systemctl disable firewalld
iptables -F && iptables -X && iptables -F -t nat && iptables -X -t nat
iptables -P FORWARD ACCEPT
swapoff -a
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
setenforce 0
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
modprobe ip_vs_rr
modprobe br_netfilter
cat > kubernetes.conf <<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0 # 禁止使用 swap 空间,只有当系统 OOM 时才允许使用它
vm.overcommit_memory=1 # 不检查物理内存是否够用
vm.panic_on_oom=0 # 开启 OOM
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF
cp kubernetes.conf /etc/sysctl.d/kubernetes.conf
sysctl -p /etc/sysctl.d/kubernetes.conf
ntpdate ntp1.aliyun.com
mkdir -p /opt/k8s/{bin,work} /etc/{kubernetes,etcd}/cert
#在之前的master节点添加
[root@test-master-1 ~]# echo "10.16.8.86 test-node-16" >>/etc/hosts
[root@test-master-1 ~]# echo "10.16.8.80 test-node-17" >>/etc/hosts
新加的俩台节点加hosts
cat >>/etc/hosts<<EOF
10.16.8.10 test-master-1
10.16.8.11 test-master-2
10.16.8.12 test-master-3
10.16.8.13 test-node-15
10.16.8.86 test-node-16
10.16.8.80 test-node-17
EOF
master节点操作
ssh-copy-id -i ~/.ssh/id_rsa.pub test-node-16
ssh-copy-id -i ~/.ssh/id_rsa.pub test-node-17
推送CA证书
cd /etc/kubernetes/cert
scp ca.pem ca-config.json test-node-16:/etc/kubernetes/cert/
scp ca.pem ca-config.json test-node-17:/etc/kubernetes/cert/
拷贝二进制文件
cd /opt/k8s/work/flannel
scp flanneld mk-docker-opts.sh test-node-16:/opt/k8s/bin/
scp flanneld mk-docker-opts.sh test-node-17:/opt/k8s/bin/
拷贝flanneld密钥
ssh test-node-16 "mkdir -p /etc/flanneld/cert"
ssh test-node-17 "mkdir -p /etc/flanneld/cert"
scp /etc/flanneld/cert/flanneld*.pem test-node-16:/etc/flanneld/cert
scp /etc/flanneld/cert/flanneld*.pem test-node-17:/etc/flanneld/cert
拷贝flannel启动文件
scp /etc/systemd/system/flanneld.service test-node-16:/etc/systemd/system/
scp /etc/systemd/system/flanneld.service test-node-17:/etc/systemd/system/
#启动flannel
ssh test-node-16 "systemctl daemon-reload && systemctl enable flanneld && systemctl restart flanneld"
ssh test-node-17 "systemctl daemon-reload && systemctl enable flanneld && systemctl restart flanneld"
yum install -y yum-utils \
device-mapper-persistent-data \
lvm2
yum-config-manager \
--add-repo \
https://download.docker.com/linux/centos/docker-ce.repo
yum install docker-ce-19.03.1-3.el7 -y
创建配置文件
mkdir -p /etc/docker/
cat > /etc/docker/daemon.json <<EOF
{
"exec-opts": ["native.cgroupdriver=systemd"],
"registry-mirrors": ["https://dockerhub.mirrors.nwafu.edu.cn/","https://docker.mirrors.ustc.edu.cn","https://hub-mirror.c.163.com"],
"max-concurrent-downloads": 20,
"live-restore": true,
"max-concurrent-uploads": 10,
"debug": true,
"log-opts": {
"max-size": "100m",
"max-file": "5"
}
}
EOF
修改docker启动配置文件
vim /usr/lib/systemd/system/docker.service
EnvironmentFile=-/run/flannel/dockerExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS -H fd:// --containerd=/run/containerd/containerd.sock
systemctl daemon-reload && systemctl enable docker && systemctl restart docker
systemctl status docker|grep Active
创建kubelet bootstrap kubeconfig文件
#master节点操作
另一台创建文件等和这个同理,这里只列举一个。
cd /opt/k8s/work
export BOOTSTRAP_TOKEN=$(kubeadm token create \
--description kubelet-bootstrap-token \
--groups system:bootstrappers:test-node-16 \
--kubeconfig ~/.kube/config)
# 设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/cert/ca.pem \
--embed-certs=true \
--server=https://10.16.8.14:6443 \
--kubeconfig=kubelet-bootstrap-test-node-16.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials kubelet-bootstrap \
--token=${BOOTSTRAP_TOKEN} \
--kubeconfig=kubelet-bootstrap-test-node-16.kubeconfig
# 设置上下文参数
kubectl config set-context default \
--cluster=kubernetes \
--user=kubelet-bootstrap \
--kubeconfig=kubelet-bootstrap-test-node-16.kubeconfig
# 设置默认上下文
kubectl config use-context default --kubeconfig=kubelet-bootstrap-test-node-16.kubeconfig
分发kubeconfig
cd /opt/k8s/work
scp kubelet-bootstrap-test-node-16.kubeconfig test-node-16:/etc/kubernetes/kubelet-bootstrap.kubeconfig
查看kubeadm为各个节点创建的token
kubeadm token list --kubeconfig ~/.kube/config
创建和分发kubelet参数配置
cd /opt/k8s/work
sed -e "s/##NODE_IP##/10.16.8.86/" kubelet-config.yaml.template > kubelet-config-10.16.8.86.yaml.template
sed -e "s/##NODE_IP##/10.16.8.80/" kubelet-config.yaml.template > kubelet-config-10.16.8.80.yaml.template
scp kubelet-config-10.16.8.86.yaml.template root@test-node-16:/etc/kubernetes/kubelet-config.yaml
scp kubelet-config-10.16.8.80.yaml.template root@test-node-17:/etc/kubernetes/kubelet-config.yaml
拷贝kubelet启动文件和命令
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
sed -e "s/##NODE_NAME##/test-node-16/" kubelet.service.template > kubelet-test-node-16.service
sed -e "s/##NODE_NAME##/test-node-17/" kubelet.service.template > kubelet-test-node-17.service
scp kubelet-test-node-16.service root@test-node-16:/etc/systemd/system/kubelet.service
scp kubelet-test-node-17.service root@test-node-17:/etc/systemd/system/kubelet.service
scp /opt/k8s/bin/kubelet test-node-16:/opt/k8s/bin/
scp /opt/k8s/bin/kubelet test-node-17:/opt/k8s/bin/
启动kubelet
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
ssh root@test-node-16 "mkdir -p ${K8S_DIR}/kubelet/kubelet-plugins/volume/exec/"
ssh root@test-node-16 "systemctl daemon-reload && systemctl enable kubelet && systemctl restart kubelet"
ssh root@test-node-17 "mkdir -p ${K8S_DIR}/kubelet/kubelet-plugins/volume/exec/"
ssh root@test-node-17 "systemctl daemon-reload && systemctl enable kubelet && systemctl restart kubelet"
签注证书
kubectl get csr | grep Pending | awk '{print $1}' | xargs kubectl certificate approve
推送kube-proxy二进制启动文件和kubeconfig文件
cd /opt/k8s/work/
scp kubernetes/server/bin/kube-proxy test-node-16:/opt/k8s/bin/
scp kubernetes/server/bin/kube-proxy test-node-17:/opt/k8s/bin/
scp kube-proxy.kubeconfig root@test-node-16:/etc/kubernetes/
scp kube-proxy.kubeconfig root@test-node-17:/etc/kubernetes/
创建和分发kube-proxy配置文件
cd /opt/k8s/work/
sed -e "s/##NODE_NAME##/test-node-16/" -e "s/##NODE_IP##/10.16.8.86/" kube-proxy-config.yaml.template > kube-proxy-config-test-node-16.yaml.template
sed -e "s/##NODE_NAME##/test-node-17/" -e "s/##NODE_IP##/10.16.8.80/" kube-proxy-config.yaml.template > kube-proxy-config-test-node-17.yaml.template
scp kube-proxy-config-test-node-16.yaml.template root@test-node-16:/etc/kubernetes/kube-proxy-config.yaml
scp kube-proxy-config-test-node-17.yaml.template root@test-node-17:/etc/kubernetes/kube-proxy-config.yaml
scp kube-proxy.service root@test-node-16:/etc/systemd/system/
scp kube-proxy.service root@test-node-17:/etc/systemd/system/
启动kube-proxy服务
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
ssh root@test-node-16 "mkdir -p ${K8S_DIR}/kube-proxy"
ssh root@test-node-16 "systemctl daemon-reload && systemctl enable kube-proxy && systemctl restart kube-proxy"
ssh root@test-node-17 "mkdir -p ${K8S_DIR}/kube-proxy"
ssh root@test-node-17 "systemctl daemon-reload && systemctl enable kube-proxy && systemctl restart kube-proxy"