我与腾讯云的故事 ：腾讯云CVM云服务器下玩转k8s集群

hostnamectl set-hostname k8s

[root@VM-0-15-centos ~]#  cat /etc/os-release
NAME="CentOS Linux"
VERSION="7 (Core)"
ID="centos"
ID_LIKE="rhel fedora"
VERSION_ID="7"
PRETTY_NAME="CentOS Linux 7 (Core)"
ANSI_COLOR="0;31"
CPE_NAME="cpe:/o:centos:centos:7"
HOME_URL="https://www.centos.org/"
BUG_REPORT_URL="https://bugs.centos.org/"

CENTOS_MANTISBT_PROJECT="CentOS-7"
CENTOS_MANTISBT_PROJECT_VERSION="7"
REDHAT_SUPPORT_PRODUCT="centos"
REDHAT_SUPPORT_PRODUCT_VERSION="7"

[root@k8s ~]#  uname -r
3.10.0-1160.99.1.el7.x86_64

[root@k8s ~]# yum repolist enabled
Loaded plugins: fastestmirror, langpacks
Loading mirror speeds from cached hostfile
repo id                                            repo name                                                           status
epel/7/x86_64                                      EPEL for redhat/centos 7 - x86_64                                   13,786
extras/7/x86_64                                    Qcloud centos extras - x86_64                                          518
os/7/x86_64                                        Qcloud centos os - x86_64                                           10,072
updates/7/x86_64                                   Qcloud centos updates - x86_64                                       5,568
repolist: 29,944

yum   install tree wget net-tools git -y

curl -fsSL https://get.docker.com -o get-docker.sh
sh get-docker.sh --mirror Aliyun

echo '{ "registry-mirrors": ["https://docker.mirrors.sjtug.sjtu.edu.cn"] }'   > /etc/docker/daemon.json 
systemctl daemon-reload  && systemctl restart docker

echo "net.ipv4.ip_forward=1" >>  /etc/sysctl.conf

sysctl -p
sysctl net.ipv4.ip_forward

[root@k8s ~]# systemctl status docker
● docker.service - Docker Application Container Engine
   Loaded: loaded (/usr/lib/systemd/system/docker.service; disabled; vendor preset: disabled)
   Active: active (running) since Wed 2024-01-24 21:21:42 CST; 49s ago
     Docs: https://docs.docker.com
 Main PID: 12324 (dockerd)
    Tasks: 10
   Memory: 31.2M
   CGroup: /system.slice/docker.service
           └─12324 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock

mkdir -p /data/minikube && cd /data/minikube

curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl

mv kubectl /usr/bin/kubectl &&  chmod +x /usr/bin/kubectl 

[root@k8s minikube]# kubectl version
Client Version: v1.29.1
Kustomize Version: v5.0.4-0.20230601165947-6ce0bf390ce3
The connection to the server localhost:8080 was refused - did you specify the right host or port?

curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64

sudo install minikube-linux-amd64 /usr/local/bin/minikube

[root@k8s minikube]# minikube version
minikube version: v1.32.0
commit: 8220a6eb95f0a4d75f7f2d7b14cef975f050512d

yum -y install conntrack-tools

wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.1/cri-dockerd-0.3.1.amd64.tgz

[root@k8s minikube]# ls
cri-dockerd-0.3.1.amd64.tgz  minikube-linux-amd64

[root@k8s minikube]# tar -xzf cri-dockerd-0.3.1.amd64.tgz
[root@k8s minikube]# cp cri-dockerd/cri-dockerd /usr/bin/

[root@k8s minikube]# cri-dockerd --version
cri-dockerd 0.3.1 (7e528b98)

vim /etc/systemd/system/cri-docker.socket

[Unit]
Description=CRI Docker Socket for the API
PartOf=cri-docker.service

[Socket]
ListenStream=%t/cri-dockerd.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker

[Install]
WantedBy=sockets.target

vim /etc/systemd/system/cri-docker.service

[Unit]
Description=CRI Interface for Docker Application Container Engine
Documentation=https://docs.mirantis.com
After=network-online.target firewalld.service docker.service
Wants=network-online.target
Requires=cri-docker.socket

[Service]
Type=notify
ExecStart=/usr/bin/cri-dockerd --container-runtime-endpoint fd:// --network-plugin= --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.6 --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always

# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
# Both the old, and new location are accepted by systemd 229 and up, so using the old location
# to make them work for either version of systemd.
StartLimitBurst=3

# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
# this option work for either version of systemd.
StartLimitInterval=60s

# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity

# Comment TasksMax if your systemd version does not support it.
# Only systemd 226 and above support this option.
TasksMax=infinity
Delegate=yes
KillMode=process

[Install]
WantedBy=multi-user.target

systemctl enable  cri-docker.service --now

[root@k8s minikube]# systemctl status cri-docker.service
● cri-docker.service - CRI Interface for Docker Application Container Engine
   Loaded: loaded (/etc/systemd/system/cri-docker.service; enabled; vendor preset: disabled)
   Active: active (running) since Wed 2024-01-24 22:05:16 CST; 28s ago
     Docs: https://docs.mirantis.com
 Main PID: 23347 (cri-dockerd)
    Tasks: 9
   Memory: 14.1M
   CGroup: /system.slice/cri-docker.service

wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.27.0/crictl-v1.27.0-linux-amd64.tar.gz

[root@k8s minikube]# ls
crictl-v1.27.0-linux-amd64.tar.gz  cri-dockerd  cri-dockerd-0.3.1.amd64.tgz  minikube-linux-amd64

tar -xzf crictl-v1.27.0-linux-amd64.tar.gz  -C /usr/bin/

[root@k8s minikube]#  crictl -version
crictl version v1.27.0

wget https://github.com/containernetworking/plugins/releases/download/v1.2.0/cni-plugins-linux-amd64-v1.2.0.tgz

[root@k8s minikube]# ls
cni-plugins-linux-amd64-v1.2.0.tgz  cri-dockerd                  minikube-linux-amd64
crictl-v1.27.0-linux-amd64.tar.gz   cri-dockerd-0.3.1.amd64.tgz

mkdir -p /opt/cni/bin/

tar -xzf  cni-plugins-linux-amd64-v1.2.0.tgz  -C /opt/cni/bin/

minikube start  --driver=none --kubernetes-version=v1.26.3

minikube start --image-mirror-country=cn --driver=none --kubernetes-version=v1.26.3

[root@k8s minikube]# minikube status
minikube
type: Control Plane
host: Running
kubelet: Running
apiserver: Running
kubeconfig: Configured

minikube node add

minikube stop/delete # 停止、删除本地集群
minikube delete --all #删除所有本地集群和配置文件

docker container ls

[root@k8s minikube]# kubectl get nodes -owide
NAME   STATUS   ROLES           AGE     VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION       CONTAINER-RUNTIME
k8s    Ready    control-plane   5m52s   v1.26.3   172.28.0.15   <none>        CentOS Linux 7 (Core)   3.10.0-1160.99.1.el7.x86_64   docker://25.0.1

[root@k8s minikube]# kubectl get pods -n kube-system
NAME                          READY   STATUS    RESTARTS   AGE
coredns-567c556887-lv6fl      1/1     Running   0          7m50s
etcd-k8s                      1/1     Running   0          8m3s
kube-apiserver-k8s            1/1     Running   0          8m3s
kube-controller-manager-k8s   1/1     Running   0          8m4s
kube-proxy-5cxbh              1/1     Running   0          7m50s
kube-scheduler-k8s            1/1     Running   0          8m3s
storage-provisioner           1/1     Running   0          8m2s

vim web.yaml 

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-testweb
  labels:
    app: nginx
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app:  nginx
    spec:
      containers:
      - name: nginx
        image: nginx:alpine
        ports:
        - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: nginx
  name: svs-web
spec:
#  clusterIP: None
  ports:
  - name: web-port
    port: 80
    protocol: TCP
    targetPort: 80
    nodePort: 30060
  selector:
    app: nginx
  type: NodePort

[root@k8s minikube]# kubectl apply -f web.yaml
deployment.apps/my-testweb created
service/svs-web created

[root@k8s minikube]# kubectl get pods
NAME                          READY   STATUS    RESTARTS   AGE
my-testweb-774f96d4d9-w8x66   1/1     Running   0          61s

[root@k8s minikube]# kubectl get service
NAME         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)        AGE
kubernetes   ClusterIP   10.96.0.1       <none>        443/TCP        14m
svs-web      NodePort    10.97.128.248   <none>        80:30060/TCP   90s

docker run --privileged -d -v /root/.minikube:/root/.minikube --name kubepi --restart=unless-stopped -p 8098:80 kubeoperator/kubepi-server

version: '3'
services:
  kubepi:
    image: kubeoperator/kubepi-server
    privileged: true
    volumes:
      - /root/.minikube:/root/.minikube
    restart: unless-stopped
    ports:
      - 8098:80

docker compose up -d  #创建KubePi容器

[root@k8s minikube]# docker compose ps
NAME                COMMAND                  SERVICE             STATUS              PORTS
minikube-kubepi-1   "tini -g -- kubepi-s…"   kubepi              running             0.0.0.0:8098->80/tcp, :::8098->80/tcp

[root@k8s minikube]# docker logs kubepi
\config file not found in , skip
time="2024-01-25T19:22:47+08:00" level=info msg="current db version: 0"
time="2024-01-25T19:22:47+08:00" level=info msg="executing db migration: [1]  Create default user and cluster"
time="2024-01-25T19:22:47+08:00" level=info msg="executing db migration: [2]  Add role repo manager"
time="2024-01-25T19:22:47+08:00" level=info msg="update db to version: 2"
Now listening on: http://0.0.0.0

sudo mv /root/.kube /root/.minikube $HOME
sudo chown -R $USER $HOME/.kube $HOME/.minikube

[root@k8s minikube]# ls /root/.kube/
cache  config