外包技能--Kubernetes证书过期

[zhangpeng@27ops ~]$ kubectl get node
Unable to connect to the server: x509: certificate has expired or is not yet valid: current time 2022-11-13T16:59:26+08:00 is after 2022-11-05T07:06:21Z
[zhangpeng@27ops ~]$ 

[zhangpeng@27ops ~]$ sudo cp -rp /etc/kubernetes /etc/kubernetes.bak
[zhangpeng@27ops ~]$ sudo cp -r /var/lib/etcd /tmp/etcd-backup/
[zhangpeng@27ops pki]$ sudo kubeadm certs check-expiration

[zhangpeng@27ops ~]$ kubeadm version
[zhangpeng@27ops ~]$ cd /etc/kubernetes/pki && openssl x509 -in apiserver.crt -text -noout
​

[zhangpeng@27ops ~]$ kubeadm version
kubeadm version: &version.Info{Major:"1", Minor:"21", GitVersion:"v1.21.3", GitCommit:"ca643a4d1f7bfe34773c74f79527be4afd95bf39", GitTreeState:"clean", BuildDate:"2021-07-15T21:03:28Z", GoVersion:"go1.16.6", Compiler:"gc", Platform:"linux/amd64"}
[zhangpeng@27ops ~]$ 
[zhangpeng@27ops ~]$ 
[zhangpeng@27ops ~]$ 
[zhangpeng@27ops ~]$ cd /etc/kubernetes/pki && openssl x509 -in apiserver.crt -text -noout
Certificate:
    Data:
        Version: 3 (0x2)
        Serial Number: 4641117883785374972 (0x4068902027bd6cfc)
        Signature Algorithm: sha256WithRSAEncryption
        Issuer: CN = kubernetes
        Validity
            Not Before: Nov  5 07:06:21 2021 GMT
            Not After : Nov  5 07:06:21 2022 GMT
        Subject: CN = kube-apiserver
        Subject Public Key Info:
            Public Key Algorithm: rsaEncryption
                RSA Public-Key: (2048 bit)
                Modulus:
                    00:be:6c:48:e2:35:97:57:97:ae:ca:9c:d8:4e:79:
                    49:67:19:f2:65:57:c2:c8:23:aa:cd:48:3b:4b:7f:
                    cc:52:2d:92:06:bf:71:15:72:61:a6:df:b2:98:97:
                    ce:cd:44:46:9b:db:59:c7:2f:25:6a:fc:1b:f7:8d:
                    9d:6a:d7:af:e1:53:b0:3e:e8:ca:f1:0f:60:05:25:
                    29:cd:c8:8d:ae:17:bc:de:3b:af:d6:b0:6a:04:1a:
                    63:05:a4:33:95:3f:ac:f1:8e:45:af:90:4a:6f:5a:
                    f0:07:01:06:39:3e:18:56:28:c9:25:6f:5d:52:eb:
                    db:1e:9c:3b:dd:dc:d3:22:6d:10:80:62:af:13:e4:
                    19:03:66:2f:54:26:7b:23:be:d4:6f:8c:1f:c1:bd:
                    84:40:df:06:37:6f:69:ec:74:ab:a5:88:12:91:da:
                    7f:0d:81:6b:81:32:12:7d:db:a6:d0:43:99:69:51:
                    19:97:9c:fb:67:69:a9:d7:43:c6:fb:53:85:07:5d:
                    a1:b9:c5:3e:2a:e7:d2:26:91:ce:7c:28:be:c7:ae:
                    f3:56:d2:dd:9f:41:f2:5c:96:ba:f3:d8:72:9a:7e:
                    7f:f5:a6:7a:87:ba:9b:26:88:be:26:85:7a:a8:9d:
                    54:b5:d8:2e:9f:31:ba:68:8e:9f:0e:68:87:ae:5e:
                    44:89
                Exponent: 65537 (0x10001)
        X509v3 extensions:
            X509v3 Key Usage: critical
                Digital Signature, Key Encipherment
            X509v3 Extended Key Usage: 
                TLS Web Server Authentication
            X509v3 Basic Constraints: critical
                CA:FALSE
            X509v3 Authority Key Identifier: 
                keyid:AA:75:0F:DF:C1:D6:12:CE:FB:AA:55:F2:99:A6:F6:CB:48:19:2F:1C
​
            X509v3 Subject Alternative Name: 
                DNS:27ops.com, DNS:kubernetes, DNS:kubernetes.default, DNS:kubernetes.default.svc, DNS:kubernetes.default.svc.cluster.local, IP Address:10.96.0.1, IP Address:10.0.20.7
    Signature Algorithm: sha256WithRSAEncryption
         55:43:0f:14:b4:b1:6b:58:65:eb:67:ef:8c:3c:25:46:0c:0e:
         42:3b:c3:24:0e:e5:be:b4:f8:be:54:03:02:88:0b:8a:46:97:
         7d:8e:c6:94:07:8a:26:a6:75:cf:cf:62:8e:a0:08:0a:08:fd:
         2b:e7:9b:5c:1b:3e:72:67:43:a3:21:bf:97:1d:74:e0:76:75:
         63:de:01:0a:a6:9b:d7:5a:db:90:72:bb:ba:50:30:c7:c4:67:
         3e:d4:bb:aa:49:5c:09:64:18:18:bf:aa:51:83:65:5c:6c:73:
         32:8c:2e:b0:5d:06:89:3f:0d:12:21:1c:dd:7b:c3:fd:ef:7f:
         cf:d8:29:31:b2:a4:94:70:64:9c:a8:b1:17:21:b4:6d:e4:2b:
         1c:b9:78:20:8f:c7:b4:98:6a:ff:66:d0:32:ea:27:26:66:e3:
         34:d1:3c:fd:8f:da:7a:f3:a8:86:ca:c9:d6:9d:e4:a6:0f:ee:
         df:a2:04:06:34:1d:85:2b:33:46:ff:50:ad:02:7c:38:25:45:
         0a:c3:3d:d9:f8:e0:6f:c8:3d:6e:5e:56:a5:79:c3:49:4b:8e:
         e4:6b:8c:62:4d:20:42:6b:d5:3c:ba:c5:37:fe:6c:49:15:a5:
         82:6d:38:16:f3:40:d3:6c:f6:19:d7:d7:ac:9d:0d:02:e8:c5:
         0e:62:07:95
[zhangpeng@27ops pki]$ 
[zhangpeng@27ops pki]$ sudo kubeadm certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[check-expiration] Error reading configuration from the Cluster. Falling back to default configuration
​
CERTIFICATE                EXPIRES                  RESIDUAL TIME   CERTIFICATE AUTHORITY   EXTERNALLY MANAGED
admin.conf                 Nov 05, 2022 07:06 UTC   <invalid>                               no      
apiserver                  Nov 05, 2022 07:06 UTC   <invalid>       ca                      no      
apiserver-etcd-client      Nov 05, 2022 07:06 UTC   <invalid>       etcd-ca                 no      
apiserver-kubelet-client   Nov 05, 2022 07:06 UTC   <invalid>       ca                      no      
controller-manager.conf    Nov 05, 2022 07:06 UTC   <invalid>                               no      
etcd-healthcheck-client    Nov 05, 2022 07:06 UTC   <invalid>       etcd-ca                 no      
etcd-peer                  Nov 05, 2022 07:06 UTC   <invalid>       etcd-ca                 no      
etcd-server                Nov 05, 2022 07:06 UTC   <invalid>       etcd-ca                 no      
front-proxy-client         Nov 05, 2022 07:06 UTC   <invalid>       front-proxy-ca          no      
scheduler.conf             Nov 05, 2022 07:06 UTC   <invalid>                               no      
​
CERTIFICATE AUTHORITY   EXPIRES                  RESIDUAL TIME   EXTERNALLY MANAGED
ca                      Nov 03, 2031 07:06 UTC   8y              no      
etcd-ca                 Nov 03, 2031 07:06 UTC   8y              no      
front-proxy-ca          Nov 03, 2031 07:06 UTC   8y              no      
[zhangpeng@27ops pki]$ 
​

[zhangpeng@27ops pki]$ kubeadm certs renew all          #更新证书命令
[zhangpeng@27ops pki]$ kubeadm certs check-expiration   #确认证书有效期

[zhangpeng@27ops pki]$ kubeadm certs renew all
failed to load kubeConfig file /etc/kubernetes/admin.conf: open /etc/kubernetes/admin.conf: permission denied
To see the stack trace of this error execute with --v=5 or higher
[zhangpeng@27ops pki]$ sudo kubeadm certs renew all
[renew] Reading configuration from the cluster...
[renew] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[renew] Error reading configuration from the Cluster. Falling back to default configuration
​
certificate embedded in the kubeconfig file for the admin to use and for kubeadm itself renewed
certificate for serving the Kubernetes API renewed
certificate the apiserver uses to access etcd renewed
certificate for the API server to connect to kubelet renewed
certificate embedded in the kubeconfig file for the controller manager to use renewed
certificate for liveness probes to healthcheck etcd renewed
certificate for etcd nodes to communicate with each other renewed
certificate for serving etcd renewed
certificate for the front proxy client renewed
certificate embedded in the kubeconfig file for the scheduler manager to use renewed
​
Done renewing certificates. You must restart the kube-apiserver, kube-controller-manager, kube-scheduler and etcd, so that they can use the new certificates.
[zhangpeng@27ops pki]$
​
[zhangpeng@27ops pki]$ kubeadm certs check-expiration
failed to load kubeConfig file /etc/kubernetes/admin.conf: open /etc/kubernetes/admin.conf: permission denied
To see the stack trace of this error execute with --v=5 or higher
[zhangpeng@27ops pki]$ sudo kubeadm certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
​
CERTIFICATE                EXPIRES                  RESIDUAL TIME   CERTIFICATE AUTHORITY   EXTERNALLY MANAGED
admin.conf                 Nov 13, 2023 09:02 UTC   364d                                    no      
apiserver                  Nov 13, 2023 09:02 UTC   364d            ca                      no      
apiserver-etcd-client      Nov 13, 2023 09:02 UTC   364d            etcd-ca                 no      
apiserver-kubelet-client   Nov 13, 2023 09:02 UTC   364d            ca                      no      
controller-manager.conf    Nov 13, 2023 09:02 UTC   364d                                    no      
etcd-healthcheck-client    Nov 13, 2023 09:02 UTC   364d            etcd-ca                 no      
etcd-peer                  Nov 13, 2023 09:02 UTC   364d            etcd-ca                 no      
etcd-server                Nov 13, 2023 09:02 UTC   364d            etcd-ca                 no      
front-proxy-client         Nov 13, 2023 09:02 UTC   364d            front-proxy-ca          no      
scheduler.conf             Nov 13, 2023 09:02 UTC   364d                                    no      
​
CERTIFICATE AUTHORITY   EXPIRES                  RESIDUAL TIME   EXTERNALLY MANAGED
ca                      Nov 03, 2031 07:06 UTC   8y              no      
etcd-ca                 Nov 03, 2031 07:06 UTC   8y              no      
front-proxy-ca          Nov 03, 2031 07:06 UTC   8y              no      
[zhangpeng@27ops pki]$

[zhangpeng@27ops pki]$ sudo systemctl restart kubelet
[zhangpeng@27ops pki]$ kubectl get nodes
error: You must be logged in to the server (Unauthorized)
[zhangpeng@27ops pki]$
[zhangpeng@27ops kubernetes]$ sudo cp admin.conf ~/.kube/config 

[zhangpeng@27ops kubernetes]$ sudo cp admin.conf ~/.kube/config 
[zhangpeng@27ops kubernetes]$ k8s
NAMESPACE     NAME                                READY   STATUS    RESTARTS   AGE    IP             NODE        NOMINATED NODE   READINESS GATES
default       docs-959c5c444-szg5x                1/1     Running   0          10d    10.244.0.170   27ops.com   <none>           <none>
default       webapp-5654c984c-wswpp              1/1     Running   0          55d    10.244.0.168   27ops.com   <none>           <none>
kube-system   coredns-558bd4d5db-bnh29            1/1     Running   0          373d   10.244.0.3     27ops.com   <none>           <none>
kube-system   coredns-558bd4d5db-pcj24            1/1     Running   0          373d   10.244.0.2     27ops.com   <none>           <none>
kube-system   etcd-27ops.com                      1/1     Running   0          373d   10.0.20.7      27ops.com   <none>           <none>
kube-system   kube-apiserver-27ops.com            1/1     Running   0          373d   10.0.20.7      27ops.com   <none>           <none>
kube-system   kube-controller-manager-27ops.com   1/1     Running   2          373d   10.0.20.7      27ops.com   <none>           <none>
kube-system   kube-flannel-ds-fs6fv               1/1     Running   0          373d   10.0.20.7      27ops.com   <none>           <none>
kube-system   kube-proxy-zw8tr                    1/1     Running   0          373d   10.0.20.7      27ops.com   <none>           <none>
kube-system   kube-scheduler-27ops.com            1/1     Running   2          373d   10.0.20.7      27ops.com   <none>           <none>
[zhangpeng@27ops kubernetes]$ kg no
NAME        STATUS   ROLES                  AGE    VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION              CONTAINER-RUNTIME
27ops.com   Ready    control-plane,master   373d   v1.21.3   10.0.20.7     <none>        CentOS Linux 8 (Core)   4.18.0-305.3.1.el8.x86_64   docker://20.10.10
[zhangpeng@27ops kubernetes]$ 
​