本文主要实践TKE集群使用nfs文件系统,包括使用cfs-csi(新创建实例,共享新实例),静态nfs挂载,已有实例共享挂载
k8s原生支持静态nfs,包括volume支持nfs,persistentvolume也支持nfs
所有yaml参考如下
apiVersion: v1
kind: PersistentVolume
metadata:
name: static-cfs-pv
spec:
accessModes:
- ReadWriteOnce
capacity:
storage: 10Gi
nfs:
path: /staticcfs1
server: 10.0.7.15
persistentVolumeReclaimPolicy: Recycle
storageClassName: slow
volumeMode: Filesystem
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: static-cfs-pvc
namespace: default
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
storageClassName: slow
volumeMode: Filesystem
volumeName: static-cfs-pv
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx
namespace: default
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: nginx
qcloud-app: nginx
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
type: RollingUpdate
template:
metadata:
creationTimestamp: null
labels:
k8s-app: nginx
qcloud-app: nginx
spec:
affinity: {}
containers:
- image: nginx
imagePullPolicy: Always
name: nginx
resources:
limits:
cpu: 500m
memory: 1Gi
requests:
cpu: 250m
memory: 256Mi
securityContext:
privileged: false
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /mnt
name: vol
dnsPolicy: ClusterFirst
imagePullSecrets:
- name: qcloudregistrykey
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds: 30
volumes:
- name: vol
persistentVolumeClaim:
claimName: static-cfs-pvc
使用上面的yaml创建之后,可以看到pod挂载正常运行,并且容器里正常挂载nfs path
$ kubectl get po
NAME READY STATUS RESTARTS AGE
nginx-7f499bd79c-8nd65 1/1 Running 0 23m
root@nginx-7f499bd79c-8nd65:/# df -h /mnt/
Filesystem Size Used Avail Use% Mounted on
10.0.7.15:/staticcfs1 10G 42M 10G 1% /mnt
yaml参考如下
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-nfs-volume
namespace: default
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: nginx-nfs-volume
qcloud-app: nginx-nfs-volume
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
type: RollingUpdate
template:
metadata:
creationTimestamp: null
labels:
k8s-app: nginx-nfs-volume
qcloud-app: nginx-nfs-volume
spec:
affinity: {}
containers:
- image: nginx
imagePullPolicy: Always
name: nginx
resources:
limits:
cpu: 500m
memory: 1Gi
requests:
cpu: 250m
memory: 256Mi
securityContext:
privileged: false
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /mnt
name: vol
dnsPolicy: ClusterFirst
imagePullSecrets:
- name: qcloudregistrykey
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds: 30
volumes:
- name: vol
nfs:
server: 10.0.7.15
path: /static-volume
使用上面的yaml创建之后,可以看到pod挂载正常运行,并且容器里正常挂载nfs path
$ kubectl get po | grep nginx-nfs-volume
nginx-nfs-volume-66db74f76f-gg52s 1/1 Running 0 4m8s
root@nginx-nfs-volume-66db74f76f-gg52s:/# df -h /mnt/
Filesystem Size Used Avail Use% Mounted on
10.0.7.15:/static-volume 10G 42M 10G 1% /mnt
已有nfs实例Provisioner推荐使用nfs-subdir-external-provisioner,开源也有nfs-client-provisioner可能会报错“selfLink was empty, can't make reference”,原因是高版本SelfLink已经删除,并且1.24以上版本也不支持设置RemoveSelfLink featuregate
参考nfs-subdir-external-provisioner部署部署方法
helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/
helm install nfs-subdir-external-provisioner nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \
--set nfs.server=x.x.x.x \
--set nfs.path=/exported/path \
--set image.repository=eipwork/nfs-subdir-external-provisioner
$ kubectl get sc nfs-client
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
nfs-client cluster.local/nfs-subdir-external-provisioner Delete Immediate true 64s
$ kubectl get po
NAME READY STATUS RESTARTS AGE
nfs-subdir-external-provisioner-745d595dfc-jxdd9 1/1 Running 0 67s
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: test-claim
annotations:
nfs.io/storage-path: "test-path" # not required, depending on whether this annotation was shown in the storage class description
spec:
storageClassName: nfs-client
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Mi
$ kubectl get pvc test-claim
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
test-claim Bound pvc-fb9f42a5-57dc-44d6-b1da-15cab95829fb 1Gi RWX nfs-client 2m28s
$ kubectl get pv pvc-fb9f42a5-57dc-44d6-b1da-15cab95829fb -o custom-columns=:.spec.nfs
map[path:/static-nfs-subdir/default-test-claim-pvc-fb9f42a5-57dc-44d6-b1da-15cab95829fb server:10.0.2.25]
apiVersion: apps/v1
kind: Deployment
metadata:
name: nfs-client-deploy
namespace: default
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: nfs-client-deploy
qcloud-app: nfs-client-deploy
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
type: RollingUpdate
template:
metadata:
creationTimestamp: null
labels:
k8s-app: nfs-client-deploy
qcloud-app: nfs-client-deploy
spec:
affinity: {}
containers:
- image: nginx
imagePullPolicy: Always
name: nginx
resources:
limits:
cpu: 500m
memory: 1Gi
requests:
cpu: 250m
memory: 256Mi
securityContext:
privileged: false
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /mnt
name: vol
dnsPolicy: ClusterFirst
imagePullSecrets:
- name: qcloudregistrykey
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds: 30
volumes:
- name: vol
persistentVolumeClaim:
claimName: test-claim
$ kubectl get po
NAME READY STATUS RESTARTS AGE
nfs-client-deploy-55f79cbc9c-zbmk8 1/1 Running 0 2m56s
root@nfs-client-deploy-55f79cbc9c-zbmk8:/# df -h /mnt/
Filesystem Size Used Avail Use% Mounted on
10.0.2.25:/static-nfs-subdir/default-test-claim-pvc-fb9f42a5-57dc-44d6-b1da-15cab95829fb 10G 32M 10G 1% /mnt
TKE支持创建cfs-csi组件,方便kubernetes快速接入腾讯云cfs,参考cfs-csi组件
storageclass根据pvc创建的pv每次会新建cfs实例
$ kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
newcfs-pvc1 Bound pvc-df26a64c-3303-4110-ada8-50615d0b3519 10Gi RWX newscf-sc 4m3s
newcfs-pvc2 Bound pvc-8b689604-62ab-4178-b94e-39ad1e98af31 10Gi RWX newscf-sc 3m53s
$ kubectl get pv pvc-df26a64c-3303-4110-ada8-50615d0b3519 -o yaml| grep volumeHandle
volumeHandle: cfs-ioien2ah
$ kubectl get pv pvc-8b689604-62ab-4178-b94e-39ad1e98af31 -o yaml| grep volumeHandle
volumeHandle: cfs-m6uiekm3
storageclass根据pvc创建的pv首次会新建cfs实例,之后所有的pv会共享使用该cfs实例
$ kubectl get pvc | grep sharecfs-pvc
sharecfs-pvc1 Bound pvc-64031d20-4346-4df2-9d2b-ab5a977334bd 10Gi RWX sharecfs-sc 68m
sharecfs-pvc2 Bound pvc-1c454b82-c95b-4148-bd86-0e1dd636861c 10Gi RWX sharecfs-sc 68m
$ kubectl get pv pvc-64031d20-4346-4df2-9d2b-ab5a977334bd -o custom-columns=:.spec.csi.volumeAttributes
map[fsid:c7e6gjwo host:10.0.2.25 path:/default-sharecfs-pvc1-pvc-64031d20-4346-4df2-9d2b-ab5a977334bd]
$ kubectl get pv pvc-1c454b82-c95b-4148-bd86-0e1dd636861c -o custom-columns=:.spec.csi.volumeAttributes
map[fsid:c7e6gjwo host:10.0.2.25 path:/default-sharecfs-pvc2-pvc-1c454b82-c95b-4148-bd86-0e1dd636861c]
原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。
如有侵权,请联系 cloudcommunity@tencent.com 删除。
原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。
如有侵权,请联系 cloudcommunity@tencent.com 删除。