k8s nfs动态存储StatefulSet

需要下载

https://github.com/kubernetes-retired/external-storage/tree/master/nfs/deploy/kubernetes

cat <<END>statefulset.yaml
apiVersion: v1
kind: Namespace
metadata:
   name: test
   labels:
     name: test
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: nfs-provisioner
  namespace: test
---
kind: Service
apiVersion: v1
metadata:
  name: nfs-provisioner
  namespace: test
  labels:
    app: nfs-provisioner
spec:
  ports:
    - name: nfs
      port: 2049
    - name: nfs-udp
      port: 2049
      protocol: UDP
    - name: nlockmgr
      port: 32803
    - name: nlockmgr-udp
      port: 32803
      protocol: UDP
    - name: mountd
      port: 20048
    - name: mountd-udp
      port: 20048
      protocol: UDP
    - name: rquotad
      port: 875
    - name: rquotad-udp
      port: 875
      protocol: UDP
    - name: rpcbind
      port: 111
    - name: rpcbind-udp
      port: 111
      protocol: UDP
    - name: statd
      port: 662
    - name: statd-udp
      port: 662
      protocol: UDP
  selector:
    app: nfs-provisioner
---
kind: StatefulSet
apiVersion: apps/v1
metadata:
  name: nfs-provisioner
  namespace: test
spec:
  selector:
    matchLabels:
      app: nfs-provisioner
  serviceName: "nfs-provisioner"
  replicas: 1
  template:
    metadata:
      labels:
        app: nfs-provisioner
    spec:
      serviceAccount: nfs-provisioner
      nodeName: node172
      terminationGracePeriodSeconds: 10
      containers:
        - name: nfs-provisioner
          #image: quay.io/kubernetes_incubator/nfs-provisioner:latest
          image: registry.cn-shenzhen.aliyuncs.com/jbjb/csi:nfs
          ports:
            - name: nfs
              containerPort: 2049
            - name: nfs-udp
              containerPort: 2049
              protocol: UDP
            - name: nlockmgr
              containerPort: 32803
            - name: nlockmgr-udp
              containerPort: 32803
              protocol: UDP
            - name: mountd
              containerPort: 20048
            - name: mountd-udp
              containerPort: 20048
              protocol: UDP
            - name: rquotad
              containerPort: 875
            - name: rquotad-udp
              containerPort: 875
              protocol: UDP
            - name: rpcbind
              containerPort: 111
            - name: rpcbind-udp
              containerPort: 111
              protocol: UDP
            - name: statd
              containerPort: 662
            - name: statd-udp
              containerPort: 662
              protocol: UDP
          securityContext:
            capabilities:
              add:
                - DAC_READ_SEARCH
                - SYS_RESOURCE
          args:
            - "-provisioner=example.com/nfs"
          env:
            - name: POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
            - name: SERVICE_NAME
              value: nfs-provisioner
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          imagePullPolicy: "IfNotPresent"
          volumeMounts:
            - name: export-volume
              mountPath: /export
      volumes:
        - name: export-volume
          hostPath:
            path: /statefulset
END

创建SC

cat <<END>sc.yaml
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: managed-nfs-storage
  namespace: test
provisioner: example.com/nfs
mountOptions:
  - vers=4.1
END

Redis集群测试

cat <<END> redis-deploy.yaml
apiVersion: v1
kind: Service
metadata:
  name: redis-service 
  namespace: test
  labels:
    app: redis
spec:
  ports:
  - port: 6379
    targetPort: 6379
    name: redis-port
  clusterIP: None
  selector:
    app: redis
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis
  namespace: test
spec:
  selector:
    matchLabels:
      app: redis
  serviceName: "redis-service"
  replicas: 6
  template:
    metadata:
      labels:
        app: redis
    spec:
      terminationGracePeriodSeconds: 20
      containers:
      - name: redis
        image: redis:latest
        imagePullPolicy: Never     
        command:
          - "redis-server"                  #redis启动命令
        args:
          - "/etc/redis/redis.conf"         #redis-server后面跟的参数,换行代表空格
          - "--protected-mode"              #允许外网访问
          - "no"
        # command: redis-server /etc/redis/redis.conf --protected-mode no
        resources:                          #资源
          requests:                         #请求的资源
            cpu: "100m"                     #m代表千分之,相当于0.1 个cpu资源
            memory: "100Mi"                 #内存100m大小
        ports:
            - name: redis
              containerPort: 6379
              protocol: "TCP"
            - name: cluster
              containerPort: 16379
              protocol: "TCP"
        volumeMounts:
          - name: "redis-conf"              #挂载configmap生成的文件
            mountPath: "/etc/redis"         #挂载到哪个路径下
          - name: "redis-data"              #挂载持久卷的路径
            mountPath: "/var/lib/redis"
      volumes:
      - name: "redis-conf"                  #引用configMap卷
        configMap:
          name: "redis-conf"
          items:
            - key: "redis.conf"             #创建configMap指定的名称
              path: "redis.conf"            #里面的那个文件--from-file参数后面的文件
  volumeClaimTemplates:
  - metadata:
      name: "redis-data"
      annotations:
        volume.beta.kubernetes.io/storage-class: "managed-nfs-storage"
    spec:
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 1Gi
END

redis配置

cat <<END> redis.conf
appendonly yes                      
cluster-enabled yes                 
cluster-config-file /var/lib/redis/nodes.conf  
cluster-node-timeout 5000           
dir /var/lib/redis                  
port 6379
END
kubectl create -n test configmap redis-conf --from-file=redis.conf
kubectl apply -f redis-deploy.yaml

[root@c169 statefulSet]# kubectl get pods -n test 
NAME                READY   STATUS    RESTARTS   AGE
busybox             1/1     Running   0          20m
nfs-provisioner-0   1/1     Running   0          35m
redis-0             1/1     Running   0          23m
redis-1             1/1     Running   0          23m
redis-2             1/1     Running   0          23m
redis-3             1/1     Running   0          22m
redis-4             1/1     Running   0          22m
redis-5             1/1     Running   0          22m

创建redis集群,再把redis串起来

kubectl   exec -it -n test  redis-0 -- redis-cli --cluster create --cluster-replicas 1  $(kubectl get pods -n test  -l app=redis -o jsonpath='{range.items[*]}{.status.podIP}:6379 ' | awk 'NF--')

最后验证

[root@c169 statefulSet]# kubectl exec -n test busybox -- nslookup redis-service
Server:    10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local

Name:      redis-service
Address 1: 10.244.1.20 redis-2.redis-service.test.svc.cluster.local
Address 2: 10.244.1.21 redis-4.redis-service.test.svc.cluster.local
Address 3: 10.244.2.24 redis-5.redis-service.test.svc.cluster.local
Address 4: 10.244.1.19 redis-0.redis-service.test.svc.cluster.local
Address 5: 10.244.2.23 redis-3.redis-service.test.svc.cluster.local
Address 6: 10.244.2.22 redis-1.redis-service.test.svc.cluster.local