kk 创建多节点集群ceph存储 Error from server (NotFound): persistentvolumeclaims

> 	解决问题时应该注意的细节：
> 	1、介质kk 应选择主节点执行操作；
> 	2、目前暂时不支持关于多节点ceph cluster + VIP(keepalived+haproxy)；
> 	3、yaml 文件的书写规范化；
> 	4、注意ceph-rbd key 需要base64转化，而ceph-csi key不必base64转化；
> 	5、安装ceph-rbd中建议执行helm repo add test https://charts.kubesphere.io/test；

#config-sample.yaml
  addons: 
  - name: rbd-provisioner
    namespace: kube-system
    sources:
      chart:
        name: rbd-provisioner
        repo: https://charts.kubesphere.io/test
        values: /root/rbd-provisioner-value.yaml
  - name: ceph-csi-rbd
    namespace: kube-system
    sources:
      chart:
        name: ceph-csi-rbd
        repo: https://ceph.github.io/csi-charts
        values: /root/ceph-csi-config.yaml
  - name: ceph-csi-rbd-sc
    sources:
      yaml:
        path:
        - /root/ceph-csi-rbd-sc.yaml

# rbd-provisioner-value.yaml 
ceph:
  adminId: admin
  adminKey: QVFDUy9rQmY4RWczT0JBQXhwdTlzWStWUU43K1VGYldudWEyY0E9PQ==  # is base64
  userId: admin
  userKey: QVFDUy9rQmY4RWczT0JBQXhwdTlzWStWUU43K1VGYldudWEyY0E9PQ==
  mon: 192.168.99.11:6789,192.168.99.12:6789,192.168.99.13:6789
  pool: kubeceph
sc:
  isDefault: true

# ceph-csi-config.yaml 
csiConfig:
  - clusterID: "cluster1"
    monitors:
      - 192.168.99.11:6789 
      - 192.168.99.12:6789 
      - 192.168.99.13:6789 
  

> 不新增容忍度则将master置为worker节点，不然则会出现如下异常
> driver name rbd.csi.ceph.com not found in the list of registered CSI drivers

nodeplugin:
  tolerations: 
    - key: node-role.kubernetes.io/master
      effect: NoSchedule 
    - key: CriticalAddonsOnly 
      operator: Exists 
    - effect: NoExecute 
      key: node.kubernetes.io/not-ready 
      operator: Exists 
      tolerationSeconds: 60 
    - effect: NoExecute 
      key: node.kubernetes.io/unreachable 
      operator: Exists 
      tolerationSeconds: 60

# ceph-csi-rbd-sc.yaml 
apiVersion: v1
kind: Secret
metadata:
  name: csi-rbd-secret
  namespace: kube-system
stringData:
  userID: admin
  userKey: AQCS/kBf8Eg3OBAAxpu9sY+VQN7+UFbWnua2cA==   #not base64
  encryptionPassphrase: test_passphrase
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
   name: csi-rbd-sc
   annotations:
     storageclass.beta.kubernetes.io/is-default-class: "false" #rbd is sc.isDefault=true
     storageclass.kubesphere.io/supported-access-modes: '["ReadWriteOnce","ReadOnlyMany","ReadWriteMany"]'
provisioner: rbd.csi.ceph.com
parameters:
   clusterID: "cluster1"
   pool: kubeceph
   imageFeatures: layering
   csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
   csi.storage.k8s.io/provisioner-secret-namespace: kube-system
   csi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secret
   csi.storage.k8s.io/controller-expand-secret-namespace: kube-system
   csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret
   csi.storage.k8s.io/node-stage-secret-namespace: kube-system
   csi.storage.k8s.io/fstype: ext4
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:
   - discard