TASK [common : Kubesphere | Creating manifests] ********************************
changed: [localhost] => (item={u'name': u'custom-values-minio', u'file': u'custom-values-minio.yaml'})

TASK [common : Kubesphere | Deploy minio] **************************************
fatal: [localhost]: FAILED! => {"changed": true, "cmd": "/usr/local/bin/helm upgrade --install ks-minio /etc/kubesphere/minio-ha -f /etc/kubesphere/custom-values-minio.yaml --set fullnameOverride=minio --namespace kubesphere-system --wait --timeout 1800\n", "delta": "0:02:09.880561", "end": "2019-11-12 07:06:50.243097", "msg": "non-zero return code", "rc": 1, "start": "2019-11-12 07:04:40.362536", "stderr": "Error: no kind \"Job\" is registered for version \"batch/v1\" in scheme \"k8s.io/kubernetes/pkg/api/legacyscheme/scheme.go:30\"", "stderr_lines": ["Error: no kind \"Job\" is registered for version \"batch/v1\" in scheme \"k8s.io/kubernetes/pkg/api/legacyscheme/scheme.go:30\""], "stdout": "Release \"ks-minio\" does not exist. Installing it now.", "stdout_lines": ["Release \"ks-minio\" does not exist. Installing it now."]}
...ignoring

TASK [common : debug] **********************************************************
ok: [localhost] => {
    "msg": [
        "1. check the storage configuration and storage server", 
        "2. execute 'helm del --purge ks-minio'", 
        "3. Restart the installer pod in kubesphere-system namespace"
    ]
}

TASK [common : fail] ***********************************************************
fatal: [localhost]: FAILED! => {"changed": false, "msg": "It is suggested to refer to the above methods for troubleshooting problems ."}

PLAY RECAP *********************************************************************
localhost                  : ok=24   changed=18   unreachable=0    failed=1    skipped=66   rescued=0    ignored=1   
    • [已注销]

    • 最佳回复Feynman 选择

    rysinal 更改calio为flannel 一般都是因为底层网络不支持ipip协议影响k8s集群的dns解析所造成的

是什么版本的 k8s 呢,看起来是 k8s 版本比较老。

    runzexia

    [root@k8s-master nfs]# kubectl  version
    Client Version: version.Info{Major:"1", Minor:"15", GitVersion:"v1.15.0", GitCommit:"e8462b5b5dc2584fdcd18e6bcfe9f1e4d970a529", GitTreeState:"clean", BuildDate:"2019-06-19T16:40:16Z", GoVersion:"go1.12.5", Compiler:"gc", Platform:"linux/amd64"}
    Server Version: version.Info{Major:"1", Minor:"15", GitVersion:"v1.15.0", GitCommit:"e8462b5b5dc2584fdcd18e6bcfe9f1e4d970a529", GitTreeState:"clean", BuildDate:"2019-06-19T16:32:14Z", GoVersion:"go1.12.5", Compiler:"gc", Platform:"linux/amd64"}
    [root@k8s-master nfs]# helm version
    Client: &version.Version{SemVer:"v2.16.0", GitCommit:"e13bc94621d4ef666270cfbe734aaabf342a49bb", GitTreeState:"clean"}
    Server: &version.Version{SemVer:"v2.16.0", GitCommit:"e13bc94621d4ef666270cfbe734aaabf342a49bb", GitTreeState:"clean"}
    [root@k8s-master nfs]# 
    [root@k8s-master nfs]# kubectl  get sc
    NAME                           PROVISIONER      AGE
    course-nfs-storage (default)   fuseim.pri/ifs   31m
    [root@k8s-master nfs]# 

      koy1619 执行下
      kubectl api-resources
      看看返回把,installer日志返回是无法创建batch/v1的Job资源

        runzexia

        [root@k8s-master deploy]# kubectl api-resources
        NAME                              SHORTNAMES   APIGROUP                       NAMESPACED   KIND
        bindings                                                                      true         Binding
        componentstatuses                 cs                                          false        ComponentStatus
        configmaps                        cm                                          true         ConfigMap
        endpoints                         ep                                          true         Endpoints
        events                            ev                                          true         Event
        limitranges                       limits                                      true         LimitRange
        namespaces                        ns                                          false        Namespace
        nodes                             no                                          false        Node
        persistentvolumeclaims            pvc                                         true         PersistentVolumeClaim
        persistentvolumes                 pv                                          false        PersistentVolume
        pods                              po                                          true         Pod
        podtemplates                                                                  true         PodTemplate
        replicationcontrollers            rc                                          true         ReplicationController
        resourcequotas                    quota                                       true         ResourceQuota
        secrets                                                                       true         Secret
        serviceaccounts                   sa                                          true         ServiceAccount
        services                          svc                                         true         Service
        mutatingwebhookconfigurations                  admissionregistration.k8s.io   false        MutatingWebhookConfiguration
        validatingwebhookconfigurations                admissionregistration.k8s.io   false        ValidatingWebhookConfiguration
        customresourcedefinitions         crd,crds     apiextensions.k8s.io           false        CustomResourceDefinition
        apiservices                                    apiregistration.k8s.io         false        APIService
        applications                                   app.k8s.io                     true         Application
        controllerrevisions                            apps                           true         ControllerRevision
        daemonsets                        ds           apps                           true         DaemonSet
        deployments                       deploy       apps                           true         Deployment
        replicasets                       rs           apps                           true         ReplicaSet
        statefulsets                      sts          apps                           true         StatefulSet
        auditsinks                                     auditregistration.k8s.io       false        AuditSink
        meshpolicies                                   authentication.istio.io        false        MeshPolicy
        policies                                       authentication.istio.io        true         Policy
        tokenreviews                                   authentication.k8s.io          false        TokenReview
        localsubjectaccessreviews                      authorization.k8s.io           true         LocalSubjectAccessReview
        selfsubjectaccessreviews                       authorization.k8s.io           false        SelfSubjectAccessReview
        selfsubjectrulesreviews                        authorization.k8s.io           false        SelfSubjectRulesReview
        subjectaccessreviews                           authorization.k8s.io           false        SubjectAccessReview
        horizontalpodautoscalers          hpa          autoscaling                    true         HorizontalPodAutoscaler
        cronjobs                          cj           batch                          true         CronJob
        jobs                                           batch                          true         Job
        certificatesigningrequests        csr          certificates.k8s.io            false        CertificateSigningRequest
        adapters                                       config.istio.io                true         adapter
        apikeys                                        config.istio.io                true         apikey
        attributemanifests                             config.istio.io                true         attributemanifest
        authorizations                                 config.istio.io                true         authorization
        bypasses                                       config.istio.io                true         bypass
        checknothings                                  config.istio.io                true         checknothing
        circonuses                                     config.istio.io                true         circonus
        cloudwatches                                   config.istio.io                true         cloudwatch
        deniers                                        config.istio.io                true         denier
        dogstatsds                                     config.istio.io                true         dogstatsd
        edges                                          config.istio.io                true         edge
        fluentds                                       config.istio.io                true         fluentd
        handlers                                       config.istio.io                true         handler
        httpapispecbindings                            config.istio.io                true         HTTPAPISpecBinding
        httpapispecs                                   config.istio.io                true         HTTPAPISpec
        instances                                      config.istio.io                true         instance
        kubernetesenvs                                 config.istio.io                true         kubernetesenv
        kuberneteses                                   config.istio.io                true         kubernetes
        listcheckers                                   config.istio.io                true         listchecker
        listentries                                    config.istio.io                true         listentry
        logentries                                     config.istio.io                true         logentry
        memquotas                                      config.istio.io                true         memquota
        metrics                                        config.istio.io                true         metric
        noops                                          config.istio.io                true         noop
        opas                                           config.istio.io                true         opa
        prometheuses                                   config.istio.io                true         prometheus
        quotas                                         config.istio.io                true         quota
        quotaspecbindings                              config.istio.io                true         QuotaSpecBinding
        quotaspecs                                     config.istio.io                true         QuotaSpec
        rbacs                                          config.istio.io                true         rbac
        redisquotas                                    config.istio.io                true         redisquota
        reportnothings                                 config.istio.io                true         reportnothing
        rules                                          config.istio.io                true         rule
        signalfxs                                      config.istio.io                true         signalfx
        solarwindses                                   config.istio.io                true         solarwinds
        stackdrivers                                   config.istio.io                true         stackdriver
        statsds                                        config.istio.io                true         statsd
        stdios                                         config.istio.io                true         stdio
        templates                                      config.istio.io                true         template
        tracespans                                     config.istio.io                true         tracespan
        zipkins                                        config.istio.io                true         zipkin
        leases                                         coordination.k8s.io            true         Lease
        s2ibinaries                                    devops.kubesphere.io           true         S2iBinary
        s2ibuilders                       s2ib         devops.kubesphere.io           true         S2iBuilder
        s2ibuildertemplates               s2ibt        devops.kubesphere.io           false        S2iBuilderTemplate
        s2iruns                           s2ir         devops.kubesphere.io           true         S2iRun
        events                            ev           events.k8s.io                  true         Event
        daemonsets                        ds           extensions                     true         DaemonSet
        deployments                       deploy       extensions                     true         Deployment
        ingresses                         ing          extensions                     true         Ingress
        networkpolicies                   netpol       extensions                     true         NetworkPolicy
        podsecuritypolicies               psp          extensions                     false        PodSecurityPolicy
        replicasets                       rs           extensions                     true         ReplicaSet
        fluentbits                                     logging.kubesphere.io          true         FluentBit
        alertmanagers                                  monitoring.coreos.com          true         Alertmanager
        prometheuses                                   monitoring.coreos.com          true         Prometheus
        prometheusrules                                monitoring.coreos.com          true         PrometheusRule
        servicemonitors                                monitoring.coreos.com          true         ServiceMonitor
        destinationrules                  dr           networking.istio.io            true         DestinationRule
        envoyfilters                                   networking.istio.io            true         EnvoyFilter
        gateways                          gw           networking.istio.io            true         Gateway
        serviceentries                    se           networking.istio.io            true         ServiceEntry
        sidecars                                       networking.istio.io            true         Sidecar
        virtualservices                   vs           networking.istio.io            true         VirtualService
        ingresses                         ing          networking.k8s.io              true         Ingress
        networkpolicies                   netpol       networking.k8s.io              true         NetworkPolicy
        runtimeclasses                                 node.k8s.io                    false        RuntimeClass
        poddisruptionbudgets              pdb          policy                         true         PodDisruptionBudget
        podsecuritypolicies               psp          policy                         false        PodSecurityPolicy
        clusterrolebindings                            rbac.authorization.k8s.io      false        ClusterRoleBinding
        clusterroles                                   rbac.authorization.k8s.io      false        ClusterRole
        rolebindings                                   rbac.authorization.k8s.io      true         RoleBinding
        roles                                          rbac.authorization.k8s.io      true         Role
        clusterrbacconfigs                             rbac.istio.io                  false        ClusterRbacConfig
        rbacconfigs                                    rbac.istio.io                  true         RbacConfig
        servicerolebindings                            rbac.istio.io                  true         ServiceRoleBinding
        serviceroles                                   rbac.istio.io                  true         ServiceRole
        priorityclasses                   pc           scheduling.k8s.io              false        PriorityClass
        servicepolicies                                servicemesh.kubesphere.io      true         ServicePolicy
        strategies                                     servicemesh.kubesphere.io      true         Strategy
        podpresets                                     settings.k8s.io                true         PodPreset
        csidrivers                                     storage.k8s.io                 false        CSIDriver
        csinodes                                       storage.k8s.io                 false        CSINode
        storageclasses                    sc           storage.k8s.io                 false        StorageClass
        volumeattachments                              storage.k8s.io                 false        VolumeAttachment
        workspaces                                     tenant.kubesphere.io           false        Workspace
        [root@k8s-master deploy]# 

          runzexia

          [root@k8s-master deploy]# kubectl explain jobs
          KIND:     Job
          VERSION:  batch/v1
          
          DESCRIPTION:
               Job represents the configuration of a single job.
          
          FIELDS:
             apiVersion   <string>
               APIVersion defines the versioned schema of this representation of an
               object. Servers should convert recognized schemas to the latest internal
               value, and may reject unrecognized values. More info:
               https://git.k8s.io/community/contributors/devel/api-conventions.md#resources
          
             kind <string>
               Kind is a string value representing the REST resource this object
               represents. Servers may infer this from the endpoint the client submits
               requests to. Cannot be updated. In CamelCase. More info:
               https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds
          
             metadata     <Object>
               Standard object's metadata. More info:
               https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata
          
             spec <Object>
               Specification of the desired behavior of a job. More info:
               https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status
          
             status       <Object>
               Current status of a job. More info:
               https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status
          
          [root@k8s-master deploy]# 

          runzexia
          helm应该没问题了,这边还是过不了

          TASK [common : Kubesphere | Creating manifests] ********************************
          changed: [localhost] => (item={u'name': u'custom-values-minio', u'file': u'custom-values-minio.yaml'})
          
          TASK [common : Kubesphere | Deploy minio] **************************************
          fatal: [localhost]: FAILED! => {"changed": true, "cmd": "/usr/local/bin/helm upgrade --install ks-minio /etc/kubesphere/minio-ha -f /etc/kubesphere/custom-values-minio.yaml --set fullnameOverride=minio --namespace kubesphere-system --wait --timeout 1800\n", "delta": "0:00:00.213862", "end": "2019-11-12 07:51:56.429823", "msg": "non-zero return code", "rc": 1, "start": "2019-11-12 07:51:56.215961", "stderr": "Error: UPGRADE FAILED: \"ks-minio\" has no deployed releases", "stderr_lines": ["Error: UPGRADE FAILED: \"ks-minio\" has no deployed releases"], "stdout": "", "stdout_lines": []}
          ...ignoring
          
          TASK [common : debug] **********************************************************
          ok: [localhost] => {
              "msg": [
                  "1. check the storage configuration and storage server", 
                  "2. execute 'helm del --purge ks-minio'", 
                  "3. Restart the installer pod in kubesphere-system namespace"
              ]
          }
          
          TASK [common : fail] ***********************************************************
          fatal: [localhost]: FAILED! => {"changed": false, "msg": "It is suggested to refer to the above methods for troubleshooting problems ."}
          
          PLAY RECAP *********************************************************************
          localhost                  : ok=24   changed=18   unreachable=0    failed=1    skipped=66   rescued=0    ignored=1   
          
          
          ^C
          [root@k8s-master deploy]# helm version
          Client: &version.Version{SemVer:"v2.15.2", GitCommit:"8dce272473e5f2a7bf58ce79bb5c3691db54c96b", GitTreeState:"clean"}
          Server: &version.Version{SemVer:"v2.15.2", GitCommit:"8dce272473e5f2a7bf58ce79bb5c3691db54c96b", GitTreeState:"clean"}
          [root@k8s-master deploy]# 

            runzexia

            kubectl  delete -f .
            sh ../scripts/kubesphere-delete.sh 
            kubectl  apply -f .

            还是上面那条报错!

              runzexia

              [root@k8s-master deploy]# helm del --purge ks-minio 
              release "ks-minio" deleted
              [root@k8s-master deploy]# 

              重新卸载部署;这段报错过去了,谢谢大佬!

                13 天 后

                koy1619 大佬,我的报错跟你一样,用kubectl get pod ks-installer-7987c659d6-vwl2t -n kubesphere-system -o yaml | kubectl replace –force -f - 这条命令重启installer后还是不行。
                # helm version
                Client: &version.Version{SemVer:"v2.14.3", GitCommit:"0e7f3b6637f7af8fcfddb3d2941fcc7cbebb0085", GitTreeState:"clean"}
                Server: &version.Version{SemVer:"v2.14.3", GitCommit:"0e7f3b6637f7af8fcfddb3d2941fcc7cbebb0085", GitTreeState:"clean"}

                系统版本centos 7.6,存储用的nfs

                  9 天 后

                  hetao runzexia 我的也是,centos 7.5 nfs存储,然后helm为2.14.3,安装2.1版本ks;试了两次都报错,卡的有点久了

                  helm del --purge ks-minio
                  kubectl get pod ks-installer-7987c659d6-sg8pv -n kubesphere-system -o yaml | kubectl replace --force -f -
                  TASK [common : Kubesphere | Deploy minio] **************************************
                  
                  fatal: [localhost]: FAILED! => {"changed": true, "cmd": "/usr/local/bin/helm upgrade --install ks-minio /etc/kubesphere/minio-ha -f /etc/kubesphere/custom-values-minio.yaml --set fullnameOverride=minio --namespace kubesphere-system --wait --timeout 1800\n", "delta": "0:30:01.145146", "end": "2019-12-04 02:43:29.052773", "msg": "non-zero return code", "rc": 1, "start": "2019-12-04 02:13:27.907627", "stderr": "Error: release ks-minio failed: timed out waiting for the condition", "stderr_lines": ["Error: release ks-minio failed: timed out waiting for the condition"], "stdout": "Release \"ks-minio\" does not exist. Installing it now.", "stdout_lines": ["Release \"ks-minio\" doesnot exist. Installing it now."]}
                  ...ignoring
                  
                  TASK [common : debug] **********************************************************
                  ok: [localhost] => {
                      "msg": [
                          "1. check the storage configuration and storage server",
                          "2. execute 'helm del --purge ks-minio'",
                          "3. Restart the installer pod in kubesphere-system namespace"
                      ]
                  }
                  
                  TASK [common : fail] ***********************************************************
                  fatal: [localhost]: FAILED! => {"changed": false, "msg": "It is suggested to refer to the above methods for troubleshooting problems ."}
                  
                  PLAY RECAP *********************************************************************
                  localhost                  : ok=24   changed=18   unreachable=0    failed=1    skipped=66   rescued=0    ignored=1

                    rysinal

                    1. 首先删除 minio helm del --purge ks-minio
                    2. 进入installer pod bash kubectl exec -n kubesphere-system ks-installer-xx -it sh
                    3. 执行minio 安装命令 /usr/local/bin/helm upgrade --install ks-minio /etc/kubesphere/minio-ha -f /etc/kubesphere/custom-values-minio.yaml --set fullnameOverride=minio --namespace kubesphere-system --wait --timeout 1800
                    4. 观察minio状态,排查为何出错

                      runzexia

                      [root@master ~]# helm del --purge ks-minio
                      release "ks-minio" deleted
                      [root@master ~]# kubectl get pods -n kubesphere-system
                      NAME                            READY   STATUS    RESTARTS   AGE
                      ks-installer-7987c659d6-rp9px   1/1     Running   0          40m
                      minio-make-bucket-job-rl9bw     1/1     Running   1          38m
                      openldap-0                      1/1     Running   0          14h
                      redis-5d4844b947-gzbsh          1/1     Running   0          14h
                      [root@master ~]# kubectl exec -n kubesphere-system ks-installer-7987c659d6-rp9px -it sh
                      / # /usr/local/bin/helm upgrade --install ks-minio /etc/kubesphere/minio-ha -f /etc/kubesphere/custom-values-minio.yaml --set fullnameOverride=minio --namespace kubesphere
                      -system --wait --timeout 1800
                      Release "ks-minio" does not exist. Installing it now.
                      Error: jobs.batch "minio-make-bucket-job" already exists
                      / #

                        rysinal

                        Events:
                          Type     Reason       Age                From               Message
                          ----     ------       ----               ----               -------
                          Normal   Scheduled    50m                default-scheduler  Successfully assigned kubesphere-system/minio-make-bucket-job-rl9bw to node2
                          Normal   Pulling      50m                kubelet, node2     Pulling image "minio/mc:RELEASE.2019-08-07T23-14-43Z"
                          Normal   Pulled       49m                kubelet, node2     Successfully pulled image "minio/mc:RELEASE.2019-08-07T23-14-43Z"
                          Normal   Created      11m (x3 over 49m)  kubelet, node2     Created container minio-mc
                          Normal   Started      11m (x3 over 49m)  kubelet, node2     Started container minio-mc
                          Normal   Pulled       11m (x2 over 30m)  kubelet, node2     Container image "minio/mc:RELEASE.2019-08-07T23-14-43Z" already present on machine
                          Warning  FailedMount  1s (x12 over 11m)  kubelet, node2     MountVolume.SetUp failed for volume "minio-configuration" : [configmap "minio" not found, secret "minio" notfound]