升级到3.0后,页面上找不到devops工程选项,日志报FAILED - RETRYING: KubeSphere | Get devops migration job status

  • apiVersion: batch/v1
    kind: Job
    metadata:
      annotations:
        kubectl.kubernetes.io/last-applied-configuration: |
          {"apiVersion":"batch/v1","kind":"Job","metadata":{"annotations":{},"labels":{"app":"ks-devops-migration","job":"ks-devops-migration","version":"v3.0.0"},"name":"ks-devops-migration","namespace":"kubesphere-system"},"spec":{"backoffLimit":6,"completions":1,"parallelism":1,"template":{"metadata":{"labels":{"app":"ks-devops-migration","job":"ks-devops-migration","version":"v3.0.0"},"name":"ks-devops-migration"},"spec":{"containers":[{"command":["ks-upgrade"],"image":"kubesphere/ks-devops:flyway-v3.0.0","imagePullPolicy":"Always","name":"ks-devops-upgrade","terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","volumeMounts":[{"mountPath":"/etc/kubesphere/","name":"kubesphere-config"}]}],"dnsPolicy":"ClusterFirst","initContainers":[{"command":["sh","-c","until wget --spider http://ks-jenkins.kubesphere-devops-system.svc/login; do echo \"waiting for myservice\"; sleep 2; done;"],"image":"alpine:3.10.4","imagePullPolicy":"IfNotPresent","name":"wait-jenkins"},{"command":["sh","-c","until nc -z mysql.kubesphere-system.svc 3306; do echo \"waiting for mysql\"; sleep 2; done;"],"image":"alpine:3.10.4","imagePullPolicy":"IfNotPresent","name":"wait-mysql"},{"command":["sh","-c","until nc -z ks-apiserver.kubesphere-system.svc 80; do echo \"waiting for apiserver\"; sleep 2; done;"],"image":"alpine:3.10.4","imagePullPolicy":"IfNotPresent","name":"wait-apiserver"}],"restartPolicy":"OnFailure","schedulerName":"default-scheduler","serviceAccount":"kubesphere","serviceAccountName":"kubesphere","terminationGracePeriodSeconds":30,"volumes":[{"configMap":{"defaultMode":420,"name":"kubesphere-config"},"name":"kubesphere-config"}]}}}}
        revisions: '{"1":{"status":"failed","reasons":["BackoffLimitExceeded"],"messages":["Job
          has reached the specified backoff limit"],"desire":1,"failed":1,"uid":"555e1e9a-f541-4738-97c5-f627ade9eaf2","start-time":"2020-09-01T10:10:44+08:00","completion-time":"0001-01-01T00:00:00Z"}}'
      creationTimestamp: "2020-09-01T02:10:44Z"
      labels:
        app: ks-devops-migration
        job: ks-devops-migration
        version: v3.0.0
      name: ks-devops-migration
      namespace: kubesphere-system
      resourceVersion: "41957538"
      selfLink: /apis/batch/v1/namespaces/kubesphere-system/jobs/ks-devops-migration
      uid: 555e1e9a-f541-4738-97c5-f627ade9eaf2
    spec:
      backoffLimit: 6
      completions: 1
      parallelism: 1
      selector:
        matchLabels:
          controller-uid: 555e1e9a-f541-4738-97c5-f627ade9eaf2
      template:
        metadata:
          creationTimestamp: null
          labels:
            app: ks-devops-migration
            controller-uid: 555e1e9a-f541-4738-97c5-f627ade9eaf2
            job: ks-devops-migration
            job-name: ks-devops-migration
            version: v3.0.0
          name: ks-devops-migration
        spec:
          containers:
          - command:
            - ks-upgrade
            image: kubesphere/ks-devops:flyway-v3.0.0
            imagePullPolicy: Always
            name: ks-devops-upgrade
            resources: {}
            terminationMessagePath: /dev/termination-log
            terminationMessagePolicy: File
            volumeMounts:
            - mountPath: /etc/kubesphere/
              name: kubesphere-config
          dnsPolicy: ClusterFirst
          initContainers:
          - command:
            - sh
            - -c
            - until wget --spider http://ks-jenkins.kubesphere-devops-system.svc/login;
              do echo "waiting for myservice"; sleep 2; done;
            image: alpine:3.10.4
            imagePullPolicy: IfNotPresent
            name: wait-jenkins
            resources: {}
            terminationMessagePath: /dev/termination-log
            terminationMessagePolicy: File
          - command:
            - sh
            - -c
            - until nc -z mysql.kubesphere-system.svc 3306; do echo "waiting for mysql";
              sleep 2; done;
            image: alpine:3.10.4
            imagePullPolicy: IfNotPresent
            name: wait-mysql
            resources: {}
            terminationMessagePath: /dev/termination-log
            terminationMessagePolicy: File
          - command:
            - sh
            - -c
            - until nc -z ks-apiserver.kubesphere-system.svc 80; do echo "waiting for
              apiserver"; sleep 2; done;
            image: alpine:3.10.4
            imagePullPolicy: IfNotPresent
            name: wait-apiserver
            resources: {}
            terminationMessagePath: /dev/termination-log
            terminationMessagePolicy: File
          restartPolicy: OnFailure
          schedulerName: default-scheduler
          securityContext: {}
          serviceAccount: kubesphere
          serviceAccountName: kubesphere
          terminationGracePeriodSeconds: 30
          volumes:
          - configMap:
              defaultMode: 420
              name: kubesphere-config
            name: kubesphere-config
    status:
      conditions:
      - lastProbeTime: "2020-09-01T02:19:03Z"
        lastTransitionTime: "2020-09-01T02:19:03Z"
        message: Job has reached the specified backoff limit
        reason: BackoffLimitExceeded
        status: "True"
        type: Failed
      failed: 1
      startTime: "2020-09-01T02:10:44Z"

kubectl get job,pod -n kubesphere-system

hqq_get
kubectl -n kubesphere-system get job ks-devops-migration -o json | jq 'del(.spec.selector)' | jq 'del(.spec.template.metadata.labels)' | kubectl replace --force -f -

然后看下ks-devops-migration这个job运行pod的日志。

hqq_get

kubectl -n kubesphere-system get job ks-devops-migration -o yaml 看下

apiVersion: batch/v1
kind: Job
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"batch/v1","kind":"Job","metadata":{"annotations":{},"labels":{"app":"ks-devops-migration","job":"ks-devops-migration","version":"v3.0.0"},"name":"ks-devops-migration","namespace":"kubesphere-system"},"spec":{"backoffLimit":6,"completions":1,"parallelism":1,"template":{"metadata":{"labels":{"app":"ks-devops-migration","job":"ks-devops-migration","version":"v3.0.0"},"name":"ks-devops-migration"},"spec":{"containers":[{"command":["ks-upgrade"],"image":"kubesphere/ks-devops:flyway-v3.0.0","imagePullPolicy":"Always","name":"ks-devops-upgrade","terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","volumeMounts":[{"mountPath":"/etc/kubesphere/","name":"kubesphere-config"}]}],"dnsPolicy":"ClusterFirst","initContainers":[{"command":["sh","-c","until wget --spider http://ks-jenkins.kubesphere-devops-system.svc/login; do echo \"waiting for myservice\"; sleep 2; done;"],"image":"alpine:3.10.4","imagePullPolicy":"IfNotPresent","name":"wait-jenkins"},{"command":["sh","-c","until nc -z mysql.kubesphere-system.svc 3306; do echo \"waiting for mysql\"; sleep 2; done;"],"image":"alpine:3.10.4","imagePullPolicy":"IfNotPresent","name":"wait-mysql"},{"command":["sh","-c","until nc -z ks-apiserver.kubesphere-system.svc 80; do echo \"waiting for apiserver\"; sleep 2; done;"],"image":"alpine:3.10.4","imagePullPolicy":"IfNotPresent","name":"wait-apiserver"}],"restartPolicy":"OnFailure","schedulerName":"default-scheduler","serviceAccount":"kubesphere","serviceAccountName":"kubesphere","terminationGracePeriodSeconds":30,"volumes":[{"configMap":{"defaultMode":420,"name":"kubesphere-config"},"name":"kubesphere-config"}]}}}}
    revisions: '{"1":{"status":"failed","reasons":["BackoffLimitExceeded"],"messages":["Job
      has reached the specified backoff limit"],"desire":1,"failed":1,"uid":"555e1e9a-f541-4738-97c5-f627ade9eaf2","start-time":"2020-09-01T10:10:44+08:00","completion-time":"0001-01-01T00:00:00Z"}}'
  creationTimestamp: "2020-09-01T02:10:44Z"
  labels:
    app: ks-devops-migration
    job: ks-devops-migration
    version: v3.0.0
  name: ks-devops-migration
  namespace: kubesphere-system
  resourceVersion: "41957538"
  selfLink: /apis/batch/v1/namespaces/kubesphere-system/jobs/ks-devops-migration
  uid: 555e1e9a-f541-4738-97c5-f627ade9eaf2
spec:
  backoffLimit: 6
  completions: 1
  parallelism: 1
  selector:
    matchLabels:
      controller-uid: 555e1e9a-f541-4738-97c5-f627ade9eaf2
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: ks-devops-migration
        controller-uid: 555e1e9a-f541-4738-97c5-f627ade9eaf2
        job: ks-devops-migration
        job-name: ks-devops-migration
        version: v3.0.0
      name: ks-devops-migration
    spec:
      containers:
      - command:
        - ks-upgrade
        image: kubesphere/ks-devops:flyway-v3.0.0
        imagePullPolicy: Always
        name: ks-devops-upgrade
        resources: {}
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /etc/kubesphere/
          name: kubesphere-config
      dnsPolicy: ClusterFirst
      initContainers:
      - command:
        - sh
        - -c
        - until wget --spider http://ks-jenkins.kubesphere-devops-system.svc/login;
          do echo "waiting for myservice"; sleep 2; done;
        image: alpine:3.10.4
        imagePullPolicy: IfNotPresent
        name: wait-jenkins
        resources: {}
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
      - command:
        - sh
        - -c
        - until nc -z mysql.kubesphere-system.svc 3306; do echo "waiting for mysql";
          sleep 2; done;
        image: alpine:3.10.4
        imagePullPolicy: IfNotPresent
        name: wait-mysql
        resources: {}
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
      - command:
        - sh
        - -c
        - until nc -z ks-apiserver.kubesphere-system.svc 80; do echo "waiting for
          apiserver"; sleep 2; done;
        image: alpine:3.10.4
        imagePullPolicy: IfNotPresent
        name: wait-apiserver
        resources: {}
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
      restartPolicy: OnFailure
      schedulerName: default-scheduler
      securityContext: {}
      serviceAccount: kubesphere
      serviceAccountName: kubesphere
      terminationGracePeriodSeconds: 30
      volumes:
      - configMap:
          defaultMode: 420
          name: kubesphere-config
        name: kubesphere-config
status:
  conditions:
  - lastProbeTime: "2020-09-01T02:19:03Z"
    lastTransitionTime: "2020-09-01T02:19:03Z"
    message: Job has reached the specified backoff limit
    reason: BackoffLimitExceeded
    status: "True"
    type: Failed
  failed: 1
  startTime: "2020-09-01T02:10:44Z"

hqq_get
kubectl -n kubesphere-system get job ks-devops-migration -o json | jq 'del(.spec.selector)' | jq 'del(.spec.template.metadata.labels)' | kubectl replace --force -f -

hqq_get
粘贴下面的命令执行

kubectl -n kubesphere-system get job ks-devops-migration -o json | jq 'del(.spec.selector)' | jq 'del(.spec.template.metadata.labels)' | kubectl replace --force -f -

如果有报错,再看 ks-devops-migration job的日志。

apiVersion: v1
data:
  kubesphere.yaml: |
    authentication:
      authenticateRateLimiterMaxTries: 10
      authenticateRateLimiterDuration: 10m0s
      loginHistoryRetentionPeriod: 168h
      maximumClockSkew: 10s
      multipleLogin: True
      kubectlImage: kubesphere/kubectl:v1.0.0
      jwtSecret: "9pYl8p3yVj5xT5Zq6AgdaYho6Ftghyde"
    ldap:
      host: openldap.kubesphere-system.svc:389
      managerDN: cn=admin,dc=kubesphere,dc=io
      managerPassword: admin
      userSearchBase: ou=Users,dc=kubesphere,dc=io
      groupSearchBase: ou=Groups,dc=kubesphere,dc=io
    redis:
      host: redis.kubesphere-system.svc
      port: 6379
      password: ""
      db: 0
    multicluster:
      enable: true
      agentImage: kubesphere/tower:v0.1.0
      proxyPublishService: tower.kubesphere-system.svc
    monitoring:
      endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090
kind: ConfigMap
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","data":{"kubesphere.yaml":"authentication:\n  authenticateRateLimiterMaxTries: 10\n  authenticateRateLimiterDuration: 10m0s\n  loginHistoryRetentionPeriod: 168h\n  maximumClockSkew: 10s\n  multipleLogin: True\n  kubectlImage: kubesphere/kubectl:v1.0.0\n  jwtSecret: \"9pYl8p3yVj5xT5Zq6AgdaYho6Ftghyde\"\nldap:\n  host: openldap.kubesphere-system.svc:389\n  managerDN: cn=admin,dc=kubesphere,dc=io\n  managerPassword: admin\n  userSearchBase: ou=Users,dc=kubesphere,dc=io\n  groupSearchBase: ou=Groups,dc=kubesphere,dc=io\nredis:\n  host: redis.kubesphere-system.svc\n  port: 6379\n  password: \"\"\n  db: 0\nmulticluster:\n  enable: true\n  agentImage: kubesphere/tower:v0.1.0\n  proxyPublishService: tower.kubesphere-system.svc\nmonitoring:\n  endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090\n"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"kubesphere-config","namespace":"kubesphere-system"}}
  creationTimestamp: "2020-05-22T11:16:52Z"
  name: kubesphere-config
  namespace: kubesphere-system
  resourceVersion: "41968045"
  selfLink: /api/v1/namespaces/kubesphere-system/configmaps/kubesphere-config
  uid: dd380454-9c32-4bed-ad20-58f700d21146

    hqq_get

        s3:
          endpoint: http://minio.kubesphere-system.svc:9000
          region: us-east-1
          disableSSL: true
          forcePathStyle: true
          accessKeyID: openpitrixminioaccesskey
          secretAccessKey: openpitrixminiosecretkey
          bucket: s2i-binaries

    看一下,有没有 minio.kubesphere-system.svc 这个服务,如果有,加上上面的配置到里面,然后重启 job

    这个 job 只是迁移数据,找不到 DevOps 工程选项是DevOps 没有开启,请参考文档开启。

    devops已经开启,但是在页面上还是找不到devops工程