使用kubekey离线安装时，在执行到初始化harbor仓库时，有两个harbor相关服务一直不能启动，后续操作无法执行。

loupandna

操作系统信息
虚拟机，Kylin v10 sp3，4C/4G

执行操作
使用kk离线安装，kk版本：v3.0.7

config-sample.yaml

apiVersion: kubekey.kubesphere.io/v1alpha2
kind: Cluster
metadata:
  name: sample
spec:
  hosts:
  - {name: master, address: 192.168.32.122, internalAddress: 192.168.32.122, user: root, password: "Gad23132"}
  - {name: node1, address: 192.168.32.123, internalAddress: 192.168.32.123, user: root, password: "Gad23132"}
  - {name: node2, address: 192.168.32.124, internalAddress: 192.168.32.124, user: root, password: "Gad23132"}
  roleGroups:
    etcd:
    - master *#这里修改为master*
    control-plane: 
    - master *#这里修改为master*
    worker:
    - node1
    - node2
    *# 设置使用 kk 自动部署镜像仓库的节点，一定要注意这个地方必须加上*
    registry:
    - master
  controlPlaneEndpoint:
    *## Internal loadbalancer for apiservers* 
    *# internalLoadbalancer: haproxy*

    domain: lb.kubesphere.local
    address: ""
    port: 6443
  kubernetes:
    version: v1.23.10
    clusterName: cluster.local
    autoRenewCerts: true
    containerManager: docker
  etcd:
    type: kubekey
  network:
    plugin: calico
    kubePodsCIDR: 10.233.64.0/18
    kubeServiceCIDR: 10.233.0.0/18
    *## multus support. https://github.com/k8snetworkplumbingwg/multus-cni*
    multusCNI:
      enabled: false
  registry:
    type: harbor
    auths:
      "dockerhub.kubekey.local":
        username: admin
        password: Harbor12345
        certsPath: "/etc/docker/certs.d/dockerhub.kubekey.local"
    privateRegistry: "dockerhub.kubekey.local"
    namespaceOverride: "kubesphereio"
    registryMirrors: []
    insecureRegistries: []
  addons: []

执行命令 ./kk init registry -f config-sample.yaml -a kubesphere.tar.gz

提示成功

之后执行docker ps查看harbor相关容器，有两个服务一直重启，goharbor/nginx-photon:v2.5.3和goharbor/registry-photon:v2.5.3，就卡在了这个地方，无法继续安装。

nginx服务报错信息：

registry服务报错信息：

docker版本：

Client:

Version:           20.10.8

API version:       1.41

Go version:        go1.16.6

Git commit:        3967b7d

Built:             Fri Jul 30 19:50:40 2021

OS/Arch:           linux/amd64

Context:           default

Experimental:      true

Server: Docker Engine - Community

Engine:

Version:          20.10.8

API version:      1.41 (minimum version 1.12)

Go version:       go1.16.6

Git commit:       75249d8

Built:            Fri Jul 30 19:55:09 2021

OS/Arch:          linux/amd64

Experimental:     false

containerd:

Version:          v1.4.9

GitCommit:        e25210fe30a0a703442421b0f60afac609f950a3

runc:

Version:          1.0.0-rc3

GitCommit:

docker-init:

Version:          0.19.0

GitCommit:        de40ad0

afumu

需要在/opt/harbor目录中把docker-compose.yml 文件替换为以下内容

主要就是给某些容器加上 user: "0:0"，否则容器内部没有权限

version: '2.3'
services:
  log:
    image: goharbor/harbor-log:v2.5.3
    container_name: harbor-log
    restart: always
    cap_drop:
      - ALL
    cap_add:
      - CHOWN
      - DAC_OVERRIDE
      - SETGID
      - SETUID
    volumes:
      - /var/log/harbor/:/var/log/docker/:z
      - type: bind
        source: ./common/config/log/logrotate.conf
        target: /etc/logrotate.d/logrotate.conf
      - type: bind
        source: ./common/config/log/rsyslog_docker.conf
        target: /etc/rsyslog.d/rsyslog_docker.conf
    ports:
      - 127.0.0.1:1514:10514
    networks:
      - harbor
  registry:
    image: goharbor/registry-photon:v2.5.3
    container_name: registry
    restart: always
    user: "0:0"
    volumes:
      - /mnt/registry/registry:/storage:z
      - ./common/config/registry/:/etc/registry/:z
      - type: bind
        source: /mnt/registry/secret/registry/root.crt
        target: /etc/registry/root.crt
      - type: bind
        source: ./common/config/shared/trust-certificates
        target: /harbor_cust_cert
    networks:
      - harbor
    depends_on:
      - log
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://localhost:1514"
        tag: "registry"
  registryctl:
    image: goharbor/harbor-registryctl:v2.5.3
    container_name: registryctl
    env_file:
      - ./common/config/registryctl/env
    restart: always
    user: "0:0"
    volumes:
      - /mnt/registry/registry:/storage:z
      - ./common/config/registry/:/etc/registry/:z
      - type: bind
        source: ./common/config/registryctl/config.yml
        target: /etc/registryctl/config.yml
      - type: bind
        source: ./common/config/shared/trust-certificates
        target: /harbor_cust_cert
    networks:
      - harbor
    depends_on:
      - log
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://localhost:1514"
        tag: "registryctl"
  postgresql:
    image: goharbor/harbor-db:v2.5.3
    container_name: harbor-db
    restart: always
    cap_drop:
      - ALL
    cap_add:
      - CHOWN
      - DAC_OVERRIDE
      - SETGID
      - SETUID
    volumes:
      - /mnt/registry/database:/var/lib/postgresql/data:z
    networks:
      harbor:
      harbor-notary:
        aliases:
          - harbor-db
    env_file:
      - ./common/config/db/env
    depends_on:
      - log
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://localhost:1514"
        tag: "postgresql"
    shm_size: '1gb'
  core:
    image: goharbor/harbor-core:v2.5.3
    container_name: harbor-core
    env_file:
      - ./common/config/core/env
    restart: always
    user: "0:0"
    volumes:
      - /mnt/registry/ca_download/:/etc/core/ca/:z
      - /mnt/registry/:/data/:z
      - ./common/config/core/certificates/:/etc/core/certificates/:z
      - type: bind
        source: ./common/config/core/app.conf
        target: /etc/core/app.conf
      - type: bind
        source: /mnt/registry/secret/core/private_key.pem
        target: /etc/core/private_key.pem
      - type: bind
        source: /mnt/registry/secret/keys/secretkey
        target: /etc/core/key
      - type: bind
        source: ./common/config/shared/trust-certificates
        target: /harbor_cust_cert
    networks:
      harbor:
      harbor-notary:
      harbor-chartmuseum:
        aliases:
          - harbor-core
    depends_on:
      - log
      - registry
      - redis
      - postgresql
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://localhost:1514"
        tag: "core"
  portal:
    image: goharbor/harbor-portal:v2.5.3
    container_name: harbor-portal
    restart: always
    cap_drop:
      - ALL
    cap_add:
      - CHOWN
      - SETGID
      - SETUID
      - NET_BIND_SERVICE
    volumes:
      - type: bind
        source: ./common/config/portal/nginx.conf
        target: /etc/nginx/nginx.conf
    networks:
      - harbor
    depends_on:
      - log
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://localhost:1514"
        tag: "portal"

  jobservice:
    image: goharbor/harbor-jobservice:v2.5.3
    container_name: harbor-jobservice
    env_file:
      - ./common/config/jobservice/env
    restart: always
    cap_drop:
      - ALL
    cap_add:
      - CHOWN
      - SETGID
      - SETUID
    volumes:
      - /mnt/registry/job_logs:/var/log/jobs:z
      - type: bind
        source: ./common/config/jobservice/config.yml
        target: /etc/jobservice/config.yml
      - type: bind
        source: ./common/config/shared/trust-certificates
        target: /harbor_cust_cert
    networks:
      - harbor
    depends_on:
      - core
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://localhost:1514"
        tag: "jobservice"
  redis:
    image: goharbor/redis-photon:v2.5.3
    container_name: redis
    restart: always
    cap_drop:
      - ALL
    cap_add:
      - CHOWN
      - SETGID
      - SETUID
    volumes:
      - /mnt/registry/redis:/var/lib/redis
    networks:
      harbor:
      harbor-chartmuseum:
        aliases:
          - redis
    depends_on:
      - log
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://localhost:1514"
        tag: "redis"
  proxy:
    image: goharbor/nginx-photon:v2.5.3
    container_name: nginx
    restart: always
    user: "0:0"
    volumes:
      - ./common/config/nginx:/etc/nginx:z
      - /mnt/registry/secret/cert:/etc/cert:z
      - type: bind
        source: ./common/config/shared/trust-certificates
        target: /harbor_cust_cert
    networks:
      - harbor
      - harbor-notary
    ports:
      - 80:8080
      - 443:8443
      - 4443:4443
    depends_on:
      - registry
      - core
      - portal
      - log
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://localhost:1514"
        tag: "proxy"
  notary-server:
    image: goharbor/notary-server-photon:v2.5.3
    container_name: notary-server
    restart: always
    user: "0:0"
    networks:
      - notary-sig
      - harbor-notary
    volumes:
      - ./common/config/notary:/etc/notary:z
      - type: bind
        source: /mnt/registry/secret/notary/notary-signer-ca.crt
        target: /etc/notary/notary-signer-ca.crt
      - type: bind
        source: /mnt/registry/secret/registry/root.crt
        target: /etc/notary/root.crt
      - type: bind
        source: ./common/config/shared/trust-certificates
        target: /harbor_cust_cert
    env_file:
      - ./common/config/notary/server_env
    depends_on:
      - postgresql
      - notary-signer
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://localhost:1514"
        tag: "notary-server"
  notary-signer:
    image: goharbor/notary-signer-photon:v2.5.3
    container_name: notary-signer
    restart: always
    user: "0:0"
    networks:
      harbor-notary:
      notary-sig:
        aliases:
          - notarysigner
    volumes:
      - ./common/config/notary:/etc/notary:z
      - type: bind
        source: /mnt/registry/secret/notary/notary-signer.crt
        target: /etc/notary/notary-signer.crt
      - type: bind
        source: /mnt/registry/secret/notary/notary-signer.key
        target: /etc/notary/notary-signer.key
      - type: bind
        source: ./common/config/shared/trust-certificates
        target: /harbor_cust_cert
    env_file:
      - ./common/config/notary/signer_env
    depends_on:
      - log
      - postgresql
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://localhost:1514"
        tag: "notary-signer"
  trivy-adapter:
    container_name: trivy-adapter
    image: goharbor/trivy-adapter-photon:v2.5.3
    restart: always
    cap_drop:
      - ALL
    depends_on:
      - log
      - redis
    networks:
      - harbor
    volumes:
      - type: bind
        source: /mnt/registry/trivy-adapter/trivy
        target: /home/scanner/.cache/trivy
      - type: bind
        source: /mnt/registry/trivy-adapter/reports
        target: /home/scanner/.cache/reports
      - type: bind
        source: ./common/config/shared/trust-certificates
        target: /harbor_cust_cert
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://localhost:1514"
        tag: "trivy-adapter"
    env_file:
      ./common/config/trivy-adapter/env
  chartmuseum:
    container_name: chartmuseum
    image: goharbor/chartmuseum-photon:v2.5.3
    restart: always
    cap_drop:
      - ALL
    cap_add:
      - CHOWN
      - DAC_OVERRIDE
      - SETGID
      - SETUID
    networks:
      - harbor-chartmuseum
    depends_on:
      - log
    volumes:
      - /mnt/registry/chart_storage:/chart_storage:z
      - ./common/config/chartserver:/etc/chartserver:z
      - type: bind
        source: ./common/config/shared/trust-certificates
        target: /harbor_cust_cert
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://localhost:1514"
        tag: "chartmuseum"
    env_file:
      ./common/config/chartserver/env
networks:
  harbor:
    external: false
  harbor-notary:
    external: false
  notary-sig:
    external: false
  harbor-chartmuseum:
    external: false

loupandna

afumu 多谢大佬，确实是容器启动时，权限不够导致相关文件读取挂载不到的问题。

afumu

loupandna 不用客气哈