[root@k8s-master deploy]# kubectl api-resources
NAME SHORTNAMES APIGROUP NAMESPACED KIND
bindings true Binding
componentstatuses cs false ComponentStatus
configmaps cm true ConfigMap
endpoints ep true Endpoints
events ev true Event
limitranges limits true LimitRange
namespaces ns false Namespace
nodes no false Node
persistentvolumeclaims pvc true PersistentVolumeClaim
persistentvolumes pv false PersistentVolume
pods po true Pod
podtemplates true PodTemplate
replicationcontrollers rc true ReplicationController
resourcequotas quota true ResourceQuota
secrets true Secret
serviceaccounts sa true ServiceAccount
services svc true Service
mutatingwebhookconfigurations admissionregistration.k8s.io false MutatingWebhookConfiguration
validatingwebhookconfigurations admissionregistration.k8s.io false ValidatingWebhookConfiguration
customresourcedefinitions crd,crds apiextensions.k8s.io false CustomResourceDefinition
apiservices apiregistration.k8s.io false APIService
applications app.k8s.io true Application
controllerrevisions apps true ControllerRevision
daemonsets ds apps true DaemonSet
deployments deploy apps true Deployment
replicasets rs apps true ReplicaSet
statefulsets sts apps true StatefulSet
auditsinks auditregistration.k8s.io false AuditSink
meshpolicies authentication.istio.io false MeshPolicy
policies authentication.istio.io true Policy
tokenreviews authentication.k8s.io false TokenReview
localsubjectaccessreviews authorization.k8s.io true LocalSubjectAccessReview
selfsubjectaccessreviews authorization.k8s.io false SelfSubjectAccessReview
selfsubjectrulesreviews authorization.k8s.io false SelfSubjectRulesReview
subjectaccessreviews authorization.k8s.io false SubjectAccessReview
horizontalpodautoscalers hpa autoscaling true HorizontalPodAutoscaler
cronjobs cj batch true CronJob
jobs batch true Job
certificatesigningrequests csr certificates.k8s.io false CertificateSigningRequest
adapters config.istio.io true adapter
apikeys config.istio.io true apikey
attributemanifests config.istio.io true attributemanifest
authorizations config.istio.io true authorization
bypasses config.istio.io true bypass
checknothings config.istio.io true checknothing
circonuses config.istio.io true circonus
cloudwatches config.istio.io true cloudwatch
deniers config.istio.io true denier
dogstatsds config.istio.io true dogstatsd
edges config.istio.io true edge
fluentds config.istio.io true fluentd
handlers config.istio.io true handler
httpapispecbindings config.istio.io true HTTPAPISpecBinding
httpapispecs config.istio.io true HTTPAPISpec
instances config.istio.io true instance
kubernetesenvs config.istio.io true kubernetesenv
kuberneteses config.istio.io true kubernetes
listcheckers config.istio.io true listchecker
listentries config.istio.io true listentry
logentries config.istio.io true logentry
memquotas config.istio.io true memquota
metrics config.istio.io true metric
noops config.istio.io true noop
opas config.istio.io true opa
prometheuses config.istio.io true prometheus
quotas config.istio.io true quota
quotaspecbindings config.istio.io true QuotaSpecBinding
quotaspecs config.istio.io true QuotaSpec
rbacs config.istio.io true rbac
redisquotas config.istio.io true redisquota
reportnothings config.istio.io true reportnothing
rules config.istio.io true rule
signalfxs config.istio.io true signalfx
solarwindses config.istio.io true solarwinds
stackdrivers config.istio.io true stackdriver
statsds config.istio.io true statsd
stdios config.istio.io true stdio
templates config.istio.io true template
tracespans config.istio.io true tracespan
zipkins config.istio.io true zipkin
leases coordination.k8s.io true Lease
s2ibinaries devops.kubesphere.io true S2iBinary
s2ibuilders s2ib devops.kubesphere.io true S2iBuilder
s2ibuildertemplates s2ibt devops.kubesphere.io false S2iBuilderTemplate
s2iruns s2ir devops.kubesphere.io true S2iRun
events ev events.k8s.io true Event
daemonsets ds extensions true DaemonSet
deployments deploy extensions true Deployment
ingresses ing extensions true Ingress
networkpolicies netpol extensions true NetworkPolicy
podsecuritypolicies psp extensions false PodSecurityPolicy
replicasets rs extensions true ReplicaSet
fluentbits logging.kubesphere.io true FluentBit
alertmanagers monitoring.coreos.com true Alertmanager
prometheuses monitoring.coreos.com true Prometheus
prometheusrules monitoring.coreos.com true PrometheusRule
servicemonitors monitoring.coreos.com true ServiceMonitor
destinationrules dr networking.istio.io true DestinationRule
envoyfilters networking.istio.io true EnvoyFilter
gateways gw networking.istio.io true Gateway
serviceentries se networking.istio.io true ServiceEntry
sidecars networking.istio.io true Sidecar
virtualservices vs networking.istio.io true VirtualService
ingresses ing networking.k8s.io true Ingress
networkpolicies netpol networking.k8s.io true NetworkPolicy
runtimeclasses node.k8s.io false RuntimeClass
poddisruptionbudgets pdb policy true PodDisruptionBudget
podsecuritypolicies psp policy false PodSecurityPolicy
clusterrolebindings rbac.authorization.k8s.io false ClusterRoleBinding
clusterroles rbac.authorization.k8s.io false ClusterRole
rolebindings rbac.authorization.k8s.io true RoleBinding
roles rbac.authorization.k8s.io true Role
clusterrbacconfigs rbac.istio.io false ClusterRbacConfig
rbacconfigs rbac.istio.io true RbacConfig
servicerolebindings rbac.istio.io true ServiceRoleBinding
serviceroles rbac.istio.io true ServiceRole
priorityclasses pc scheduling.k8s.io false PriorityClass
servicepolicies servicemesh.kubesphere.io true ServicePolicy
strategies servicemesh.kubesphere.io true Strategy
podpresets settings.k8s.io true PodPreset
csidrivers storage.k8s.io false CSIDriver
csinodes storage.k8s.io false CSINode
storageclasses sc storage.k8s.io false StorageClass
volumeattachments storage.k8s.io false VolumeAttachment
workspaces tenant.kubesphere.io false Workspace
[root@k8s-master deploy]# 2.1安装失败，log如下，
[root@k8s-master deploy]# kubectl explain jobs
KIND: Job
VERSION: batch/v1
DESCRIPTION:
Job represents the configuration of a single job.
FIELDS:
apiVersion <string>
APIVersion defines the versioned schema of this representation of an
object. Servers should convert recognized schemas to the latest internal
value, and may reject unrecognized values. More info:
https://git.k8s.io/community/contributors/devel/api-conventions.md#resources
kind <string>
Kind is a string value representing the REST resource this object
represents. Servers may infer this from the endpoint the client submits
requests to. Cannot be updated. In CamelCase. More info:
https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds
metadata <Object>
Standard object's metadata. More info:
https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata
spec <Object>
Specification of the desired behavior of a job. More info:
https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status
status <Object>
Current status of a job. More info:
https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status
[root@k8s-master deploy]# 
runzexiaK零S
https://github.com/helm/helm/issues/6894
是 helm v2.16.0 的问题。尝试安装一个其他版本的helm，再继续安装
- runzexiaK零S
koy1619 https://github.com/helm/helm/issues/6894#issuecomment-551903514
参考issue 2.14.3/2.15.2 是没问题的
runzexia
helm应该没问题了，这边还是过不了
TASK [common : Kubesphere | Creating manifests] ********************************
changed: [localhost] => (item={u'name': u'custom-values-minio', u'file': u'custom-values-minio.yaml'})
TASK [common : Kubesphere | Deploy minio] **************************************
fatal: [localhost]: FAILED! => {"changed": true, "cmd": "/usr/local/bin/helm upgrade --install ks-minio /etc/kubesphere/minio-ha -f /etc/kubesphere/custom-values-minio.yaml --set fullnameOverride=minio --namespace kubesphere-system --wait --timeout 1800\n", "delta": "0:00:00.213862", "end": "2019-11-12 07:51:56.429823", "msg": "non-zero return code", "rc": 1, "start": "2019-11-12 07:51:56.215961", "stderr": "Error: UPGRADE FAILED: \"ks-minio\" has no deployed releases", "stderr_lines": ["Error: UPGRADE FAILED: \"ks-minio\" has no deployed releases"], "stdout": "", "stdout_lines": []}
...ignoring
TASK [common : debug] **********************************************************
ok: [localhost] => {
"msg": [
"1. check the storage configuration and storage server",
"2. execute 'helm del --purge ks-minio'",
"3. Restart the installer pod in kubesphere-system namespace"
]
}
TASK [common : fail] ***********************************************************
fatal: [localhost]: FAILED! => {"changed": false, "msg": "It is suggested to refer to the above methods for troubleshooting problems ."}
PLAY RECAP *********************************************************************
localhost : ok=24 changed=18 unreachable=0 failed=1 skipped=66 rescued=0 ignored=1
^C
[root@k8s-master deploy]# helm version
Client: &version.Version{SemVer:"v2.15.2", GitCommit:"8dce272473e5f2a7bf58ce79bb5c3691db54c96b", GitTreeState:"clean"}
Server: &version.Version{SemVer:"v2.15.2", GitCommit:"8dce272473e5f2a7bf58ce79bb5c3691db54c96b", GitTreeState:"clean"}
[root@k8s-master deploy]# 13 天 后
HhetaoK零S
koy1619 大佬，我的报错跟你一样，用kubectl get pod ks-installer-7987c659d6-vwl2t -n kubesphere-system -o yaml | kubectl replace –force -f - 这条命令重启installer后还是不行。
# helm version
Client: &version.Version{SemVer:"v2.14.3", GitCommit:"0e7f3b6637f7af8fcfddb3d2941fcc7cbebb0085", GitTreeState:"clean"}
Server: &version.Version{SemVer:"v2.14.3", GitCommit:"0e7f3b6637f7af8fcfddb3d2941fcc7cbebb0085", GitTreeState:"clean"}
系统版本centos 7.6，存储用的nfs
9 天 后
rysinalK零S
hetao runzexia 我的也是，centos 7.5 nfs存储，然后helm为2.14.3，安装2.1版本ks；试了两次都报错,卡的有点久了
helm del --purge ks-minio
kubectl get pod ks-installer-7987c659d6-sg8pv -n kubesphere-system -o yaml | kubectl replace --force -f -TASK [common : Kubesphere | Deploy minio] **************************************
fatal: [localhost]: FAILED! => {"changed": true, "cmd": "/usr/local/bin/helm upgrade --install ks-minio /etc/kubesphere/minio-ha -f /etc/kubesphere/custom-values-minio.yaml --set fullnameOverride=minio --namespace kubesphere-system --wait --timeout 1800\n", "delta": "0:30:01.145146", "end": "2019-12-04 02:43:29.052773", "msg": "non-zero return code", "rc": 1, "start": "2019-12-04 02:13:27.907627", "stderr": "Error: release ks-minio failed: timed out waiting for the condition", "stderr_lines": ["Error: release ks-minio failed: timed out waiting for the condition"], "stdout": "Release \"ks-minio\" does not exist. Installing it now.", "stdout_lines": ["Release \"ks-minio\" doesnot exist. Installing it now."]}
...ignoring
TASK [common : debug] **********************************************************
ok: [localhost] => {
"msg": [
"1. check the storage configuration and storage server",
"2. execute 'helm del --purge ks-minio'",
"3. Restart the installer pod in kubesphere-system namespace"
]
}
TASK [common : fail] ***********************************************************
fatal: [localhost]: FAILED! => {"changed": false, "msg": "It is suggested to refer to the above methods for troubleshooting problems ."}
PLAY RECAP *********************************************************************
localhost : ok=24 changed=18 unreachable=0 failed=1 skipped=66 rescued=0 ignored=1- runzexiaK零S
- 首先删除 minio
helm del --purge ks-minio - 进入installer pod bash
kubectl exec -n kubesphere-system ks-installer-xx -it sh - 执行minio 安装命令
/usr/local/bin/helm upgrade --install ks-minio /etc/kubesphere/minio-ha -f /etc/kubesphere/custom-values-minio.yaml --set fullnameOverride=minio --namespace kubesphere-system --wait --timeout 1800 - 观察minio状态，排查为何出错
- rysinalK零S
[root@master ~]# helm del --purge ks-minio
release "ks-minio" deleted
[root@master ~]# kubectl get pods -n kubesphere-system
NAME READY STATUS RESTARTS AGE
ks-installer-7987c659d6-rp9px 1/1 Running 0 40m
minio-make-bucket-job-rl9bw 1/1 Running 1 38m
openldap-0 1/1 Running 0 14h
redis-5d4844b947-gzbsh 1/1 Running 0 14h
[root@master ~]# kubectl exec -n kubesphere-system ks-installer-7987c659d6-rp9px -it sh
/ # /usr/local/bin/helm upgrade --install ks-minio /etc/kubesphere/minio-ha -f /etc/kubesphere/custom-values-minio.yaml --set fullnameOverride=minio --namespace kubesphere
-system --wait --timeout 1800
Release "ks-minio" does not exist. Installing it now.
Error: jobs.batch "minio-make-bucket-job" already exists
/ #- rysinalK零S
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled 50m default-scheduler Successfully assigned kubesphere-system/minio-make-bucket-job-rl9bw to node2
Normal Pulling 50m kubelet, node2 Pulling image "minio/mc:RELEASE.2019-08-07T23-14-43Z"
Normal Pulled 49m kubelet, node2 Successfully pulled image "minio/mc:RELEASE.2019-08-07T23-14-43Z"
Normal Created 11m (x3 over 49m) kubelet, node2 Created container minio-mc
Normal Started 11m (x3 over 49m) kubelet, node2 Started container minio-mc
Normal Pulled 11m (x2 over 30m) kubelet, node2 Container image "minio/mc:RELEASE.2019-08-07T23-14-43Z" already present on machine
Warning FailedMount 1s (x12 over 11m) kubelet, node2 MountVolume.SetUp failed for volume "minio-configuration" : [configmap "minio" not found, secret "minio" notfound]- rysinalK零S
- 已编辑
看日志是存储问题，现在在每个节点安装了yum install -y nfs-utils
runzexia 大佬，现在报错为新的
TASK [common : Kubesphere | Deploy minio] **************************************
fatal: [localhost]: FAILED! => {"changed": true, "cmd": "/usr/local/bin/helm upgrade --install ks-minio /etc/kubesphere/minio-ha -f /etc/kubesphere/custom-values-minio.yaml --set fullnameOverride=minio --namespace kubesphere-system --wait --timeout 1800\n", "delta": "0:00:23.077751", "end": "2019-12-04 05:10:12.887874", "msg": "non-zero return code", "rc": 1, "start": "2019-12-04 05:09:49.810123", "stderr": "Error: jobs.batch \"minio-make-bucket-job\" already exists", "stderr_lines": ["Error: jobs.batch \"minio-make-bucket-job\" already exists"], "stdout": "Release \"ks-minio\" does not exist. Installing it now.", "stdout_lines": ["Release \"ks-minio\" does not exist. Installing it now."]}删除了minio 的deveop和pod，包括job的pod，然后重新安装也是这样报错
kubectl delete deployment minio -n kubesphere-system
kubectl delete pod minio-8cd46c8d9-287gw -n kubesphere-system
kubectl delete pod minio-make-bucket-job-mvtzz -n kubesphere-system
helm del --purge ks-minio
kubectl get pod ks-installer-7987c659d6-ttprq -n kubesphere-system -o yaml | kubectl replace --force -f -