3.4.1外接ES，启动报错

DehaoCheng · 2023年12月26日

你是外置es？https://kubesphere.io/zh/blogs/kubesphere-elasticsearch/ 参考一下这个。另外你把kubesphere-config这个configmap以及clusterconfigration发一下，敏感信息打码

DehaoCheng · 2023年12月26日

至于fluentbit的警告，你可以将output的buffer改大一点

DehaoCheng · 2023年12月26日

调整这些 https://github.com/kubesphere/ks-installer/blob/master/roles/ks-logging/templates/custom-output-elasticsearch-logging.yaml.j2#L31

Ssword913 · 2023年12月26日

DehaoCheng

ClusterConfiguration的配置如下：

apiVersion: installer.kubesphere.io/v1alpha1

kind: ClusterConfiguration

metadata:

annotations:

kubectl.kubernetes.io/last-applied-configuration: >

{"apiVersion":"installer.kubesphere.io/v1alpha1","kind":"ClusterConfiguration","metadata":{"annotations":{},"labels":{"version":"v3.4.1"},"name":"ks-installer","namespace":"kubesphere-system"},"spec":{"alerting":{"enabled":true},"auditing":{"enabled":true},"authentication":{"jwtSecret":""},"common":{"core":{"console":{"enableMultiLogin":true,"port":30880,"type":"NodePort"}},"es":{"basicAuth":{"enabled":false,"password":"","username":""},"elkPrefix":"dev","enabled":true,"externalElasticsearchHost":"10.203.98.104","externalElasticsearchPort":"9200","logMaxAge":7},"gpu":{"kinds":[{"default":true,"resourceName":"nvidia.com/gpu","resourceType":"GPU"}]},"minio":{"volumeSize":"20Gi"},"monitoring":{"GPUMonitoring":{"enabled":false},"endpoint":"http://prometheus-operated.kubesphere-monitoring-system.svc:9090"},"openldap":{"enabled":false,"volumeSize":"2Gi"},"opensearch":{"basicAuth":{"enabled":true,"password":"admin","username":"admin"},"dashboard":{"enabled":false},"enabled":false,"externalOpensearchHost":"","externalOpensearchPort":"","logMaxAge":7,"opensearchPrefix":"whizard"},"redis":{"enableHA":false,"enabled":false,"volumeSize":"2Gi"}},"devops":{"enabled":true,"jenkinsCpuLim":1,"jenkinsCpuReq":0.5,"jenkinsMemoryLim":"4Gi","jenkinsMemoryReq":"4Gi","jenkinsVolumeSize":"16Gi"},"edgeruntime":{"enabled":false,"kubeedge":{"cloudCore":{"cloudHub":{"advertiseAddress":[""]},"service":{"cloudhubHttpsNodePort":"30002","cloudhubNodePort":"30000","cloudhubQuicNodePort":"30001","cloudstreamNodePort":"30003","tunnelNodePort":"30004"}},"enabled":false,"iptables-manager":{"enabled":true,"mode":"external"}}},"etcd":{"endpointIps":"10.203.99.101,10.203.99.102,10.203.99.103","monitoring":true,"port":2379,"tlsEnable":true},"events":{"enabled":true,"ruler":{"enabled":true,"replicas":2}},"gatekeeper":{"enabled":false},"logging":{"containerruntime":"containerd","enabled":true,"logsidecar":{"enabled":true,"replicas":2}},"metrics_server":{"enabled":true},"monitoring":{"gpu":{"nvidia_dcgm_exporter":{"enabled":false}},"node_exporter":{"port":9100},"storageClass":""},"multicluster":{"clusterRole":"none"},"network":{"ippool":{"type":"calico"},"networkpolicy":{"enabled":true},"topology":{"type":"weave-scope"}},"openpitrix":{"store":{"enabled":false}},"persistence":{"storageClass":""},"servicemesh":{"enabled":true,"istio":{"components":{"cni":{"enabled":false},"ingressGateways":[{"enabled":false,"name":"istio-ingressgateway"}]}}},"terminal":{"timeout":600}}}

labels:

version: v3.4.1

name: ks-installer

namespace: kubesphere-system

spec:

alerting:

enabled: true

auditing:

enabled: true

authentication:

jwtSecret: ''

common:

core:

console:

enableMultiLogin: true

port: 30880

type: NodePort

es:

basicAuth:

enabled: false

password: ''

username: ''

elkPrefix: dev

enabled: true

externalElasticsearchHost: 10.203.98.104

externalElasticsearchPort: '9200'

logMaxAge: 7

gpu:

kinds:

- default: true

resourceName: nvidia.com/gpu

resourceType: GPU

minio:

volumeSize: 20Gi

monitoring:

GPUMonitoring:

enabled: false

endpoint: 'http://prometheus-operated.kubesphere-monitoring-system.svc:9090'

openldap:

enabled: false

volumeSize: 2Gi

opensearch:

basicAuth:

enabled: true

password: admin

username: admin

dashboard:

enabled: false

externalOpensearchHost: ''

externalOpensearchPort: ''

logMaxAge: 7

opensearchPrefix: whizard

redis:

enableHA: false

enabled: false

volumeSize: 2Gi

devops:

enabled: true

jenkinsCpuLim: 1

jenkinsCpuReq: 0.5

jenkinsMemoryLim: 4Gi

jenkinsMemoryReq: 4Gi

jenkinsVolumeSize: 16Gi

edgeruntime:

enabled: false

kubeedge:

cloudCore:

cloudHub:

advertiseAddress:

- ''

service:

cloudhubHttpsNodePort: '30002'

cloudhubNodePort: '30000'

cloudhubQuicNodePort: '30001'

cloudstreamNodePort: '30003'

tunnelNodePort: '30004'

enabled: false

iptables-manager:

enabled: true

mode: external

etcd:

endpointIps: '10.203.99.101,10.203.99.102,10.203.99.103'

monitoring: true

port: 2379

tlsEnable: true

events:

enabled: true

ruler:

enabled: true

replicas: 2

gatekeeper:

enabled: false

logging:

containerruntime: containerd

enabled: true

logsidecar:

enabled: true

replicas: 2

metrics_server:

enabled: true

monitoring:

gpu:

nvidia_dcgm_exporter:

enabled: false

node_exporter:

port: 9100

storageClass: ''

multicluster:

clusterRole: none

network:

ippool:

type: calico

networkpolicy:

enabled: true

topology:

type: weave-scope

openpitrix:

store:

enabled: false

persistence:

storageClass: ''

servicemesh:

enabled: true

istio:

components:

cni:

enabled: false

ingressGateways:

- enabled: false

name: istio-ingressgateway

terminal:

timeout: 600

status:

alerting:

enabledTime: '2023-12-26T14:55:05CST'

status: enabled

auditing:

enabledTime: '2023-12-26T14:42:15CST'

status: enabled

clusterId: 9563212c-01b3-48a0-8aec-74b230735b61-1703573825

core:

enabledTime: '2023-12-26T14:41:44CST'

status: enabled

version: v3.4.1

events:

enabledTime: '2023-12-26T14:42:19CST'

ruler:

enabledTime: '2023-12-26T14:42:20CST'

status: enabled

fluentbit:

enabledTime: '2023-12-26T14:40:31CST'

status: enabled

logging:

enabledTime: '2023-12-26T14:42:20CST'

status: enabled

metricsServer:

enabledTime: '2023-12-26T14:38:28CST'

status: enabled

minio:

enabledTime: '2023-12-26T14:40:18CST'

status: enabled

monitoring:

enabledTime: '2023-12-26T14:54:59CST'

status: enabled

network:

topology:

enabledTime: '2023-12-26T14:41:58CST'

status: enabled

openldap:

enabledTime: '2023-12-26T14:38:59CST'

status: enabled

servicemesh:

enabledTime: '2023-12-26T14:43:12CST'

status: enabled

Ssword913 · 2023年12月26日

DehaoCheng

kubesphere-v3.4.1.yaml的配置如下

apiVersion: kubekey.kubesphere.io/v1alpha2

kind: Cluster

metadata:

name: sample

spec:

hosts:

- {name: k8s-master01, address: 10.203.99.101, internalAddress: 10.203.99.101, user: root, password: "yoening@123"}

- {name: k8s-master02, address: 10.203.99.102, internalAddress: 10.203.99.102, user: root, password: "yoening@123"}

- {name: k8s-master03, address: 10.203.99.103, internalAddress: 10.203.99.103, user: root, password: "yoening@123"}

- {name: k8s-node01, address: 10.203.99.104, internalAddress: 10.203.99.104, user: root, password: "yoening@123"}

- {name: k8s-node02, address: 10.203.99.105, internalAddress: 10.203.99.105, user: root, password: "yoening@123"}

- {name: k8s-node03, address: 10.203.99.106, internalAddress: 10.203.99.106, user: root, password: "yoening@123"}

roleGroups:

etcd:

- k8s-master01

- k8s-master02

- k8s-master03

control-plane:

- k8s-master01

- k8s-master02

- k8s-master03

worker:

- k8s-node01

- k8s-node02

- k8s-node03

controlPlaneEndpoint:

## Internal loadbalancer for apiservers

## internalLoadbalancer: haproxy

domain: lb.kubesphere.local

address: 10.203.99.107

port: 16443

kubernetes:

version: v1.26.5

clusterName: cluster.local

autoRenewCerts: true

containerManager: containerd

etcd:

type: kubekey

network:

plugin: calico

kubePodsCIDR: 172.17.64.0/18

kubeServiceCIDR: 172.17.0.0/18

## multus support. https://github.com/k8snetworkplumbingwg/multus-cni

multusCNI:

enabled: false

registry:

privateRegistry: ""

namespaceOverride: ""

registryMirrors: []

insecureRegistries: []

addons: []

---

apiVersion: installer.kubesphere.io/v1alpha1

kind: ClusterConfiguration

metadata:

name: ks-installer

namespace: kubesphere-system

labels:

version: v3.4.1

spec:

persistence:

storageClass: ""

authentication:

jwtSecret: ""

local_registry: ""

# dev_tag: ""

etcd:

monitoring: true

endpointIps: localhost

port: 2379

tlsEnable: true

common:

core:

console:

enableMultiLogin: true

port: 30880

type: NodePort

# apiserver:

# resources: {}

# controllerManager:

# resources: {}

redis:

enabled: false

enableHA: false

volumeSize: 2Gi

openldap:

enabled: false

volumeSize: 2Gi

minio:

volumeSize: 20Gi

monitoring:

# type: external

endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090

GPUMonitoring:

enabled: false

gpu:

kinds:

- resourceName: "nvidia.com/gpu"

resourceType: "GPU"

default: true

es:

# master:

# volumeSize: 4Gi

# replicas: 1

# resources: {}

# data:

# volumeSize: 20Gi

# replicas: 1

# resources: {}

enabled: true

logMaxAge: 7

elkPrefix: dev

basicAuth:

enabled: false

username: ""

password: ""

externalElasticsearchHost: "10.203.98.104"

externalElasticsearchPort: "9200"

opensearch:

# master:

# volumeSize: 4Gi

# replicas: 1

# resources: {}

# data:

# volumeSize: 20Gi

# replicas: 1

# resources: {}

enabled: false

logMaxAge: 7

opensearchPrefix: whizard

basicAuth:

enabled: true

username: "admin"

password: "admin"

externalOpensearchHost: ""

externalOpensearchPort: ""

dashboard:

enabled: false

alerting:

enabled: true

# thanosruler:

# replicas: 1

# resources: {}

auditing:

enabled: true

# operator:

# resources: {}

# webhook:

# resources: {}

devops:

enabled: true

jenkinsCpuReq: 0.5

jenkinsCpuLim: 1

jenkinsMemoryReq: 4Gi

jenkinsMemoryLim: 4Gi

jenkinsVolumeSize: 16Gi

events:

enabled: true

# operator:

# resources: {}

# exporter:

# resources: {}

ruler:

enabled: true

replicas: 2

# resources: {}

logging:

enabled: true

containerruntime: containerd

logsidecar:

enabled: true

replicas: 2

# resources: {}

metrics_server:

enabled: true

monitoring:

storageClass: ""

node_exporter:

port: 9100

# resources: {}

# kube_rbac_proxy:

# resources: {}

# kube_state_metrics:

# resources: {}

# prometheus:

# replicas: 1

# volumeSize: 20Gi

# resources: {}

# operator:

# resources: {}

# alertmanager:

# replicas: 1

# resources: {}

# notification_manager:

# resources: {}

# operator:

# resources: {}

# proxy:

# resources: {}

gpu:

nvidia_dcgm_exporter:

enabled: false

# resources: {}

multicluster:

clusterRole: none

network:

networkpolicy:

enabled: true

ippool:

type: calico

topology:

type: weave-scope

openpitrix:

store:

enabled: false

servicemesh:

enabled: true

istio:

components:

ingressGateways:

- name: istio-ingressgateway

enabled: false

cni:

enabled: false

edgeruntime:

enabled: false

kubeedge:

enabled: false

cloudCore:

cloudHub:

advertiseAddress:

- ""

service:

cloudhubNodePort: "30000"

cloudhubQuicNodePort: "30001"

cloudhubHttpsNodePort: "30002"

cloudstreamNodePort: "30003"

tunnelNodePort: "30004"

# resources: {}

# hostNetWork: false

iptables-manager:

enabled: true

mode: "external"

# resources: {}

# edgeService:

# resources: {}

gatekeeper:

enabled: false

# controller_manager:

# resources: {}

# audit:

# resources: {}

terminal:

timeout: 600

Ssword913 · 2023年12月26日

DehaoCheng

config 配置如下：

[root@k8s-master01 kubekey]# kubectl get configmap kubesphere-config -n kubesphere-system -o yaml

apiVersion: v1

data:

kubesphere.yaml: |

authentication:

authenticateRateLimiterMaxTries: 10

authenticateRateLimiterDuration: 10m0s

loginHistoryRetentionPeriod: 168h

maximumClockSkew: 10s

multipleLogin: True

kubectlImage: kubesphere/kubectl:v1.22.0

jwtSecret: "1W9EADsHAmqMJ8szthOeXPiTQ51KfHfh"

oauthOptions:

clients:

- name: kubesphere

secret: kubesphere

redirectURIs:

- '*'

ldap:

host: openldap.kubesphere-system.svc:389

managerDN: cn=admin,dc=kubesphere,dc=io

managerPassword: admin

userSearchBase: ou=Users,dc=kubesphere,dc=io

groupSearchBase: ou=Groups,dc=kubesphere,dc=io

s3:

endpoint: http://minio.kubesphere-system.svc:9000

region: us-east-1

disableSSL: True

forcePathStyle: True

accessKeyID: openpitrixminioaccesskey

secretAccessKey: openpitrixminiosecretkey

bucket: s2i-binaries

network:

enableNetworkPolicy: true

ippoolType: calico

weaveScopeHost: weave-scope-app.weave

devops:

host: http://devops-jenkins.kubesphere-devops-system.svc/

username: admin

password: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImFkbWluQGt1YmVzcGhlcmUuaW8iLCJ1c2VybmFtZSI6ImFkbWluIiwidG9rZW5fdHlwZSI6InN0YXRpY190b2tlbiJ9.AVHKZgFyYk_DQAV2rgPybZqnXJBs5EyVgQuNrg5SxWI

maxConnections: 100

endpoint: http://devops-apiserver.kubesphere-devops-system:9090

servicemesh:

istioPilotHost: http://istiod.istio-system.svc:8080/version

jaegerQueryHost: http://jaeger-query.istio-system.svc:16686

servicemeshPrometheusHost: http://prometheus-k8s.kubesphere-monitoring-system.svc:9090

kialiQueryHost: http://kiali.istio-system:20001

multicluster:

clusterRole: none

monitoring:

endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090

enableGPUMonitoring: false

gpu:

kinds:

- resourceName: nvidia.com/gpu

resourceType: GPU

default: True

notification:

endpoint: http://notification-manager-svc.kubesphere-monitoring-system.svc:19093

logging:

host: http://10.203.98.104:9200

basicAuth: True

username: "admin"

password: "admin"

indexPrefix: ks-whizard-logging

events:

host: http://10.203.98.104:9200

basicAuth: True

username: "admin"

password: "admin"

indexPrefix: ks-whizard-events

auditing:

enable: true

webhookURL: https://kube-auditing-webhook-svc.kubesphere-logging-system.svc:6443/audit/webhook/event

host: http://10.203.98.104:9200

basicAuth: True

username: "admin"

password: "admin"

indexPrefix: ks-whizard-auditing

alerting:

prometheusEndpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090

thanosRulerEndpoint: http://thanos-ruler-operated.kubesphere-monitoring-system.svc:10902

thanosRuleResourceLabels: thanos-ruler=kubesphere,role=alert-rules

terminal:

image: alpine:3.14

timeout: 600

gateway:

watchesPath: /var/helm-charts/watches.yaml

repository: kubesphere/nginx-ingress-controller

tag: v1.3.1

namespace: kubesphere-controls-system

kind: ConfigMap

metadata:

annotations:

kubectl.kubernetes.io/last-applied-configuration: |

{"apiVersion":"v1","data":{"kubesphere.yaml":"authentication:\n authenticateRateLimiterMaxTries: 10\n authenticateRateLimiterDuration: 10m0s\n loginHistoryRetentionPeriod: 168h\n maximumClockSkew: 10s\n multipleLogin: True\n kubectlImage: kubesphere/kubectl:v1.22.0\n jwtSecret: \"1W9EADsHAmqMJ8szthOeXPiTQ51KfHfh\"\n oauthOptions:\n clients:\n - name: kubesphere\n secret: kubesphere\n redirectURIs:\n - '*'\nldap:\n host: openldap.kubesphere-system.svc:389\n managerDN: cn=admin,dc=kubesphere,dc=io\n managerPassword: admin\n userSearchBase: ou=Users,dc=kubesphere,dc=io\n groupSearchBase: ou=Groups,dc=kubesphere,dc=io\ns3:\n endpoint: http://minio.kubesphere-system.svc:9000\n region: us-east-1\n disableSSL: True\n forcePathStyle: True\n accessKeyID: openpitrixminioaccesskey\n secretAccessKey: openpitrixminiosecretkey\n bucket: s2i-binaries\nnetwork:\n enableNetworkPolicy: true\n ippoolType: calico\n weaveScopeHost: weave-scope-app.weave\ndevops:\n host: http://devops-jenkins.kubesphere-devops-system.svc/\n username: admin\n password: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImFkbWluQGt1YmVzcGhlcmUuaW8iLCJ1c2VybmFtZSI6ImFkbWluIiwidG9rZW5fdHlwZSI6InN0YXRpY190b2tlbiJ9.AVHKZgFyYk_DQAV2rgPybZqnXJBs5EyVgQuNrg5SxWI\n maxConnections: 100\n endpoint: http://devops-apiserver.kubesphere-devops-system:9090\nservicemesh:\n istioPilotHost: http://istiod.istio-system.svc:8080/version\n jaegerQueryHost: http://jaeger-query.istio-system.svc:16686\n servicemeshPrometheusHost: http://prometheus-k8s.kubesphere-monitoring-system.svc:9090\n kialiQueryHost: http://kiali.istio-system:20001\nmulticluster:\n clusterRole: none\nmonitoring:\n endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090\n enableGPUMonitoring: false\ngpu:\n kinds:\n - resourceName: nvidia.com/gpu\n resourceType: GPU\n default: True\nnotification:\n endpoint: http://notification-manager-svc.kubesphere-monitoring-system.svc:19093\nlogging:\n host: http://10.203.98.104:9200\n basicAuth: True\n username: \"admin\"\n password: \"admin\"\n indexPrefix: ks-whizard-logging\nevents:\n host: http://10.203.98.104:9200\n basicAuth: True\n username: \"admin\"\n password: \"admin\"\n indexPrefix: ks-whizard-events\nauditing:\n enable: true\n webhookURL: https://kube-auditing-webhook-svc.kubesphere-logging-system.svc:6443/audit/webhook/event\n host: http://10.203.98.104:9200\n basicAuth: True\n username: \"admin\"\n password: \"admin\"\n indexPrefix: ks-whizard-auditing\nalerting:\n prometheusEndpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090\n thanosRulerEndpoint: http://thanos-ruler-operated.kubesphere-monitoring-system.svc:10902\n thanosRuleResourceLabels: thanos-ruler=kubesphere,role=alert-rules\n\n\nterminal:\n image: alpine:3.14\n timeout: 600\ngateway:\n watchesPath: /var/helm-charts/watches.yaml\n repository: kubesphere/nginx-ingress-controller\n tag: v1.3.1\n namespace: kubesphere-controls-system\n"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"kubesphere-config","namespace":"kubesphere-system"}}

creationTimestamp: "2023-12-26T06:55:17Z"

name: kubesphere-config

namespace: kubesphere-system

resourceVersion: "7586"

uid: 0c183d51-ae2e-4e2c-bc71-75878a08b2e9

Ssword913 · 2023年12月26日

DehaoCheng

另外我的外部ES是没有设置账号密码的，可以直接使用9200免密码登录

DehaoCheng · 2023年12月27日

logging:

host: http://10.203.98.104:9200

basicAuth: True

username: "admin"

password: "admin"

indexPrefix: ks-whizard-logging

events:

host: http://10.203.98.104:9200

basicAuth: True

username: "admin"

password: "admin"

indexPrefix: ks-whizard-events

auditing:

enable: true

webhookURL: https://kube-auditing-webhook-svc.kubesphere-logging-system.svc:6443/audit/webhook/event

host: http://10.203.98.104:9200

basicAuth: True

username: "admin"

password: "admin"

indexPrefix: ks-whizard-auditing

你把kubesphere-config中的这些配置删除一下，这里默认引用了opensearch的配置，没有加载进es的配置。改成类似


events:

host: http://xxx:9200

indexPrefix: ks-dev-events

就可以了，其余修改类似，然后重启ks-apiserver。

DehaoCheng · 2023年12月27日

然后看看自定义资源定义里的output，看一下配置，如果有也需要删除相应字段，如果你拿不准，可以发出来看看

Ssword913 · 2023年12月27日

DehaoCheng

也使用 kubectl -n kubesphere-system rollout restart deploy ks-apiserver

重启后多了个es的pod：如下图：

但是ES里面没有新创建索引（我在config里面换了个indexPrefix名字），同样kubesphere也没有日志

盼回复

DehaoCheng · 2023年12月27日

escurator这个pod作用是定时任务清理es索引的，至于kubesphere-config你为什么将indexPrefix设置为ks-devnew-events你外接es里面索引不是ks-dev-events，吗？

DehaoCheng · 2023年12月27日

sword913 设置成ks-dev-events,ks-dev-log保持和你外置es的索引一样，这样才可以查询

Ssword913 · 2023年12月27日

DehaoCheng

我设置成ks-devnew-events是为了跟之前的ks-dev-events做区分，

我现在在config里面重新改成了ks-dev-events。

然后重启apiserver

结果还是没有日志

并且还报错了

同时有一个疑问点是：我部署了一个mysql，我想确定一下操作mysql的哪些东西，可以上传到es并且在kubesphere上面查到日志

DehaoCheng · 2023年12月27日

其他审计、日志的字段类似你也应该修改，你打开事件页面看看报什么错

Ssword913 · 2023年12月27日

DehaoCheng

上面的错忽略，我重新配置了config和output

目前fluent-bit报错如下：

我的外接ES版本如下：

请问是版本不兼容吗？

DehaoCheng · 2023年12月27日

应该不是，你可以调整参数试试，参考这个issue

Ssword913 · 2023年12月27日

DehaoCheng 哪个issue？

3.4.1外接ES，启动报错

Ssword913

Ssword913

Ssword913

DehaoChengK零S

DehaoChengK零S

DehaoChengK零S

Ssword913

Ssword913

Ssword913

Ssword913

DehaoChengK零S

DehaoChengK零S

Ssword913

DehaoChengK零S

DehaoChengK零S

Ssword913

DehaoChengK零S

Ssword913

DehaoChengK零S

Ssword913