环境 ks3.0
最近发现一个ks集群的系统日志量明显过高（每半小时大概40W），经排查是kubesphere-logging-system下的 elasticsearch-logging-discovery节点的大量刷报错：

[2020-11-05T02:42:26,308][WARN ][o.e.c.r.a.AllocationService] [elasticsearch-logging-discovery-2] failing shard [failed shard, shard [ks-logstash-log-2020.11.05][1], node[p4qSYifaQy-bjnxM5zLV7Q], relocating [msZhxFHhShWjnNi-VEIjWQ], [P], recovery_source[peer recovery], s[INITIALIZING], a[id=3V_DF_SGRNC0dKniEjFYEA, rId=fGakFS9_RuOsnfv-rG9LYA], expected_shard_size[591323645], message [failed to create shard], failure [IllegalStateException[environment is not locked]; nested: IOException[Stale file handle]; ], markAsStale [true]]

java.lang.IllegalStateException: environment is not locked

at org.elasticsearch.env.NodeEnvironment.assertEnvIsLocked(NodeEnvironment.java:1025) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.env.NodeEnvironment.availableShardPaths(NodeEnvironment.java:840) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.index.shard.ShardPath.loadShardPath(ShardPath.java:120) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.index.IndexService.createShard(IndexService.java:340) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.indices.IndicesService.createShard(IndicesService.java:623) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.indices.IndicesService.createShard(IndicesService.java:158) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.indices.cluster.IndicesClusterStateService.createShard(IndicesClusterStateService.java:597) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.indices.cluster.IndicesClusterStateService.createOrUpdateShards(IndicesClusterStateService.java:573) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:270) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.cluster.service.ClusterApplierService.lambda$callClusterStateAppliers$6(ClusterApplierService.java:484) ~[elasticsearch-6.7.0.jar:6.7.0]

at java.lang.Iterable.forEach(Iterable.java:75) ~[?:?]

at org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:481) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:468) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:419) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:163) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) [elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:252) [elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:215) [elasticsearch-6.7.0.jar:6.7.0]

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]

at java.lang.Thread.run(Thread.java:835) [?:?]

Suppressed: java.lang.IllegalStateException: environment is not locked

at org.elasticsearch.env.NodeEnvironment.assertEnvIsLocked(NodeEnvironment.java:1025) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.env.NodeEnvironment.availableShardPaths(NodeEnvironment.java:840) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.index.shard.ShardPath.deleteLeftoverShardDirectory(ShardPath.java:176) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.index.IndexService.createShard(IndexService.java:344) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.indices.IndicesService.createShard(IndicesService.java:623) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.indices.IndicesService.createShard(IndicesService.java:158) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.indices.cluster.IndicesClusterStateService.createShard(IndicesClusterStateService.java:597) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.indices.cluster.IndicesClusterStateService.createOrUpdateShards(IndicesClusterStateService.java:573) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:270) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.cluster.service.ClusterApplierService.lambda$callClusterStateAppliers$6(ClusterApplierService.java:484) ~[elasticsearch-6.7.0.jar:6.7.0]

at java.lang.Iterable.forEach(Iterable.java:75) ~[?:?]

at org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:481) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:468) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:419) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:163) ~[elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) [elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:252) [elasticsearch-6.7.0.jar:6.7.0]

at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:215) [elasticsearch-6.7.0.jar:6.7.0]

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]

at java.lang.Thread.run(Thread.java:835) [?:?]

Caused by: java.io.IOException: Stale file handle

at sun.nio.ch.FileDispatcherImpl.size0(Native Method) ~[?:?]

at sun.nio.ch.FileDispatcherImpl.size(FileDispatcherImpl.java:90) ~[?:?]

at sun.nio.ch.FileChannelImpl.size(FileChannelImpl.java:383) ~[?:?]

at org.apache.lucene.store.NativeFSLockFactory$NativeFSLock.ensureValid(NativeFSLockFactory.java:182) ~[lucene-core-7.7.0.jar:7.7.0 8c831daf4eb41153c25ddb152501ab5bae3ea3d5 - jimczi - 2019-02-04 23:16:28]

at org.elasticsearch.env.NodeEnvironment.assertEnvIsLocked(NodeEnvironment.java:1022) ~[elasticsearch-6.7.0.jar:6.7.0]

... 20 more

Caused by: java.io.IOException: Stale file handle

at sun.nio.ch.FileDispatcherImpl.size0(Native Method) ~[?:?]

at sun.nio.ch.FileDispatcherImpl.size(FileDispatcherImpl.java:90) ~[?:?]

at sun.nio.ch.FileChannelImpl.size(FileChannelImpl.java:383) ~[?:?]

at org.apache.lucene.store.NativeFSLockFactory$NativeFSLock.ensureValid(NativeFSLockFactory.java:182) ~[lucene-core-7.7.0.jar:7.7.0 8c831daf4eb41153c25ddb152501ab5bae3ea3d5 - jimczi - 2019-02-04 23:16:28]

at org.elasticsearch.env.NodeEnvironment.assertEnvIsLocked(NodeEnvironment.java:1022) ~[elasticsearch-6.7.0.jar:6.7.0]

... 20 more

经过简单查看es集群状态没发现有啥影响，搜索后也没找到合适的解决方法，对es不是很了解，请教如何解决上述刷异常的问题呢？

sh-4.2$ curl localhost:9200/_cluster/health?pretty
{
  "cluster_name" : "elasticsearch",
  "status" : "green",
  "timed_out" : false,
  "number_of_nodes" : 8,
  "number_of_data_nodes" : 5,
  "active_primary_shards" : 150,
  "active_shards" : 300,
  "relocating_shards" : 2,
  "initializing_shards" : 0,
  "unassigned_shards" : 0,
  "delayed_unassigned_shards" : 0,
  "number_of_pending_tasks" : 4,
  "number_of_in_flight_fetch" : 0,
  "task_max_waiting_in_queue_millis" : 77,
  "active_shards_percent_as_number" : 100.0
}

sh-4.2$ curl localhost:9200/_cat/indices?v
health status index                           uuid                   pri rep docs.count docs.deleted store.size pri.store.size
green  open   ks-logstash-log-2020.10.27      _YknKv4NQUS-lWpLXPi90Q   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-auditing-2020.10.30 pDCyC8oDStWSD-sJbVH_OQ   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-log-2020.11.04      VtK3ceLoRGqpoEGKoznbGw   5   1   63526638            0     17.7gb          8.8gb
green  open   ks-logstash-auditing-2020.10.31 Edxt-oNJSWKBScVce1paYg   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-log-2020.11.03      MwGAcaHJSh6r5Lor8MpZRg   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-events-2020.11.01   2Iul-sPTR9W7BSZBzbtDcA   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-log-2020.11.05      gXp13nYaSySTWyJejRFkmw   5   1   14919861            0      4.7gb          2.3gb
green  open   ks-logstash-auditing-2020.11.05 5-VGMR3bR8aFREtAQe80UQ   5   1         66            0    758.2kb        379.1kb
green  open   ks-logstash-events-2020.10.29   cJhZmSoBTzKv6yPce0e0XQ   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-events-2020.10.31   x7xrf0jSRy2NN0dEjbr1Tw   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-auditing-2020.11.04 K9QAgxeaSSOoRvhoaBSlfw   5   1        328            0      1.2mb        688.4kb
green  open   ks-logstash-events-2020.11.03   CdL1A2SpRuqA0D3eQYvm6w   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-log-2020.11.02      Dk17sceQSk6Gk3gQqlbEpg   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-events-2020.11.02   w7-HXDP_SgCV-i5GWY-RvA   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-events-2020.11.04   5ICGq9neQ1SqzwW8No8V-A   5   1      13573            0     14.2mb            7mb
green  open   ks-logstash-events-2020.10.30   tIqV8MbsTeagTgOat6VJQg   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-auditing-2020.10.28 QEvDduBvTA2iu4TDtWrP0Q   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-auditing-2020.11.03 u3VIPtIsTcqTBK23s2YR1Q   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-auditing-2020.10.29 g1FPdSJ9QUmlc0n7Da79sQ   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-log-2020.10.30      71GDd76KRf-VzUFLLaTWqg   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-auditing-2020.11.01 XZ5CGYNGTuSqA5fHeIwIEg   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-events-2020.10.28   c52UWfe7QuK00lziLPp-1A   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-log-2020.10.31      Artfha8JQwufr2H0ETbHYQ   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-log-2020.11.01      s3Oe-w0rTJeSo5I8XuFOSg   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-events-2020.11.05   u5EVxtwQTHegj0mM0FrlUw   5   1       2830            0        6mb          3.1mb
green  open   ks-logstash-log-2020.10.26      v4LR0eIxSSeK6P4rhf8CWQ   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-log-2020.10.29      hQRe-R2gQvuq6t4VLCdsqQ   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-auditing-2020.11.02 qzGPU3mVRhmsgMDht1bDOg   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-log-2020.10.28      FXsns61TRpanQtLgwmxqtw   5   1          0            0      2.5kb          1.2kb
green  open   ks-logstash-auditing-2020.10.27 6wiVDDYmSiixzPwjTEfRxQ   5   1          0            0      2.5kb          1.2kb

#这里有两条RELOCATING状态的如何处理呢？
ks-logstash-auditing-2020.11.05 1     p      STARTED          13  52.7kb 10.234.92.59  elasticsearch-logging-data-2
ks-logstash-auditing-2020.11.05 1     r      RELOCATING                  10.234.90.56  elasticsearch-logging-data-4 -> 10.234.105.98 p4qSYifaQy-bjnxM5zLV7Q elasticsearch-logging-data-1
ks-logstash-auditing-2020.11.05 4     p      STARTED           9 131.2kb 10.234.96.48  elasticsearch-logging-data-3
ks-logstash-auditing-2020.11.05 4     r      STARTED           9 131.2kb 10.234.90.56  elasticsearch-logging-data-4
ks-logstash-auditing-2020.11.05 0     p      STARTED          20 120.9kb 10.234.90.55  elasticsearch-logging-data-0
ks-logstash-auditing-2020.11.05 0     r      RELOCATING                  10.234.92.59  elasticsearch-logging-data-2 -> 10.234.105.98 p4qSYifaQy-bjnxM5zLV7Q elasticsearch-logging-data-1