devops jenkins 通过 webhooks没办法触发构建

TobeSayaka

因将持续集成迁移到了另外一个集群后出现的问题。
在原集群V3版本下的devops是可以正常触发的，但将devops迁移到同网络环境下的另外一个集群V3.1的时候,使用同样的配置，但是webhooks就是无法触发他开始运行流水线

webhooks调用ks-jenkins后的相关日志如下

`2021-07-01 02:47:07.962+0000 [id=743] WARNING i.f.k.c.d.i.WatchConnectionManager$1#onFailure: Exec Failure

java.util.concurrent.RejectedExecutionException: Task okhttp3.RealCall$AsyncCall@35193aa rejected from java.util.concurrent.ThreadPoolExecutor@c47ae7d[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 106]

at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2063)

at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830)

at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379)

at okhttp3.RealCall$AsyncCall.executeOn(RealCall.java:183)

Caused: java.io.InterruptedIOException: executor rejected

at okhttp3.RealCall$AsyncCall.executeOn(RealCall.java:186)

at okhttp3.Dispatcher.promoteAndExecute(Dispatcher.java:186)

at okhttp3.Dispatcher.enqueue(Dispatcher.java:137)

at okhttp3.RealCall.enqueue(RealCall.java:127)

at okhttp3.internal.ws.RealWebSocket.connect(RealWebSocket.java:193)

at okhttp3.OkHttpClient.newWebSocket(OkHttpClient.java:435)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager.runWatch(WatchConnectionManager.java:156)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager.access$1200(WatchConnectionManager.java:50)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$2$1.execute(WatchConnectionManager.java:320)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$NamedRunnable.run(WatchConnectionManager.java:409)

at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

at java.util.concurrent.FutureTask.run(FutureTask.java:266)

at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)

at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

at java.lang.Thread.run(Thread.java:748)

2021-07-01 02:47:39.962+0000 [id=743] WARNING i.f.k.c.d.i.WatchConnectionManager$1#onFailure: Exec Failure

java.util.concurrent.RejectedExecutionException: Task okhttp3.RealCall$AsyncCall@5b0d844b rejected from java.util.concurrent.ThreadPoolExecutor@c47ae7d[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 106]

at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2063)

at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830)

at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379)

at okhttp3.RealCall$AsyncCall.executeOn(RealCall.java:183)

Caused: java.io.InterruptedIOException: executor rejected

at okhttp3.RealCall$AsyncCall.executeOn(RealCall.java:186)

at okhttp3.Dispatcher.promoteAndExecute(Dispatcher.java:186)

at okhttp3.Dispatcher.enqueue(Dispatcher.java:137)

at okhttp3.RealCall.enqueue(RealCall.java:127)

at okhttp3.internal.ws.RealWebSocket.connect(RealWebSocket.java:193)

at okhttp3.OkHttpClient.newWebSocket(OkHttpClient.java:435)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager.runWatch(WatchConnectionManager.java:156)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager.access$1200(WatchConnectionManager.java:50)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$2$1.execute(WatchConnectionManager.java:320)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$NamedRunnable.run(WatchConnectionManager.java:409)

at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

at java.util.concurrent.FutureTask.run(FutureTask.java:266)

at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)

at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

at java.lang.Thread.run(Thread.java:748)

2021-07-01 02:48:11.963+0000 [id=743] WARNING i.f.k.c.d.i.WatchConnectionManager$1#onFailure: Exec Failure

java.util.concurrent.RejectedExecutionException: Task okhttp3.RealCall$AsyncCall@38006a50 rejected from java.util.concurrent.ThreadPoolExecutor@c47ae7d[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 106]

at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2063)

at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830)

at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379)

at okhttp3.RealCall$AsyncCall.executeOn(RealCall.java:183)

Caused: java.io.InterruptedIOException: executor rejected

at okhttp3.RealCall$AsyncCall.executeOn(RealCall.java:186)

at okhttp3.Dispatcher.promoteAndExecute(Dispatcher.java:186)

at okhttp3.Dispatcher.enqueue(Dispatcher.java:137)

at okhttp3.RealCall.enqueue(RealCall.java:127)

at okhttp3.internal.ws.RealWebSocket.connect(RealWebSocket.java:193)

at okhttp3.OkHttpClient.newWebSocket(OkHttpClient.java:435)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager.runWatch(WatchConnectionManager.java:156)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager.access$1200(WatchConnectionManager.java:50)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$2$1.execute(WatchConnectionManager.java:320)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$NamedRunnable.run(WatchConnectionManager.java:409)

at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

at java.util.concurrent.FutureTask.run(FutureTask.java:266)

at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)

at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

at java.lang.Thread.run(Thread.java:748)

2021-07-01 02:48:43.963+0000 [id=743] WARNING i.f.k.c.d.i.WatchConnectionManager$1#onFailure: Exec Failure

java.util.concurrent.RejectedExecutionException: Task okhttp3.RealCall$AsyncCall@4e159da2 rejected from java.util.concurrent.ThreadPoolExecutor@c47ae7d[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 106]

at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2063)

at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830)

at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379)

at okhttp3.RealCall$AsyncCall.executeOn(RealCall.java:183)

Caused: java.io.InterruptedIOException: executor rejected

at okhttp3.RealCall$AsyncCall.executeOn(RealCall.java:186)

at okhttp3.Dispatcher.promoteAndExecute(Dispatcher.java:186)

at okhttp3.Dispatcher.enqueue(Dispatcher.java:137)

at okhttp3.RealCall.enqueue(RealCall.java:127)

at okhttp3.internal.ws.RealWebSocket.connect(RealWebSocket.java:193)

at okhttp3.OkHttpClient.newWebSocket(OkHttpClient.java:435)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager.runWatch(WatchConnectionManager.java:156)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager.access$1200(WatchConnectionManager.java:50)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$2$1.execute(WatchConnectionManager.java:320)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$NamedRunnable.run(WatchConnectionManager.java:409)

at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

at java.util.concurrent.FutureTask.run(FutureTask.java:266)

at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)

at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

at java.lang.Thread.run(Thread.java:748)

2021-07-01 02:49:01.340+0000 [id=231696] INFO j.b.MultiBranchProject$BranchIndexing#run: devvdgt2/interaction-service #20210701.024900 branch indexing action completed: SUCCESS in 1.2 sec

2021-07-01 02:49:15.965+0000 [id=743] WARNING i.f.k.c.d.i.WatchConnectionManager$1#onFailure: Exec Failure

java.util.concurrent.RejectedExecutionException: Task okhttp3.RealCall$AsyncCall@33ffe583 rejected from java.util.concurrent.ThreadPoolExecutor@c47ae7d[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 106]

at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2063)

at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830)

at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379)

at okhttp3.RealCall$AsyncCall.executeOn(RealCall.java:183)

Caused: java.io.InterruptedIOException: executor rejected

at okhttp3.RealCall$AsyncCall.executeOn(RealCall.java:186)

at okhttp3.Dispatcher.promoteAndExecute(Dispatcher.java:186)

at okhttp3.Dispatcher.enqueue(Dispatcher.java:137)

at okhttp3.RealCall.enqueue(RealCall.java:127)

at okhttp3.internal.ws.RealWebSocket.connect(RealWebSocket.java:193)

at okhttp3.OkHttpClient.newWebSocket(OkHttpClient.java:435)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager.runWatch(WatchConnectionManager.java:156)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager.access$1200(WatchConnectionManager.java:50)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$2$1.execute(WatchConnectionManager.java:320)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$NamedRunnable.run(WatchConnectionManager.java:409)

at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

at java.util.concurrent.FutureTask.run(FutureTask.java:266)

at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)

at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

at java.lang.Thread.run(Thread.java:748)

2021-07-01 02:49:47.967+0000 [id=743] WARNING i.f.k.c.d.i.WatchConnectionManager$1#onFailure: Exec Failure

java.util.concurrent.RejectedExecutionException: Task okhttp3.RealCall$AsyncCall@3493313a rejected from java.util.concurrent.ThreadPoolExecutor@c47ae7d[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 106]

at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2063)

at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830)

at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379)

at okhttp3.RealCall$AsyncCall.executeOn(RealCall.java:183)

Caused: java.io.InterruptedIOException: executor rejected

at okhttp3.RealCall$AsyncCall.executeOn(RealCall.java:186)

at okhttp3.Dispatcher.promoteAndExecute(Dispatcher.java:186)

at okhttp3.Dispatcher.enqueue(Dispatcher.java:137)

at okhttp3.RealCall.enqueue(RealCall.java:127)

at okhttp3.internal.ws.RealWebSocket.connect(RealWebSocket.java:193)

at okhttp3.OkHttpClient.newWebSocket(OkHttpClient.java:435)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager.runWatch(WatchConnectionManager.java:156)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager.access$1200(WatchConnectionManager.java:50)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$2$1.execute(WatchConnectionManager.java:320)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$NamedRunnable.run(WatchConnectionManager.java:409)

at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

at java.util.concurrent.FutureTask.run(FutureTask.java:266)

at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)

at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

at java.lang.Thread.run(Thread.java:748)

2021-07-01 02:50:01.426+0000 [id=231736] INFO j.b.MultiBranchProject$BranchIndexing#run: devvdgt2/user-service #20210701.025000 branch indexing action completed: SUCCESS in 1.3 sec

2021-07-01 02:50:01.498+0000 [id=231735] INFO j.b.MultiBranchProject$BranchIndexing#run: devvdgt2/support-service #20210701.025000 branch indexing action completed: SUCCESS in 1.4 sec

2021-07-01 02:50:01.611+0000 [id=231734] INFO j.b.MultiBranchProject$BranchIndexing#run: devvdgt2/open-gateway #20210701.025000 branch indexing action completed: SUCCESS in 1.5 sec

2021-07-01 02:50:19.967+0000 [id=743] WARNING i.f.k.c.d.i.WatchConnectionManager$1#onFailure: Exec Failure

java.util.concurrent.RejectedExecutionException: Task okhttp3.RealCall$AsyncCall@1f85d4d5 rejected from java.util.concurrent.ThreadPoolExecutor@c47ae7d[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 106]

at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2063)

at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830)

at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379)

at okhttp3.RealCall$AsyncCall.executeOn(RealCall.java:183)

Caused: java.io.InterruptedIOException: executor rejected

at okhttp3.RealCall$AsyncCall.executeOn(RealCall.java:186)

at okhttp3.Dispatcher.promoteAndExecute(Dispatcher.java:186)

at okhttp3.Dispatcher.enqueue(Dispatcher.java:137)

at okhttp3.RealCall.enqueue(RealCall.java:127)

at okhttp3.internal.ws.RealWebSocket.connect(RealWebSocket.java:193)

at okhttp3.OkHttpClient.newWebSocket(OkHttpClient.java:435)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager.runWatch(WatchConnectionManager.java:156)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager.access$1200(WatchConnectionManager.java:50)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$2$1.execute(WatchConnectionManager.java:320)

at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$NamedRunnable.run(WatchConnectionManager.java:409)

at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)`

Rick

帮忙补充如下的信息：

你这个是什么流水线
webhook 地址是什么
是多集群还是单集群
- 多集群的话，是 host 还是 member

TobeSayaka

Rick
java项目的流水线

webhook地址是复制下面的，配置到了gitlab对应项目的webhook配置中

多集群，host

Rick

这个地址信息很关键，你可以把host部分屏蔽了

kubesphere/console#1289

TobeSayaka

Rick
http://xxxxxxxxx/cluster/host/devops_webhook/git/?url=https://xxxxxxxxxxxx.git

TobeSayaka

Rick 是这样的，我们是加了域名解析，通过域名去调用的，图中打码的是我们的域名

Rick

根据上面我给的issue，应该能解决这个问题

Rick

你把 /cluster/host 去掉试试

TobeSayaka

Rick 真的是这个问题，解决了，感谢