Add active http workers check
Currently only number of active ssh worker threads is checked during
the healthcheck. Adding check for number of active http worker threads
prevents situation when instance is out of http worker threads but
healthcheck is successful.
Feature: Issue 11651
Change-Id: Idee5c405d2c2026f4f968e2dcb201731e3176b6c
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java
index 7a64c57..11692f4 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java
@@ -21,6 +21,7 @@
import com.googlesource.gerrit.plugins.healthcheck.check.AuthHealthCheck;
import com.googlesource.gerrit.plugins.healthcheck.check.DeadlockCheck;
import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheck;
+import com.googlesource.gerrit.plugins.healthcheck.check.HttpActiveWorkersCheck;
import com.googlesource.gerrit.plugins.healthcheck.check.JGitHealthCheck;
import com.googlesource.gerrit.plugins.healthcheck.check.ProjectsListHealthCheck;
import com.googlesource.gerrit.plugins.healthcheck.check.QueryChangesHealthCheck;
@@ -36,6 +37,7 @@
bindChecker(QueryChangesHealthCheck.class);
bindChecker(AuthHealthCheck.class);
bindChecker(ActiveWorkersCheck.class);
+ bindChecker(HttpActiveWorkersCheck.class);
bindChecker(DeadlockCheck.class);
bind(LifecycleListener.class).to(HealthCheckMetrics.class);
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/AbstractWorkersHealthCheck.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/AbstractWorkersHealthCheck.java
new file mode 100644
index 0000000..8e5f8fb
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/AbstractWorkersHealthCheck.java
@@ -0,0 +1,70 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.healthcheck.check;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.MetricRegistry;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.googlesource.gerrit.plugins.healthcheck.HealthCheckConfig;
+import java.util.Optional;
+
+public abstract class AbstractWorkersHealthCheck extends AbstractHealthCheck {
+
+ private final String metricName;
+ private final Integer maxPoolSize;
+ private final Integer threshold;
+ private final MetricRegistry metricRegistry;
+
+ protected AbstractWorkersHealthCheck(
+ ListeningExecutorService executor,
+ HealthCheckConfig config,
+ MetricRegistry metricRegistry,
+ String name,
+ String metricName,
+ Integer maxPoolSize) {
+ super(executor, config, name);
+ this.metricRegistry = metricRegistry;
+ this.metricName = metricName;
+ this.maxPoolSize = maxPoolSize;
+ this.threshold = config.getActiveWorkersThreshold(name);
+ }
+
+ @Override
+ protected Result doCheck() throws Exception {
+ return Optional.ofNullable(metricRegistry.getGauges().get(metricName))
+ .map(
+ metric -> {
+ float currentThreadsPercentage = (getMetricValue(metric) * 100) / maxPoolSize;
+ return (currentThreadsPercentage <= threshold) ? Result.PASSED : Result.FAILED;
+ })
+ .orElse(Result.PASSED);
+ }
+
+ private Long getMetricValue(Gauge<?> metric) {
+ Object value = metric.getValue();
+ if (value instanceof Long) {
+ return (Long) value;
+ }
+
+ if (value instanceof Integer) {
+ return ((Integer) value).longValue();
+ }
+
+ throw new IllegalArgumentException(
+ String.format(
+ "Workers metric value must be of type java.lang.Long or java.lang.Integer but was %s ",
+ value.getClass().getName()));
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/ActiveWorkersCheck.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/ActiveWorkersCheck.java
index f3afc99..84b8e9a 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/ActiveWorkersCheck.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/ActiveWorkersCheck.java
@@ -22,18 +22,13 @@
import com.google.gerrit.server.config.ThreadSettingsConfig;
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.healthcheck.HealthCheckConfig;
-import java.util.Optional;
import org.eclipse.jgit.lib.Config;
-public class ActiveWorkersCheck extends AbstractHealthCheck {
+public class ActiveWorkersCheck extends AbstractWorkersHealthCheck {
public static final String ACTIVE_WORKERS_METRIC_NAME =
"queue/ssh_interactive_worker/active_threads";
- private Integer threshold;
- private Integer interactiveThreadsMaxPoolSize;
- private MetricRegistry metricRegistry;
-
@Inject
public ActiveWorkersCheck(
@GerritServerConfig Config gerritConfig,
@@ -41,25 +36,14 @@
HealthCheckConfig healthCheckConfig,
ThreadSettingsConfig threadSettingsConfig,
MetricRegistry metricRegistry) {
- super(executor, healthCheckConfig, ACTIVEWORKERS);
- this.threshold = healthCheckConfig.getActiveWorkersThreshold(ACTIVEWORKERS);
- this.metricRegistry = metricRegistry;
- this.interactiveThreadsMaxPoolSize =
- getInteractiveThreadsMaxPoolSize(threadSettingsConfig, gerritConfig);
- }
- @Override
- protected Result doCheck() throws Exception {
- return Optional.ofNullable(metricRegistry.getGauges().get(ACTIVE_WORKERS_METRIC_NAME))
- .map(
- metric -> {
- float currentInteractiveThreadsPercentage =
- ((long) metric.getValue() * 100) / interactiveThreadsMaxPoolSize;
- return (currentInteractiveThreadsPercentage <= threshold)
- ? Result.PASSED
- : Result.FAILED;
- })
- .orElse(Result.PASSED);
+ super(
+ executor,
+ healthCheckConfig,
+ metricRegistry,
+ ACTIVEWORKERS,
+ ACTIVE_WORKERS_METRIC_NAME,
+ getInteractiveThreadsMaxPoolSize(threadSettingsConfig, gerritConfig));
}
/**
@@ -70,7 +54,7 @@
*
* @return max number of allowed threads in interactive work queue
*/
- private Integer getInteractiveThreadsMaxPoolSize(
+ private static Integer getInteractiveThreadsMaxPoolSize(
ThreadSettingsConfig threadSettingsConfig, Config gerritConfig) {
int poolSize = threadSettingsConfig.getSshdThreads();
int batchThreads =
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java
index ce97a1b..862d2de 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java
@@ -21,5 +21,6 @@
String QUERYCHANGES = "querychanges";
String AUTH = "auth";
String ACTIVEWORKERS = "activeworkers";
+ String HTTPACTIVEWORKERS = "httpactiveworkers";
String DEADLOCK = "deadlock";
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HttpActiveWorkersCheck.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HttpActiveWorkersCheck.java
new file mode 100644
index 0000000..0690596
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HttpActiveWorkersCheck.java
@@ -0,0 +1,43 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.healthcheck.check;
+
+import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.HTTPACTIVEWORKERS;
+
+import com.codahale.metrics.MetricRegistry;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.gerrit.server.config.ThreadSettingsConfig;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.healthcheck.HealthCheckConfig;
+
+public class HttpActiveWorkersCheck extends AbstractWorkersHealthCheck {
+ public static final String HTTP_WORKERS_METRIC_NAME =
+ "http/server/jetty/threadpool/active_threads";
+
+ @Inject
+ public HttpActiveWorkersCheck(
+ ListeningExecutorService executor,
+ HealthCheckConfig healthCheckConfig,
+ ThreadSettingsConfig threadSettingsConfig,
+ MetricRegistry metricRegistry) {
+ super(
+ executor,
+ healthCheckConfig,
+ metricRegistry,
+ HTTPACTIVEWORKERS,
+ HTTP_WORKERS_METRIC_NAME,
+ threadSettingsConfig.getHttpdMaxThreads());
+ }
+}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 3738b9b..e53b8fb 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -49,6 +49,8 @@
- `projectslist` : check the ability to list projects with their descriptions
- `auth`: check the ability to authenticate with username and password
- `activeworkers`: check the number of active worker threads and the ability to create a new one
+- `httpactiveworkers`: check the number of active HTTP worker threads and the ability
+ to create a new one
- `deadlock` : check if Java deadlocks are reported by the JVM
Each check name can be disabled by setting the `enabled` parameter to **false**,
@@ -88,4 +90,9 @@
- `healthcheck.activeworkers.threshold` : Percent of queue occupancy above which queue is consider
as full.
+ Default: 80
+
+ - `healthcheck.httpactiveworkers.threshold` : Percent of queue occupancy above which queue is
+ considered as full.
+
Default: 80
\ No newline at end of file
diff --git a/src/test/java/com/googlesource/gerrit/plugins/healthcheck/ActiveWorkersCheckTest.java b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/ActiveWorkersCheckTest.java
index 71bef43..df5a754 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/healthcheck/ActiveWorkersCheckTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/ActiveWorkersCheckTest.java
@@ -112,6 +112,27 @@
assertThat(check.run().result).isEqualTo(Result.PASSED);
}
+ @Test
+ public void shouldFailCheckWhenMetricValueIsNotLongOrInteger() {
+
+ MetricRegistry metricRegistry = new MetricRegistry();
+ metricRegistry.register(
+ ActiveWorkersCheck.ACTIVE_WORKERS_METRIC_NAME,
+ new Gauge<String>() {
+ @Override
+ public String getValue() {
+ return "55";
+ }
+ });
+ Config gerritConfig = new Config();
+ gerritConfig.setInt("sshd", null, "threads", 12);
+
+ Injector injector = testInjector(new TestModule(gerritConfig, metricRegistry));
+
+ ActiveWorkersCheck check = createCheck(injector);
+ assertThat(check.run().result).isEqualTo(Result.FAILED);
+ }
+
private Injector testInjector(AbstractModule testModule) {
return Guice.createInjector(new HealthCheckModule(), testModule);
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/healthcheck/HttpActiveWorkersCheckTest.java b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/HttpActiveWorkersCheckTest.java
new file mode 100644
index 0000000..b1557e2
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/HttpActiveWorkersCheckTest.java
@@ -0,0 +1,142 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.healthcheck;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.googlesource.gerrit.plugins.healthcheck.HealthCheckConfig.DEFAULT_CONFIG;
+import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.HTTPACTIVEWORKERS;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.MetricRegistry;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.gerrit.server.config.GerritServerConfig;
+import com.google.gerrit.server.config.ThreadSettingsConfig;
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheck.Result;
+import com.googlesource.gerrit.plugins.healthcheck.check.HttpActiveWorkersCheck;
+import org.eclipse.jgit.lib.Config;
+import org.junit.Test;
+
+public class HttpActiveWorkersCheckTest {
+
+ @Test
+ public void shouldPassCheckWhenActiveHttpWorkersLessThanThreshold() {
+
+ int thresholdPerc = 50;
+ int maxThreads = 12;
+ int usedThreads = maxThreads * thresholdPerc / 100 - 1;
+
+ MetricRegistry metricRegistry = createHttpMetricRegistry(usedThreads);
+
+ Config gerritConfig = new Config();
+ gerritConfig.setInt("httpd", null, "maxThreads", maxThreads);
+
+ Injector injector = testInjector(new TestModule(gerritConfig, metricRegistry));
+
+ HealthCheckConfig healthCheckConfig =
+ new HealthCheckConfig(
+ "[healthcheck \"" + HTTPACTIVEWORKERS + "\"]\n" + " threshold = " + thresholdPerc);
+ HttpActiveWorkersCheck check = createCheck(injector, healthCheckConfig);
+ assertThat(check.run().result).isEqualTo(Result.PASSED);
+ }
+
+ @Test
+ public void shouldFailCheckWhenActiveHttpWorkersMoreThanThreshold() {
+ int thresholdPerc = 50;
+ int maxThreads = 12;
+ int usedThreads = maxThreads * thresholdPerc / 100 + 1;
+
+ MetricRegistry metricRegistry = createHttpMetricRegistry(usedThreads);
+
+ Config gerritConfig = new Config();
+ gerritConfig.setInt("httpd", null, "maxThreads", maxThreads);
+
+ Injector injector = testInjector(new TestModule(gerritConfig, metricRegistry));
+
+ HealthCheckConfig healthCheckConfig =
+ new HealthCheckConfig(
+ "[healthcheck \"" + HTTPACTIVEWORKERS + "\"]\n" + " threshold = " + thresholdPerc);
+ HttpActiveWorkersCheck check = createCheck(injector, healthCheckConfig);
+ assertThat(check.run().result).isEqualTo(Result.FAILED);
+ }
+
+ @Test
+ public void shouldUseHttpWorkersDefaultThreshold() {
+ HealthCheckConfig healthCheckConfig =
+ new HealthCheckConfig("[healthcheck \"" + HTTPACTIVEWORKERS + "\"]\n");
+ assertThat(healthCheckConfig.getActiveWorkersThreshold(HTTPACTIVEWORKERS)).isEqualTo(80);
+ }
+
+ @Test
+ public void shouldPassCheckWhenNoActiveHttpWorkers() {
+
+ MetricRegistry metricRegistry = createHttpMetricRegistry(0);
+
+ Injector injector = testInjector(new TestModule(new Config(), metricRegistry));
+
+ HttpActiveWorkersCheck check = createCheck(injector);
+ assertThat(check.run().result).isEqualTo(Result.PASSED);
+ }
+
+ private MetricRegistry createHttpMetricRegistry(Integer value) {
+ MetricRegistry metricRegistry = new MetricRegistry();
+
+ metricRegistry.register(
+ HttpActiveWorkersCheck.HTTP_WORKERS_METRIC_NAME,
+ new Gauge<Integer>() {
+ @Override
+ public Integer getValue() {
+ return value;
+ }
+ });
+ return metricRegistry;
+ }
+
+ private HttpActiveWorkersCheck createCheck(Injector injector) {
+ return createCheck(injector, DEFAULT_CONFIG);
+ }
+
+ private HttpActiveWorkersCheck createCheck(
+ Injector injector, HealthCheckConfig healtchCheckConfig) {
+ return new HttpActiveWorkersCheck(
+ injector.getInstance(ListeningExecutorService.class),
+ healtchCheckConfig,
+ injector.getInstance(ThreadSettingsConfig.class),
+ injector.getInstance(MetricRegistry.class));
+ }
+
+ private Injector testInjector(AbstractModule testModule) {
+ return Guice.createInjector(new HealthCheckModule(), testModule);
+ }
+
+ private class TestModule extends AbstractModule {
+ Config gerritConfig;
+ MetricRegistry metricRegistry;
+
+ public TestModule(Config gerritConfig, MetricRegistry metricRegistry) {
+ this.gerritConfig = gerritConfig;
+ this.metricRegistry = metricRegistry;
+ }
+
+ @Override
+ protected void configure() {
+ bind(Config.class).annotatedWith(GerritServerConfig.class).toInstance(gerritConfig);
+ bind(ThreadSettingsConfig.class);
+ bind(MetricRegistry.class).toInstance(metricRegistry);
+ }
+ }
+}
