Add blocked threads check
Introduce check that detects the blocked threads (enabled by default).
In case when it reaches above the configured percent of all threads (by
default 50%) check fails and instance is considered unhealthy.
Check can:
* be disabled with 'healthcheck.blockedthreads.enabled = false'
* have threshold reconfigured with 'healthcheck.blocked.threshold=XX'
Note that one can fine tune the check to certain group of threads by
configuring threshold for threads with name starting wiht prefix.
For instance in case when Gerrit instance should be considered
unhealthy when 33% of SSH-Interactive-Worker threads are 'BLOCKED'
one can configure it with
[healthcheck "blockedthreads"]
threshold = SSH-Interactive-Threads=33
Note that multiple checks for multiple thread group could be
configured (for details with examples see the config.md file).
Bug: Issue 14401
Change-Id: I7d0c4b70decf49465a7318abe4a555cde7318833
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java
index 3e5306c..651dbe9 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java
@@ -15,6 +15,7 @@
package com.googlesource.gerrit.plugins.healthcheck;
import static com.google.common.base.Preconditions.checkNotNull;
+import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.BLOCKEDTHREADS;
import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.QUERYCHANGES;
import com.google.common.annotations.VisibleForTesting;
@@ -142,6 +143,10 @@
HEALTHCHECK, checkNotNull(healthCheckName), "enabled", HEALTH_CHECK_ENABLED_DEFAULT);
}
+ public String[] getListOfBlockedThreadsThresholds() {
+ return config.getStringList(HEALTHCHECK, BLOCKEDTHREADS, "threshold");
+ }
+
private String getStringWithFallback(
String parameter, String healthCheckName, String defaultValue) {
String fallbackDefault =
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java
index 304a40c..3cda7b6 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java
@@ -18,6 +18,7 @@
import com.google.gerrit.extensions.registration.DynamicSet;
import com.googlesource.gerrit.plugins.healthcheck.check.ActiveWorkersCheck;
import com.googlesource.gerrit.plugins.healthcheck.check.AuthHealthCheck;
+import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsCheck;
import com.googlesource.gerrit.plugins.healthcheck.check.DeadlockCheck;
import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheck;
import com.googlesource.gerrit.plugins.healthcheck.check.JGitHealthCheck;
@@ -34,8 +35,10 @@
bindChecker(AuthHealthCheck.class);
bindChecker(ActiveWorkersCheck.class);
bindChecker(DeadlockCheck.class);
+ bindChecker(BlockedThreadsCheck.class);
factory(HealthCheckMetrics.Factory.class);
+ install(BlockedThreadsCheck.SUB_CHECKS);
}
private void bindChecker(Class<? extends HealthCheck> healthCheckClass) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsCheck.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsCheck.java
new file mode 100644
index 0000000..f2cbb70
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsCheck.java
@@ -0,0 +1,123 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.healthcheck.check;
+
+import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.BLOCKEDTHREADS;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Suppliers;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.gerrit.extensions.config.FactoryModule;
+import com.google.inject.Inject;
+import com.google.inject.Module;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.healthcheck.HealthCheckConfig;
+import com.googlesource.gerrit.plugins.healthcheck.HealthCheckMetrics;
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+
+@Singleton
+public class BlockedThreadsCheck extends AbstractHealthCheck {
+ public static Module SUB_CHECKS =
+ new FactoryModule() {
+ @Override
+ protected void configure() {
+ factory(BlockedThreadsSubCheck.Factory.class);
+ }
+ };
+
+ private final ThreadMXBean threads;
+ private final Supplier<List<Collector>> collectorsSupplier;
+
+ @Inject
+ public BlockedThreadsCheck(
+ ListeningExecutorService executor,
+ HealthCheckConfig healthCheckConfig,
+ HealthCheckMetrics.Factory healthCheckMetricsFactory,
+ ThreadBeanProvider threadBeanProvider,
+ Provider<BlockedThreadsConfigurator> checksConfig) {
+ super(executor, healthCheckConfig, BLOCKEDTHREADS, healthCheckMetricsFactory);
+ this.threads = threadBeanProvider.get();
+ this.collectorsSupplier = Suppliers.memoize(() -> checksConfig.get().collectors());
+ }
+
+ @Override
+ protected Result doCheck() throws Exception {
+ List<Collector> collectors = collectorsSupplier.get();
+ dumpAllThreads().forEach(info -> collectors.forEach(c -> c.collect(info)));
+
+ // call check on all sub-checks so that metrics are populated
+ collectors.forEach(Collector::check);
+
+ // report unhealthy instance if any of sub-checks failed
+ return collectors.stream()
+ .map(Collector::result)
+ .filter(r -> Result.FAILED == r)
+ .findAny()
+ .orElse(Result.PASSED);
+ }
+
+ private Stream<ThreadInfo> dumpAllThreads() {
+ // getting all thread ids and translating it into thread infos is noticeably faster then call to
+ // ThreadMXBean.dumpAllThreads as it doesn't calculate StackTrace. Note that some threads could
+ // be already finished (between call to get all ids and translate them to ThreadInfo objects
+ // hence they have to be filtered out).
+ return Arrays.stream(threads.getThreadInfo(threads.getAllThreadIds(), 0))
+ .filter(Objects::nonNull);
+ }
+
+ @VisibleForTesting
+ public static class ThreadBeanProvider {
+ public ThreadMXBean get() {
+ return ManagementFactory.getThreadMXBean();
+ }
+ }
+
+ static class Collector {
+ protected final Integer threshold;
+
+ protected int blocked;
+ protected int total;
+ protected Result result;
+
+ Collector(Integer threshold) {
+ this.threshold = threshold;
+ }
+
+ void collect(ThreadInfo info) {
+ total += 1;
+ if (Thread.State.BLOCKED == info.getThreadState()) {
+ blocked += 1;
+ }
+ }
+
+ void check() {
+ result = blocked * 100 <= threshold * total ? Result.PASSED : Result.FAILED;
+ }
+
+ Result result() {
+ return result;
+ }
+ }
+
+ interface CollectorProvider<T extends Collector> extends Provider<T> {}
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfigurator.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfigurator.java
new file mode 100644
index 0000000..ddc67f9
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfigurator.java
@@ -0,0 +1,205 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.healthcheck.check;
+
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.toList;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.healthcheck.HealthCheckConfig;
+import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsCheck.Collector;
+import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsCheck.CollectorProvider;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@VisibleForTesting
+public class BlockedThreadsConfigurator {
+ private static final Logger log = LoggerFactory.getLogger(BlockedThreadsConfigurator.class);
+ private static final Pattern THRESHOLD_PATTERN = Pattern.compile("^(\\d\\d?)$");
+
+ static final int DEFAULT_BLOCKED_THREADS_THRESHOLD = 50;
+
+ private final List<CollectorProvider<?>> providers;
+
+ @Inject
+ BlockedThreadsConfigurator(
+ BlockedThreadsSubCheck.Factory subchecks, HealthCheckConfig healthCheckConfig) {
+ this.providers = getProviders(subchecks, healthCheckConfig);
+ }
+
+ List<Collector> collectors() {
+ return providers.stream().map(CollectorProvider::get).collect(toList());
+ }
+
+ private static List<CollectorProvider<?>> getProviders(
+ BlockedThreadsSubCheck.Factory subchecksFactory, HealthCheckConfig healthCheckConfig) {
+ return getConfig(healthCheckConfig.getListOfBlockedThreadsThresholds()).stream()
+ .map(spec -> collectorProvider(subchecksFactory, spec))
+ .collect(toList());
+ }
+
+ private static CollectorProvider<?> collectorProvider(
+ BlockedThreadsSubCheck.Factory subchecksFactory, Threshold spec) {
+ return spec.prefix.isPresent()
+ ? subchecksFactory.create(spec.prefix.get(), spec.value)
+ : () -> new BlockedThreadsCheck.Collector(spec.value);
+ }
+
+ @VisibleForTesting
+ static Collection<Threshold> getConfig(String[] thresholds) {
+ // Threshold can be defined as a sole value e.g
+ // threshold = 80
+ // and would become a default one for all blocked threads check or as a set of specific thread
+ // groups checks defined like
+ // threshold = foo=30
+ // threshold = bar=40
+ // ...
+ // they are mutually exclusive which means that one either checks all threads or groups
+ Map<Boolean, List<Threshold>> specsClassified =
+ Arrays.stream(thresholds)
+ .filter(spec -> !spec.isEmpty())
+ .map(BlockedThreadsConfigurator::getSpec)
+ .filter(Optional::isPresent)
+ .map(Optional::get)
+ .collect(groupingBy(Threshold::hasPrefix));
+
+ // check configuration consistency
+ if (specsClassified.size() > 1) {
+ Collection<Threshold> specs = deduplicatePrefixes(specsClassified.get(true));
+ log.warn(
+ "Global and specific thresholds were configured for blocked threads check. Specific"
+ + " configuration is used {}.",
+ specs);
+ return specs;
+ }
+
+ if (specsClassified.size() == 1) {
+ Map.Entry<Boolean, List<Threshold>> entry = specsClassified.entrySet().iterator().next();
+ return Boolean.TRUE == entry.getKey()
+ ? deduplicatePrefixes(entry.getValue())
+ : deduplicateGlobal(entry.getValue());
+ }
+
+ log.info(
+ "Default blocked threads check is configured with {}% threshold",
+ DEFAULT_BLOCKED_THREADS_THRESHOLD);
+ return ImmutableSet.of(new Threshold(DEFAULT_BLOCKED_THREADS_THRESHOLD));
+ }
+
+ private static Collection<Threshold> deduplicateGlobal(List<Threshold> input) {
+ if (input.size() > 1) {
+ Threshold spec = input.get(input.size() - 1);
+ log.warn("Multiple threshold values were configured. Using {}", spec);
+ return ImmutableSet.of(spec);
+ }
+ return input;
+ }
+
+ private static Collection<Threshold> deduplicatePrefixes(Collection<Threshold> input) {
+ Map<String, Threshold> deduplicated = new HashMap<>();
+ input.forEach(t -> deduplicated.put(t.prefix.get(), t));
+ if (deduplicated.size() != input.size()) {
+ log.warn(
+ "The same prefixes were configured multiple times. The following configuration is used"
+ + " {}",
+ deduplicated.values());
+ }
+ return deduplicated.values();
+ }
+
+ private static Optional<Threshold> getSpec(String spec) {
+ int equals = spec.lastIndexOf('=');
+ if (equals != -1) {
+ Optional<Integer> maybeThreshold = isThresholdDefined(spec.substring(equals + 1));
+ if (maybeThreshold.isPresent()) {
+ return Optional.of(new Threshold(spec.substring(0, equals).trim(), maybeThreshold.get()));
+ }
+ } else {
+ Optional<Integer> maybeThreshold = isThresholdDefined(spec);
+ if (maybeThreshold.isPresent()) {
+ return Optional.of(new Threshold(maybeThreshold.get()));
+ }
+ }
+
+ log.warn("Invalid configuration of blocked threads threshold [{}]", spec);
+ return Optional.empty();
+ }
+
+ private static Optional<Integer> isThresholdDefined(String input) {
+ Matcher value = THRESHOLD_PATTERN.matcher(input.trim());
+ if (value.matches()) {
+ return Optional.of(Integer.valueOf(value.group(1)));
+ }
+ return Optional.empty();
+ }
+
+ @VisibleForTesting
+ static class Threshold {
+ final Optional<String> prefix;
+ final Integer value;
+
+ Threshold(int value) {
+ this(null, value);
+ }
+
+ Threshold(String prefix, int value) {
+ this.prefix = Optional.ofNullable(prefix);
+ this.value = value;
+ }
+
+ boolean hasPrefix() {
+ return prefix.isPresent();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(prefix, value);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ Threshold other = (Threshold) obj;
+ return Objects.equals(prefix, other.prefix) && Objects.equals(value, other.value);
+ }
+
+ @Override
+ public String toString() {
+ return new StringBuilder()
+ .append("Threshold [prefix=")
+ .append(prefix)
+ .append(", value=")
+ .append(value)
+ .append("]")
+ .toString();
+ }
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsSubCheck.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsSubCheck.java
new file mode 100644
index 0000000..d4f0fe8
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsSubCheck.java
@@ -0,0 +1,83 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.healthcheck.check;
+
+import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.BLOCKEDTHREADS;
+
+import com.google.common.base.Strings;
+import com.google.gerrit.metrics.Counter0;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import com.googlesource.gerrit.plugins.healthcheck.HealthCheckMetrics;
+import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheck.Result;
+import java.lang.management.ThreadInfo;
+import java.util.function.Supplier;
+
+class BlockedThreadsSubCheck
+ implements BlockedThreadsCheck.CollectorProvider<BlockedThreadsSubCheck.SubCheckCollector> {
+ interface Factory {
+ BlockedThreadsSubCheck create(String prefix, Integer threshold);
+ }
+
+ static class SubCheckCollector extends BlockedThreadsCheck.Collector {
+ private final String prefix;
+ private final Counter0 failureCounterMetric;
+
+ SubCheckCollector(String prefix, Integer threshold, Counter0 failureCounterMetric) {
+ super(threshold);
+ this.prefix = prefix;
+ this.failureCounterMetric = failureCounterMetric;
+ }
+
+ @Override
+ void collect(ThreadInfo info) {
+ String threadName = info.getThreadName();
+ if (!Strings.isNullOrEmpty(threadName) && threadName.startsWith(prefix)) {
+ total += 1;
+ if (Thread.State.BLOCKED == info.getThreadState()) {
+ blocked += 1;
+ }
+ }
+ }
+
+ @Override
+ void check() {
+ super.check();
+ if (Result.FAILED == result) {
+ failureCounterMetric.increment();
+ }
+ }
+ }
+
+ private final Supplier<SubCheckCollector> collector;
+
+ @Inject
+ BlockedThreadsSubCheck(
+ HealthCheckMetrics.Factory healthCheckMetricsFactory,
+ @Assisted String prefix,
+ @Assisted Integer threshold) {
+ HealthCheckMetrics healthCheckMetrics =
+ healthCheckMetricsFactory.create(
+ String.format(
+ "%s-%s", BLOCKEDTHREADS, prefix.toLowerCase().replaceAll("[^\\w-/]", "_")));
+ Counter0 failureCounterMetric = healthCheckMetrics.getFailureCounterMetric();
+ this.collector = () -> new SubCheckCollector(prefix, threshold, failureCounterMetric);
+ }
+
+ @Override
+ public SubCheckCollector get() {
+ return collector.get();
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java
index 97e06d0..4771942 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java
@@ -21,5 +21,6 @@
String AUTH = "auth";
String ACTIVEWORKERS = "activeworkers";
String DEADLOCK = "deadlock";
+ String BLOCKEDTHREADS = "blockedthreads";
String GLOBAL = "global";
}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 47acb01..0b4f05f 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -13,7 +13,12 @@
)]}'
{
"ts": 139402910202,
- "elapsed": 100,
+ "elapsed": 120,
+ "blockedthreads": {
+ "ts": 139402910202,
+ "elapsed": 30,
+ "result": "passed"
+ },
"querychanges": {
"ts": 139402910202,
"elapsed": 20,
@@ -21,12 +26,12 @@
},
"projectslist": {
"ts": 139402910202,
- "elapsed": 100,
+ "elapsed": 40,
"result": "passed"
},
"auth": {
"ts": 139402910202,
- "elapsed": 80,
+ "elapsed": 30,
"result": "passed"
}
}
@@ -50,6 +55,7 @@
- `auth`: check the ability to authenticate with username and password
- `activeworkers`: check the number of active worker threads and the ability to create a new one
- `deadlock` : check if Java deadlocks are reported by the JVM
+- `blockedthreads` : check the number of blocked threads
Each check name can be disabled by setting the `enabled` parameter to **false**,
by default this parameter is set to **true**
@@ -88,4 +94,46 @@
- `healthcheck.activeworkers.threshold` : Percent of queue occupancy above which queue is consider
as full.
- Default: 80
\ No newline at end of file
+ Default: 80
+
+ - `healthcheck.blockedthreads.threshold` : Percent of all threads that are blocked above which instance
+ is considered as unhealthy.
+
+ Default: 50
+
+By default `healthcheck.blockedthreads` is calculated as ratio of BLOCKED threads against the all
+Gerrit threads. It might be not sufficient for instance in case of `SSH-Interactive-Worker` threads
+that could be all blocked making effectively a Gerrit instance unhealthy (neither fetch nor push
+would succeed) but the threshold could be still not reached. Therefore one can fine tune the check
+by putting detailed configuration for one or more thread groups (all threads that have the name
+starting with a given prefix) to be checked according to the following template:
+
+```
+[healthcheck "blockedthreads"]
+ threshold = [prefix]=[XX]
+```
+
+Note that in case when specific thread groups are configured all threads are no longer checked.
+
+* **Example 1:** _check if BLOCKED threads are above the limit of 70_
+
+ ```
+ [healthcheck "blockedthreads"]
+ threshold = 70
+ ```
+
+* **Example 2:** _check if BLOCKED `foo` threads are above the 33 limit_
+
+ ```
+ [healthcheck "blockedthreads"]
+ threshold = foo=45
+ ```
+
+* **Example 3:** _check if BLOCKED `foo` threads are above the 33 limit and if BLOCKED `bar`_
+ _threads are above the the 60 limit_
+
+ ```
+ [healthcheck "blockedthreads"]
+ threshold = foo=33
+ threshold = bar=60
+ ```
diff --git a/src/resources/Documentation/config.md b/src/resources/Documentation/config.md
index aa59c29..76e6eb5 100644
--- a/src/resources/Documentation/config.md
+++ b/src/resources/Documentation/config.md
@@ -48,6 +48,9 @@
# TYPE plugins_healthcheck_projectslist_latest_measured_latency gauge
plugins_healthcheck_projectslist_latest_measured_latency 5.0
+# HELP plugins_healthcheck_blockedthreads_latest_measured_latency Generated from Dropwizard metric import (metric=plugins/healthcheck/blockedthreads/latency, type=com.google.gerrit.metrics.dropwizard.CallbackMetricImpl0$1)
+# TYPE plugins_healthcheck_blockedthreads_latest_measured_latency gauge
+plugins_healthcheck_blockedthreads_latest_measured_latency 6.0
# HELP plugins_healthcheck_jgit_failure_total Generated from Dropwizard metric import (metric=plugins/healthcheck/jgit/failure, type=com.codahale.metrics.Meter)
# TYPE plugins_healthcheck_jgit_failure_total counter
@@ -56,6 +59,29 @@
# HELP plugins_healthcheck_projectslist_failure_total Generated from Dropwizard metric import (metric=plugins/healthcheck/projectslist/failure, type=com.codahale.metrics.Meter)
# TYPE plugins_healthcheck_projectslist_failure_total counter
plugins_healthcheck_projectslist_failure_total 0.0
+
+# HELP plugins_healthcheck_blockedthreads_failure_total Generated from Dropwizard metric import (metric=plugins/healthcheck/blockedthreads/failure, type=com.codahale.metrics.Meter)
+# TYPE plugins_healthcheck_blockedthreads_failure_total counter
+plugins_healthcheck_blockedthreads_failure_total 1.0
```
+Note that additionally to the default `blockedthreads` metrics pair failures counter will reported for
+each configured prefix. For given config:
+
+```
+[healthcheck "blockedthreads"]
+ threshold = Foo=33
+```
+
+the following additional metric will be exposed and populated:
+
+```
+# HELP plugins_healthcheck_blockedthreads_foo_failure_total Generated from Dropwizard metric import (metric=plugins/healthcheck/blockedthreads-foo/failure, type=com.codahale.metrics.Meter)
+# TYPE plugins_healthcheck_blockedthreads_foo_failure_total counter
+plugins_healthcheck_blockedthreads_foo_failure_total 2.0
+```
+
+Note that prefix is used as postfix for a metric name but it is lower-cased and sanitized as only
+`a-zA-Z0-9_-/` chars are allowed to be a metric name (chars outside this set are turned to `_`).
+
Metrics will be exposed to prometheus by the [metrics-reporter-prometheus](https://gerrit.googlesource.com/plugins/metrics-reporter-prometheus/) plugin.
diff --git a/src/test/java/com/googlesource/gerrit/plugins/healthcheck/BlockedThreadsCheckTest.java b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/BlockedThreadsCheckTest.java
new file mode 100644
index 0000000..d50d12a
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/BlockedThreadsCheckTest.java
@@ -0,0 +1,206 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.healthcheck;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.BLOCKEDTHREADS;
+import static java.util.Collections.nCopies;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.util.Providers;
+import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsCheck;
+import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsConfigurator;
+import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheck.Result;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class BlockedThreadsCheckTest {
+ @Mock BlockedThreadsCheck.ThreadBeanProvider threadBeanProviderMock;
+
+ @Mock ThreadMXBean beanMock;
+
+ @Before
+ public void setUp() {
+ when(threadBeanProviderMock.get()).thenReturn(beanMock);
+ }
+
+ @Test
+ public void shouldPassCheckWhenNoThreadsAreReturned() {
+ BlockedThreadsCheck objectUnderTest = createCheck(HealthCheckConfig.DEFAULT_CONFIG);
+ when(beanMock.getThreadInfo(null, 0)).thenReturn(new ThreadInfo[0]);
+ assertThat(objectUnderTest.run().result).isEqualTo(Result.PASSED);
+ }
+
+ @Test
+ public void shouldPassCheckWhenNoThreadsAreBlocked() {
+ int running = 1;
+ int blocked = 0;
+ mockThreadsAndCheckResult(running, blocked, Result.PASSED);
+ }
+
+ @Test
+ public void shouldPassCheckWhenBlockedThreadsAreLessThanDefaultThreshold() {
+ int running = 2;
+ int blocked = 1;
+ mockThreadsAndCheckResult(running, blocked, Result.PASSED);
+ }
+
+ @Test
+ public void shouldPassCheckWhenBlockedThreadsAreEqualToDefaultThreshold() {
+ int running = 1;
+ int blocked = 1;
+ mockThreadsAndCheckResult(running, blocked, Result.PASSED);
+ }
+
+ @Test
+ public void shouldFailCheckWhenBlockedThreadsAreAboveTheDefaultThreshold() {
+ int running = 1;
+ int blocked = 2;
+ mockThreadsAndCheckResult(running, blocked, Result.FAILED);
+ }
+
+ @Test
+ public void shouldPassCheckWhenBlockedThreadsAreLessThenThreshold() {
+ int running = 3;
+ int blocked = 1;
+ HealthCheckConfig config =
+ new HealthCheckConfig("[healthcheck \"" + BLOCKEDTHREADS + "\"]\n" + " threshold = 25");
+ mockThreadsAndCheckResult(running, blocked, Result.PASSED, config);
+ }
+
+ @Test
+ public void shouldFailCheckWhenBlockedThreadsAreAboveTheThreshold() {
+ int running = 1;
+ int blocked = 1;
+ HealthCheckConfig config =
+ new HealthCheckConfig("[healthcheck \"" + BLOCKEDTHREADS + "\"]\n" + " threshold = 33");
+ mockThreadsAndCheckResult(running, blocked, Result.FAILED, config);
+ }
+
+ @Test
+ public void shouldPassCheckWhenBlockedThreadsWithPrefixAreLessThenThreshold() {
+ int running = 3;
+ int blocked = 1;
+ String prefix = "blocked-threads-prefix";
+ HealthCheckConfig config =
+ new HealthCheckConfig(
+ "[healthcheck \"" + BLOCKEDTHREADS + "\"]\n" + " threshold = " + prefix + " = 25");
+ mockThreadsAndCheckResult(running, blocked, Result.PASSED, prefix, config);
+ }
+
+ @Test
+ public void shouldFailCheckWhenBlockedThreadsWithPrefixAreAboveTheThreshold() {
+ int running = 1;
+ int blocked = 1;
+ String prefix = "blocked-threads-prefix";
+ HealthCheckConfig config =
+ new HealthCheckConfig(
+ "[healthcheck \"" + BLOCKEDTHREADS + "\"]\n" + " threshold = " + prefix + " = 33");
+ mockThreadsAndCheckResult(running, blocked, Result.FAILED, prefix, config);
+ }
+
+ @Test
+ public void shouldFailCheckWhenAnyOfTheBlockedThreadsWithPrefixAreAboveTheThreshold() {
+ int running = 1;
+ int blocked = 1;
+ String blockedPrefix = "blocked-threads-prefix";
+ String notBlockedPrefix = "running-threads";
+ HealthCheckConfig config =
+ new HealthCheckConfig(
+ "[healthcheck \""
+ + BLOCKEDTHREADS
+ + "\"]\n"
+ + " threshold = "
+ + blockedPrefix
+ + " = 33"
+ + "\nthreshold = "
+ + notBlockedPrefix
+ + "=33");
+ List<ThreadInfo> infos = new ArrayList<>(running + blocked + running);
+ infos.addAll(nCopies(running, mockInfo(Thread.State.RUNNABLE, blockedPrefix)));
+ infos.addAll(nCopies(blocked, mockInfo(Thread.State.BLOCKED, blockedPrefix)));
+ infos.addAll(nCopies(running, mockInfo(Thread.State.RUNNABLE, notBlockedPrefix)));
+ when(beanMock.getThreadInfo(null, 0)).thenReturn(infos.toArray(new ThreadInfo[infos.size()]));
+ checkResult(Result.FAILED, config);
+ }
+
+ private void mockThreadsAndCheckResult(int running, int blocked, Result expected) {
+ mockThreadsAndCheckResult(running, blocked, expected, HealthCheckConfig.DEFAULT_CONFIG);
+ }
+
+ private void mockThreadsAndCheckResult(
+ int running, int blocked, Result expected, HealthCheckConfig config) {
+ mockThreadsAndCheckResult(running, blocked, expected, "some-prefix", config);
+ }
+
+ private void mockThreadsAndCheckResult(
+ int running, int blocked, Result expected, String prefix, HealthCheckConfig config) {
+ mockThreads(running, blocked, prefix);
+ checkResult(expected, config);
+ }
+
+ private void checkResult(Result expected, HealthCheckConfig config) {
+ BlockedThreadsCheck objectUnderTest = createCheck(config);
+ assertThat(objectUnderTest.run().result).isEqualTo(expected);
+ }
+
+ private void mockThreads(int running, int blocked, String prefix) {
+ List<ThreadInfo> infos = new ArrayList<>(running + blocked);
+ infos.addAll(nCopies(running, mockInfo(Thread.State.RUNNABLE, prefix)));
+ infos.addAll(nCopies(blocked, mockInfo(Thread.State.BLOCKED, prefix)));
+ when(beanMock.getThreadInfo(null, 0)).thenReturn(infos.toArray(new ThreadInfo[infos.size()]));
+ }
+
+ private ThreadInfo mockInfo(Thread.State state, String prefix) {
+ ThreadInfo infoMock = mock(ThreadInfo.class);
+ when(infoMock.getThreadState()).thenReturn(state);
+ when(infoMock.getThreadName()).thenReturn(prefix);
+ return infoMock;
+ }
+
+ private BlockedThreadsCheck createCheck(HealthCheckConfig config) {
+ DummyHealthCheckMetricsFactory checkMetricsFactory = new DummyHealthCheckMetricsFactory();
+ Injector injector =
+ Guice.createInjector(
+ new HealthCheckModule(),
+ new AbstractModule() {
+ @Override
+ protected void configure() {
+ bind(HealthCheckConfig.class).toInstance(config);
+ bind(HealthCheckMetrics.Factory.class).toInstance(checkMetricsFactory);
+ }
+ },
+ BlockedThreadsCheck.SUB_CHECKS);
+ return new BlockedThreadsCheck(
+ injector.getInstance(ListeningExecutorService.class),
+ config,
+ checkMetricsFactory,
+ threadBeanProviderMock,
+ Providers.of(injector.getInstance(BlockedThreadsConfigurator.class)));
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfiguratorConfigsTest.java b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfiguratorConfigsTest.java
new file mode 100644
index 0000000..d22ebbd
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfiguratorConfigsTest.java
@@ -0,0 +1,77 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.healthcheck.check;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsConfigurator.DEFAULT_BLOCKED_THREADS_THRESHOLD;
+
+import java.util.Arrays;
+import java.util.Collection;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class BlockedThreadsConfiguratorConfigsTest {
+ private final String[] input;
+ private final Collection<BlockedThreadsConfigurator.Threshold> expected;
+
+ public BlockedThreadsConfiguratorConfigsTest(
+ String[] input, Collection<BlockedThreadsConfigurator.Threshold> expected) {
+ this.input = input;
+ this.expected = expected;
+ }
+
+ @Test
+ public void shouldReturnExpectedConfig() {
+ Collection<BlockedThreadsConfigurator.Threshold> result =
+ BlockedThreadsConfigurator.getConfig(input);
+ assertThat(result).containsExactlyElementsIn(expected);
+ }
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> configs() {
+ return Arrays.asList(
+ new Object[][] {
+ {new String[] {}, specs(threshold(DEFAULT_BLOCKED_THREADS_THRESHOLD))},
+ {new String[] {"30"}, specs(threshold(30))},
+ {
+ new String[] {"prefix1=40", "prefix2=70", "prefix3 = 80"},
+ specs(threshold("prefix1", 40), threshold("prefix2", 70), threshold("prefix3", 80))
+ },
+ // the latter configuration is selected
+ {new String[] {"30", "40"}, specs(threshold(40))},
+ // the latter configuration is selected
+ {new String[] {"prefix1=40", "prefix1=70"}, specs(threshold("prefix1", 70))},
+ // specific prefix configuration is favored over the global one
+ {new String[] {"30", "prefix1=40"}, specs(threshold("prefix1", 40))},
+ // specific prefix configuration is favored over the global one and it is deduplicated
+ {new String[] {"30", "prefix1=40", "prefix1=70"}, specs(threshold("prefix1", 70))},
+ });
+ }
+
+ private static BlockedThreadsConfigurator.Threshold threshold(int value) {
+ return threshold(null, value);
+ }
+
+ private static BlockedThreadsConfigurator.Threshold threshold(String prefix, int value) {
+ return new BlockedThreadsConfigurator.Threshold(prefix, value);
+ }
+
+ private static Collection<BlockedThreadsConfigurator.Threshold> specs(
+ BlockedThreadsConfigurator.Threshold... thresholds) {
+ return Arrays.asList(thresholds);
+ }
+}
