Merge branch 'stable-3.2' into stable-3.3
* stable-3.2:
Add blocked threads check
Change-Id: Icf3fca96bd1df7817c547243dd7603da18e3cb77
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java
index 3e5306c..651dbe9 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java
@@ -15,6 +15,7 @@
package com.googlesource.gerrit.plugins.healthcheck;
import static com.google.common.base.Preconditions.checkNotNull;
+import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.BLOCKEDTHREADS;
import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.QUERYCHANGES;
import com.google.common.annotations.VisibleForTesting;
@@ -142,6 +143,10 @@
HEALTHCHECK, checkNotNull(healthCheckName), "enabled", HEALTH_CHECK_ENABLED_DEFAULT);
}
+ public String[] getListOfBlockedThreadsThresholds() {
+ return config.getStringList(HEALTHCHECK, BLOCKEDTHREADS, "threshold");
+ }
+
private String getStringWithFallback(
String parameter, String healthCheckName, String defaultValue) {
String fallbackDefault =
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java
index 304a40c..3cda7b6 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java
@@ -18,6 +18,7 @@
import com.google.gerrit.extensions.registration.DynamicSet;
import com.googlesource.gerrit.plugins.healthcheck.check.ActiveWorkersCheck;
import com.googlesource.gerrit.plugins.healthcheck.check.AuthHealthCheck;
+import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsCheck;
import com.googlesource.gerrit.plugins.healthcheck.check.DeadlockCheck;
import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheck;
import com.googlesource.gerrit.plugins.healthcheck.check.JGitHealthCheck;
@@ -34,8 +35,10 @@
bindChecker(AuthHealthCheck.class);
bindChecker(ActiveWorkersCheck.class);
bindChecker(DeadlockCheck.class);
+ bindChecker(BlockedThreadsCheck.class);
factory(HealthCheckMetrics.Factory.class);
+ install(BlockedThreadsCheck.SUB_CHECKS);
}
private void bindChecker(Class<? extends HealthCheck> healthCheckClass) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsCheck.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsCheck.java
new file mode 100644
index 0000000..f2cbb70
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsCheck.java
@@ -0,0 +1,123 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.healthcheck.check;
+
+import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.BLOCKEDTHREADS;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Suppliers;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.gerrit.extensions.config.FactoryModule;
+import com.google.inject.Inject;
+import com.google.inject.Module;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.healthcheck.HealthCheckConfig;
+import com.googlesource.gerrit.plugins.healthcheck.HealthCheckMetrics;
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+
+@Singleton
+public class BlockedThreadsCheck extends AbstractHealthCheck {
+ public static Module SUB_CHECKS =
+ new FactoryModule() {
+ @Override
+ protected void configure() {
+ factory(BlockedThreadsSubCheck.Factory.class);
+ }
+ };
+
+ private final ThreadMXBean threads;
+ private final Supplier<List<Collector>> collectorsSupplier;
+
+ @Inject
+ public BlockedThreadsCheck(
+ ListeningExecutorService executor,
+ HealthCheckConfig healthCheckConfig,
+ HealthCheckMetrics.Factory healthCheckMetricsFactory,
+ ThreadBeanProvider threadBeanProvider,
+ Provider<BlockedThreadsConfigurator> checksConfig) {
+ super(executor, healthCheckConfig, BLOCKEDTHREADS, healthCheckMetricsFactory);
+ this.threads = threadBeanProvider.get();
+ this.collectorsSupplier = Suppliers.memoize(() -> checksConfig.get().collectors());
+ }
+
+ @Override
+ protected Result doCheck() throws Exception {
+ List<Collector> collectors = collectorsSupplier.get();
+ dumpAllThreads().forEach(info -> collectors.forEach(c -> c.collect(info)));
+
+ // call check on all sub-checks so that metrics are populated
+ collectors.forEach(Collector::check);
+
+ // report unhealthy instance if any of sub-checks failed
+ return collectors.stream()
+ .map(Collector::result)
+ .filter(r -> Result.FAILED == r)
+ .findAny()
+ .orElse(Result.PASSED);
+ }
+
+ private Stream<ThreadInfo> dumpAllThreads() {
+ // getting all thread ids and translating it into thread infos is noticeably faster then call to
+ // ThreadMXBean.dumpAllThreads as it doesn't calculate StackTrace. Note that some threads could
+ // be already finished (between call to get all ids and translate them to ThreadInfo objects
+ // hence they have to be filtered out).
+ return Arrays.stream(threads.getThreadInfo(threads.getAllThreadIds(), 0))
+ .filter(Objects::nonNull);
+ }
+
+ @VisibleForTesting
+ public static class ThreadBeanProvider {
+ public ThreadMXBean get() {
+ return ManagementFactory.getThreadMXBean();
+ }
+ }
+
+ static class Collector {
+ protected final Integer threshold;
+
+ protected int blocked;
+ protected int total;
+ protected Result result;
+
+ Collector(Integer threshold) {
+ this.threshold = threshold;
+ }
+
+ void collect(ThreadInfo info) {
+ total += 1;
+ if (Thread.State.BLOCKED == info.getThreadState()) {
+ blocked += 1;
+ }
+ }
+
+ void check() {
+ result = blocked * 100 <= threshold * total ? Result.PASSED : Result.FAILED;
+ }
+
+ Result result() {
+ return result;
+ }
+ }
+
+ interface CollectorProvider<T extends Collector> extends Provider<T> {}
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfigurator.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfigurator.java
new file mode 100644
index 0000000..ddc67f9
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfigurator.java
@@ -0,0 +1,205 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.healthcheck.check;
+
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.toList;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.healthcheck.HealthCheckConfig;
+import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsCheck.Collector;
+import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsCheck.CollectorProvider;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@VisibleForTesting
+public class BlockedThreadsConfigurator {
+ private static final Logger log = LoggerFactory.getLogger(BlockedThreadsConfigurator.class);
+ private static final Pattern THRESHOLD_PATTERN = Pattern.compile("^(\\d\\d?)$");
+
+ static final int DEFAULT_BLOCKED_THREADS_THRESHOLD = 50;
+
+ private final List<CollectorProvider<?>> providers;
+
+ @Inject
+ BlockedThreadsConfigurator(
+ BlockedThreadsSubCheck.Factory subchecks, HealthCheckConfig healthCheckConfig) {
+ this.providers = getProviders(subchecks, healthCheckConfig);
+ }
+
+ List<Collector> collectors() {
+ return providers.stream().map(CollectorProvider::get).collect(toList());
+ }
+
+ private static List<CollectorProvider<?>> getProviders(
+ BlockedThreadsSubCheck.Factory subchecksFactory, HealthCheckConfig healthCheckConfig) {
+ return getConfig(healthCheckConfig.getListOfBlockedThreadsThresholds()).stream()
+ .map(spec -> collectorProvider(subchecksFactory, spec))
+ .collect(toList());
+ }
+
+ private static CollectorProvider<?> collectorProvider(
+ BlockedThreadsSubCheck.Factory subchecksFactory, Threshold spec) {
+ return spec.prefix.isPresent()
+ ? subchecksFactory.create(spec.prefix.get(), spec.value)
+ : () -> new BlockedThreadsCheck.Collector(spec.value);
+ }
+
+ @VisibleForTesting
+ static Collection<Threshold> getConfig(String[] thresholds) {
+ // Threshold can be defined as a sole value e.g
+ // threshold = 80
+ // and would become a default one for all blocked threads check or as a set of specific thread
+ // groups checks defined like
+ // threshold = foo=30
+ // threshold = bar=40
+ // ...
+ // they are mutually exclusive which means that one either checks all threads or groups
+ Map<Boolean, List<Threshold>> specsClassified =
+ Arrays.stream(thresholds)
+ .filter(spec -> !spec.isEmpty())
+ .map(BlockedThreadsConfigurator::getSpec)
+ .filter(Optional::isPresent)
+ .map(Optional::get)
+ .collect(groupingBy(Threshold::hasPrefix));
+
+ // check configuration consistency
+ if (specsClassified.size() > 1) {
+ Collection<Threshold> specs = deduplicatePrefixes(specsClassified.get(true));
+ log.warn(
+ "Global and specific thresholds were configured for blocked threads check. Specific"
+ + " configuration is used {}.",
+ specs);
+ return specs;
+ }
+
+ if (specsClassified.size() == 1) {
+ Map.Entry<Boolean, List<Threshold>> entry = specsClassified.entrySet().iterator().next();
+ return Boolean.TRUE == entry.getKey()
+ ? deduplicatePrefixes(entry.getValue())
+ : deduplicateGlobal(entry.getValue());
+ }
+
+ log.info(
+ "Default blocked threads check is configured with {}% threshold",
+ DEFAULT_BLOCKED_THREADS_THRESHOLD);
+ return ImmutableSet.of(new Threshold(DEFAULT_BLOCKED_THREADS_THRESHOLD));
+ }
+
+ private static Collection<Threshold> deduplicateGlobal(List<Threshold> input) {
+ if (input.size() > 1) {
+ Threshold spec = input.get(input.size() - 1);
+ log.warn("Multiple threshold values were configured. Using {}", spec);
+ return ImmutableSet.of(spec);
+ }
+ return input;
+ }
+
+ private static Collection<Threshold> deduplicatePrefixes(Collection<Threshold> input) {
+ Map<String, Threshold> deduplicated = new HashMap<>();
+ input.forEach(t -> deduplicated.put(t.prefix.get(), t));
+ if (deduplicated.size() != input.size()) {
+ log.warn(
+ "The same prefixes were configured multiple times. The following configuration is used"
+ + " {}",
+ deduplicated.values());
+ }
+ return deduplicated.values();
+ }
+
+ private static Optional<Threshold> getSpec(String spec) {
+ int equals = spec.lastIndexOf('=');
+ if (equals != -1) {
+ Optional<Integer> maybeThreshold = isThresholdDefined(spec.substring(equals + 1));
+ if (maybeThreshold.isPresent()) {
+ return Optional.of(new Threshold(spec.substring(0, equals).trim(), maybeThreshold.get()));
+ }
+ } else {
+ Optional<Integer> maybeThreshold = isThresholdDefined(spec);
+ if (maybeThreshold.isPresent()) {
+ return Optional.of(new Threshold(maybeThreshold.get()));
+ }
+ }
+
+ log.warn("Invalid configuration of blocked threads threshold [{}]", spec);
+ return Optional.empty();
+ }
+
+ private static Optional<Integer> isThresholdDefined(String input) {
+ Matcher value = THRESHOLD_PATTERN.matcher(input.trim());
+ if (value.matches()) {
+ return Optional.of(Integer.valueOf(value.group(1)));
+ }
+ return Optional.empty();
+ }
+
+ @VisibleForTesting
+ static class Threshold {
+ final Optional<String> prefix;
+ final Integer value;
+
+ Threshold(int value) {
+ this(null, value);
+ }
+
+ Threshold(String prefix, int value) {
+ this.prefix = Optional.ofNullable(prefix);
+ this.value = value;
+ }
+
+ boolean hasPrefix() {
+ return prefix.isPresent();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(prefix, value);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ Threshold other = (Threshold) obj;
+ return Objects.equals(prefix, other.prefix) && Objects.equals(value, other.value);
+ }
+
+ @Override
+ public String toString() {
+ return new StringBuilder()
+ .append("Threshold [prefix=")
+ .append(prefix)
+ .append(", value=")
+ .append(value)
+ .append("]")
+ .toString();
+ }
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsSubCheck.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsSubCheck.java
new file mode 100644
index 0000000..d4f0fe8
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsSubCheck.java
@@ -0,0 +1,83 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.healthcheck.check;
+
+import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.BLOCKEDTHREADS;
+
+import com.google.common.base.Strings;
+import com.google.gerrit.metrics.Counter0;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import com.googlesource.gerrit.plugins.healthcheck.HealthCheckMetrics;
+import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheck.Result;
+import java.lang.management.ThreadInfo;
+import java.util.function.Supplier;
+
+class BlockedThreadsSubCheck
+ implements BlockedThreadsCheck.CollectorProvider<BlockedThreadsSubCheck.SubCheckCollector> {
+ interface Factory {
+ BlockedThreadsSubCheck create(String prefix, Integer threshold);
+ }
+
+ static class SubCheckCollector extends BlockedThreadsCheck.Collector {
+ private final String prefix;
+ private final Counter0 failureCounterMetric;
+
+ SubCheckCollector(String prefix, Integer threshold, Counter0 failureCounterMetric) {
+ super(threshold);
+ this.prefix = prefix;
+ this.failureCounterMetric = failureCounterMetric;
+ }
+
+ @Override
+ void collect(ThreadInfo info) {
+ String threadName = info.getThreadName();
+ if (!Strings.isNullOrEmpty(threadName) && threadName.startsWith(prefix)) {
+ total += 1;
+ if (Thread.State.BLOCKED == info.getThreadState()) {
+ blocked += 1;
+ }
+ }
+ }
+
+ @Override
+ void check() {
+ super.check();
+ if (Result.FAILED == result) {
+ failureCounterMetric.increment();
+ }
+ }
+ }
+
+ private final Supplier<SubCheckCollector> collector;
+
+ @Inject
+ BlockedThreadsSubCheck(
+ HealthCheckMetrics.Factory healthCheckMetricsFactory,
+ @Assisted String prefix,
+ @Assisted Integer threshold) {
+ HealthCheckMetrics healthCheckMetrics =
+ healthCheckMetricsFactory.create(
+ String.format(
+ "%s-%s", BLOCKEDTHREADS, prefix.toLowerCase().replaceAll("[^\\w-/]", "_")));
+ Counter0 failureCounterMetric = healthCheckMetrics.getFailureCounterMetric();
+ this.collector = () -> new SubCheckCollector(prefix, threshold, failureCounterMetric);
+ }
+
+ @Override
+ public SubCheckCollector get() {
+ return collector.get();
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java
index 97e06d0..4771942 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java
@@ -21,5 +21,6 @@
String AUTH = "auth";
String ACTIVEWORKERS = "activeworkers";
String DEADLOCK = "deadlock";
+ String BLOCKEDTHREADS = "blockedthreads";
String GLOBAL = "global";
}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 47acb01..0b4f05f 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -13,7 +13,12 @@
)]}'
{
"ts": 139402910202,
- "elapsed": 100,
+ "elapsed": 120,
+ "blockedthreads": {
+ "ts": 139402910202,
+ "elapsed": 30,
+ "result": "passed"
+ },
"querychanges": {
"ts": 139402910202,
"elapsed": 20,
@@ -21,12 +26,12 @@
},
"projectslist": {
"ts": 139402910202,
- "elapsed": 100,
+ "elapsed": 40,
"result": "passed"
},
"auth": {
"ts": 139402910202,
- "elapsed": 80,
+ "elapsed": 30,
"result": "passed"
}
}
@@ -50,6 +55,7 @@
- `auth`: check the ability to authenticate with username and password
- `activeworkers`: check the number of active worker threads and the ability to create a new one
- `deadlock` : check if Java deadlocks are reported by the JVM
+- `blockedthreads` : check the number of blocked threads
Each check name can be disabled by setting the `enabled` parameter to **false**,
by default this parameter is set to **true**
@@ -88,4 +94,46 @@
- `healthcheck.activeworkers.threshold` : Percent of queue occupancy above which queue is consider
as full.
- Default: 80
\ No newline at end of file
+ Default: 80
+
+ - `healthcheck.blockedthreads.threshold` : Percent of all threads that are blocked above which instance
+ is considered as unhealthy.
+
+ Default: 50
+
+By default `healthcheck.blockedthreads` is calculated as ratio of BLOCKED threads against the all
+Gerrit threads. It might be not sufficient for instance in case of `SSH-Interactive-Worker` threads
+that could be all blocked making effectively a Gerrit instance unhealthy (neither fetch nor push
+would succeed) but the threshold could be still not reached. Therefore one can fine tune the check
+by putting detailed configuration for one or more thread groups (all threads that have the name
+starting with a given prefix) to be checked according to the following template:
+
+```
+[healthcheck "blockedthreads"]
+ threshold = [prefix]=[XX]
+```
+
+Note that in case when specific thread groups are configured all threads are no longer checked.
+
+* **Example 1:** _check if BLOCKED threads are above the limit of 70_
+
+ ```
+ [healthcheck "blockedthreads"]
+ threshold = 70
+ ```
+
+* **Example 2:** _check if BLOCKED `foo` threads are above the 33 limit_
+
+ ```
+ [healthcheck "blockedthreads"]
+ threshold = foo=45
+ ```
+
+* **Example 3:** _check if BLOCKED `foo` threads are above the 33 limit and if BLOCKED `bar`_
+ _threads are above the the 60 limit_
+
+ ```
+ [healthcheck "blockedthreads"]
+ threshold = foo=33
+ threshold = bar=60
+ ```
diff --git a/src/resources/Documentation/config.md b/src/resources/Documentation/config.md
index aa59c29..76e6eb5 100644
--- a/src/resources/Documentation/config.md
+++ b/src/resources/Documentation/config.md
@@ -48,6 +48,9 @@
# TYPE plugins_healthcheck_projectslist_latest_measured_latency gauge
plugins_healthcheck_projectslist_latest_measured_latency 5.0
+# HELP plugins_healthcheck_blockedthreads_latest_measured_latency Generated from Dropwizard metric import (metric=plugins/healthcheck/blockedthreads/latency, type=com.google.gerrit.metrics.dropwizard.CallbackMetricImpl0$1)
+# TYPE plugins_healthcheck_blockedthreads_latest_measured_latency gauge
+plugins_healthcheck_blockedthreads_latest_measured_latency 6.0
# HELP plugins_healthcheck_jgit_failure_total Generated from Dropwizard metric import (metric=plugins/healthcheck/jgit/failure, type=com.codahale.metrics.Meter)
# TYPE plugins_healthcheck_jgit_failure_total counter
@@ -56,6 +59,29 @@
# HELP plugins_healthcheck_projectslist_failure_total Generated from Dropwizard metric import (metric=plugins/healthcheck/projectslist/failure, type=com.codahale.metrics.Meter)
# TYPE plugins_healthcheck_projectslist_failure_total counter
plugins_healthcheck_projectslist_failure_total 0.0
+
+# HELP plugins_healthcheck_blockedthreads_failure_total Generated from Dropwizard metric import (metric=plugins/healthcheck/blockedthreads/failure, type=com.codahale.metrics.Meter)
+# TYPE plugins_healthcheck_blockedthreads_failure_total counter
+plugins_healthcheck_blockedthreads_failure_total 1.0
```
+Note that additionally to the default `blockedthreads` metrics pair failures counter will reported for
+each configured prefix. For given config:
+
+```
+[healthcheck "blockedthreads"]
+ threshold = Foo=33
+```
+
+the following additional metric will be exposed and populated:
+
+```
+# HELP plugins_healthcheck_blockedthreads_foo_failure_total Generated from Dropwizard metric import (metric=plugins/healthcheck/blockedthreads-foo/failure, type=com.codahale.metrics.Meter)
+# TYPE plugins_healthcheck_blockedthreads_foo_failure_total counter
+plugins_healthcheck_blockedthreads_foo_failure_total 2.0
+```
+
+Note that prefix is used as postfix for a metric name but it is lower-cased and sanitized as only
+`a-zA-Z0-9_-/` chars are allowed to be a metric name (chars outside this set are turned to `_`).
+
Metrics will be exposed to prometheus by the [metrics-reporter-prometheus](https://gerrit.googlesource.com/plugins/metrics-reporter-prometheus/) plugin.
diff --git a/src/test/java/com/googlesource/gerrit/plugins/healthcheck/BlockedThreadsCheckTest.java b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/BlockedThreadsCheckTest.java
new file mode 100644
index 0000000..d50d12a
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/BlockedThreadsCheckTest.java
@@ -0,0 +1,206 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.healthcheck;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.BLOCKEDTHREADS;
+import static java.util.Collections.nCopies;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.util.Providers;
+import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsCheck;
+import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsConfigurator;
+import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheck.Result;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class BlockedThreadsCheckTest {
+ @Mock BlockedThreadsCheck.ThreadBeanProvider threadBeanProviderMock;
+
+ @Mock ThreadMXBean beanMock;
+
+ @Before
+ public void setUp() {
+ when(threadBeanProviderMock.get()).thenReturn(beanMock);
+ }
+
+ @Test
+ public void shouldPassCheckWhenNoThreadsAreReturned() {
+ BlockedThreadsCheck objectUnderTest = createCheck(HealthCheckConfig.DEFAULT_CONFIG);
+ when(beanMock.getThreadInfo(null, 0)).thenReturn(new ThreadInfo[0]);
+ assertThat(objectUnderTest.run().result).isEqualTo(Result.PASSED);
+ }
+
+ @Test
+ public void shouldPassCheckWhenNoThreadsAreBlocked() {
+ int running = 1;
+ int blocked = 0;
+ mockThreadsAndCheckResult(running, blocked, Result.PASSED);
+ }
+
+ @Test
+ public void shouldPassCheckWhenBlockedThreadsAreLessThanDefaultThreshold() {
+ int running = 2;
+ int blocked = 1;
+ mockThreadsAndCheckResult(running, blocked, Result.PASSED);
+ }
+
+ @Test
+ public void shouldPassCheckWhenBlockedThreadsAreEqualToDefaultThreshold() {
+ int running = 1;
+ int blocked = 1;
+ mockThreadsAndCheckResult(running, blocked, Result.PASSED);
+ }
+
+ @Test
+ public void shouldFailCheckWhenBlockedThreadsAreAboveTheDefaultThreshold() {
+ int running = 1;
+ int blocked = 2;
+ mockThreadsAndCheckResult(running, blocked, Result.FAILED);
+ }
+
+ @Test
+ public void shouldPassCheckWhenBlockedThreadsAreLessThenThreshold() {
+ int running = 3;
+ int blocked = 1;
+ HealthCheckConfig config =
+ new HealthCheckConfig("[healthcheck \"" + BLOCKEDTHREADS + "\"]\n" + " threshold = 25");
+ mockThreadsAndCheckResult(running, blocked, Result.PASSED, config);
+ }
+
+ @Test
+ public void shouldFailCheckWhenBlockedThreadsAreAboveTheThreshold() {
+ int running = 1;
+ int blocked = 1;
+ HealthCheckConfig config =
+ new HealthCheckConfig("[healthcheck \"" + BLOCKEDTHREADS + "\"]\n" + " threshold = 33");
+ mockThreadsAndCheckResult(running, blocked, Result.FAILED, config);
+ }
+
+ @Test
+ public void shouldPassCheckWhenBlockedThreadsWithPrefixAreLessThenThreshold() {
+ int running = 3;
+ int blocked = 1;
+ String prefix = "blocked-threads-prefix";
+ HealthCheckConfig config =
+ new HealthCheckConfig(
+ "[healthcheck \"" + BLOCKEDTHREADS + "\"]\n" + " threshold = " + prefix + " = 25");
+ mockThreadsAndCheckResult(running, blocked, Result.PASSED, prefix, config);
+ }
+
+ @Test
+ public void shouldFailCheckWhenBlockedThreadsWithPrefixAreAboveTheThreshold() {
+ int running = 1;
+ int blocked = 1;
+ String prefix = "blocked-threads-prefix";
+ HealthCheckConfig config =
+ new HealthCheckConfig(
+ "[healthcheck \"" + BLOCKEDTHREADS + "\"]\n" + " threshold = " + prefix + " = 33");
+ mockThreadsAndCheckResult(running, blocked, Result.FAILED, prefix, config);
+ }
+
+ @Test
+ public void shouldFailCheckWhenAnyOfTheBlockedThreadsWithPrefixAreAboveTheThreshold() {
+ int running = 1;
+ int blocked = 1;
+ String blockedPrefix = "blocked-threads-prefix";
+ String notBlockedPrefix = "running-threads";
+ HealthCheckConfig config =
+ new HealthCheckConfig(
+ "[healthcheck \""
+ + BLOCKEDTHREADS
+ + "\"]\n"
+ + " threshold = "
+ + blockedPrefix
+ + " = 33"
+ + "\nthreshold = "
+ + notBlockedPrefix
+ + "=33");
+ List<ThreadInfo> infos = new ArrayList<>(running + blocked + running);
+ infos.addAll(nCopies(running, mockInfo(Thread.State.RUNNABLE, blockedPrefix)));
+ infos.addAll(nCopies(blocked, mockInfo(Thread.State.BLOCKED, blockedPrefix)));
+ infos.addAll(nCopies(running, mockInfo(Thread.State.RUNNABLE, notBlockedPrefix)));
+ when(beanMock.getThreadInfo(null, 0)).thenReturn(infos.toArray(new ThreadInfo[infos.size()]));
+ checkResult(Result.FAILED, config);
+ }
+
+ private void mockThreadsAndCheckResult(int running, int blocked, Result expected) {
+ mockThreadsAndCheckResult(running, blocked, expected, HealthCheckConfig.DEFAULT_CONFIG);
+ }
+
+ private void mockThreadsAndCheckResult(
+ int running, int blocked, Result expected, HealthCheckConfig config) {
+ mockThreadsAndCheckResult(running, blocked, expected, "some-prefix", config);
+ }
+
+ private void mockThreadsAndCheckResult(
+ int running, int blocked, Result expected, String prefix, HealthCheckConfig config) {
+ mockThreads(running, blocked, prefix);
+ checkResult(expected, config);
+ }
+
+ private void checkResult(Result expected, HealthCheckConfig config) {
+ BlockedThreadsCheck objectUnderTest = createCheck(config);
+ assertThat(objectUnderTest.run().result).isEqualTo(expected);
+ }
+
+ private void mockThreads(int running, int blocked, String prefix) {
+ List<ThreadInfo> infos = new ArrayList<>(running + blocked);
+ infos.addAll(nCopies(running, mockInfo(Thread.State.RUNNABLE, prefix)));
+ infos.addAll(nCopies(blocked, mockInfo(Thread.State.BLOCKED, prefix)));
+ when(beanMock.getThreadInfo(null, 0)).thenReturn(infos.toArray(new ThreadInfo[infos.size()]));
+ }
+
+ private ThreadInfo mockInfo(Thread.State state, String prefix) {
+ ThreadInfo infoMock = mock(ThreadInfo.class);
+ when(infoMock.getThreadState()).thenReturn(state);
+ when(infoMock.getThreadName()).thenReturn(prefix);
+ return infoMock;
+ }
+
+ private BlockedThreadsCheck createCheck(HealthCheckConfig config) {
+ DummyHealthCheckMetricsFactory checkMetricsFactory = new DummyHealthCheckMetricsFactory();
+ Injector injector =
+ Guice.createInjector(
+ new HealthCheckModule(),
+ new AbstractModule() {
+ @Override
+ protected void configure() {
+ bind(HealthCheckConfig.class).toInstance(config);
+ bind(HealthCheckMetrics.Factory.class).toInstance(checkMetricsFactory);
+ }
+ },
+ BlockedThreadsCheck.SUB_CHECKS);
+ return new BlockedThreadsCheck(
+ injector.getInstance(ListeningExecutorService.class),
+ config,
+ checkMetricsFactory,
+ threadBeanProviderMock,
+ Providers.of(injector.getInstance(BlockedThreadsConfigurator.class)));
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfiguratorConfigsTest.java b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfiguratorConfigsTest.java
new file mode 100644
index 0000000..d22ebbd
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfiguratorConfigsTest.java
@@ -0,0 +1,77 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.healthcheck.check;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsConfigurator.DEFAULT_BLOCKED_THREADS_THRESHOLD;
+
+import java.util.Arrays;
+import java.util.Collection;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class BlockedThreadsConfiguratorConfigsTest {
+ private final String[] input;
+ private final Collection<BlockedThreadsConfigurator.Threshold> expected;
+
+ public BlockedThreadsConfiguratorConfigsTest(
+ String[] input, Collection<BlockedThreadsConfigurator.Threshold> expected) {
+ this.input = input;
+ this.expected = expected;
+ }
+
+ @Test
+ public void shouldReturnExpectedConfig() {
+ Collection<BlockedThreadsConfigurator.Threshold> result =
+ BlockedThreadsConfigurator.getConfig(input);
+ assertThat(result).containsExactlyElementsIn(expected);
+ }
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> configs() {
+ return Arrays.asList(
+ new Object[][] {
+ {new String[] {}, specs(threshold(DEFAULT_BLOCKED_THREADS_THRESHOLD))},
+ {new String[] {"30"}, specs(threshold(30))},
+ {
+ new String[] {"prefix1=40", "prefix2=70", "prefix3 = 80"},
+ specs(threshold("prefix1", 40), threshold("prefix2", 70), threshold("prefix3", 80))
+ },
+ // the latter configuration is selected
+ {new String[] {"30", "40"}, specs(threshold(40))},
+ // the latter configuration is selected
+ {new String[] {"prefix1=40", "prefix1=70"}, specs(threshold("prefix1", 70))},
+ // specific prefix configuration is favored over the global one
+ {new String[] {"30", "prefix1=40"}, specs(threshold("prefix1", 40))},
+ // specific prefix configuration is favored over the global one and it is deduplicated
+ {new String[] {"30", "prefix1=40", "prefix1=70"}, specs(threshold("prefix1", 70))},
+ });
+ }
+
+ private static BlockedThreadsConfigurator.Threshold threshold(int value) {
+ return threshold(null, value);
+ }
+
+ private static BlockedThreadsConfigurator.Threshold threshold(String prefix, int value) {
+ return new BlockedThreadsConfigurator.Threshold(prefix, value);
+ }
+
+ private static Collection<BlockedThreadsConfigurator.Threshold> specs(
+ BlockedThreadsConfigurator.Threshold... thresholds) {
+ return Arrays.asList(thresholds);
+ }
+}