Merge branch 'stable-3.2' into stable-3.3

* stable-3.2:
  Add blocked threads check

Change-Id: Icf3fca96bd1df7817c547243dd7603da18e3cb77
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java
index 3e5306c..651dbe9 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java
@@ -15,6 +15,7 @@
 package com.googlesource.gerrit.plugins.healthcheck;
 
 import static com.google.common.base.Preconditions.checkNotNull;
+import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.BLOCKEDTHREADS;
 import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.QUERYCHANGES;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -142,6 +143,10 @@
         HEALTHCHECK, checkNotNull(healthCheckName), "enabled", HEALTH_CHECK_ENABLED_DEFAULT);
   }
 
+  public String[] getListOfBlockedThreadsThresholds() {
+    return config.getStringList(HEALTHCHECK, BLOCKEDTHREADS, "threshold");
+  }
+
   private String getStringWithFallback(
       String parameter, String healthCheckName, String defaultValue) {
     String fallbackDefault =
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java
index 304a40c..3cda7b6 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java
@@ -18,6 +18,7 @@
 import com.google.gerrit.extensions.registration.DynamicSet;
 import com.googlesource.gerrit.plugins.healthcheck.check.ActiveWorkersCheck;
 import com.googlesource.gerrit.plugins.healthcheck.check.AuthHealthCheck;
+import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsCheck;
 import com.googlesource.gerrit.plugins.healthcheck.check.DeadlockCheck;
 import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheck;
 import com.googlesource.gerrit.plugins.healthcheck.check.JGitHealthCheck;
@@ -34,8 +35,10 @@
     bindChecker(AuthHealthCheck.class);
     bindChecker(ActiveWorkersCheck.class);
     bindChecker(DeadlockCheck.class);
+    bindChecker(BlockedThreadsCheck.class);
 
     factory(HealthCheckMetrics.Factory.class);
+    install(BlockedThreadsCheck.SUB_CHECKS);
   }
 
   private void bindChecker(Class<? extends HealthCheck> healthCheckClass) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsCheck.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsCheck.java
new file mode 100644
index 0000000..f2cbb70
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsCheck.java
@@ -0,0 +1,123 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.healthcheck.check;
+
+import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.BLOCKEDTHREADS;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Suppliers;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.gerrit.extensions.config.FactoryModule;
+import com.google.inject.Inject;
+import com.google.inject.Module;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.healthcheck.HealthCheckConfig;
+import com.googlesource.gerrit.plugins.healthcheck.HealthCheckMetrics;
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+
+@Singleton
+public class BlockedThreadsCheck extends AbstractHealthCheck {
+  public static Module SUB_CHECKS =
+      new FactoryModule() {
+        @Override
+        protected void configure() {
+          factory(BlockedThreadsSubCheck.Factory.class);
+        }
+      };
+
+  private final ThreadMXBean threads;
+  private final Supplier<List<Collector>> collectorsSupplier;
+
+  @Inject
+  public BlockedThreadsCheck(
+      ListeningExecutorService executor,
+      HealthCheckConfig healthCheckConfig,
+      HealthCheckMetrics.Factory healthCheckMetricsFactory,
+      ThreadBeanProvider threadBeanProvider,
+      Provider<BlockedThreadsConfigurator> checksConfig) {
+    super(executor, healthCheckConfig, BLOCKEDTHREADS, healthCheckMetricsFactory);
+    this.threads = threadBeanProvider.get();
+    this.collectorsSupplier = Suppliers.memoize(() -> checksConfig.get().collectors());
+  }
+
+  @Override
+  protected Result doCheck() throws Exception {
+    List<Collector> collectors = collectorsSupplier.get();
+    dumpAllThreads().forEach(info -> collectors.forEach(c -> c.collect(info)));
+
+    // call check on all sub-checks so that metrics are populated
+    collectors.forEach(Collector::check);
+
+    // report unhealthy instance if any of sub-checks failed
+    return collectors.stream()
+        .map(Collector::result)
+        .filter(r -> Result.FAILED == r)
+        .findAny()
+        .orElse(Result.PASSED);
+  }
+
+  private Stream<ThreadInfo> dumpAllThreads() {
+    // getting all thread ids and translating it into thread infos is noticeably faster then call to
+    // ThreadMXBean.dumpAllThreads as it doesn't calculate StackTrace. Note that some threads could
+    // be already finished (between call to get all ids and translate them to ThreadInfo objects
+    // hence they have to be filtered out).
+    return Arrays.stream(threads.getThreadInfo(threads.getAllThreadIds(), 0))
+        .filter(Objects::nonNull);
+  }
+
+  @VisibleForTesting
+  public static class ThreadBeanProvider {
+    public ThreadMXBean get() {
+      return ManagementFactory.getThreadMXBean();
+    }
+  }
+
+  static class Collector {
+    protected final Integer threshold;
+
+    protected int blocked;
+    protected int total;
+    protected Result result;
+
+    Collector(Integer threshold) {
+      this.threshold = threshold;
+    }
+
+    void collect(ThreadInfo info) {
+      total += 1;
+      if (Thread.State.BLOCKED == info.getThreadState()) {
+        blocked += 1;
+      }
+    }
+
+    void check() {
+      result = blocked * 100 <= threshold * total ? Result.PASSED : Result.FAILED;
+    }
+
+    Result result() {
+      return result;
+    }
+  }
+
+  interface CollectorProvider<T extends Collector> extends Provider<T> {}
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfigurator.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfigurator.java
new file mode 100644
index 0000000..ddc67f9
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfigurator.java
@@ -0,0 +1,205 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.healthcheck.check;
+
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.toList;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.healthcheck.HealthCheckConfig;
+import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsCheck.Collector;
+import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsCheck.CollectorProvider;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@VisibleForTesting
+public class BlockedThreadsConfigurator {
+  private static final Logger log = LoggerFactory.getLogger(BlockedThreadsConfigurator.class);
+  private static final Pattern THRESHOLD_PATTERN = Pattern.compile("^(\\d\\d?)$");
+
+  static final int DEFAULT_BLOCKED_THREADS_THRESHOLD = 50;
+
+  private final List<CollectorProvider<?>> providers;
+
+  @Inject
+  BlockedThreadsConfigurator(
+      BlockedThreadsSubCheck.Factory subchecks, HealthCheckConfig healthCheckConfig) {
+    this.providers = getProviders(subchecks, healthCheckConfig);
+  }
+
+  List<Collector> collectors() {
+    return providers.stream().map(CollectorProvider::get).collect(toList());
+  }
+
+  private static List<CollectorProvider<?>> getProviders(
+      BlockedThreadsSubCheck.Factory subchecksFactory, HealthCheckConfig healthCheckConfig) {
+    return getConfig(healthCheckConfig.getListOfBlockedThreadsThresholds()).stream()
+        .map(spec -> collectorProvider(subchecksFactory, spec))
+        .collect(toList());
+  }
+
+  private static CollectorProvider<?> collectorProvider(
+      BlockedThreadsSubCheck.Factory subchecksFactory, Threshold spec) {
+    return spec.prefix.isPresent()
+        ? subchecksFactory.create(spec.prefix.get(), spec.value)
+        : () -> new BlockedThreadsCheck.Collector(spec.value);
+  }
+
+  @VisibleForTesting
+  static Collection<Threshold> getConfig(String[] thresholds) {
+    // Threshold can be defined as a sole value e.g
+    //  threshold = 80
+    // and would become a default one for all blocked threads check or as a set of specific thread
+    // groups checks defined like
+    //  threshold = foo=30
+    //  threshold = bar=40
+    //  ...
+    // they are mutually exclusive which means that one either checks all threads or groups
+    Map<Boolean, List<Threshold>> specsClassified =
+        Arrays.stream(thresholds)
+            .filter(spec -> !spec.isEmpty())
+            .map(BlockedThreadsConfigurator::getSpec)
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .collect(groupingBy(Threshold::hasPrefix));
+
+    // check configuration consistency
+    if (specsClassified.size() > 1) {
+      Collection<Threshold> specs = deduplicatePrefixes(specsClassified.get(true));
+      log.warn(
+          "Global and specific thresholds were configured for blocked threads check. Specific"
+              + " configuration is used {}.",
+          specs);
+      return specs;
+    }
+
+    if (specsClassified.size() == 1) {
+      Map.Entry<Boolean, List<Threshold>> entry = specsClassified.entrySet().iterator().next();
+      return Boolean.TRUE == entry.getKey()
+          ? deduplicatePrefixes(entry.getValue())
+          : deduplicateGlobal(entry.getValue());
+    }
+
+    log.info(
+        "Default blocked threads check is configured with {}% threshold",
+        DEFAULT_BLOCKED_THREADS_THRESHOLD);
+    return ImmutableSet.of(new Threshold(DEFAULT_BLOCKED_THREADS_THRESHOLD));
+  }
+
+  private static Collection<Threshold> deduplicateGlobal(List<Threshold> input) {
+    if (input.size() > 1) {
+      Threshold spec = input.get(input.size() - 1);
+      log.warn("Multiple threshold values were configured. Using {}", spec);
+      return ImmutableSet.of(spec);
+    }
+    return input;
+  }
+
+  private static Collection<Threshold> deduplicatePrefixes(Collection<Threshold> input) {
+    Map<String, Threshold> deduplicated = new HashMap<>();
+    input.forEach(t -> deduplicated.put(t.prefix.get(), t));
+    if (deduplicated.size() != input.size()) {
+      log.warn(
+          "The same prefixes were configured multiple times. The following configuration is used"
+              + " {}",
+          deduplicated.values());
+    }
+    return deduplicated.values();
+  }
+
+  private static Optional<Threshold> getSpec(String spec) {
+    int equals = spec.lastIndexOf('=');
+    if (equals != -1) {
+      Optional<Integer> maybeThreshold = isThresholdDefined(spec.substring(equals + 1));
+      if (maybeThreshold.isPresent()) {
+        return Optional.of(new Threshold(spec.substring(0, equals).trim(), maybeThreshold.get()));
+      }
+    } else {
+      Optional<Integer> maybeThreshold = isThresholdDefined(spec);
+      if (maybeThreshold.isPresent()) {
+        return Optional.of(new Threshold(maybeThreshold.get()));
+      }
+    }
+
+    log.warn("Invalid configuration of blocked threads threshold [{}]", spec);
+    return Optional.empty();
+  }
+
+  private static Optional<Integer> isThresholdDefined(String input) {
+    Matcher value = THRESHOLD_PATTERN.matcher(input.trim());
+    if (value.matches()) {
+      return Optional.of(Integer.valueOf(value.group(1)));
+    }
+    return Optional.empty();
+  }
+
+  @VisibleForTesting
+  static class Threshold {
+    final Optional<String> prefix;
+    final Integer value;
+
+    Threshold(int value) {
+      this(null, value);
+    }
+
+    Threshold(String prefix, int value) {
+      this.prefix = Optional.ofNullable(prefix);
+      this.value = value;
+    }
+
+    boolean hasPrefix() {
+      return prefix.isPresent();
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(prefix, value);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (this == obj) {
+        return true;
+      }
+      if (obj == null || getClass() != obj.getClass()) {
+        return false;
+      }
+      Threshold other = (Threshold) obj;
+      return Objects.equals(prefix, other.prefix) && Objects.equals(value, other.value);
+    }
+
+    @Override
+    public String toString() {
+      return new StringBuilder()
+          .append("Threshold [prefix=")
+          .append(prefix)
+          .append(", value=")
+          .append(value)
+          .append("]")
+          .toString();
+    }
+  }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsSubCheck.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsSubCheck.java
new file mode 100644
index 0000000..d4f0fe8
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsSubCheck.java
@@ -0,0 +1,83 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.healthcheck.check;
+
+import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.BLOCKEDTHREADS;
+
+import com.google.common.base.Strings;
+import com.google.gerrit.metrics.Counter0;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import com.googlesource.gerrit.plugins.healthcheck.HealthCheckMetrics;
+import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheck.Result;
+import java.lang.management.ThreadInfo;
+import java.util.function.Supplier;
+
+class BlockedThreadsSubCheck
+    implements BlockedThreadsCheck.CollectorProvider<BlockedThreadsSubCheck.SubCheckCollector> {
+  interface Factory {
+    BlockedThreadsSubCheck create(String prefix, Integer threshold);
+  }
+
+  static class SubCheckCollector extends BlockedThreadsCheck.Collector {
+    private final String prefix;
+    private final Counter0 failureCounterMetric;
+
+    SubCheckCollector(String prefix, Integer threshold, Counter0 failureCounterMetric) {
+      super(threshold);
+      this.prefix = prefix;
+      this.failureCounterMetric = failureCounterMetric;
+    }
+
+    @Override
+    void collect(ThreadInfo info) {
+      String threadName = info.getThreadName();
+      if (!Strings.isNullOrEmpty(threadName) && threadName.startsWith(prefix)) {
+        total += 1;
+        if (Thread.State.BLOCKED == info.getThreadState()) {
+          blocked += 1;
+        }
+      }
+    }
+
+    @Override
+    void check() {
+      super.check();
+      if (Result.FAILED == result) {
+        failureCounterMetric.increment();
+      }
+    }
+  }
+
+  private final Supplier<SubCheckCollector> collector;
+
+  @Inject
+  BlockedThreadsSubCheck(
+      HealthCheckMetrics.Factory healthCheckMetricsFactory,
+      @Assisted String prefix,
+      @Assisted Integer threshold) {
+    HealthCheckMetrics healthCheckMetrics =
+        healthCheckMetricsFactory.create(
+            String.format(
+                "%s-%s", BLOCKEDTHREADS, prefix.toLowerCase().replaceAll("[^\\w-/]", "_")));
+    Counter0 failureCounterMetric = healthCheckMetrics.getFailureCounterMetric();
+    this.collector = () -> new SubCheckCollector(prefix, threshold, failureCounterMetric);
+  }
+
+  @Override
+  public SubCheckCollector get() {
+    return collector.get();
+  }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java
index 97e06d0..4771942 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java
@@ -21,5 +21,6 @@
   String AUTH = "auth";
   String ACTIVEWORKERS = "activeworkers";
   String DEADLOCK = "deadlock";
+  String BLOCKEDTHREADS = "blockedthreads";
   String GLOBAL = "global";
 }
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 47acb01..0b4f05f 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -13,7 +13,12 @@
 )]}'
 {
   "ts": 139402910202,
-  "elapsed": 100,
+  "elapsed": 120,
+  "blockedthreads": {
+    "ts": 139402910202,
+    "elapsed": 30,
+    "result": "passed"
+  },
   "querychanges": {
     "ts": 139402910202,
     "elapsed": 20,
@@ -21,12 +26,12 @@
   },
   "projectslist": {
     "ts": 139402910202,
-    "elapsed": 100,
+    "elapsed": 40,
     "result": "passed"
   },
   "auth": {
     "ts": 139402910202,
-    "elapsed": 80,
+    "elapsed": 30,
     "result": "passed"
   }
 }
@@ -50,6 +55,7 @@
 - `auth`: check the ability to authenticate with username and password
 - `activeworkers`: check the number of active worker threads and the ability to create a new one
 - `deadlock` : check if Java deadlocks are reported by the JVM
+- `blockedthreads` : check the number of blocked threads
 
 Each check name can be disabled by setting the `enabled` parameter to **false**,
 by default this parameter is set to **true**
@@ -88,4 +94,46 @@
  - `healthcheck.activeworkers.threshold` : Percent of queue occupancy above which queue is consider 
     as full.
 
-   Default: 80
\ No newline at end of file
+   Default: 80
+
+ - `healthcheck.blockedthreads.threshold` : Percent of all threads that are blocked above which instance
+   is considered as unhealthy.
+
+   Default: 50
+
+By default `healthcheck.blockedthreads` is calculated as ratio of BLOCKED threads against the all
+Gerrit threads. It might be not sufficient for instance in case of `SSH-Interactive-Worker` threads
+that could be all blocked making effectively a Gerrit instance unhealthy (neither fetch nor push
+would succeed) but the threshold could be still not reached. Therefore one can fine tune the check
+by putting detailed configuration for one or more thread groups (all threads that have the name
+starting with a given prefix) to be checked according to the following template:
+
+```
+[healthcheck "blockedthreads"]
+    threshold = [prefix]=[XX]
+```
+
+Note that in case when specific thread groups are configured all threads are no longer checked.
+
+* **Example 1:** _check if BLOCKED threads are above the limit of 70_
+
+   ```
+   [healthcheck "blockedthreads"]
+       threshold = 70
+   ```
+
+* **Example 2:** _check if BLOCKED `foo` threads are above the 33 limit_
+
+   ```
+   [healthcheck "blockedthreads"]
+       threshold = foo=45
+   ```
+
+* **Example 3:** _check if BLOCKED `foo` threads are above the 33 limit and if BLOCKED `bar`_
+  _threads are above the the 60 limit_
+
+   ```
+   [healthcheck "blockedthreads"]
+      threshold = foo=33
+      threshold = bar=60
+   ```
diff --git a/src/resources/Documentation/config.md b/src/resources/Documentation/config.md
index aa59c29..76e6eb5 100644
--- a/src/resources/Documentation/config.md
+++ b/src/resources/Documentation/config.md
@@ -48,6 +48,9 @@
 # TYPE plugins_healthcheck_projectslist_latest_measured_latency gauge
 plugins_healthcheck_projectslist_latest_measured_latency 5.0
 
+# HELP plugins_healthcheck_blockedthreads_latest_measured_latency Generated from Dropwizard metric import (metric=plugins/healthcheck/blockedthreads/latency, type=com.google.gerrit.metrics.dropwizard.CallbackMetricImpl0$1)
+# TYPE plugins_healthcheck_blockedthreads_latest_measured_latency gauge
+plugins_healthcheck_blockedthreads_latest_measured_latency 6.0
 
 # HELP plugins_healthcheck_jgit_failure_total Generated from Dropwizard metric import (metric=plugins/healthcheck/jgit/failure, type=com.codahale.metrics.Meter)
 # TYPE plugins_healthcheck_jgit_failure_total counter
@@ -56,6 +59,29 @@
 # HELP plugins_healthcheck_projectslist_failure_total Generated from Dropwizard metric import (metric=plugins/healthcheck/projectslist/failure, type=com.codahale.metrics.Meter)
 # TYPE plugins_healthcheck_projectslist_failure_total counter
 plugins_healthcheck_projectslist_failure_total 0.0
+
+# HELP plugins_healthcheck_blockedthreads_failure_total Generated from Dropwizard metric import (metric=plugins/healthcheck/blockedthreads/failure, type=com.codahale.metrics.Meter)
+# TYPE plugins_healthcheck_blockedthreads_failure_total counter
+plugins_healthcheck_blockedthreads_failure_total 1.0
 ```
 
+Note that additionally to the default `blockedthreads` metrics pair failures counter will reported for
+each configured prefix. For given config:
+
+```
+[healthcheck "blockedthreads"]
+    threshold = Foo=33
+```
+
+the following additional metric will be exposed and populated:
+
+```
+# HELP plugins_healthcheck_blockedthreads_foo_failure_total Generated from Dropwizard metric import (metric=plugins/healthcheck/blockedthreads-foo/failure, type=com.codahale.metrics.Meter)
+# TYPE plugins_healthcheck_blockedthreads_foo_failure_total counter
+plugins_healthcheck_blockedthreads_foo_failure_total 2.0
+```
+
+Note that prefix is used as postfix for a metric name but it is lower-cased and sanitized as only
+`a-zA-Z0-9_-/` chars are allowed to be a metric name (chars outside this set are turned to `_`).
+
 Metrics will be exposed to prometheus by the [metrics-reporter-prometheus](https://gerrit.googlesource.com/plugins/metrics-reporter-prometheus/) plugin.
diff --git a/src/test/java/com/googlesource/gerrit/plugins/healthcheck/BlockedThreadsCheckTest.java b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/BlockedThreadsCheckTest.java
new file mode 100644
index 0000000..d50d12a
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/BlockedThreadsCheckTest.java
@@ -0,0 +1,206 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.healthcheck;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.BLOCKEDTHREADS;
+import static java.util.Collections.nCopies;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.util.Providers;
+import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsCheck;
+import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsConfigurator;
+import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheck.Result;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class BlockedThreadsCheckTest {
+  @Mock BlockedThreadsCheck.ThreadBeanProvider threadBeanProviderMock;
+
+  @Mock ThreadMXBean beanMock;
+
+  @Before
+  public void setUp() {
+    when(threadBeanProviderMock.get()).thenReturn(beanMock);
+  }
+
+  @Test
+  public void shouldPassCheckWhenNoThreadsAreReturned() {
+    BlockedThreadsCheck objectUnderTest = createCheck(HealthCheckConfig.DEFAULT_CONFIG);
+    when(beanMock.getThreadInfo(null, 0)).thenReturn(new ThreadInfo[0]);
+    assertThat(objectUnderTest.run().result).isEqualTo(Result.PASSED);
+  }
+
+  @Test
+  public void shouldPassCheckWhenNoThreadsAreBlocked() {
+    int running = 1;
+    int blocked = 0;
+    mockThreadsAndCheckResult(running, blocked, Result.PASSED);
+  }
+
+  @Test
+  public void shouldPassCheckWhenBlockedThreadsAreLessThanDefaultThreshold() {
+    int running = 2;
+    int blocked = 1;
+    mockThreadsAndCheckResult(running, blocked, Result.PASSED);
+  }
+
+  @Test
+  public void shouldPassCheckWhenBlockedThreadsAreEqualToDefaultThreshold() {
+    int running = 1;
+    int blocked = 1;
+    mockThreadsAndCheckResult(running, blocked, Result.PASSED);
+  }
+
+  @Test
+  public void shouldFailCheckWhenBlockedThreadsAreAboveTheDefaultThreshold() {
+    int running = 1;
+    int blocked = 2;
+    mockThreadsAndCheckResult(running, blocked, Result.FAILED);
+  }
+
+  @Test
+  public void shouldPassCheckWhenBlockedThreadsAreLessThenThreshold() {
+    int running = 3;
+    int blocked = 1;
+    HealthCheckConfig config =
+        new HealthCheckConfig("[healthcheck \"" + BLOCKEDTHREADS + "\"]\n" + "  threshold = 25");
+    mockThreadsAndCheckResult(running, blocked, Result.PASSED, config);
+  }
+
+  @Test
+  public void shouldFailCheckWhenBlockedThreadsAreAboveTheThreshold() {
+    int running = 1;
+    int blocked = 1;
+    HealthCheckConfig config =
+        new HealthCheckConfig("[healthcheck \"" + BLOCKEDTHREADS + "\"]\n" + "  threshold = 33");
+    mockThreadsAndCheckResult(running, blocked, Result.FAILED, config);
+  }
+
+  @Test
+  public void shouldPassCheckWhenBlockedThreadsWithPrefixAreLessThenThreshold() {
+    int running = 3;
+    int blocked = 1;
+    String prefix = "blocked-threads-prefix";
+    HealthCheckConfig config =
+        new HealthCheckConfig(
+            "[healthcheck \"" + BLOCKEDTHREADS + "\"]\n" + "  threshold = " + prefix + " = 25");
+    mockThreadsAndCheckResult(running, blocked, Result.PASSED, prefix, config);
+  }
+
+  @Test
+  public void shouldFailCheckWhenBlockedThreadsWithPrefixAreAboveTheThreshold() {
+    int running = 1;
+    int blocked = 1;
+    String prefix = "blocked-threads-prefix";
+    HealthCheckConfig config =
+        new HealthCheckConfig(
+            "[healthcheck \"" + BLOCKEDTHREADS + "\"]\n" + "  threshold = " + prefix + " = 33");
+    mockThreadsAndCheckResult(running, blocked, Result.FAILED, prefix, config);
+  }
+
+  @Test
+  public void shouldFailCheckWhenAnyOfTheBlockedThreadsWithPrefixAreAboveTheThreshold() {
+    int running = 1;
+    int blocked = 1;
+    String blockedPrefix = "blocked-threads-prefix";
+    String notBlockedPrefix = "running-threads";
+    HealthCheckConfig config =
+        new HealthCheckConfig(
+            "[healthcheck \""
+                + BLOCKEDTHREADS
+                + "\"]\n"
+                + "  threshold = "
+                + blockedPrefix
+                + " = 33"
+                + "\nthreshold = "
+                + notBlockedPrefix
+                + "=33");
+    List<ThreadInfo> infos = new ArrayList<>(running + blocked + running);
+    infos.addAll(nCopies(running, mockInfo(Thread.State.RUNNABLE, blockedPrefix)));
+    infos.addAll(nCopies(blocked, mockInfo(Thread.State.BLOCKED, blockedPrefix)));
+    infos.addAll(nCopies(running, mockInfo(Thread.State.RUNNABLE, notBlockedPrefix)));
+    when(beanMock.getThreadInfo(null, 0)).thenReturn(infos.toArray(new ThreadInfo[infos.size()]));
+    checkResult(Result.FAILED, config);
+  }
+
+  private void mockThreadsAndCheckResult(int running, int blocked, Result expected) {
+    mockThreadsAndCheckResult(running, blocked, expected, HealthCheckConfig.DEFAULT_CONFIG);
+  }
+
+  private void mockThreadsAndCheckResult(
+      int running, int blocked, Result expected, HealthCheckConfig config) {
+    mockThreadsAndCheckResult(running, blocked, expected, "some-prefix", config);
+  }
+
+  private void mockThreadsAndCheckResult(
+      int running, int blocked, Result expected, String prefix, HealthCheckConfig config) {
+    mockThreads(running, blocked, prefix);
+    checkResult(expected, config);
+  }
+
+  private void checkResult(Result expected, HealthCheckConfig config) {
+    BlockedThreadsCheck objectUnderTest = createCheck(config);
+    assertThat(objectUnderTest.run().result).isEqualTo(expected);
+  }
+
+  private void mockThreads(int running, int blocked, String prefix) {
+    List<ThreadInfo> infos = new ArrayList<>(running + blocked);
+    infos.addAll(nCopies(running, mockInfo(Thread.State.RUNNABLE, prefix)));
+    infos.addAll(nCopies(blocked, mockInfo(Thread.State.BLOCKED, prefix)));
+    when(beanMock.getThreadInfo(null, 0)).thenReturn(infos.toArray(new ThreadInfo[infos.size()]));
+  }
+
+  private ThreadInfo mockInfo(Thread.State state, String prefix) {
+    ThreadInfo infoMock = mock(ThreadInfo.class);
+    when(infoMock.getThreadState()).thenReturn(state);
+    when(infoMock.getThreadName()).thenReturn(prefix);
+    return infoMock;
+  }
+
+  private BlockedThreadsCheck createCheck(HealthCheckConfig config) {
+    DummyHealthCheckMetricsFactory checkMetricsFactory = new DummyHealthCheckMetricsFactory();
+    Injector injector =
+        Guice.createInjector(
+            new HealthCheckModule(),
+            new AbstractModule() {
+              @Override
+              protected void configure() {
+                bind(HealthCheckConfig.class).toInstance(config);
+                bind(HealthCheckMetrics.Factory.class).toInstance(checkMetricsFactory);
+              }
+            },
+            BlockedThreadsCheck.SUB_CHECKS);
+    return new BlockedThreadsCheck(
+        injector.getInstance(ListeningExecutorService.class),
+        config,
+        checkMetricsFactory,
+        threadBeanProviderMock,
+        Providers.of(injector.getInstance(BlockedThreadsConfigurator.class)));
+  }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfiguratorConfigsTest.java b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfiguratorConfigsTest.java
new file mode 100644
index 0000000..d22ebbd
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfiguratorConfigsTest.java
@@ -0,0 +1,77 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.healthcheck.check;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsConfigurator.DEFAULT_BLOCKED_THREADS_THRESHOLD;
+
+import java.util.Arrays;
+import java.util.Collection;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class BlockedThreadsConfiguratorConfigsTest {
+  private final String[] input;
+  private final Collection<BlockedThreadsConfigurator.Threshold> expected;
+
+  public BlockedThreadsConfiguratorConfigsTest(
+      String[] input, Collection<BlockedThreadsConfigurator.Threshold> expected) {
+    this.input = input;
+    this.expected = expected;
+  }
+
+  @Test
+  public void shouldReturnExpectedConfig() {
+    Collection<BlockedThreadsConfigurator.Threshold> result =
+        BlockedThreadsConfigurator.getConfig(input);
+    assertThat(result).containsExactlyElementsIn(expected);
+  }
+
+  @Parameterized.Parameters
+  public static Collection<Object[]> configs() {
+    return Arrays.asList(
+        new Object[][] {
+          {new String[] {}, specs(threshold(DEFAULT_BLOCKED_THREADS_THRESHOLD))},
+          {new String[] {"30"}, specs(threshold(30))},
+          {
+            new String[] {"prefix1=40", "prefix2=70", "prefix3 = 80"},
+            specs(threshold("prefix1", 40), threshold("prefix2", 70), threshold("prefix3", 80))
+          },
+          // the latter configuration is selected
+          {new String[] {"30", "40"}, specs(threshold(40))},
+          // the latter configuration is selected
+          {new String[] {"prefix1=40", "prefix1=70"}, specs(threshold("prefix1", 70))},
+          // specific prefix configuration is favored over the global one
+          {new String[] {"30", "prefix1=40"}, specs(threshold("prefix1", 40))},
+          // specific prefix configuration is favored over the global one and it is deduplicated
+          {new String[] {"30", "prefix1=40", "prefix1=70"}, specs(threshold("prefix1", 70))},
+        });
+  }
+
+  private static BlockedThreadsConfigurator.Threshold threshold(int value) {
+    return threshold(null, value);
+  }
+
+  private static BlockedThreadsConfigurator.Threshold threshold(String prefix, int value) {
+    return new BlockedThreadsConfigurator.Threshold(prefix, value);
+  }
+
+  private static Collection<BlockedThreadsConfigurator.Threshold> specs(
+      BlockedThreadsConfigurator.Threshold... thresholds) {
+    return Arrays.asList(thresholds);
+  }
+}