Merge branch 'stable-3.3' * stable-3.3: Add blocked threads check Change-Id: I4dbe09c3ba43ea1ee85c73fd2e8ff5b9fb115536
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java index 3e5306c..651dbe9 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java +++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java
@@ -15,6 +15,7 @@ package com.googlesource.gerrit.plugins.healthcheck; import static com.google.common.base.Preconditions.checkNotNull; +import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.BLOCKEDTHREADS; import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.QUERYCHANGES; import com.google.common.annotations.VisibleForTesting; @@ -142,6 +143,10 @@ HEALTHCHECK, checkNotNull(healthCheckName), "enabled", HEALTH_CHECK_ENABLED_DEFAULT); } + public String[] getListOfBlockedThreadsThresholds() { + return config.getStringList(HEALTHCHECK, BLOCKEDTHREADS, "threshold"); + } + private String getStringWithFallback( String parameter, String healthCheckName, String defaultValue) { String fallbackDefault =
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java index 304a40c..3cda7b6 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckSubsystemsModule.java
@@ -18,6 +18,7 @@ import com.google.gerrit.extensions.registration.DynamicSet; import com.googlesource.gerrit.plugins.healthcheck.check.ActiveWorkersCheck; import com.googlesource.gerrit.plugins.healthcheck.check.AuthHealthCheck; +import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsCheck; import com.googlesource.gerrit.plugins.healthcheck.check.DeadlockCheck; import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheck; import com.googlesource.gerrit.plugins.healthcheck.check.JGitHealthCheck; @@ -34,8 +35,10 @@ bindChecker(AuthHealthCheck.class); bindChecker(ActiveWorkersCheck.class); bindChecker(DeadlockCheck.class); + bindChecker(BlockedThreadsCheck.class); factory(HealthCheckMetrics.Factory.class); + install(BlockedThreadsCheck.SUB_CHECKS); } private void bindChecker(Class<? extends HealthCheck> healthCheckClass) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsCheck.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsCheck.java new file mode 100644 index 0000000..f2cbb70 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsCheck.java
@@ -0,0 +1,123 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.healthcheck.check; + +import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.BLOCKEDTHREADS; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Suppliers; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.gerrit.extensions.config.FactoryModule; +import com.google.inject.Inject; +import com.google.inject.Module; +import com.google.inject.Provider; +import com.google.inject.Singleton; +import com.googlesource.gerrit.plugins.healthcheck.HealthCheckConfig; +import com.googlesource.gerrit.plugins.healthcheck.HealthCheckMetrics; +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.function.Supplier; +import java.util.stream.Stream; + +@Singleton +public class BlockedThreadsCheck extends AbstractHealthCheck { + public static Module SUB_CHECKS = + new FactoryModule() { + @Override + protected void configure() { + factory(BlockedThreadsSubCheck.Factory.class); + } + }; + + private final ThreadMXBean threads; + private final Supplier<List<Collector>> collectorsSupplier; + + @Inject + public BlockedThreadsCheck( + ListeningExecutorService executor, + HealthCheckConfig healthCheckConfig, + HealthCheckMetrics.Factory healthCheckMetricsFactory, + ThreadBeanProvider threadBeanProvider, + Provider<BlockedThreadsConfigurator> checksConfig) { + super(executor, healthCheckConfig, BLOCKEDTHREADS, healthCheckMetricsFactory); + this.threads = threadBeanProvider.get(); + this.collectorsSupplier = Suppliers.memoize(() -> checksConfig.get().collectors()); + } + + @Override + protected Result doCheck() throws Exception { + List<Collector> collectors = collectorsSupplier.get(); + dumpAllThreads().forEach(info -> collectors.forEach(c -> c.collect(info))); + + // call check on all sub-checks so that metrics are populated + collectors.forEach(Collector::check); + + // report unhealthy instance if any of sub-checks failed + return collectors.stream() + .map(Collector::result) + .filter(r -> Result.FAILED == r) + .findAny() + .orElse(Result.PASSED); + } + + private Stream<ThreadInfo> dumpAllThreads() { + // getting all thread ids and translating it into thread infos is noticeably faster then call to + // ThreadMXBean.dumpAllThreads as it doesn't calculate StackTrace. Note that some threads could + // be already finished (between call to get all ids and translate them to ThreadInfo objects + // hence they have to be filtered out). + return Arrays.stream(threads.getThreadInfo(threads.getAllThreadIds(), 0)) + .filter(Objects::nonNull); + } + + @VisibleForTesting + public static class ThreadBeanProvider { + public ThreadMXBean get() { + return ManagementFactory.getThreadMXBean(); + } + } + + static class Collector { + protected final Integer threshold; + + protected int blocked; + protected int total; + protected Result result; + + Collector(Integer threshold) { + this.threshold = threshold; + } + + void collect(ThreadInfo info) { + total += 1; + if (Thread.State.BLOCKED == info.getThreadState()) { + blocked += 1; + } + } + + void check() { + result = blocked * 100 <= threshold * total ? Result.PASSED : Result.FAILED; + } + + Result result() { + return result; + } + } + + interface CollectorProvider<T extends Collector> extends Provider<T> {} +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfigurator.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfigurator.java new file mode 100644 index 0000000..ddc67f9 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfigurator.java
@@ -0,0 +1,205 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.healthcheck.check; + +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.toList; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; +import com.google.inject.Inject; +import com.googlesource.gerrit.plugins.healthcheck.HealthCheckConfig; +import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsCheck.Collector; +import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsCheck.CollectorProvider; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@VisibleForTesting +public class BlockedThreadsConfigurator { + private static final Logger log = LoggerFactory.getLogger(BlockedThreadsConfigurator.class); + private static final Pattern THRESHOLD_PATTERN = Pattern.compile("^(\\d\\d?)$"); + + static final int DEFAULT_BLOCKED_THREADS_THRESHOLD = 50; + + private final List<CollectorProvider<?>> providers; + + @Inject + BlockedThreadsConfigurator( + BlockedThreadsSubCheck.Factory subchecks, HealthCheckConfig healthCheckConfig) { + this.providers = getProviders(subchecks, healthCheckConfig); + } + + List<Collector> collectors() { + return providers.stream().map(CollectorProvider::get).collect(toList()); + } + + private static List<CollectorProvider<?>> getProviders( + BlockedThreadsSubCheck.Factory subchecksFactory, HealthCheckConfig healthCheckConfig) { + return getConfig(healthCheckConfig.getListOfBlockedThreadsThresholds()).stream() + .map(spec -> collectorProvider(subchecksFactory, spec)) + .collect(toList()); + } + + private static CollectorProvider<?> collectorProvider( + BlockedThreadsSubCheck.Factory subchecksFactory, Threshold spec) { + return spec.prefix.isPresent() + ? subchecksFactory.create(spec.prefix.get(), spec.value) + : () -> new BlockedThreadsCheck.Collector(spec.value); + } + + @VisibleForTesting + static Collection<Threshold> getConfig(String[] thresholds) { + // Threshold can be defined as a sole value e.g + // threshold = 80 + // and would become a default one for all blocked threads check or as a set of specific thread + // groups checks defined like + // threshold = foo=30 + // threshold = bar=40 + // ... + // they are mutually exclusive which means that one either checks all threads or groups + Map<Boolean, List<Threshold>> specsClassified = + Arrays.stream(thresholds) + .filter(spec -> !spec.isEmpty()) + .map(BlockedThreadsConfigurator::getSpec) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(groupingBy(Threshold::hasPrefix)); + + // check configuration consistency + if (specsClassified.size() > 1) { + Collection<Threshold> specs = deduplicatePrefixes(specsClassified.get(true)); + log.warn( + "Global and specific thresholds were configured for blocked threads check. Specific" + + " configuration is used {}.", + specs); + return specs; + } + + if (specsClassified.size() == 1) { + Map.Entry<Boolean, List<Threshold>> entry = specsClassified.entrySet().iterator().next(); + return Boolean.TRUE == entry.getKey() + ? deduplicatePrefixes(entry.getValue()) + : deduplicateGlobal(entry.getValue()); + } + + log.info( + "Default blocked threads check is configured with {}% threshold", + DEFAULT_BLOCKED_THREADS_THRESHOLD); + return ImmutableSet.of(new Threshold(DEFAULT_BLOCKED_THREADS_THRESHOLD)); + } + + private static Collection<Threshold> deduplicateGlobal(List<Threshold> input) { + if (input.size() > 1) { + Threshold spec = input.get(input.size() - 1); + log.warn("Multiple threshold values were configured. Using {}", spec); + return ImmutableSet.of(spec); + } + return input; + } + + private static Collection<Threshold> deduplicatePrefixes(Collection<Threshold> input) { + Map<String, Threshold> deduplicated = new HashMap<>(); + input.forEach(t -> deduplicated.put(t.prefix.get(), t)); + if (deduplicated.size() != input.size()) { + log.warn( + "The same prefixes were configured multiple times. The following configuration is used" + + " {}", + deduplicated.values()); + } + return deduplicated.values(); + } + + private static Optional<Threshold> getSpec(String spec) { + int equals = spec.lastIndexOf('='); + if (equals != -1) { + Optional<Integer> maybeThreshold = isThresholdDefined(spec.substring(equals + 1)); + if (maybeThreshold.isPresent()) { + return Optional.of(new Threshold(spec.substring(0, equals).trim(), maybeThreshold.get())); + } + } else { + Optional<Integer> maybeThreshold = isThresholdDefined(spec); + if (maybeThreshold.isPresent()) { + return Optional.of(new Threshold(maybeThreshold.get())); + } + } + + log.warn("Invalid configuration of blocked threads threshold [{}]", spec); + return Optional.empty(); + } + + private static Optional<Integer> isThresholdDefined(String input) { + Matcher value = THRESHOLD_PATTERN.matcher(input.trim()); + if (value.matches()) { + return Optional.of(Integer.valueOf(value.group(1))); + } + return Optional.empty(); + } + + @VisibleForTesting + static class Threshold { + final Optional<String> prefix; + final Integer value; + + Threshold(int value) { + this(null, value); + } + + Threshold(String prefix, int value) { + this.prefix = Optional.ofNullable(prefix); + this.value = value; + } + + boolean hasPrefix() { + return prefix.isPresent(); + } + + @Override + public int hashCode() { + return Objects.hash(prefix, value); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + Threshold other = (Threshold) obj; + return Objects.equals(prefix, other.prefix) && Objects.equals(value, other.value); + } + + @Override + public String toString() { + return new StringBuilder() + .append("Threshold [prefix=") + .append(prefix) + .append(", value=") + .append(value) + .append("]") + .toString(); + } + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsSubCheck.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsSubCheck.java new file mode 100644 index 0000000..d4f0fe8 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsSubCheck.java
@@ -0,0 +1,83 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.healthcheck.check; + +import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.BLOCKEDTHREADS; + +import com.google.common.base.Strings; +import com.google.gerrit.metrics.Counter0; +import com.google.inject.Inject; +import com.google.inject.assistedinject.Assisted; +import com.googlesource.gerrit.plugins.healthcheck.HealthCheckMetrics; +import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheck.Result; +import java.lang.management.ThreadInfo; +import java.util.function.Supplier; + +class BlockedThreadsSubCheck + implements BlockedThreadsCheck.CollectorProvider<BlockedThreadsSubCheck.SubCheckCollector> { + interface Factory { + BlockedThreadsSubCheck create(String prefix, Integer threshold); + } + + static class SubCheckCollector extends BlockedThreadsCheck.Collector { + private final String prefix; + private final Counter0 failureCounterMetric; + + SubCheckCollector(String prefix, Integer threshold, Counter0 failureCounterMetric) { + super(threshold); + this.prefix = prefix; + this.failureCounterMetric = failureCounterMetric; + } + + @Override + void collect(ThreadInfo info) { + String threadName = info.getThreadName(); + if (!Strings.isNullOrEmpty(threadName) && threadName.startsWith(prefix)) { + total += 1; + if (Thread.State.BLOCKED == info.getThreadState()) { + blocked += 1; + } + } + } + + @Override + void check() { + super.check(); + if (Result.FAILED == result) { + failureCounterMetric.increment(); + } + } + } + + private final Supplier<SubCheckCollector> collector; + + @Inject + BlockedThreadsSubCheck( + HealthCheckMetrics.Factory healthCheckMetricsFactory, + @Assisted String prefix, + @Assisted Integer threshold) { + HealthCheckMetrics healthCheckMetrics = + healthCheckMetricsFactory.create( + String.format( + "%s-%s", BLOCKEDTHREADS, prefix.toLowerCase().replaceAll("[^\\w-/]", "_"))); + Counter0 failureCounterMetric = healthCheckMetrics.getFailureCounterMetric(); + this.collector = () -> new SubCheckCollector(prefix, threshold, failureCounterMetric); + } + + @Override + public SubCheckCollector get() { + return collector.get(); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java index 97e06d0..4771942 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java +++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/check/HealthCheckNames.java
@@ -21,5 +21,6 @@ String AUTH = "auth"; String ACTIVEWORKERS = "activeworkers"; String DEADLOCK = "deadlock"; + String BLOCKEDTHREADS = "blockedthreads"; String GLOBAL = "global"; }
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index 47acb01..0b4f05f 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md
@@ -13,7 +13,12 @@ )]}' { "ts": 139402910202, - "elapsed": 100, + "elapsed": 120, + "blockedthreads": { + "ts": 139402910202, + "elapsed": 30, + "result": "passed" + }, "querychanges": { "ts": 139402910202, "elapsed": 20, @@ -21,12 +26,12 @@ }, "projectslist": { "ts": 139402910202, - "elapsed": 100, + "elapsed": 40, "result": "passed" }, "auth": { "ts": 139402910202, - "elapsed": 80, + "elapsed": 30, "result": "passed" } } @@ -50,6 +55,7 @@ - `auth`: check the ability to authenticate with username and password - `activeworkers`: check the number of active worker threads and the ability to create a new one - `deadlock` : check if Java deadlocks are reported by the JVM +- `blockedthreads` : check the number of blocked threads Each check name can be disabled by setting the `enabled` parameter to **false**, by default this parameter is set to **true** @@ -88,4 +94,46 @@ - `healthcheck.activeworkers.threshold` : Percent of queue occupancy above which queue is consider as full. - Default: 80 \ No newline at end of file + Default: 80 + + - `healthcheck.blockedthreads.threshold` : Percent of all threads that are blocked above which instance + is considered as unhealthy. + + Default: 50 + +By default `healthcheck.blockedthreads` is calculated as ratio of BLOCKED threads against the all +Gerrit threads. It might be not sufficient for instance in case of `SSH-Interactive-Worker` threads +that could be all blocked making effectively a Gerrit instance unhealthy (neither fetch nor push +would succeed) but the threshold could be still not reached. Therefore one can fine tune the check +by putting detailed configuration for one or more thread groups (all threads that have the name +starting with a given prefix) to be checked according to the following template: + +``` +[healthcheck "blockedthreads"] + threshold = [prefix]=[XX] +``` + +Note that in case when specific thread groups are configured all threads are no longer checked. + +* **Example 1:** _check if BLOCKED threads are above the limit of 70_ + + ``` + [healthcheck "blockedthreads"] + threshold = 70 + ``` + +* **Example 2:** _check if BLOCKED `foo` threads are above the 33 limit_ + + ``` + [healthcheck "blockedthreads"] + threshold = foo=45 + ``` + +* **Example 3:** _check if BLOCKED `foo` threads are above the 33 limit and if BLOCKED `bar`_ + _threads are above the the 60 limit_ + + ``` + [healthcheck "blockedthreads"] + threshold = foo=33 + threshold = bar=60 + ```
diff --git a/src/resources/Documentation/config.md b/src/resources/Documentation/config.md index aa59c29..76e6eb5 100644 --- a/src/resources/Documentation/config.md +++ b/src/resources/Documentation/config.md
@@ -48,6 +48,9 @@ # TYPE plugins_healthcheck_projectslist_latest_measured_latency gauge plugins_healthcheck_projectslist_latest_measured_latency 5.0 +# HELP plugins_healthcheck_blockedthreads_latest_measured_latency Generated from Dropwizard metric import (metric=plugins/healthcheck/blockedthreads/latency, type=com.google.gerrit.metrics.dropwizard.CallbackMetricImpl0$1) +# TYPE plugins_healthcheck_blockedthreads_latest_measured_latency gauge +plugins_healthcheck_blockedthreads_latest_measured_latency 6.0 # HELP plugins_healthcheck_jgit_failure_total Generated from Dropwizard metric import (metric=plugins/healthcheck/jgit/failure, type=com.codahale.metrics.Meter) # TYPE plugins_healthcheck_jgit_failure_total counter @@ -56,6 +59,29 @@ # HELP plugins_healthcheck_projectslist_failure_total Generated from Dropwizard metric import (metric=plugins/healthcheck/projectslist/failure, type=com.codahale.metrics.Meter) # TYPE plugins_healthcheck_projectslist_failure_total counter plugins_healthcheck_projectslist_failure_total 0.0 + +# HELP plugins_healthcheck_blockedthreads_failure_total Generated from Dropwizard metric import (metric=plugins/healthcheck/blockedthreads/failure, type=com.codahale.metrics.Meter) +# TYPE plugins_healthcheck_blockedthreads_failure_total counter +plugins_healthcheck_blockedthreads_failure_total 1.0 ``` +Note that additionally to the default `blockedthreads` metrics pair failures counter will reported for +each configured prefix. For given config: + +``` +[healthcheck "blockedthreads"] + threshold = Foo=33 +``` + +the following additional metric will be exposed and populated: + +``` +# HELP plugins_healthcheck_blockedthreads_foo_failure_total Generated from Dropwizard metric import (metric=plugins/healthcheck/blockedthreads-foo/failure, type=com.codahale.metrics.Meter) +# TYPE plugins_healthcheck_blockedthreads_foo_failure_total counter +plugins_healthcheck_blockedthreads_foo_failure_total 2.0 +``` + +Note that prefix is used as postfix for a metric name but it is lower-cased and sanitized as only +`a-zA-Z0-9_-/` chars are allowed to be a metric name (chars outside this set are turned to `_`). + Metrics will be exposed to prometheus by the [metrics-reporter-prometheus](https://gerrit.googlesource.com/plugins/metrics-reporter-prometheus/) plugin.
diff --git a/src/test/java/com/googlesource/gerrit/plugins/healthcheck/BlockedThreadsCheckTest.java b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/BlockedThreadsCheckTest.java new file mode 100644 index 0000000..d50d12a --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/BlockedThreadsCheckTest.java
@@ -0,0 +1,206 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.healthcheck; + +import static com.google.common.truth.Truth.assertThat; +import static com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames.BLOCKEDTHREADS; +import static java.util.Collections.nCopies; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.util.Providers; +import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsCheck; +import com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsConfigurator; +import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheck.Result; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; +import java.util.ArrayList; +import java.util.List; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class BlockedThreadsCheckTest { + @Mock BlockedThreadsCheck.ThreadBeanProvider threadBeanProviderMock; + + @Mock ThreadMXBean beanMock; + + @Before + public void setUp() { + when(threadBeanProviderMock.get()).thenReturn(beanMock); + } + + @Test + public void shouldPassCheckWhenNoThreadsAreReturned() { + BlockedThreadsCheck objectUnderTest = createCheck(HealthCheckConfig.DEFAULT_CONFIG); + when(beanMock.getThreadInfo(null, 0)).thenReturn(new ThreadInfo[0]); + assertThat(objectUnderTest.run().result).isEqualTo(Result.PASSED); + } + + @Test + public void shouldPassCheckWhenNoThreadsAreBlocked() { + int running = 1; + int blocked = 0; + mockThreadsAndCheckResult(running, blocked, Result.PASSED); + } + + @Test + public void shouldPassCheckWhenBlockedThreadsAreLessThanDefaultThreshold() { + int running = 2; + int blocked = 1; + mockThreadsAndCheckResult(running, blocked, Result.PASSED); + } + + @Test + public void shouldPassCheckWhenBlockedThreadsAreEqualToDefaultThreshold() { + int running = 1; + int blocked = 1; + mockThreadsAndCheckResult(running, blocked, Result.PASSED); + } + + @Test + public void shouldFailCheckWhenBlockedThreadsAreAboveTheDefaultThreshold() { + int running = 1; + int blocked = 2; + mockThreadsAndCheckResult(running, blocked, Result.FAILED); + } + + @Test + public void shouldPassCheckWhenBlockedThreadsAreLessThenThreshold() { + int running = 3; + int blocked = 1; + HealthCheckConfig config = + new HealthCheckConfig("[healthcheck \"" + BLOCKEDTHREADS + "\"]\n" + " threshold = 25"); + mockThreadsAndCheckResult(running, blocked, Result.PASSED, config); + } + + @Test + public void shouldFailCheckWhenBlockedThreadsAreAboveTheThreshold() { + int running = 1; + int blocked = 1; + HealthCheckConfig config = + new HealthCheckConfig("[healthcheck \"" + BLOCKEDTHREADS + "\"]\n" + " threshold = 33"); + mockThreadsAndCheckResult(running, blocked, Result.FAILED, config); + } + + @Test + public void shouldPassCheckWhenBlockedThreadsWithPrefixAreLessThenThreshold() { + int running = 3; + int blocked = 1; + String prefix = "blocked-threads-prefix"; + HealthCheckConfig config = + new HealthCheckConfig( + "[healthcheck \"" + BLOCKEDTHREADS + "\"]\n" + " threshold = " + prefix + " = 25"); + mockThreadsAndCheckResult(running, blocked, Result.PASSED, prefix, config); + } + + @Test + public void shouldFailCheckWhenBlockedThreadsWithPrefixAreAboveTheThreshold() { + int running = 1; + int blocked = 1; + String prefix = "blocked-threads-prefix"; + HealthCheckConfig config = + new HealthCheckConfig( + "[healthcheck \"" + BLOCKEDTHREADS + "\"]\n" + " threshold = " + prefix + " = 33"); + mockThreadsAndCheckResult(running, blocked, Result.FAILED, prefix, config); + } + + @Test + public void shouldFailCheckWhenAnyOfTheBlockedThreadsWithPrefixAreAboveTheThreshold() { + int running = 1; + int blocked = 1; + String blockedPrefix = "blocked-threads-prefix"; + String notBlockedPrefix = "running-threads"; + HealthCheckConfig config = + new HealthCheckConfig( + "[healthcheck \"" + + BLOCKEDTHREADS + + "\"]\n" + + " threshold = " + + blockedPrefix + + " = 33" + + "\nthreshold = " + + notBlockedPrefix + + "=33"); + List<ThreadInfo> infos = new ArrayList<>(running + blocked + running); + infos.addAll(nCopies(running, mockInfo(Thread.State.RUNNABLE, blockedPrefix))); + infos.addAll(nCopies(blocked, mockInfo(Thread.State.BLOCKED, blockedPrefix))); + infos.addAll(nCopies(running, mockInfo(Thread.State.RUNNABLE, notBlockedPrefix))); + when(beanMock.getThreadInfo(null, 0)).thenReturn(infos.toArray(new ThreadInfo[infos.size()])); + checkResult(Result.FAILED, config); + } + + private void mockThreadsAndCheckResult(int running, int blocked, Result expected) { + mockThreadsAndCheckResult(running, blocked, expected, HealthCheckConfig.DEFAULT_CONFIG); + } + + private void mockThreadsAndCheckResult( + int running, int blocked, Result expected, HealthCheckConfig config) { + mockThreadsAndCheckResult(running, blocked, expected, "some-prefix", config); + } + + private void mockThreadsAndCheckResult( + int running, int blocked, Result expected, String prefix, HealthCheckConfig config) { + mockThreads(running, blocked, prefix); + checkResult(expected, config); + } + + private void checkResult(Result expected, HealthCheckConfig config) { + BlockedThreadsCheck objectUnderTest = createCheck(config); + assertThat(objectUnderTest.run().result).isEqualTo(expected); + } + + private void mockThreads(int running, int blocked, String prefix) { + List<ThreadInfo> infos = new ArrayList<>(running + blocked); + infos.addAll(nCopies(running, mockInfo(Thread.State.RUNNABLE, prefix))); + infos.addAll(nCopies(blocked, mockInfo(Thread.State.BLOCKED, prefix))); + when(beanMock.getThreadInfo(null, 0)).thenReturn(infos.toArray(new ThreadInfo[infos.size()])); + } + + private ThreadInfo mockInfo(Thread.State state, String prefix) { + ThreadInfo infoMock = mock(ThreadInfo.class); + when(infoMock.getThreadState()).thenReturn(state); + when(infoMock.getThreadName()).thenReturn(prefix); + return infoMock; + } + + private BlockedThreadsCheck createCheck(HealthCheckConfig config) { + DummyHealthCheckMetricsFactory checkMetricsFactory = new DummyHealthCheckMetricsFactory(); + Injector injector = + Guice.createInjector( + new HealthCheckModule(), + new AbstractModule() { + @Override + protected void configure() { + bind(HealthCheckConfig.class).toInstance(config); + bind(HealthCheckMetrics.Factory.class).toInstance(checkMetricsFactory); + } + }, + BlockedThreadsCheck.SUB_CHECKS); + return new BlockedThreadsCheck( + injector.getInstance(ListeningExecutorService.class), + config, + checkMetricsFactory, + threadBeanProviderMock, + Providers.of(injector.getInstance(BlockedThreadsConfigurator.class))); + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfiguratorConfigsTest.java b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfiguratorConfigsTest.java new file mode 100644 index 0000000..d22ebbd --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/check/BlockedThreadsConfiguratorConfigsTest.java
@@ -0,0 +1,77 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.healthcheck.check; + +import static com.google.common.truth.Truth.assertThat; +import static com.googlesource.gerrit.plugins.healthcheck.check.BlockedThreadsConfigurator.DEFAULT_BLOCKED_THREADS_THRESHOLD; + +import java.util.Arrays; +import java.util.Collection; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class BlockedThreadsConfiguratorConfigsTest { + private final String[] input; + private final Collection<BlockedThreadsConfigurator.Threshold> expected; + + public BlockedThreadsConfiguratorConfigsTest( + String[] input, Collection<BlockedThreadsConfigurator.Threshold> expected) { + this.input = input; + this.expected = expected; + } + + @Test + public void shouldReturnExpectedConfig() { + Collection<BlockedThreadsConfigurator.Threshold> result = + BlockedThreadsConfigurator.getConfig(input); + assertThat(result).containsExactlyElementsIn(expected); + } + + @Parameterized.Parameters + public static Collection<Object[]> configs() { + return Arrays.asList( + new Object[][] { + {new String[] {}, specs(threshold(DEFAULT_BLOCKED_THREADS_THRESHOLD))}, + {new String[] {"30"}, specs(threshold(30))}, + { + new String[] {"prefix1=40", "prefix2=70", "prefix3 = 80"}, + specs(threshold("prefix1", 40), threshold("prefix2", 70), threshold("prefix3", 80)) + }, + // the latter configuration is selected + {new String[] {"30", "40"}, specs(threshold(40))}, + // the latter configuration is selected + {new String[] {"prefix1=40", "prefix1=70"}, specs(threshold("prefix1", 70))}, + // specific prefix configuration is favored over the global one + {new String[] {"30", "prefix1=40"}, specs(threshold("prefix1", 40))}, + // specific prefix configuration is favored over the global one and it is deduplicated + {new String[] {"30", "prefix1=40", "prefix1=70"}, specs(threshold("prefix1", 70))}, + }); + } + + private static BlockedThreadsConfigurator.Threshold threshold(int value) { + return threshold(null, value); + } + + private static BlockedThreadsConfigurator.Threshold threshold(String prefix, int value) { + return new BlockedThreadsConfigurator.Threshold(prefix, value); + } + + private static Collection<BlockedThreadsConfigurator.Threshold> specs( + BlockedThreadsConfigurator.Threshold... thresholds) { + return Arrays.asList(thresholds); + } +}