Add combinedRefsSha1 to metrics
Expose new metric providing a "combined refs SHA-1", which could be
used to ensure eventual consistency of repository refs.
It's based on the `Sha1AllRefs`[1] Groovy script, which will iterate
over all repository refs and combine them into a single SHA-1. Then the
resulting SHA-1 will be truncated to an integer value so that it can
be included in metrics.
Exposing this metric should enable Gerrit admins to detect potential
inconsistencies between nodes. All git repositories are **eventually
consistent** which means that detecting different values for combined
refs SHA-1 is **NOT** equivalent to data inconsistency. But not
reporting a consistent state (when the combined refs SHA-1 across nodes
is the same) for a _reasonable time_ should raise an eyebrow and lead to
an investigation if any of the replication events were lost.
The _reasonable time_ will differ from repository to repository, for
some it may be 2 hours, and for others 24 hours. Meaning, that within
that time the repositories should be consistent at least once. It's on
the Gerrit admin to find the time window that works best for their
repositories.
[1]
https://gerrit.googlesource.com/plugins/scripts/+/refs/heads/master/multi-primary/localrefdb.groovy#142
Bug: Issue 324975825
Change-Id: I2e4dbea0892776069dea371803653b7dd239f7f8
diff --git a/src/main/java/com/googlesource/gerrit/plugins/gitrepometrics/Module.java b/src/main/java/com/googlesource/gerrit/plugins/gitrepometrics/Module.java
index 10c116a..6bbf8f7 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/gitrepometrics/Module.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/gitrepometrics/Module.java
@@ -19,6 +19,7 @@
import com.google.gerrit.server.events.EventListener;
import com.google.inject.Scopes;
import com.googlesource.gerrit.plugins.gitrepometrics.collectors.FSMetricsCollector;
+import com.googlesource.gerrit.plugins.gitrepometrics.collectors.GitRefsMetricsCollector;
import com.googlesource.gerrit.plugins.gitrepometrics.collectors.GitStatsMetricsCollector;
import com.googlesource.gerrit.plugins.gitrepometrics.collectors.MetricsCollector;
import java.util.concurrent.ExecutorService;
@@ -37,6 +38,7 @@
DynamicSet.setOf(binder(), MetricsCollector.class);
DynamicSet.bind(binder(), MetricsCollector.class).to(GitStatsMetricsCollector.class);
DynamicSet.bind(binder(), MetricsCollector.class).to(FSMetricsCollector.class);
+ DynamicSet.bind(binder(), MetricsCollector.class).to(GitRefsMetricsCollector.class);
install(new UpdateGitMetricsTaskModule());
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/gitrepometrics/collectors/GitRefsMetricsCollector.java b/src/main/java/com/googlesource/gerrit/plugins/gitrepometrics/collectors/GitRefsMetricsCollector.java
new file mode 100644
index 0000000..96e8350
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/gitrepometrics/collectors/GitRefsMetricsCollector.java
@@ -0,0 +1,99 @@
+// Copyright (C) 2024 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.gitrepometrics.collectors;
+
+import static com.google.gerrit.entities.RefNames.REFS_USERS;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.flogger.FluentLogger;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.gitrepometrics.UpdateGitMetricsExecutor;
+import java.io.IOException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.function.Consumer;
+import org.eclipse.jgit.internal.storage.file.FileRepository;
+import org.eclipse.jgit.lib.Ref;
+
+public class GitRefsMetricsCollector implements MetricsCollector {
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+ @VisibleForTesting
+ protected static final GitRepoMetric combinedRefsSha1 =
+ new GitRepoMetric("combinedRefsSha1", "Numeric value of combined refs SHA-1's", "Number");
+
+ private static final ImmutableList<GitRepoMetric> availableMetrics =
+ ImmutableList.of(combinedRefsSha1);
+
+ private final ExecutorService executorService;
+
+ @Inject
+ GitRefsMetricsCollector(@UpdateGitMetricsExecutor ExecutorService executorService) {
+ this.executorService = executorService;
+ }
+
+ @Override
+ public void collect(
+ FileRepository repository,
+ String projectName,
+ Consumer<HashMap<GitRepoMetric, Long>> populateMetrics) {
+ executorService.submit(
+ () -> {
+ try {
+ MessageDigest md = MessageDigest.getInstance("SHA-1");
+ repository.getRefDatabase().getRefs().stream()
+ .filter(ref -> !ref.isSymbolic() && !ref.getName().startsWith(REFS_USERS))
+ .sorted(Comparator.comparing(Ref::getName))
+ .forEach(ref -> md.update(ref.getObjectId().toString().getBytes(UTF_8)));
+ int sha1Int = truncateHashToInt(md.digest());
+
+ HashMap<GitRepoMetric, Long> metrics = new HashMap<>();
+ metrics.put(combinedRefsSha1, (long) sha1Int);
+ populateMetrics.accept(metrics);
+ } catch (NoSuchAlgorithmException e) {
+ logger.atSevere().withCause(e).log(
+ "Could not obtain SHA-1 implementation will not compute the combinedRefsSha1 metric");
+ } catch (IOException e) {
+ logger.atSevere().withCause(e).log(
+ "Computing combinedRefsSha1 failed. Will retry next time");
+ }
+ });
+ }
+
+ @Override
+ public String getMetricsCollectorName() {
+ return "repo-ref-statistics";
+ }
+
+ @Override
+ public ImmutableList<GitRepoMetric> availableMetrics() {
+ return availableMetrics;
+ }
+
+ // Source
+ // http://www.java2s.com/example/java-utility-method/sha1/sha1hashint-string-text-d6c0e.html
+ private static int truncateHashToInt(byte[] bytes) {
+ int offset = bytes[bytes.length - 1] & 0x0f;
+ return (bytes[offset] & (0x7f << 24))
+ | (bytes[offset + 1] & (0xff << 16))
+ | (bytes[offset + 2] & (0xff << 8))
+ | (bytes[offset + 3] & 0xff);
+ }
+}
diff --git a/src/resources/Documentation/config.md b/src/resources/Documentation/config.md
index acda594..803839a 100644
--- a/src/resources/Documentation/config.md
+++ b/src/resources/Documentation/config.md
@@ -19,6 +19,7 @@
plugins_git_repo_metrics_numberoffiles_<repo_name>
plugins_git_repo_metrics_numberofdirectories_<repo_name>
plugins_git_repo_metrics_numberofemptydirectories_<repo_name>
+plugins_git_repo_metrics_combinedrefssha1_<repo_name>
```
Settings
diff --git a/src/test/java/com/googlesource/gerrit/plugins/gitrepometrics/GitRepoMetricsCacheIT.java b/src/test/java/com/googlesource/gerrit/plugins/gitrepometrics/GitRepoMetricsCacheIT.java
index 7397132..4eee736 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/gitrepometrics/GitRepoMetricsCacheIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/gitrepometrics/GitRepoMetricsCacheIT.java
@@ -25,6 +25,7 @@
import com.google.gerrit.entities.Project;
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.gitrepometrics.collectors.FSMetricsCollector;
+import com.googlesource.gerrit.plugins.gitrepometrics.collectors.GitRefsMetricsCollector;
import com.googlesource.gerrit.plugins.gitrepometrics.collectors.GitStatsMetricsCollector;
import java.time.Duration;
import java.util.Arrays;
@@ -41,6 +42,7 @@
@Inject MetricRegistry metricRegistry;
private FSMetricsCollector fsMetricsCollector;
private GitStatsMetricsCollector gitStatsMetricsCollector;
+ private GitRefsMetricsCollector gitRefsMetricsCollector;
private GitRepoMetricsCache gitRepoMetricsCache;
private final Project.NameKey testProject1 = Project.nameKey("testProject1");
@@ -55,6 +57,7 @@
gitRepoMetricsCache = plugin.getSysInjector().getInstance(GitRepoMetricsCache.class);
fsMetricsCollector = plugin.getSysInjector().getInstance(FSMetricsCollector.class);
gitStatsMetricsCollector = plugin.getSysInjector().getInstance(GitStatsMetricsCollector.class);
+ gitRefsMetricsCollector = plugin.getSysInjector().getInstance(GitRefsMetricsCollector.class);
}
@Test
@@ -70,7 +73,8 @@
int expectedMetricsCount =
fsMetricsCollector.availableMetrics().size()
- + gitStatsMetricsCollector.availableMetrics().size();
+ + gitStatsMetricsCollector.availableMetrics().size()
+ + gitRefsMetricsCollector.availableMetrics().size();
try {
WaitUtil.waitUntil(
diff --git a/src/test/java/com/googlesource/gerrit/plugins/gitrepometrics/collectors/GitRefsMetricsCollectorTest.java b/src/test/java/com/googlesource/gerrit/plugins/gitrepometrics/collectors/GitRefsMetricsCollectorTest.java
new file mode 100644
index 0000000..d1ed3ef
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/gitrepometrics/collectors/GitRefsMetricsCollectorTest.java
@@ -0,0 +1,73 @@
+// Copyright (C) 2024 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.gitrepometrics.collectors;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.internal.storage.file.FileRepository;
+import org.eclipse.jgit.junit.TestRepository;
+import org.eclipse.jgit.lib.PersonIdent;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class GitRefsMetricsCollectorTest {
+ private static final String REPO_NAME = "test-repo";
+ private static final long EXPECTED_COMBINED_SHA1_REF = 16777109;
+
+ @Rule public TemporaryFolder dir = new TemporaryFolder();
+
+ private TestRepository<FileRepository> repo;
+
+ @Before
+ public void setUp() throws Exception {
+ File gitRoot = dir.newFolder(REPO_NAME);
+ try (Git git = Git.init().setDirectory(gitRoot).call()) {
+ repo = new TestRepository<>((FileRepository) git.getRepository());
+ repo.commit().author(new PersonIdent("repo-metrics", "repo@metrics.com")).create();
+ }
+ }
+
+ @Test
+ public void shouldComputeCombinedRefsSha1() throws Exception {
+ HashMap<GitRepoMetric, Long> result = new HashMap<>();
+
+ CountDownLatch latch = new CountDownLatch(1);
+ new GitRefsMetricsCollector(Executors.newFixedThreadPool(1))
+ .collect(
+ repo.getRepository(),
+ REPO_NAME,
+ m -> {
+ result.putAll(m);
+ latch.countDown();
+ });
+ latch.await();
+
+ assertThat(result.get(GitRefsMetricsCollector.combinedRefsSha1))
+ .isEqualTo(EXPECTED_COMBINED_SHA1_REF);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ repo.close();
+ }
+}
