Introduce persistent cache for replication-status ReplicationStatus keeps track of the replication lags associated to projects in a multi-site set-up, so that it is possible to understand how much behind a particular project is falling due to replication. This information however is lost upon gerrit restart. Persist replication status information by using a persistent cache to populate in-memory replication-status at startup. Bug: Issue 15310 Change-Id: I86c40684439ec5a457866f56f4cb9157c4af0092
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/PluginModule.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/PluginModule.java index 81a0d8d..20aaef6 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/multisite/PluginModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/PluginModule.java
@@ -22,6 +22,7 @@ import com.google.inject.Scopes; import com.googlesource.gerrit.plugins.multisite.broker.BrokerApiWrapper; import com.googlesource.gerrit.plugins.multisite.consumer.MultiSiteConsumerRunner; +import com.googlesource.gerrit.plugins.multisite.consumer.ReplicationStatusModule; import com.googlesource.gerrit.plugins.multisite.consumer.SubscriberModule; import com.googlesource.gerrit.plugins.multisite.forwarder.broker.BrokerForwarderModule; @@ -41,6 +42,7 @@ install(new BrokerForwarderModule()); listener().to(MultiSiteConsumerRunner.class); + install(new ReplicationStatusModule()); if (config.getSharedRefDbConfiguration().getSharedRefDb().isEnabled()) { listener().to(PluginStartup.class); DynamicSet.bind(binder(), ProjectDeletedListener.class)
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ReplicationStatus.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ReplicationStatus.java index 45e66a6..c024e67 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ReplicationStatus.java +++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ReplicationStatus.java
@@ -15,10 +15,18 @@ package com.googlesource.gerrit.plugins.multisite.consumer; import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.Cache; import com.google.common.flogger.FluentLogger; import com.google.gerrit.entities.Project; +import com.google.gerrit.extensions.events.LifecycleListener; +import com.google.gerrit.server.cache.CacheModule; +import com.google.gerrit.server.cache.serialize.JavaCacheSerializer; +import com.google.gerrit.server.cache.serialize.StringCacheSerializer; +import com.google.gerrit.server.project.ProjectCache; import com.google.inject.Inject; +import com.google.inject.Module; import com.google.inject.Singleton; +import com.google.inject.name.Named; import com.googlesource.gerrit.plugins.multisite.ProjectVersionLogger; import com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate; import java.util.Collection; @@ -27,22 +35,44 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; @Singleton -public class ReplicationStatus { +public class ReplicationStatus implements LifecycleListener { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private final Map<String, Long> replicationStatusPerProject = new HashMap<>(); + static final String REPLICATION_STATUS_CACHE = "replication_status"; + + public static Module cacheModule() { + return new CacheModule() { + @Override + protected void configure() { + persist(REPLICATION_STATUS_CACHE, String.class, Long.class) + .version(1) + .keySerializer(StringCacheSerializer.INSTANCE) + .valueSerializer(new JavaCacheSerializer<>()); + } + }; + } + private final Map<String, Long> localVersionPerProject = new HashMap<>(); + private final Cache<String, Long> cache; private final ProjectVersionRefUpdate projectVersionRefUpdate; private final ProjectVersionLogger verLogger; + private final ProjectCache projectCache; @Inject public ReplicationStatus( - ProjectVersionRefUpdate projectVersionRefUpdate, ProjectVersionLogger verLogger) { + @Named(REPLICATION_STATUS_CACHE) Cache<String, Long> cache, + ProjectVersionRefUpdate projectVersionRefUpdate, + ProjectVersionLogger verLogger, + ProjectCache projectCache) { + this.cache = cache; this.projectVersionRefUpdate = projectVersionRefUpdate; this.verLogger = verLogger; + this.projectCache = projectCache; } public Long getMaxLag() { @@ -90,6 +120,21 @@ @VisibleForTesting public void doUpdateLag(Project.NameKey projectName, Long lag) { + cache.put(projectName.get(), lag); replicationStatusPerProject.put(projectName.get(), lag); } + + @Override + public void start() { + loadAllFromCache(); + } + + @Override + public void stop() {} + + private void loadAllFromCache() { + Set<String> cachedProjects = + projectCache.all().stream().map(Project.NameKey::get).collect(Collectors.toSet()); + replicationStatusPerProject.putAll(cache.getAllPresent(cachedProjects)); + } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ReplicationStatusModule.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ReplicationStatusModule.java new file mode 100644 index 0000000..e954bc4 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ReplicationStatusModule.java
@@ -0,0 +1,27 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.multisite.consumer; + +import com.google.gerrit.lifecycle.LifecycleModule; +import com.google.inject.Scopes; + +public class ReplicationStatusModule extends LifecycleModule { + @Override + protected void configure() { + bind(ReplicationStatus.class).in(Scopes.SINGLETON); + install(ReplicationStatus.cacheModule()); + listener().to(ReplicationStatus.class); + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/consumer/ReplicationStatusTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/consumer/ReplicationStatusTest.java new file mode 100644 index 0000000..beb93dd --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/consumer/ReplicationStatusTest.java
@@ -0,0 +1,89 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.multisite.consumer; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.when; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.collect.ImmutableSortedSet; +import com.google.gerrit.entities.Project; +import com.google.gerrit.server.project.ProjectCache; +import com.googlesource.gerrit.plugins.multisite.ProjectVersionLogger; +import com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class ReplicationStatusTest { + + @Mock private ProjectVersionLogger verLogger; + @Mock private ProjectCache projectCache; + @Mock private ProjectVersionRefUpdate projectVersionRefUpdate; + private ReplicationStatus objectUnderTest; + private Cache<String, Long> replicationStatusCache; + + @Before + public void setup() throws Exception { + when(projectCache.all()) + .thenReturn( + ImmutableSortedSet.of(Project.nameKey("projectA"), Project.nameKey("projectB"))); + replicationStatusCache = CacheBuilder.newBuilder().build(); + objectUnderTest = + new ReplicationStatus( + replicationStatusCache, projectVersionRefUpdate, verLogger, projectCache); + } + + @Test + public void shouldPopulateLagsFromPersistedCacheOnStart() { + replicationStatusCache.put("projectA", 10L); + replicationStatusCache.put("projectB", 3L); + + objectUnderTest.start(); + assertThat(objectUnderTest.getMaxLag()).isEqualTo(10L); + } + + @Test + public void shouldBeAbleToUpdatePersistedCacheValues() { + replicationStatusCache.put("projectA", 3L); + + objectUnderTest.start(); + + objectUnderTest.doUpdateLag(Project.nameKey("projectA"), 20L); + assertThat(objectUnderTest.getMaxLag()).isEqualTo(20L); + } + + @Test + public void shouldCombinePersistedProjectsWithNewEntries() { + replicationStatusCache.put("projectA", 3L); + objectUnderTest.start(); + + objectUnderTest.doUpdateLag(Project.nameKey("projectB"), 20L); + + assertThat(objectUnderTest.getReplicationLags(2).keySet()) + .containsExactly("projectA", "projectB"); + } + + @Test + public void shouldUpdatePersistedCacheWhenUpdatingLagValue() { + objectUnderTest.doUpdateLag(Project.nameKey("projectA"), 20L); + + assertThat(replicationStatusCache.getIfPresent("projectA")).isEqualTo(20L); + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetricsTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetricsTest.java index 0c2f750..f648f6b 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetricsTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetricsTest.java
@@ -20,11 +20,13 @@ import com.gerritforge.gerrit.eventbroker.EventMessage; import com.gerritforge.gerrit.globalrefdb.validation.SharedRefDatabaseWrapper; import com.google.common.base.Suppliers; +import com.google.common.cache.CacheBuilder; import com.google.gerrit.entities.Project; import com.google.gerrit.metrics.MetricMaker; import com.google.gerrit.server.data.RefUpdateAttribute; import com.google.gerrit.server.events.RefUpdatedEvent; import com.google.gerrit.server.extensions.events.GitReferenceUpdated; +import com.google.gerrit.server.project.ProjectCache; import com.googlesource.gerrit.plugins.multisite.ProjectVersionLogger; import com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate; import java.util.Optional; @@ -45,6 +47,7 @@ @Mock private GitReferenceUpdated gitReferenceUpdated; @Mock private MetricMaker metricMaker; @Mock private ProjectVersionLogger verLogger; + @Mock private ProjectCache projectCache; @Mock private ProjectVersionRefUpdate projectVersionRefUpdate; private SubscriberMetrics metrics; private EventMessage.Header msgHeader; @@ -54,7 +57,12 @@ msgHeader = new EventMessage.Header(UUID.randomUUID(), UUID.randomUUID()); metrics = new SubscriberMetrics( - metricMaker, new ReplicationStatus(projectVersionRefUpdate, verLogger)); + metricMaker, + new ReplicationStatus( + CacheBuilder.newBuilder().build(), + projectVersionRefUpdate, + verLogger, + projectCache)); } @Test
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/http/ReplicationStatusServletIT.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/http/ReplicationStatusServletIT.java index 3b5f9a6..da02d02 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/multisite/http/ReplicationStatusServletIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/http/ReplicationStatusServletIT.java
@@ -31,6 +31,7 @@ import com.googlesource.gerrit.plugins.multisite.ProjectVersionLogger; import com.googlesource.gerrit.plugins.multisite.cache.CacheModule; import com.googlesource.gerrit.plugins.multisite.consumer.ReplicationStatus; +import com.googlesource.gerrit.plugins.multisite.consumer.ReplicationStatusModule; import com.googlesource.gerrit.plugins.multisite.forwarder.ForwarderModule; import com.googlesource.gerrit.plugins.multisite.forwarder.router.RouterModule; import com.googlesource.gerrit.plugins.multisite.index.IndexModule; @@ -57,6 +58,7 @@ install(new CacheModule()); install(new RouterModule()); install(new IndexModule()); + install(new ReplicationStatusModule()); SharedRefDbConfiguration sharedRefDbConfig = new SharedRefDbConfiguration(new Config(), "multi-site"); bind(SharedRefDbConfiguration.class).toInstance(sharedRefDbConfig);