Store project versioning
The version of the local repo will be the timestamp of the last
RefUpdate, stored in this custom ref: "refs/multi-site/version"
Feature: Issue 12163
Change-Id: I05be56c8f02bfbb691c0755257a8aa9fe1455b16
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/event/EventModule.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/event/EventModule.java
index 8889935..1c0c644 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/event/EventModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/event/EventModule.java
@@ -17,6 +17,7 @@
import com.google.gerrit.extensions.registration.DynamicSet;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.server.events.EventListener;
+import com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate;
import java.util.concurrent.Executor;
public class EventModule extends LifecycleModule {
@@ -26,5 +27,6 @@
bind(Executor.class).annotatedWith(EventExecutor.class).toProvider(EventExecutorProvider.class);
listener().to(EventExecutorProvider.class);
DynamicSet.bind(binder(), EventListener.class).to(EventHandler.class);
+ DynamicSet.bind(binder(), EventListener.class).to(ProjectVersionRefUpdate.class);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdate.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdate.java
new file mode 100644
index 0000000..a4866ea
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdate.java
@@ -0,0 +1,235 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.validation;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
+
+import com.gerritforge.gerrit.globalrefdb.GlobalRefDbSystemError;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.flogger.FluentLogger;
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.server.events.Event;
+import com.google.gerrit.server.events.EventListener;
+import com.google.gerrit.server.events.RefUpdatedEvent;
+import com.google.gerrit.server.git.GitRepositoryManager;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
+import com.googlesource.gerrit.plugins.multisite.forwarder.Context;
+import com.googlesource.gerrit.plugins.replication.RefReplicatedEvent;
+import java.io.IOException;
+import java.util.Set;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectIdRef;
+import org.eclipse.jgit.lib.ObjectInserter;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.RefUpdate;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.revwalk.RevWalk;
+import org.eclipse.jgit.transport.RemoteRefUpdate;
+
+public class ProjectVersionRefUpdate implements EventListener {
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+ public static final String MULTI_SITE_VERSIONING_REF = "refs/multi-site/version";
+ public static final String SEQUENCE_REF_PREFIX = "refs/sequences/";
+ private static final Ref NULL_PROJECT_VERSION_REF =
+ new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, MULTI_SITE_VERSIONING_REF, ObjectId.zeroId());
+ private static final Set<RefUpdate.Result> SUCCESSFUL_RESULTS =
+ ImmutableSet.of(RefUpdate.Result.NEW, RefUpdate.Result.FORCED, RefUpdate.Result.NO_CHANGE);
+
+ protected final SharedRefDatabaseWrapper sharedRefDb;
+ private final GitRepositoryManager gitRepositoryManager;
+
+ @Inject
+ public ProjectVersionRefUpdate(
+ GitRepositoryManager gitRepositoryManager, SharedRefDatabaseWrapper sharedRefDb) {
+ this.gitRepositoryManager = gitRepositoryManager;
+ this.sharedRefDb = sharedRefDb;
+ }
+
+ @Override
+ public void onEvent(Event event) {
+ logger.atFine().log("Processing event type: " + event.type);
+ // Producer of the Event use RefUpdatedEvent to trigger the version update
+ if (!Context.isForwardedEvent() && event instanceof RefUpdatedEvent) {
+ updateProducerProjectVersionUpdate((RefUpdatedEvent) event);
+ }
+
+ // Consumers of the Event use RefReplicatedEvent to trigger the version update
+ if (Context.isForwardedEvent() && event instanceof RefReplicatedEvent) {
+ updateConsumerProjectVersion((RefReplicatedEvent) event);
+ }
+ }
+
+ private void updateConsumerProjectVersion(RefReplicatedEvent refReplicatedEvent) {
+ Project.NameKey projectNameKey = refReplicatedEvent.getProjectNameKey();
+ if (!refReplicatedEvent.refStatus.equals(RemoteRefUpdate.Status.OK)) {
+ logger.atFine().log(
+ String.format(
+ "Skipping version update for %s. RefReplicatedEvent failed with %s",
+ projectNameKey.get(), refReplicatedEvent.refStatus));
+ return;
+ }
+ if (refReplicatedEvent.getRefName().startsWith(SEQUENCE_REF_PREFIX)) {
+ logger.atFine().log("Found Sequence ref, skipping update for " + projectNameKey.get());
+ return;
+ }
+ try {
+ updateLocalProjectVersion(projectNameKey, refReplicatedEvent.getRefName());
+ } catch (LocalProjectVersionUpdateException e) {
+ logger.atSevere().withCause(e).log(
+ "Issue encountered when updating version for project " + projectNameKey);
+ }
+ }
+
+ private void updateProducerProjectVersionUpdate(RefUpdatedEvent refUpdatedEvent) {
+ if (refUpdatedEvent.getRefName().startsWith(SEQUENCE_REF_PREFIX)) {
+ logger.atFine().log(
+ "Found Sequence ref, skipping update for " + refUpdatedEvent.getProjectNameKey().get());
+ return;
+ }
+ try {
+ Project.NameKey projectNameKey = refUpdatedEvent.getProjectNameKey();
+ Ref currentProjectVersionRef = getLocalProjectVersionRef(refUpdatedEvent.getProjectNameKey());
+ ObjectId newProjectVersionObjectId =
+ updateLocalProjectVersion(projectNameKey, refUpdatedEvent.getRefName());
+ updateSharedProjectVersion(
+ projectNameKey, currentProjectVersionRef, newProjectVersionObjectId);
+ } catch (LocalProjectVersionUpdateException | SharedProjectVersionUpdateException e) {
+ logger.atSevere().withCause(e).log(
+ "Issue encountered when updating version for project "
+ + refUpdatedEvent.getProjectNameKey());
+ }
+ }
+
+ private RefUpdate getProjectVersionRefUpdate(Repository repository, Long version)
+ throws IOException {
+ RefUpdate refUpdate = repository.getRefDatabase().newUpdate(MULTI_SITE_VERSIONING_REF, false);
+ refUpdate.setNewObjectId(getNewId(repository, version));
+ refUpdate.setForceUpdate(true);
+ return refUpdate;
+ }
+
+ private ObjectId getNewId(Repository repository, Long version) throws IOException {
+ ObjectInserter ins = repository.newObjectInserter();
+ ObjectId newId = ins.insert(OBJ_BLOB, Long.toString(version).getBytes(UTF_8));
+ ins.flush();
+ return newId;
+ }
+
+ private Ref getLocalProjectVersionRef(Project.NameKey projectNameKey)
+ throws LocalProjectVersionUpdateException {
+ try (Repository repository = gitRepositoryManager.openRepository(projectNameKey)) {
+ Ref ref = repository.findRef(MULTI_SITE_VERSIONING_REF);
+ return ref != null ? ref : NULL_PROJECT_VERSION_REF;
+ } catch (IOException e) {
+ String message =
+ String.format("Error while getting current version for %s", projectNameKey.get());
+ logger.atSevere().withCause(e).log(message);
+ throw new LocalProjectVersionUpdateException(message);
+ }
+ }
+
+ private Long getLastRefUpdatedTimestamp(Project.NameKey projectNameKey, String refName)
+ throws LocalProjectVersionUpdateException {
+ logger.atFine().log(
+ String.format(
+ "Getting last ref updated time for project %s, ref %s", projectNameKey.get(), refName));
+ try (Repository repository = gitRepositoryManager.openRepository(projectNameKey)) {
+ Ref ref = repository.findRef(refName);
+ try (RevWalk walk = new RevWalk(repository)) {
+ RevCommit commit = walk.parseCommit(ref.getObjectId());
+ return Integer.toUnsignedLong(commit.getCommitTime());
+ }
+ } catch (IOException ioe) {
+ String message =
+ String.format(
+ "Error while getting last ref updated time for project %s, ref %s",
+ projectNameKey.get(), refName);
+ logger.atSevere().withCause(ioe).log(message);
+ throw new LocalProjectVersionUpdateException(message);
+ }
+ }
+
+ private void updateSharedProjectVersion(
+ Project.NameKey projectNameKey, Ref currentRef, ObjectId newObjectId)
+ throws SharedProjectVersionUpdateException {
+ logger.atFine().log(
+ String.format(
+ "Updating shared project version for %s. Current value %s, new value: %s",
+ projectNameKey.get(), currentRef.getObjectId(), newObjectId));
+ try {
+ boolean success = sharedRefDb.compareAndPut(projectNameKey, currentRef, newObjectId);
+ String message =
+ String.format(
+ "Project version update failed for %s. Current value %s, new value: %s",
+ projectNameKey.get(), currentRef.getObjectId(), newObjectId);
+ if (!success) {
+ logger.atSevere().log(message);
+ throw new SharedProjectVersionUpdateException(message);
+ }
+ } catch (GlobalRefDbSystemError refDbSystemError) {
+ String message =
+ String.format(
+ "Error while updating shared project version for %s. Current value %s, new value: %s. Error: %s",
+ projectNameKey.get(),
+ currentRef.getObjectId(),
+ newObjectId,
+ refDbSystemError.getMessage());
+ logger.atSevere().withCause(refDbSystemError).log(message);
+ throw new SharedProjectVersionUpdateException(message);
+ }
+ }
+
+ private ObjectId updateLocalProjectVersion(Project.NameKey projectNameKey, String refName)
+ throws LocalProjectVersionUpdateException {
+ Long lastRefUpdatedTimestamp = getLastRefUpdatedTimestamp(projectNameKey, refName);
+ logger.atFine().log("Updating local version for project " + projectNameKey.get());
+ try (Repository repository = gitRepositoryManager.openRepository(projectNameKey)) {
+ RefUpdate refUpdate = getProjectVersionRefUpdate(repository, lastRefUpdatedTimestamp);
+ RefUpdate.Result result = refUpdate.update();
+ if (!isSuccessful(result)) {
+ String message =
+ String.format(
+ "RefUpdate failed with result %s for: project=%s, version=%d",
+ result.name(), projectNameKey.get(), lastRefUpdatedTimestamp);
+ logger.atSevere().log(message);
+ throw new LocalProjectVersionUpdateException(message);
+ }
+ return refUpdate.getNewObjectId();
+ } catch (IOException e) {
+ String message = "Cannot create versioning command for " + projectNameKey.get();
+ logger.atSevere().withCause(e).log(message);
+ throw new LocalProjectVersionUpdateException(message);
+ }
+ }
+
+ private Boolean isSuccessful(RefUpdate.Result result) {
+ return SUCCESSFUL_RESULTS.contains(result);
+ }
+
+ public static class LocalProjectVersionUpdateException extends Exception {
+ public LocalProjectVersionUpdateException(String projectName) {
+ super("Cannot update local project version of " + projectName);
+ }
+ }
+
+ public static class SharedProjectVersionUpdateException extends Exception {
+ public SharedProjectVersionUpdateException(String projectName) {
+ super("Cannot update shared project version of " + projectName);
+ }
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidator.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidator.java
index 1f87b77..b23b65a 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidator.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/RefUpdateValidator.java
@@ -86,7 +86,8 @@
public RefUpdate.Result executeRefUpdate(
RefUpdate refUpdate, NoParameterFunction<RefUpdate.Result> refUpdateFunction)
throws IOException {
- if (refEnforcement.getPolicy(projectName) == EnforcePolicy.IGNORED) {
+ if (isProjectVersionUpdate(refUpdate.getName())
+ || refEnforcement.getPolicy(projectName) == EnforcePolicy.IGNORED) {
return refUpdateFunction.invoke();
}
@@ -105,6 +106,13 @@
return null;
}
+ private Boolean isProjectVersionUpdate(String refName) {
+ Boolean isProjectVersionUpdate =
+ refName.equals(ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF);
+ logger.atFine().log("Is project version update? " + isProjectVersionUpdate);
+ return isProjectVersionUpdate;
+ }
+
private <T extends Throwable> void softFailBasedOnEnforcement(T e, EnforcePolicy policy)
throws T {
logger.atWarning().withCause(e).log(
@@ -257,7 +265,9 @@
@Override
public void close() {
- elements.values().stream()
+ elements
+ .values()
+ .stream()
.forEach(
closeable -> {
try {
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateTest.java
new file mode 100644
index 0000000..ff1c31c
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateTest.java
@@ -0,0 +1,187 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.validation;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.*;
+
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.server.events.RefUpdatedEvent;
+import com.google.gerrit.server.project.ProjectConfig;
+import com.google.gerrit.testing.InMemoryRepositoryManager;
+import com.google.gerrit.testing.InMemoryTestEnvironment;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.multisite.SharedRefDatabaseWrapper;
+import com.googlesource.gerrit.plugins.multisite.forwarder.Context;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.RefFixture;
+import com.googlesource.gerrit.plugins.replication.RefReplicatedEvent;
+import com.googlesource.gerrit.plugins.replication.ReplicationState;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import org.apache.commons.io.IOUtils;
+import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
+import org.eclipse.jgit.junit.TestRepository;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectLoader;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.transport.RemoteRefUpdate;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ProjectVersionRefUpdateTest implements RefFixture {
+
+ @Rule public InMemoryTestEnvironment testEnvironment = new InMemoryTestEnvironment();
+
+ @Mock RefUpdatedEvent refUpdatedEvent;
+ @Mock SharedRefDatabaseWrapper sharedRefDb;
+
+ @Inject private ProjectConfig.Factory projectConfigFactory;
+ @Inject private InMemoryRepositoryManager repoManager;
+
+ private TestRepository<InMemoryRepository> repo;
+ private ProjectConfig project;
+ private RevCommit masterCommit;
+
+ @Before
+ public void setUp() throws Exception {
+ InMemoryRepository inMemoryRepo = repoManager.createRepository(A_TEST_PROJECT_NAME_KEY);
+ project = projectConfigFactory.create(A_TEST_PROJECT_NAME_KEY);
+ project.load(inMemoryRepo);
+ repo = new TestRepository<>(inMemoryRepo);
+ masterCommit = repo.branch("master").commit().create();
+ }
+
+ @After
+ public void tearDown() {
+ Context.unsetForwardedEvent();
+ }
+
+ @Test
+ public void producerShouldUpdateProjectVersionUponRefUpdatedEvent() throws IOException {
+ Context.setForwardedEvent(false);
+ when(sharedRefDb.compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class)))
+ .thenReturn(true);
+ when(refUpdatedEvent.getProjectNameKey()).thenReturn(A_TEST_PROJECT_NAME_KEY);
+ when(refUpdatedEvent.getRefName()).thenReturn(A_TEST_REF_NAME);
+
+ new ProjectVersionRefUpdate(repoManager, sharedRefDb).onEvent(refUpdatedEvent);
+
+ Ref ref = repo.getRepository().findRef(ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF);
+
+ verify(sharedRefDb, atMost(1))
+ .compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class));
+
+ assertThat(ref).isNotNull();
+
+ ObjectLoader loader = repo.getRepository().open(ref.getObjectId());
+ String storedVersion = IOUtils.toString(loader.openStream(), StandardCharsets.UTF_8.name());
+ assertThat(Long.parseLong(storedVersion)).isEqualTo(masterCommit.getCommitTime());
+ }
+
+ @Test
+ public void producerShouldNotUpdateProjectVersionUponSequenceRefUpdatedEvent()
+ throws IOException {
+ Context.setForwardedEvent(false);
+ when(refUpdatedEvent.getProjectNameKey()).thenReturn(A_TEST_PROJECT_NAME_KEY);
+ when(refUpdatedEvent.getRefName()).thenReturn("refs/sequences/changes");
+
+ new ProjectVersionRefUpdate(repoManager, sharedRefDb).onEvent(refUpdatedEvent);
+
+ Ref ref = repo.getRepository().findRef(ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF);
+ assertThat(ref).isNull();
+ }
+
+ @Test
+ public void shouldNotUpdateProjectVersionWhenProjectDoesntExist() throws IOException {
+ Context.setForwardedEvent(false);
+ when(refUpdatedEvent.getProjectNameKey())
+ .thenReturn(new Project.NameKey("aNonExistentProject"));
+ when(refUpdatedEvent.getRefName()).thenReturn(A_TEST_REF_NAME);
+
+ new ProjectVersionRefUpdate(repoManager, sharedRefDb).onEvent(refUpdatedEvent);
+
+ Ref ref = repo.getRepository().findRef(ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF);
+ assertThat(ref).isNull();
+ }
+
+ @Test
+ public void consumerShouldUpdateProjectVersionUponRefReplicatedEvent() throws IOException {
+ Context.setForwardedEvent(true);
+ RefReplicatedEvent refReplicatedEvent =
+ new RefReplicatedEvent(
+ A_TEST_PROJECT_NAME,
+ A_TEST_REF_NAME,
+ "targetNode",
+ ReplicationState.RefPushResult.SUCCEEDED,
+ RemoteRefUpdate.Status.OK);
+
+ new ProjectVersionRefUpdate(repoManager, sharedRefDb).onEvent(refReplicatedEvent);
+
+ Ref ref = repo.getRepository().findRef(ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF);
+ assertThat(ref).isNotNull();
+
+ verify(sharedRefDb, never())
+ .compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class));
+
+ ObjectLoader loader = repo.getRepository().open(ref.getObjectId());
+ String storedVersion = IOUtils.toString(loader.openStream(), StandardCharsets.UTF_8.name());
+ assertThat(Long.parseLong(storedVersion))
+ .isEqualTo(Integer.toUnsignedLong(masterCommit.getCommitTime()));
+ }
+
+ @Test
+ public void consumerShouldNotUpdateProjectVersionUponFailedRefReplicatedEvent()
+ throws IOException {
+ Context.setForwardedEvent(true);
+ RefReplicatedEvent refReplicatedEvent =
+ new RefReplicatedEvent(
+ A_TEST_PROJECT_NAME,
+ A_TEST_REF_NAME,
+ "targetNode",
+ ReplicationState.RefPushResult.SUCCEEDED,
+ RemoteRefUpdate.Status.REJECTED_OTHER_REASON);
+
+ new ProjectVersionRefUpdate(repoManager, sharedRefDb).onEvent(refReplicatedEvent);
+
+ Ref ref = repo.getRepository().findRef(ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF);
+ assertThat(ref).isNull();
+ }
+
+ @Test
+ public void consumerShouldNotUpdateProjectVersionUponSequenceRefReplicatedEvent()
+ throws IOException {
+ Context.setForwardedEvent(true);
+ RefReplicatedEvent refReplicatedEvent =
+ new RefReplicatedEvent(
+ A_TEST_PROJECT_NAME,
+ "refs/sequences/groups",
+ "targetNode",
+ ReplicationState.RefPushResult.SUCCEEDED,
+ RemoteRefUpdate.Status.OK);
+
+ new ProjectVersionRefUpdate(repoManager, sharedRefDb).onEvent(refReplicatedEvent);
+
+ Ref ref = repo.getRepository().findRef(ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF);
+ assertThat(ref).isNull();
+ }
+}
