Merge branch 'stable-3.10' into stable-3.11
* stable-3.10:
FetchOne: consider order of create/delete events when replicating deletions
Consume only relevant stream events
Use stream events to delete repositories
Change-Id: Ib3318cee50d8c43a3ff44bf5f50c2b73174a7c73
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java
index e75c4c3..5e7fda2 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java
@@ -17,6 +17,7 @@
import static com.googlesource.gerrit.plugins.replication.pull.PullReplicationLogger.repLog;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Throwables;
import com.google.common.collect.LinkedListMultimap;
import com.google.common.collect.ListMultimap;
@@ -476,11 +477,7 @@
try {
List<FetchRefSpec> toFetch =
fetchRefSpecs.stream().filter(rs -> rs.getSource() != null).toList();
- Set<String> toDelete =
- fetchRefSpecs.stream()
- .filter(rs -> rs.getSource() == null)
- .map(RefSpec::getDestination)
- .collect(Collectors.toSet());
+ Set<String> toDelete = refsToDelete(fetchRefSpecs);
updateStates(fetch.fetch(toFetch));
// JGit doesn't support a fetch of <empty> to a ref (e.g. :refs/to/delete) therefore we have
@@ -512,6 +509,21 @@
return fetchRefSpecs;
}
+ @VisibleForTesting
+ static Set<String> refsToDelete(List<FetchRefSpec> fetchRefSpecs) {
+ final Set<String> refsToDelete = new HashSet<>();
+ fetchRefSpecs.forEach(
+ rs -> {
+ String refName = rs.refName();
+ if (rs.isDelete()) {
+ refsToDelete.add(refName);
+ } else {
+ refsToDelete.remove(refName);
+ }
+ });
+ return refsToDelete;
+ }
+
/**
* Return the list of refSpecs to fetch, possibly after having been filtered.
*
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchRefSpec.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchRefSpec.java
index f85f22a..e41e194 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchRefSpec.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchRefSpec.java
@@ -44,6 +44,10 @@
return MoreObjects.firstNonNull(getSource(), getDestination());
}
+ public boolean isDelete() {
+ return getSource() == null;
+ }
+
@Override
public String toString() {
return getSource() == null ? "<<DELETED>>:" + getDestination() : super.toString();
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionAction.java
index f9165ad..18574fd 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionAction.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionAction.java
@@ -29,6 +29,7 @@
import com.google.gerrit.server.project.ProjectResource;
import com.google.inject.Inject;
import com.google.inject.Provider;
+import com.google.inject.Singleton;
import com.googlesource.gerrit.plugins.deleteproject.cache.CacheDeleteHandler;
import com.googlesource.gerrit.plugins.deleteproject.fs.RepositoryDelete;
import com.googlesource.gerrit.plugins.replication.pull.GerritConfigOps;
@@ -37,12 +38,13 @@
import org.eclipse.jgit.errors.RepositoryNotFoundException;
import org.eclipse.jgit.transport.URIish;
-class ProjectDeletionAction
+@Singleton
+public class ProjectDeletionAction
implements RestModifyView<ProjectResource, ProjectDeletionAction.DeleteInput> {
private static final PluginPermission DELETE_PROJECT =
new PluginPermission("delete-project", "deleteProject");
- static class DeleteInput {}
+ public static class DeleteInput {}
private final Provider<CurrentUser> userProvider;
private final GerritConfigOps gerritConfigOps;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
index f863148..eb2d7e0 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
@@ -25,7 +25,9 @@
import com.google.gerrit.entities.Project.NameKey;
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.extensions.restapi.AuthException;
+import com.google.gerrit.extensions.restapi.IdString;
import com.google.gerrit.extensions.restapi.ResourceNotFoundException;
+import com.google.gerrit.extensions.restapi.TopLevelResource;
import com.google.gerrit.extensions.restapi.UnprocessableEntityException;
import com.google.gerrit.server.config.GerritInstanceId;
import com.google.gerrit.server.data.RefUpdateAttribute;
@@ -37,9 +39,12 @@
import com.google.gerrit.server.events.RefUpdatedEvent;
import com.google.gerrit.server.git.WorkQueue;
import com.google.gerrit.server.permissions.PermissionBackendException;
+import com.google.gerrit.server.project.ProjectResource;
+import com.google.gerrit.server.restapi.project.ProjectsCollection;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.name.Named;
+import com.googlesource.gerrit.plugins.deleteproject.ProjectDeletedEvent;
import com.googlesource.gerrit.plugins.replication.pull.ApplyObjectsCacheKey;
import com.googlesource.gerrit.plugins.replication.pull.FetchOne;
import com.googlesource.gerrit.plugins.replication.pull.Source;
@@ -47,6 +52,7 @@
import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchJob;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchJob.Factory;
+import com.googlesource.gerrit.plugins.replication.pull.api.ProjectDeletionAction;
import com.googlesource.gerrit.plugins.replication.pull.api.ProjectInitializationAction;
import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics;
import com.googlesource.gerrit.plugins.replication.pull.api.UpdateHeadCommand;
@@ -68,6 +74,8 @@
private final String instanceId;
private final WorkQueue workQueue;
private final Cache<ApplyObjectsCacheKey, Long> refUpdatesSucceededCache;
+ private final ProjectDeletionAction projectDeletionAction;
+ private final ProjectsCollection projectsCollection;
@Inject
public StreamEventListener(
@@ -79,7 +87,9 @@
Provider<PullReplicationApiRequestMetrics> metricsProvider,
SourcesCollection sources,
ExcludedRefsFilter excludedRefsFilter,
- @Named(APPLY_OBJECTS_CACHE) Cache<ApplyObjectsCacheKey, Long> refUpdatesSucceededCache) {
+ @Named(APPLY_OBJECTS_CACHE) Cache<ApplyObjectsCacheKey, Long> refUpdatesSucceededCache,
+ ProjectDeletionAction projectDeletionAction,
+ ProjectsCollection projectsCollection) {
this.instanceId = instanceId;
this.updateHeadCommand = updateHeadCommand;
this.projectInitializationAction = projectInitializationAction;
@@ -89,6 +99,8 @@
this.sources = sources;
this.refsFilter = excludedRefsFilter;
this.refUpdatesSucceededCache = refUpdatesSucceededCache;
+ this.projectDeletionAction = projectDeletionAction;
+ this.projectsCollection = projectsCollection;
requireNonNull(
Strings.emptyToNull(this.instanceId), "gerrit.instanceId cannot be null or empty");
@@ -173,6 +185,25 @@
"Failed to update HEAD on project: %s", headUpdatedEvent.projectName);
throw e;
}
+ } else if (event instanceof ProjectDeletedEvent) {
+ deleteProject((ProjectDeletedEvent) event);
+ }
+ }
+
+ protected void deleteProject(ProjectEvent projectDeletedEvent) {
+ try {
+ ProjectResource projectResource =
+ projectsCollection.parse(
+ TopLevelResource.INSTANCE,
+ IdString.fromDecoded(projectDeletedEvent.getProjectNameKey().get()));
+ projectDeletionAction.apply(projectResource, new ProjectDeletionAction.DeleteInput());
+ } catch (ResourceNotFoundException e) {
+ logger.atFine().withCause(e).log(
+ "Repository not found whilst trying to delete project:%s",
+ projectDeletedEvent.getProjectNameKey().get());
+ } catch (Exception e) {
+ logger.atSevere().withCause(e).log(
+ "Cannot delete project:%s", projectDeletedEvent.getProjectNameKey().get());
}
}
@@ -181,7 +212,7 @@
}
private boolean shouldReplicateProject(Event event) {
- if (!(event instanceof ProjectEvent)) {
+ if (!isInterestingEventType(event)) {
return false;
}
@@ -202,10 +233,23 @@
&& source.wouldCreateProject(projectCreatedEvent.getProjectNameKey());
}
+ if (event instanceof ProjectDeletedEvent) {
+ ProjectDeletedEvent projectDeletedEvent = (ProjectDeletedEvent) event;
+
+ return source.wouldDeleteProject(projectDeletedEvent.getProjectNameKey());
+ }
+
ProjectEvent projectEvent = (ProjectEvent) event;
return source.wouldFetchProject(projectEvent.getProjectNameKey());
}
+ private static boolean isInterestingEventType(Event event) {
+ return event instanceof ProjectDeletedEvent
+ || event instanceof ProjectCreatedEvent
+ || event instanceof RefUpdatedEvent
+ || event instanceof ProjectHeadUpdatedEvent;
+ }
+
private boolean isRefDelete(RefUpdatedEvent event) {
return ZERO_ID_NAME.equals(event.refUpdate.get().newRev);
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchOneTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchOneTest.java
index 49e8d6f..73715d6 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchOneTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchOneTest.java
@@ -15,6 +15,7 @@
package com.googlesource.gerrit.plugins.replication.pull;
import static com.google.common.truth.Truth.assertThat;
+import static com.googlesource.gerrit.plugins.replication.pull.FetchOne.refsToDelete;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.anyList;
import static org.mockito.Mockito.argThat;
@@ -850,6 +851,36 @@
assertThat(testMetricMaker.getTimer("replication_latency")).isGreaterThan(0);
}
+ @Test
+ public void shouldSkipDeletionWhenDeleteAndCreateOfSameRef() {
+ List<FetchRefSpec> fetchRefSpecs =
+ List.of(
+ FetchRefSpec.fromRef(":refs/something/someref"),
+ FetchRefSpec.fromRef("refs/something/someref"));
+ assertThat(refsToDelete(fetchRefSpecs)).isEmpty();
+ }
+
+ @Test
+ public void shouldDeleteWhenCreateAndDeleteOfSameRef() {
+ List<FetchRefSpec> fetchRefSpecs =
+ List.of(
+ FetchRefSpec.fromRef("refs/something/someref"),
+ FetchRefSpec.fromRef(":refs/something/someref"));
+ assertThat(refsToDelete(fetchRefSpecs)).isEqualTo(Set.of("refs/something/someref"));
+ }
+
+ @Test
+ public void shouldNotDeleteWhenCreateRef() {
+ List<FetchRefSpec> fetchRefSpecs = List.of(FetchRefSpec.fromRef("refs/something/someref"));
+ assertThat(refsToDelete(fetchRefSpecs)).isEmpty();
+ }
+
+ @Test
+ public void shouldDeleteWhenDeleteRef() {
+ List<FetchRefSpec> fetchRefSpecs = List.of(FetchRefSpec.fromRef(":refs/something/someref"));
+ assertThat(refsToDelete(fetchRefSpecs)).isEqualTo(Set.of("refs/something/someref"));
+ }
+
private void setupRequestScopeMock() {
when(scoper.scope(any()))
.thenAnswer(
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java
index a60fed6..980c509 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java
@@ -31,12 +31,14 @@
import com.google.gerrit.server.events.ProjectHeadUpdatedEvent;
import com.google.gerrit.server.events.RefUpdatedEvent;
import com.google.gerrit.server.git.WorkQueue;
+import com.google.gerrit.server.restapi.project.ProjectsCollection;
import com.googlesource.gerrit.plugins.replication.pull.ApplyObjectsCacheKey;
import com.googlesource.gerrit.plugins.replication.pull.FetchOne;
import com.googlesource.gerrit.plugins.replication.pull.Source;
import com.googlesource.gerrit.plugins.replication.pull.SourcesCollection;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchJob;
+import com.googlesource.gerrit.plugins.replication.pull.api.ProjectDeletionAction;
import com.googlesource.gerrit.plugins.replication.pull.api.ProjectInitializationAction;
import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics;
import com.googlesource.gerrit.plugins.replication.pull.api.UpdateHeadCommand;
@@ -72,6 +74,9 @@
@Mock private SourcesCollection sources;
@Mock private Source source;
@Mock private ExcludedRefsFilter refsFilter;
+ @Mock private ProjectDeletionAction projectDeletionAction;
+ @Mock private ProjectsCollection projectsCollection;
+
private Cache<ApplyObjectsCacheKey, Long> cache;
private StreamEventListener objectUnderTest;
@@ -98,7 +103,9 @@
() -> metrics,
sources,
refsFilter,
- cache);
+ cache,
+ projectDeletionAction,
+ projectsCollection);
}
@Test
