Allow project deletion on primaries Currently, the plugin doesn't allow project deletion. A follow up change will enable project deletion for replicas as well. Bug: Issue 15247 Change-Id: I27c8d791cb5c4e0603f63adf519533f8aeb45a91
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/DeleteProjectTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/DeleteProjectTask.java new file mode 100644 index 0000000..ed584b7 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/DeleteProjectTask.java
@@ -0,0 +1,79 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull; + +import static com.googlesource.gerrit.plugins.replication.pull.ReplicationQueue.repLog; + +import com.google.common.flogger.FluentLogger; +import com.google.gerrit.entities.Project; +import com.google.gerrit.server.ioutil.HexFormat; +import com.google.gerrit.server.util.IdGenerator; +import com.google.inject.Inject; +import com.google.inject.assistedinject.Assisted; +import com.googlesource.gerrit.plugins.replication.pull.client.FetchRestApiClient; +import com.googlesource.gerrit.plugins.replication.pull.client.HttpResult; +import java.io.IOException; +import java.net.URISyntaxException; +import org.eclipse.jgit.transport.URIish; + +public class DeleteProjectTask implements Runnable { + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + interface Factory { + DeleteProjectTask create(Source source, String uri, Project.NameKey project); + } + + private final int id; + private final Source source; + private final String uri; + private final Project.NameKey project; + private final FetchRestApiClient.Factory fetchClientFactory; + + @Inject + DeleteProjectTask( + FetchRestApiClient.Factory fetchClientFactory, + IdGenerator ig, + @Assisted Source source, + @Assisted String uri, + @Assisted Project.NameKey project) { + this.fetchClientFactory = fetchClientFactory; + this.id = ig.next(); + this.uri = uri; + this.source = source; + this.project = project; + } + + @Override + public void run() { + try { + URIish urIish = new URIish(uri); + HttpResult httpResult = fetchClientFactory.create(source).deleteProject(project, urIish); + if (!httpResult.isSuccessful()) { + throw new IOException(httpResult.getMessage().orElse("Unknown")); + } + logger.atFine().log("Successfully deleted project {} on remote {}", project.get(), uri); + } catch (URISyntaxException | IOException e) { + String errorMessage = + String.format("Cannot delete project %s on remote site %s.", project, uri); + logger.atWarning().withCause(e).log(errorMessage); + repLog.warn(errorMessage); + } + } + + @Override + public String toString() { + return String.format("[%s] delete-project %s at %s", HexFormat.fromInt(id), project.get(), uri); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java index 50ab913..647a2f7 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java
@@ -21,6 +21,7 @@ import com.google.gerrit.extensions.config.CapabilityDefinition; import com.google.gerrit.extensions.events.GitReferenceUpdatedListener; import com.google.gerrit.extensions.events.LifecycleListener; +import com.google.gerrit.extensions.events.ProjectDeletedListener; import com.google.gerrit.extensions.registration.DynamicSet; import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.events.EventTypes; @@ -101,6 +102,7 @@ bind(EventBus.class).in(Scopes.SINGLETON); bind(ReplicationSources.class).to(SourcesCollection.class); + DynamicSet.bind(binder(), ProjectDeletedListener.class).to(ReplicationQueue.class); bind(ReplicationQueue.class).in(Scopes.SINGLETON); bind(ObservableQueue.class).to(ReplicationQueue.class);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java index 2a77e09..b7b509d 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
@@ -20,6 +20,7 @@ import com.google.gerrit.entities.Project.NameKey; import com.google.gerrit.extensions.events.GitReferenceUpdatedListener; import com.google.gerrit.extensions.events.LifecycleListener; +import com.google.gerrit.extensions.events.ProjectDeletedListener; import com.google.gerrit.extensions.registration.DynamicItem; import com.google.gerrit.server.events.EventDispatcher; import com.google.gerrit.server.git.WorkQueue; @@ -51,7 +52,10 @@ import org.slf4j.LoggerFactory; public class ReplicationQueue - implements ObservableQueue, LifecycleListener, GitReferenceUpdatedListener { + implements ObservableQueue, + LifecycleListener, + GitReferenceUpdatedListener, + ProjectDeletedListener { static final String PULL_REPLICATION_LOG_NAME = "pull_replication_log"; static final Logger repLog = LoggerFactory.getLogger(PULL_REPLICATION_LOG_NAME); @@ -131,6 +135,16 @@ } } + @Override + public void onProjectDeleted(ProjectDeletedListener.Event event) { + Project.NameKey project = Project.nameKey(event.getProjectName()); + sources.get().getAll().stream() + .filter((Source s) -> s.wouldDeleteProject(project)) + .forEach( + source -> + source.getApis().forEach(apiUrl -> source.scheduleDeleteProject(apiUrl, project))); + } + private Boolean isRefToBeReplicated(String refName) { return !refsFilter.match(refName); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java index 45713b8..e223036 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
@@ -80,6 +80,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import org.apache.commons.io.FilenameUtils; @@ -114,6 +115,7 @@ private final SourceConfiguration config; private final DynamicItem<EventDispatcher> eventDispatcher; private CloseableHttpClient httpClient; + private final DeleteProjectTask.Factory deleteProjectFactory; protected enum RetryReason { TRANSPORT_ERROR, @@ -180,6 +182,7 @@ bind(Source.class).toInstance(Source.this); bind(SourceConfiguration.class).toInstance(config); install(new FactoryModuleBuilder().build(FetchOne.Factory.class)); + install(new FactoryModuleBuilder().build(DeleteProjectTask.Factory.class)); Class<? extends Fetch> clientClass = cfg.useCGitClient() ? CGitFetch.class : JGitFetch.class; install( @@ -210,6 +213,7 @@ child.getBinding(FetchFactory.class).acceptTargetVisitor(new CGitFetchValidator()); opFactory = child.getInstance(FetchOne.Factory.class); threadScoper = child.getInstance(PerThreadRequestScope.Scoper.class); + deleteProjectFactory = child.getInstance(DeleteProjectTask.Factory.class); } public synchronized CloseableHttpClient memoize( @@ -445,6 +449,12 @@ } } + void scheduleDeleteProject(String uri, Project.NameKey project) { + @SuppressWarnings("unused") + ScheduledFuture<?> ignored = + pool.schedule(deleteProjectFactory.create(this, uri, project), 0, TimeUnit.SECONDS); + } + void fetchWasCanceled(FetchOne fetchOp) { synchronized (stateLock) { URIish uri = fetchOp.getURI(); @@ -592,11 +602,22 @@ return false; } + public boolean wouldDeleteProject(Project.NameKey project) { + if (isReplicateProjectDeletions()) { + return configSettingsAllowReplication(project); + } + return false; + } + public boolean wouldFetchProject(Project.NameKey project) { if (!shouldReplicate(project)) { return false; } + return configSettingsAllowReplication(project); + } + + private boolean configSettingsAllowReplication(Project.NameKey project) { // by default fetch all projects List<String> projects = config.getProjects(); if (projects.isEmpty()) { @@ -737,6 +758,10 @@ return config.createMissingRepositories(); } + public boolean isReplicateProjectDeletions() { + return config.replicateProjectDeletions(); + } + private static boolean matches(URIish uri, String urlMatch) { if (urlMatch == null || urlMatch.equals("") || urlMatch.equals("*")) { return true;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java index 4858b17..ab9c634 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java
@@ -39,6 +39,7 @@ private final boolean replicatePermissions; private final boolean replicateHiddenProjects; private final boolean createMissingRepositories; + private final boolean replicateProjectDeletions; private final String remoteNameStyle; private final ImmutableList<String> urls; private final ImmutableList<String> projects; @@ -76,6 +77,7 @@ lockErrorMaxRetries = cfg.getInt("replication", "lockErrorMaxRetries", 0); createMissingRepositories = cfg.getBoolean("remote", name, "createMissingRepositories", true); + replicateProjectDeletions = cfg.getBoolean("remote", name, "replicateProjectDeletions", true); replicatePermissions = cfg.getBoolean("remote", name, "replicatePermissions", true); replicateHiddenProjects = cfg.getBoolean("remote", name, "replicateHiddenProjects", false); useCGitClient = cfg.getBoolean("replication", "useCGitClient", false); @@ -197,6 +199,10 @@ return createMissingRepositories; } + public boolean replicateProjectDeletions() { + return replicateProjectDeletions; + } + private static int getInt(RemoteConfig rc, Config cfg, String name, int defValue) { return cfg.getInt("remote", rc.getName(), name, defValue); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java index 811d064..a7cc3e7 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
@@ -44,6 +44,7 @@ import org.apache.http.client.ClientProtocolException; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.ResponseHandler; +import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.apache.http.client.protocol.HttpClientContext; @@ -121,6 +122,13 @@ return httpClientFactory.create(source).execute(put, this, getContext(uri)); } + public HttpResult deleteProject(Project.NameKey project, URIish apiUri) throws IOException { + String url = + String.format("%s/%s", apiUri.toASCIIString(), getProjectDeletionUrl(project.get())); + HttpDelete delete = new HttpDelete(url); + return httpClientFactory.create(source).execute(delete, this, getContext(apiUri)); + } + public HttpResult callSendObject( Project.NameKey project, String refName, RevisionData revisionData, URIish targetUri) throws ClientProtocolException, IOException { @@ -168,4 +176,8 @@ } return null; } + + String getProjectDeletionUrl(String projectName) { + return String.format("a/projects/%s", Url.encode(projectName)); + } }
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index f2a596b..68c9001 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md
@@ -354,6 +354,12 @@ By default, true. +remote.NAME.replicateProjectDeletions +: If true, project deletions will also be replicated to the +remote site. + + By default, false, do *not* replicate project deletions. + remote.NAME.authGroup : Specifies the name of a group that the remote should use to access the repositories. Multiple authGroups may be specified
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java index 9557025..c8c294f 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java
@@ -15,6 +15,7 @@ package com.googlesource.gerrit.plugins.replication.pull; import static com.google.common.truth.Truth.assertThat; +import static com.google.gerrit.server.project.ProjectResource.PROJECT_KIND; import static java.util.stream.Collectors.toList; import com.google.common.flogger.FluentLogger; @@ -25,10 +26,23 @@ import com.google.gerrit.acceptance.UseLocalDisk; import com.google.gerrit.acceptance.testsuite.project.ProjectOperations; import com.google.gerrit.entities.Project; +import com.google.gerrit.extensions.api.changes.NotifyHandling; import com.google.gerrit.extensions.api.projects.BranchInput; +import com.google.gerrit.extensions.common.Input; import com.google.gerrit.extensions.events.GitReferenceUpdatedListener; +import com.google.gerrit.extensions.events.ProjectDeletedListener; +import com.google.gerrit.extensions.registration.DynamicSet; +import com.google.gerrit.extensions.restapi.AuthException; +import com.google.gerrit.extensions.restapi.BadRequestException; +import com.google.gerrit.extensions.restapi.ResourceConflictException; +import com.google.gerrit.extensions.restapi.Response; +import com.google.gerrit.extensions.restapi.RestApiModule; +import com.google.gerrit.extensions.restapi.RestModifyView; import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.project.ProjectResource; +import com.google.inject.AbstractModule; import com.google.inject.Inject; +import com.google.inject.Singleton; import com.googlesource.gerrit.plugins.replication.AutoReloadConfigDecorator; import java.io.IOException; import java.nio.file.Path; @@ -62,14 +76,33 @@ @Inject private SitePaths sitePaths; @Inject private ProjectOperations projectOperations; + @Inject private DynamicSet<ProjectDeletedListener> deletedListeners; private Path gitPath; private FileBasedConfig config; private FileBasedConfig secureConfig; + static FakeDeleteProjectPlugin fakeDeleteProjectPlugin; + + static class FakeDeleteModule extends AbstractModule { + + @Override + public void configure() { + install( + new RestApiModule() { + @Override + public void configure() { + fakeDeleteProjectPlugin = new FakeDeleteProjectPlugin(); + delete(PROJECT_KIND).toInstance(fakeDeleteProjectPlugin); + } + }); + } + } + @Override public void setUpTestPlugin() throws Exception { gitPath = sitePaths.site_path.resolve("git"); + installPlugin("fakeDeleteProjectPlugin", FakeDeleteModule.class, null, null); config = new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED); setReplicationSource( @@ -222,6 +255,35 @@ } } + @Test + public void shouldReplicateProjectDeletion() throws Exception { + String projectToDelete = project.get(); + setReplicationSource(TEST_REPLICATION_REMOTE, "", Optional.of(projectToDelete)); + config.save(); + AutoReloadConfigDecorator autoReloadConfigDecorator = + getInstance(AutoReloadConfigDecorator.class); + autoReloadConfigDecorator.reload(); + + ProjectDeletedListener.Event event = + new ProjectDeletedListener.Event() { + @Override + public String getProjectName() { + return projectToDelete; + } + + @Override + public NotifyHandling getNotify() { + return NotifyHandling.NONE; + } + }; + + for (ProjectDeletedListener l : deletedListeners) { + l.onProjectDeleted(event); + } + + waitUntil(() -> fakeDeleteProjectPlugin.getDeleteEndpointCalls() == 1); + } + private Ref getRef(Repository repo, String branchName) throws IOException { return repo.getRefDatabase().exactRef(branchName); } @@ -278,4 +340,24 @@ private Project.NameKey createTestProject(String name) throws Exception { return projectOperations.newProject().name(name).create(); } + + @Singleton + public static class FakeDeleteProjectPlugin implements RestModifyView<ProjectResource, Input> { + private int deleteEndpointCalls; + + FakeDeleteProjectPlugin() { + this.deleteEndpointCalls = 0; + } + + @Override + public Response<?> apply(ProjectResource resource, Input input) + throws AuthException, BadRequestException, ResourceConflictException, Exception { + deleteEndpointCalls += 1; + return Response.ok(); + } + + int getDeleteEndpointCalls() { + return deleteEndpointCalls; + } + } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java index 286fe24..4779d63 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
@@ -14,20 +14,24 @@ package com.googlesource.gerrit.plugins.replication.pull; +import static com.google.common.truth.Truth.assertThat; import static java.nio.file.Files.createTempDirectory; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import com.google.gerrit.entities.Project; import com.google.gerrit.extensions.api.changes.NotifyHandling; import com.google.gerrit.extensions.common.AccountInfo; import com.google.gerrit.extensions.events.GitReferenceUpdatedListener; import com.google.gerrit.extensions.events.GitReferenceUpdatedListener.Event; +import com.google.gerrit.extensions.events.ProjectDeletedListener; import com.google.gerrit.extensions.registration.DynamicItem; import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.events.EventDispatcher; @@ -51,6 +55,8 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; @@ -71,6 +77,9 @@ @Mock RevisionData revisionData; @Mock HttpResult httpResult; + @Captor ArgumentCaptor<String> stringCaptor; + @Captor ArgumentCaptor<Project.NameKey> projectNameKeyCaptor; + private ExcludedRefsFilter refsFilter; private ReplicationQueue objectUnderTest; private SitePaths sitePaths; @@ -261,6 +270,34 @@ verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo); } + @Test + public void shouldCallDeleteWhenReplicateProjectDeletionsTrue() throws IOException { + when(source.wouldDeleteProject(any())).thenReturn(true); + + String projectName = "testProject"; + FakeProjectDeletedEvent event = new FakeProjectDeletedEvent(projectName); + + objectUnderTest.start(); + objectUnderTest.onProjectDeleted(event); + + verify(source, times(1)) + .scheduleDeleteProject(stringCaptor.capture(), projectNameKeyCaptor.capture()); + assertThat(stringCaptor.getValue()).isEqualTo(source.getApis().get(0)); + assertThat(projectNameKeyCaptor.getValue()).isEqualTo(Project.NameKey.parse(projectName)); + } + + @Test + public void shouldNotCallDeleteWhenProjectNotToDelete() throws IOException { + when(source.wouldDeleteProject(any())).thenReturn(false); + + FakeProjectDeletedEvent event = new FakeProjectDeletedEvent("testProject"); + + objectUnderTest.start(); + objectUnderTest.onProjectDeleted(event); + + verify(source, never()).scheduleDeleteProject(any(), any()); + } + protected static Path createTempPath(String prefix) throws IOException { return createTempDirectory(prefix); } @@ -325,4 +362,22 @@ return null; } } + + private class FakeProjectDeletedEvent implements ProjectDeletedListener.Event { + private String projectName; + + public FakeProjectDeletedEvent(String projectName) { + this.projectName = projectName; + } + + @Override + public NotifyHandling getNotify() { + return null; + } + + @Override + public String getProjectName() { + return projectName; + } + } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java index 39a8f2d..bde86fc 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java
@@ -38,6 +38,7 @@ import java.util.Optional; import org.apache.http.Header; import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.apache.http.message.BasicHeader; @@ -69,6 +70,7 @@ @Mock Source source; @Captor ArgumentCaptor<HttpPost> httpPostCaptor; @Captor ArgumentCaptor<HttpPut> httpPutCaptor; + @Captor ArgumentCaptor<HttpDelete> httpDeleteCaptor; String api = "http://gerrit-host"; String pluginName = "pull-replication"; @@ -362,6 +364,18 @@ .isEqualTo("/a/plugins/pull-replication/init-project/test_repo.git"); } + @Test + public void shouldCallDeleteProjectEndpoint() throws IOException, URISyntaxException { + + objectUnderTest.deleteProject(Project.nameKey("test_repo"), new URIish(api)); + + verify(httpClient, times(1)).execute(httpDeleteCaptor.capture(), any(), any()); + + HttpDelete httpDelete = httpDeleteCaptor.getValue(); + assertThat(httpDelete.getURI().getHost()).isEqualTo("gerrit-host"); + assertThat(httpDelete.getURI().getPath()).isEqualTo("/a/projects/test_repo"); + } + public String readPayload(HttpPost entity) throws UnsupportedOperationException, IOException { ByteBuffer buf = IO.readWholeStream(entity.getEntity().getContent(), 1024); return RawParseUtils.decode(buf.array(), buf.arrayOffset(), buf.limit()).trim();