Merge "Allow batch-fetch endpoint to delete refs"
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/AutoValueTypeAdapterFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/AutoValueTypeAdapterFactory.java new file mode 100644 index 0000000..264b87e --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/AutoValueTypeAdapterFactory.java
@@ -0,0 +1,26 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull; + +import com.google.gson.TypeAdapterFactory; +import com.ryanharter.auto.value.gson.GsonTypeAdapterFactory; + +@GsonTypeAdapterFactory +public abstract class AutoValueTypeAdapterFactory implements TypeAdapterFactory { + + public static TypeAdapterFactory create() { + return new AutoValueGson_AutoValueTypeAdapterFactory(); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java index a058426..ddc6258 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java
@@ -40,6 +40,7 @@ import com.googlesource.gerrit.plugins.replication.ObservableQueue; import com.googlesource.gerrit.plugins.replication.ReplicationConfigModule; import com.googlesource.gerrit.plugins.replication.StartReplicationCapability; +import com.googlesource.gerrit.plugins.replication.pull.api.DeleteRefJob; import com.googlesource.gerrit.plugins.replication.pull.api.FetchApiCapability; import com.googlesource.gerrit.plugins.replication.pull.api.FetchJob; import com.googlesource.gerrit.plugins.replication.pull.auth.PullReplicationGroupModule; @@ -82,6 +83,7 @@ bind(RevisionReader.class).in(Scopes.SINGLETON); bind(ApplyObject.class); install(new FactoryModuleBuilder().build(FetchJob.Factory.class)); + install(new FactoryModuleBuilder().build(DeleteRefJob.Factory.class)); install(new ApplyObjectCacheModule()); install(
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java index baeb330..d9b47d3 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
@@ -40,6 +40,7 @@ import com.google.inject.Singleton; import com.googlesource.gerrit.plugins.replication.ObservableQueue; import com.googlesource.gerrit.plugins.replication.pull.FetchResultProcessing.GitUpdateProcessing; +import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.RefInput; import com.googlesource.gerrit.plugins.replication.pull.api.data.BatchApplyObjectData; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException; @@ -611,13 +612,13 @@ boolean resultIsSuccessful = true; - List<String> filteredRefs = + List<RefInput> filteredRefs = refs.stream() - .map(ReferenceUpdatedEvent::refName) - .filter(refName -> source.wouldFetchProject(project) && source.wouldFetchRef(refName)) + .map(ref -> RefInput.create(ref.refName(), ref.isDelete())) + .filter(ref -> source.wouldFetchProject(project) && source.wouldFetchRef(ref.refName())) .collect(Collectors.toList()); - String refsStr = String.join(",", filteredRefs); + String refsStr = filteredRefs.stream().map(RefInput::refName).collect(Collectors.joining(",")); FetchApiClient fetchClient = fetchClientFactory.create(source); for (String apiUrl : source.getApis()) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java index 3a7d0ff..df13944 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java
@@ -38,6 +38,7 @@ import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState; import java.io.IOException; import java.util.Optional; +import java.util.Set; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefUpdate; @@ -67,6 +68,18 @@ this.gitManager = gitManagerProvider.get(); } + public void deleteRefsSync( + Project.NameKey name, Set<String> deletedRefNames, String sourceLabel) { + deletedRefNames.forEach( + r -> { + try { + deleteRef(name, r, sourceLabel); + } catch (RestApiException | IOException e) { + repLog.error("Could not delete ref {}:{} from source {}", name.get(), r, sourceLabel); + } + }); + } + public void deleteRef(Project.NameKey name, String refName, String sourceLabel) throws IOException, RestApiException { Source source =
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefJob.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefJob.java new file mode 100644 index 0000000..acc40cb --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefJob.java
@@ -0,0 +1,45 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api; + +import com.google.gerrit.entities.Project; +import com.google.inject.Inject; +import com.google.inject.assistedinject.Assisted; +import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.BatchInput; + +public class DeleteRefJob implements Runnable { + public interface Factory { + DeleteRefJob create(Project.NameKey project, BatchInput input); + } + + private final DeleteRefCommand command; + private final Project.NameKey project; + private final BatchInput batchInput; + + @Inject + public DeleteRefJob( + DeleteRefCommand command, + @Assisted Project.NameKey project, + @Assisted BatchInput batchInput) { + this.command = command; + this.project = project; + this.batchInput = batchInput; + } + + @Override + public void run() { + command.deleteRefsSync(project, batchInput.getDeletedRefNames(), batchInput.label); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java index 38d0cdd..9cb9285 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java
@@ -16,7 +16,9 @@ import static com.google.common.base.Preconditions.checkState; +import com.google.auto.value.AutoValue; import com.google.common.base.Strings; +import com.google.gerrit.common.Nullable; import com.google.gerrit.entities.Project; import com.google.gerrit.extensions.registration.DynamicItem; import com.google.gerrit.extensions.restapi.AuthException; @@ -30,6 +32,9 @@ import com.google.gerrit.server.git.WorkQueue.Task; import com.google.gerrit.server.ioutil.HexFormat; import com.google.gerrit.server.project.ProjectResource; +import com.google.gson.Gson; +import com.google.gson.TypeAdapter; +import com.google.gson.annotations.SerializedName; import com.google.inject.Inject; import com.google.inject.Singleton; import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.Input; @@ -39,6 +44,7 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; import org.eclipse.jgit.errors.TransportException; @@ -46,43 +52,92 @@ @Singleton public class FetchAction implements RestModifyView<ProjectResource, Input> { private final FetchCommand command; + private final DeleteRefCommand deleteRefCommand; private final WorkQueue workQueue; private final DynamicItem<UrlFormatter> urlFormatter; private final FetchPreconditions preConditions; private final Factory fetchJobFactory; + private final DeleteRefJob.Factory deleteJobFactory; @Inject public FetchAction( FetchCommand command, + DeleteRefCommand deleteRefCommand, WorkQueue workQueue, DynamicItem<UrlFormatter> urlFormatter, FetchPreconditions preConditions, - FetchJob.Factory fetchJobFactory) { + FetchJob.Factory fetchJobFactory, + DeleteRefJob.Factory deleteJobFactory) { this.command = command; + this.deleteRefCommand = deleteRefCommand; this.workQueue = workQueue; this.urlFormatter = urlFormatter; this.preConditions = preConditions; this.fetchJobFactory = fetchJobFactory; + this.deleteJobFactory = deleteJobFactory; } public static class Input { public String label; public String refName; public boolean async; + public boolean isDelete; + } + + @AutoValue + public abstract static class RefInput { + public static final Predicate<RefInput> IS_DELETE = RefInput::isDelete; + + @Nullable + @SerializedName("ref_name") + public abstract String refName(); + + @SerializedName("is_delete") + public abstract boolean isDelete(); + + public static RefInput create(@Nullable String refName, boolean isDelete) { + return new AutoValue_FetchAction_RefInput(refName, isDelete); + } + + public static RefInput create(@Nullable String refName) { + return new AutoValue_FetchAction_RefInput(refName, false); + } + + public static TypeAdapter<RefInput> typeAdapter(Gson gson) { + return new AutoValue_FetchAction_RefInput.GsonTypeAdapter(gson); + } } public static class BatchInput { public String label; - public Set<String> refsNames; + public Set<RefInput> refInputs; public boolean async; public static BatchInput fromInput(Input... input) { BatchInput batchInput = new BatchInput(); batchInput.async = input[0].async; batchInput.label = input[0].label; - batchInput.refsNames = Stream.of(input).map(i -> i.refName).collect(Collectors.toSet()); + batchInput.refInputs = + Stream.of(input) + .map(i -> RefInput.create(i.refName, i.isDelete)) + .collect(Collectors.toSet()); return batchInput; } + + private Set<String> getFilteredRefNames(Predicate<RefInput> filterFunc) { + return refInputs.stream() + .filter(filterFunc) + .map(RefInput::refName) + .collect(Collectors.toSet()); + } + + public Set<String> getNonDeletedRefNames() { + return getFilteredRefNames(RefInput.IS_DELETE.negate()); + } + + public Set<String> getDeletedRefNames() { + return getFilteredRefNames(RefInput.IS_DELETE); + } } @Override @@ -101,12 +156,12 @@ throw new BadRequestException("Source label cannot be null or empty"); } - if (batchInput.refsNames.isEmpty()) { + if (batchInput.refInputs.isEmpty()) { throw new BadRequestException("Ref-update refname cannot be null or empty"); } - for (String refName : batchInput.refsNames) { - if (Strings.isNullOrEmpty(refName)) { + for (RefInput input : batchInput.refInputs) { + if (Strings.isNullOrEmpty(input.refName())) { throw new BadRequestException("Ref-update refname cannot be null or empty"); } } @@ -129,7 +184,16 @@ private Response<?> applySync(Project.NameKey project, BatchInput input) throws InterruptedException, ExecutionException, RemoteConfigurationMissingException, TimeoutException, TransportException { - command.fetchSync(project, input.label, input.refsNames); + command.fetchSync(project, input.label, input.getNonDeletedRefNames()); + + /* git fetches and deletes cannot be handled atomically within the same transaction. + Here we choose to handle fetches first and then deletes: + - If the fetch fails delete is not even attempted. + - If the delete fails after the fetch then the client is left with some extra refs. + */ + if (!input.getDeletedRefNames().isEmpty()) { + deleteRefCommand.deleteRefsSync(project, input.getDeletedRefNames(), input.label); + } return Response.created(input); } @@ -146,6 +210,10 @@ urlFormatter .get() .getRestUrl("a/config/server/tasks/" + HexFormat.fromInt(task.getTaskId())); + + if (!batchInput.getDeletedRefNames().isEmpty()) { + workQueue.getDefaultQueue().submit(deleteJobFactory.create(project, batchInput)); + } // We're in a HTTP handler, so must be present. checkState(url.isPresent()); return Response.accepted(url.get());
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchJob.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchJob.java index 0975045..d30ba74 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchJob.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchJob.java
@@ -52,7 +52,7 @@ @Override public void run() { try { - command.fetchAsync(project, batchInput.label, batchInput.refsNames, metrics); + command.fetchAsync(project, batchInput.label, batchInput.getNonDeletedRefNames(), metrics); } catch (InterruptedException | ExecutionException | RemoteConfigurationMissingException @@ -60,7 +60,7 @@ | TransportException e) { log.atSevere().withCause(e).log( "Exception during the async fetch call for project %s, label %s and ref(s) name(s) %s", - project.get(), batchInput.label, batchInput.refsNames); + project.get(), batchInput.label, batchInput.getNonDeletedRefNames()); } } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/HttpPayloadGsonProvider.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/HttpPayloadGsonProvider.java new file mode 100644 index 0000000..cf1bd86 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/HttpPayloadGsonProvider.java
@@ -0,0 +1,29 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api; + +import com.google.gerrit.json.OutputFormat; +import com.google.gson.Gson; +import com.googlesource.gerrit.plugins.replication.pull.AutoValueTypeAdapterFactory; + +public class HttpPayloadGsonProvider { + + public static Gson get() { + return OutputFormat.JSON + .newGsonBuilder() + .registerTypeAdapterFactory(AutoValueTypeAdapterFactory.create()) + .create(); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java index 368e61a..9618168 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java
@@ -38,7 +38,6 @@ import com.google.gerrit.extensions.restapi.UnprocessableEntityException; import com.google.gerrit.httpd.AllRequestFilter; import com.google.gerrit.httpd.restapi.RestApiServlet; -import com.google.gerrit.json.OutputFormat; import com.google.gerrit.server.AnonymousUser; import com.google.gerrit.server.CurrentUser; import com.google.gerrit.server.permissions.PermissionBackendException; @@ -117,7 +116,7 @@ this.projectDeletionAction = projectDeletionAction; this.projectCache = projectCache; this.pluginName = pluginName; - this.gson = OutputFormat.JSON.newGsonBuilder().create(); + this.gson = HttpPayloadGsonProvider.get(); this.currentUserProvider = currentUserProvider; }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/util/PayloadSerDes.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/util/PayloadSerDes.java index 414af37..d188247 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/util/PayloadSerDes.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/util/PayloadSerDes.java
@@ -22,11 +22,11 @@ import com.google.gerrit.extensions.restapi.BadRequestException; import com.google.gerrit.extensions.restapi.Response; import com.google.gerrit.httpd.restapi.RestApiServlet; -import com.google.gerrit.json.OutputFormat; import com.google.gson.Gson; import com.google.gson.stream.JsonReader; import com.google.inject.TypeLiteral; import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction; +import com.googlesource.gerrit.plugins.replication.pull.api.HttpPayloadGsonProvider; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionsInput; import java.io.BufferedReader; @@ -38,7 +38,7 @@ public class PayloadSerDes { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private static final Gson gson = OutputFormat.JSON.newGsonBuilder().create(); + private static final Gson gson = HttpPayloadGsonProvider.get(); public static RevisionInput parseRevisionInput(HttpServletRequest httpRequest) throws BadRequestException, IOException {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java index 28c74c9..b908435 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java
@@ -19,6 +19,7 @@ import com.google.gerrit.entities.Project; import com.google.gerrit.entities.Project.NameKey; import com.googlesource.gerrit.plugins.replication.pull.Source; +import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.RefInput; import com.googlesource.gerrit.plugins.replication.pull.api.data.BatchApplyObjectData; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; import java.io.IOException; @@ -46,11 +47,11 @@ } HttpResult callBatchFetch( - Project.NameKey project, List<String> refsInBatch, URIish targetUri, long startTimeNanos) + Project.NameKey project, List<RefInput> refsInBatch, URIish targetUri, long startTimeNanos) throws IOException; default HttpResult callBatchFetch( - Project.NameKey project, List<String> refsInBatch, URIish targetUri) throws IOException { + Project.NameKey project, List<RefInput> refsInBatch, URIish targetUri) throws IOException { return callBatchFetch( project, refsInBatch, targetUri, MILLISECONDS.toNanos(System.currentTimeMillis())); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java index 614774d..cb03606 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
@@ -36,6 +36,7 @@ import com.googlesource.gerrit.plugins.replication.ReplicationConfig; import com.googlesource.gerrit.plugins.replication.pull.BearerTokenProvider; import com.googlesource.gerrit.plugins.replication.pull.Source; +import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.RefInput; import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics; import com.googlesource.gerrit.plugins.replication.pull.api.data.BatchApplyObjectData; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; @@ -151,19 +152,22 @@ return executeRequest(post, bearerTokenProvider.get(), targetUri); } - private Boolean containsSyncFetchRef(List<String> refsInBatch) { - return refsInBatch.stream().anyMatch(syncRefsFilter::match); + private Boolean containsSyncFetchRef(List<RefInput> refsInBatch) { + return refsInBatch.stream().anyMatch(r -> syncRefsFilter.match(r.refName())); } @Override public HttpResult callBatchFetch( - NameKey project, List<String> refsInBatch, URIish targetUri, long startTimeNanos) + NameKey project, List<RefInput> refsInBatch, URIish targetUri, long startTimeNanos) throws IOException { boolean callAsync = !containsSyncFetchRef(refsInBatch); - String refsNamesBody = refsInBatch.stream().collect(Collectors.joining("\",\"", "\"", "\"")); + String refsNamesBody = + refsInBatch.stream() + .map(r -> "{\"ref_name\":\"" + r.refName() + "\", \"is_delete\":" + r.isDelete() + "}") + .collect(Collectors.joining(",")); String msgBody = String.format( - "{\"label\":\"%s\", \"refs_names\": [ %s ], \"async\":%s}", + "{\"label\":\"%s\", \"ref_inputs\": [ %s ], \"async\":%s}", instanceId, refsNamesBody, callAsync); String url = formatUrl(targetUri.toString(), project, "batch-fetch");
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index abc96ca..3b76be9 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md
@@ -609,6 +609,10 @@ > *NOTE*: if any ref from a single batch matches `replication.syncRefs` > filter, all refs in that batch are going to be fetched synchronously as > a single git fetch operation. + +> *NOTE*: Should ref deletions over apply-object/HTTP, they will +> be attempted over fetch/HTTP endpoint only when `enableBatchedRefs` is +> enabled. > > By default, true.
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java index 84fee55..f494c2f 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
@@ -51,6 +51,7 @@ import com.googlesource.gerrit.plugins.replication.MergedConfigResource; import com.googlesource.gerrit.plugins.replication.ReplicationConfig; import com.googlesource.gerrit.plugins.replication.ReplicationConfigImpl; +import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.RefInput; import com.googlesource.gerrit.plugins.replication.pull.api.data.BatchApplyObjectData; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; import com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient; @@ -65,6 +66,7 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.storage.file.FileBasedConfig; @@ -503,7 +505,7 @@ verify(fetchRestApiClient) .callBatchFetch( PROJECT, - List.of("refs/changes/01/1/1", "refs/changes/02/1/1"), + Stream.of("refs/changes/01/1/1", "refs/changes/02/1/1").map(RefInput::create).toList(), new URIish("http://localhost:18080")); }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchActionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchActionTest.java index 9dc736f..23b0fe1 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchActionTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchActionTest.java
@@ -22,11 +22,14 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.common.annotations.VisibleForTesting; import com.google.gerrit.extensions.restapi.MergeConflictException; import com.google.gerrit.extensions.restapi.Response; import com.google.gerrit.extensions.restapi.RestApiException; import com.google.gerrit.server.project.ProjectResource; -import java.util.Set; +import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.RefInput; +import java.util.Arrays; +import java.util.stream.Collectors; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -103,10 +106,12 @@ return input; } + @VisibleForTesting private FetchAction.BatchInput createBatchInput(String... refNames) { FetchAction.BatchInput batchInput = new FetchAction.BatchInput(); batchInput.label = label; - batchInput.refsNames = Set.of(refNames); + batchInput.refInputs = + Arrays.stream(refNames).map(RefInput::create).collect(Collectors.toSet()); return batchInput; } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionTest.java index 9653209..a4642ba 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionTest.java
@@ -19,7 +19,9 @@ import static org.apache.http.HttpStatus.SC_CREATED; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.google.gerrit.extensions.registration.DynamicItem; @@ -32,6 +34,8 @@ import com.google.gerrit.server.git.WorkQueue; import com.google.gerrit.server.git.WorkQueue.Task; import com.google.gerrit.server.project.ProjectResource; +import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.BatchInput; +import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.RefInput; import com.googlesource.gerrit.plugins.replication.pull.api.exception.RemoteConfigurationMissingException; import java.util.Optional; import java.util.Set; @@ -57,8 +61,11 @@ int taskId = 1234; @Mock FetchCommand fetchCommand; + @Mock DeleteRefCommand deleteRefCommand; @Mock FetchJob fetchJob; + @Mock DeleteRefJob deleteRefJob; @Mock FetchJob.Factory fetchJobFactory; + @Mock DeleteRefJob.Factory deleteRefJobFactory; @Mock ProjectResource projectResource; @Mock WorkQueue workQueue; @Mock ScheduledExecutorService exceutorService; @@ -70,6 +77,7 @@ @Before public void setup() throws Exception { when(fetchJobFactory.create(any(), any(), any())).thenReturn(fetchJob); + when(deleteRefJobFactory.create(any(), any())).thenReturn(deleteRefJob); when(workQueue.getDefaultQueue()).thenReturn(exceutorService); when(urlFormatter.getRestUrl(anyString())).thenReturn(Optional.of(location)); when(exceutorService.submit(any(Runnable.class))) @@ -86,7 +94,13 @@ fetchAction = new FetchAction( - fetchCommand, workQueue, urlFormatterDynamicItem, preConditions, fetchJobFactory); + fetchCommand, + deleteRefCommand, + workQueue, + urlFormatterDynamicItem, + preConditions, + fetchJobFactory, + deleteRefJobFactory); } @Test @@ -104,13 +118,38 @@ public void shouldReturnCreatedResponseCodeForBatchRefFetchAction() throws Exception { FetchAction.BatchInput batchInputParams = new FetchAction.BatchInput(); batchInputParams.label = label; - batchInputParams.refsNames = Set.of(refName, altRefName); + batchInputParams.refInputs = Set.of(RefInput.create(refName), RefInput.create(altRefName)); Response<?> response = fetchAction.apply(projectResource, batchInputParams); assertThat(response.statusCode()).isEqualTo(SC_CREATED); } + @Test + public void shouldDeleteRefSync() throws Exception { + FetchAction.BatchInput batchInputParams = new FetchAction.BatchInput(); + batchInputParams.label = label; + batchInputParams.refInputs = Set.of(RefInput.create(refName, true)); + + Response<?> response = fetchAction.apply(projectResource, batchInputParams); + verify(deleteRefCommand).deleteRefsSync(any(), eq(Set.of(refName)), eq(label)); + + assertThat(response.statusCode()).isEqualTo(SC_CREATED); + } + + @Test + public void shouldDeleteRefAsync() throws Exception { + FetchAction.BatchInput batchInputParams = new FetchAction.BatchInput(); + batchInputParams.label = label; + batchInputParams.async = true; + batchInputParams.refInputs = Set.of(RefInput.create(refName, true)); + + Response<?> response = fetchAction.apply(projectResource, batchInputParams); + verify(deleteRefJobFactory).create(any(), eq(batchInputParams)); + + assertThat(response.statusCode()).isEqualTo(SC_ACCEPTED); + } + @SuppressWarnings("cast") @Test public void shouldReturnSourceUrlAndrefNameAsAResponseBody() throws Exception { @@ -120,11 +159,11 @@ Response<?> response = fetchAction.apply(projectResource, inputParams); - FetchAction.BatchInput responseBatchInput = (FetchAction.BatchInput) response.value(); + BatchInput responseBatchInput = (BatchInput) response.value(); assertThat(responseBatchInput.label).isEqualTo(inputParams.label); assertThat(responseBatchInput.async).isEqualTo(inputParams.async); - assertThat(responseBatchInput.refsNames).containsExactly(inputParams.refName); + assertThat(responseBatchInput.refInputs).containsExactly(RefInput.create(inputParams.refName)); } @Test(expected = BadRequestException.class)
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilterTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilterTest.java index b193cd7..bc9e870 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilterTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilterTest.java
@@ -145,7 +145,7 @@ byte[] payloadBatchFetch = ("{" + "\"label\":\"Replication\", " - + "\"refs_names\": [ \"refs/heads/master\" , \"refs/heads/test\" ], " + + "\"ref_inputs\": [ {\"ref_name\":\"refs/heads/master\", \"is_delete\":false}, {\"ref_name\":\"refs/heads/test\", \"is_delete\":false} ], " + "\"async\":false" + "}") .getBytes(StandardCharsets.UTF_8);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java index 3aa5b5a..1c7b553 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java
@@ -31,6 +31,7 @@ import com.googlesource.gerrit.plugins.replication.ReplicationConfig; import com.googlesource.gerrit.plugins.replication.pull.BearerTokenProvider; import com.googlesource.gerrit.plugins.replication.pull.Source; +import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.RefInput; import com.googlesource.gerrit.plugins.replication.pull.api.data.BatchApplyObjectData; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData; @@ -42,6 +43,7 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.http.Header; import org.apache.http.HttpHeaders; import org.apache.http.client.methods.HttpDelete; @@ -185,7 +187,7 @@ objectUnderTest.callBatchFetch( Project.nameKey("test_repo"), - List.of(refName, RefNames.REFS_HEADS + "test"), + List.of(RefInput.create(refName), RefInput.create(RefNames.REFS_HEADS + "test")), new URIish(api)); verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any()); @@ -253,19 +255,21 @@ source); String testRef = RefNames.REFS_HEADS + "test"; - List<String> refs = List.of(refName, testRef); + List<RefInput> refs = refInputs(refName, testRef); objectUnderTest.callBatchFetch(Project.nameKey("test_repo"), refs, new URIish(api)); verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any()); HttpPost httpPost = httpPostCaptor.getValue(); String expectedPayload = - "{\"label\":\"Replication\", \"refs_names\": [ " - + '"' + "{\"label\":\"Replication\", \"ref_inputs\": [" + + " {\"ref_name\":\"" + refName - + "\",\"" + + "\", \"is_delete\":false}" + + ",{\"ref_name\":\"" + testRef - + "\" ]" + + "\", \"is_delete\":false}" + + " ]" + ", \"async\":true}"; assertThat(readPayload(httpPost)).isEqualTo(expectedPayload); } @@ -305,7 +309,10 @@ public void shouldCallSyncBatchFetchOnlyForMetaRef() throws Exception { String metaRefName = "refs/changes/01/101/meta"; String expectedMetaRefPayload = - "{\"label\":\"Replication\", \"refs_names\": [ \"" + metaRefName + "\" ], \"async\":false}"; + "{\"label\":\"Replication\", \"ref_inputs\": [ " + + "{\"ref_name\":\"" + + metaRefName + + "\", \"is_delete\":false} ], \"async\":false}"; when(config.getStringList("replication", null, "syncRefs")) .thenReturn(new String[] {"^refs\\/changes\\/.*\\/meta"}); @@ -322,7 +329,7 @@ source); objectUnderTest.callBatchFetch( - Project.nameKey("test_repo"), List.of(metaRefName), new URIish(api)); + Project.nameKey("test_repo"), List.of(RefInput.create(metaRefName)), new URIish(api)); verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any()); HttpPost httpPost = httpPostCaptor.getValue(); assertThat(readPayload(httpPost)).isEqualTo(expectedMetaRefPayload); @@ -343,19 +350,23 @@ public void shouldCallBatchFetchEndpointWithPayload() throws Exception { String testRef = RefNames.REFS_HEADS + "test"; - List<String> refs = List.of(refName, testRef); + List<RefInput> refs = refInputs(refName, testRef); objectUnderTest.callBatchFetch(Project.nameKey("test_repo"), refs, new URIish(api)); verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any()); HttpPost httpPost = httpPostCaptor.getValue(); String expectedPayload = - "{\"label\":\"Replication\", \"refs_names\": [ " - + '"' + "{\"label\":\"Replication\", \"ref_inputs\": [ " + + "{\"ref_name\":\"" + refName - + "\",\"" - + refs.get(1) - + "\" ], \"async\":false}"; + + "\", \"is_delete\":false}" + + ",{\"ref_name\":\"" + + refs.get(1).refName() + + "\", \"is_delete\":" + + refs.get(1).isDelete() + + "}" + + " ], \"async\":false}"; assertThat(readPayload(httpPost)).isEqualTo(expectedPayload); } @@ -366,7 +377,7 @@ .thenReturn(new String[] {"^refs\\/heads\\/test"}); syncRefsFilter = new SyncRefsFilter(replicationConfig); String testRef = RefNames.REFS_HEADS + "test"; - List<String> refs = List.of(refName, testRef); + List<RefInput> refs = refInputs(refName, testRef); objectUnderTest = new FetchRestApiClient( credentials, @@ -383,8 +394,16 @@ HttpPost httpPosts = httpPostCaptor.getValue(); String expectedSyncPayload = - "{\"label\":\"Replication\", \"refs_names\": [ " - + refs.stream().map(r -> '"' + r + '"').collect(Collectors.joining(",")) + "{\"label\":\"Replication\", \"ref_inputs\": [ " + + refs.stream() + .map( + r -> + "{\"ref_name\":\"" + + r.refName() + + "\", \"is_delete\":" + + r.isDelete() + + "}") + .collect(Collectors.joining(",")) + " ], \"async\":false}"; assertThat(readPayload(httpPosts)).isEqualTo(expectedSyncPayload); @@ -405,7 +424,8 @@ @Test public void shouldSetContentTypeHeaderInBatchFetch() throws Exception { - objectUnderTest.callBatchFetch(Project.nameKey("test_repo"), List.of(refName), new URIish(api)); + objectUnderTest.callBatchFetch( + Project.nameKey("test_repo"), refInputs(refName), new URIish(api)); verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any()); @@ -733,4 +753,8 @@ return createSampleRevisionData( commitObjectId, commitObject, treeObjectId, treeObject, blobObjectId, blobObject); } + + private List<RefInput> refInputs(String... refs) { + return Stream.of(refs).map(RefInput::create).collect(Collectors.toList()); + } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java index 0f4bd33..c81820e 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java
@@ -186,7 +186,7 @@ FetchAction.BatchInput batchInput = batchInputCaptor.getValue(); assertThat(batchInput.label).isEqualTo(REMOTE_INSTANCE_ID); - assertThat(batchInput.refsNames).contains(TEST_REF_NAME); + assertThat(batchInput.refInputs).contains(FetchAction.RefInput.create(TEST_REF_NAME)); verify(executor).submit(any(FetchJob.class)); } @@ -259,7 +259,7 @@ FetchAction.BatchInput input = batchInputCaptor.getValue(); assertThat(input.label).isEqualTo(REMOTE_INSTANCE_ID); - assertThat(input.refsNames).contains(FetchOne.ALL_REFS); + assertThat(input.refInputs).contains(FetchAction.RefInput.create(FetchOne.ALL_REFS)); verify(executor).submit(any(FetchJob.class)); }