Use native BatchInput for the sync batch-fetch REST-API Use the BatchInput data type for the batch-fetch REST-API for reducing the payload size, keeping the correct granularity of the batch ref-update data and disallowing at protocol level the possibility to split a batch into sync and async executions. Also avoid scheduing a batch as a series of individual fetch operations of the individual refs, as a follow-up of the review of Ifd2a28a6. Change-Id: I1b8fcb0b803cd441f4fac21940c205767d0eb3a9
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchAction.java index b7b1ab1..ac3df43 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchAction.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchAction.java
@@ -21,10 +21,9 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.BatchInput; -import java.util.List; @Singleton -public class BatchFetchAction implements RestModifyView<ProjectResource, List<FetchAction.Input>> { +public class BatchFetchAction implements RestModifyView<ProjectResource, BatchInput> { private final FetchAction fetchAction; @Inject @@ -33,10 +32,8 @@ } @Override - public Response<?> apply(ProjectResource resource, List<FetchAction.Input> inputs) + public Response<?> apply(ProjectResource resource, BatchInput batchInput) throws RestApiException { - return Response.ok( - fetchAction.apply( - resource, BatchInput.fromInput(inputs.toArray(new FetchAction.Input[0])))); + return Response.ok(fetchAction.apply(resource, batchInput)); } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java index d57ae0c..38d0cdd 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java
@@ -76,7 +76,7 @@ public Set<String> refsNames; public boolean async; - static BatchInput fromInput(Input... input) { + public static BatchInput fromInput(Input... input) { BatchInput batchInput = new BatchInput(); batchInput.async = input[0].async; batchInput.label = input[0].label; @@ -135,23 +135,14 @@ @SuppressWarnings("unchecked") private Response.Accepted applyAsync(Project.NameKey project, BatchInput batchInput) { - WorkQueue.Task<Void> task = null; - Optional<String> url; - - for (String refName : batchInput.refsNames) { - Input input = new Input(); - input.label = batchInput.label; - input.async = batchInput.async; - input.refName = refName; - task = - (Task<Void>) - workQueue - .getDefaultQueue() - .submit( - fetchJobFactory.create( - project, input, PullReplicationApiRequestMetrics.get())); - } - url = + WorkQueue.Task<Void> task = + (Task<Void>) + workQueue + .getDefaultQueue() + .submit( + fetchJobFactory.create( + project, batchInput, PullReplicationApiRequestMetrics.get())); + Optional<String> url = urlFormatter .get() .getRestUrl("a/config/server/tasks/" + HexFormat.fromInt(task.getTaskId()));
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java index 8b86965..47d5391 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java
@@ -64,11 +64,11 @@ public void fetchAsync( Project.NameKey name, String label, - String refName, + Set<String> refsNames, PullReplicationApiRequestMetrics apiRequestMetrics) throws InterruptedException, ExecutionException, RemoteConfigurationMissingException, TimeoutException, TransportException { - fetch(name, label, Set.of(refName), ASYNC, Optional.of(apiRequestMetrics)); + fetch(name, label, refsNames, ASYNC, Optional.of(apiRequestMetrics)); } public void fetchSync(Project.NameKey name, String label, Set<String> refsNames)
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchJob.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchJob.java index a613c0e..0975045 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchJob.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchJob.java
@@ -18,6 +18,7 @@ import com.google.gerrit.entities.Project; import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; +import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.BatchInput; import com.googlesource.gerrit.plugins.replication.pull.api.exception.RemoteConfigurationMissingException; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; @@ -28,38 +29,38 @@ public interface Factory { FetchJob create( - Project.NameKey project, FetchAction.Input input, PullReplicationApiRequestMetrics metrics); + Project.NameKey project, BatchInput input, PullReplicationApiRequestMetrics metrics); } private FetchCommand command; private Project.NameKey project; - private FetchAction.Input input; + private BatchInput batchInput; private final PullReplicationApiRequestMetrics metrics; @Inject public FetchJob( FetchCommand command, @Assisted Project.NameKey project, - @Assisted FetchAction.Input input, + @Assisted BatchInput batchInput, @Assisted PullReplicationApiRequestMetrics metrics) { this.command = command; this.project = project; - this.input = input; + this.batchInput = batchInput; this.metrics = metrics; } @Override public void run() { try { - command.fetchAsync(project, input.label, input.refName, metrics); + command.fetchAsync(project, batchInput.label, batchInput.refsNames, metrics); } catch (InterruptedException | ExecutionException | RemoteConfigurationMissingException | TimeoutException | TransportException e) { log.atSevere().withCause(e).log( - "Exception during the async fetch call for project %s, label %s and ref name %s", - project.get(), input.label, input.refName); + "Exception during the async fetch call for project %s, label %s and ref(s) name(s) %s", + project.get(), batchInput.label, batchInput.refsNames); } } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java index 86fba22..368e61a 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java
@@ -52,6 +52,7 @@ import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.TypeLiteral; +import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.BatchInput; import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.Input; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionsInput; @@ -274,11 +275,11 @@ @SuppressWarnings("unchecked") private Response<Map<String, Object>> doBatchFetch(HttpServletRequest httpRequest) throws IOException, RestApiException { - TypeLiteral<List<Input>> collectionType = new TypeLiteral<>() {}; - List<Input> inputs = readJson(httpRequest, collectionType.getType()); + BatchInput batchInput = readJson(httpRequest, BatchInput.class); IdString id = getProjectName(httpRequest).get(); - return (Response<Map<String, Object>>) batchFetchAction.apply(parseProjectResource(id), inputs); + return (Response<Map<String, Object>>) + batchFetchAction.apply(parseProjectResource(id), batchInput); } private <T> void writeResponse(HttpServletResponse httpResponse, Response<T> response)
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java index e682b71..614774d 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
@@ -160,17 +160,14 @@ NameKey project, List<String> refsInBatch, URIish targetUri, long startTimeNanos) throws IOException { boolean callAsync = !containsSyncFetchRef(refsInBatch); + String refsNamesBody = refsInBatch.stream().collect(Collectors.joining("\",\"", "\"", "\"")); String msgBody = - refsInBatch.stream() - .map( - refName -> - String.format( - "{\"label\":\"%s\", \"ref_name\": \"%s\", \"async\":%s}", - instanceId, refName, callAsync)) - .collect(Collectors.joining(",")); + String.format( + "{\"label\":\"%s\", \"refs_names\": [ %s ], \"async\":%s}", + instanceId, refsNamesBody, callAsync); String url = formatUrl(targetUri.toString(), project, "batch-fetch"); - HttpPost post = createPostRequest(url, "[" + msgBody + "]", startTimeNanos); + HttpPost post = createPostRequest(url, msgBody, startTimeNanos); return executeRequest(post, bearerTokenProvider.get(), targetUri); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java index 2f992cd..74bf51d 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
@@ -15,6 +15,7 @@ package com.googlesource.gerrit.plugins.replication.pull.event; import static com.googlesource.gerrit.plugins.replication.pull.ApplyObjectCacheModule.APPLY_OBJECTS_CACHE; +import static com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.BatchInput.fromInput; import static java.util.Objects.requireNonNull; import com.google.common.base.Strings; @@ -242,7 +243,9 @@ FetchAction.Input input = new FetchAction.Input(); input.refName = refName; input.label = sourceInstanceId; - workQueue.getDefaultQueue().submit(fetchJobFactory.create(projectNameKey, input, metrics)); + workQueue + .getDefaultQueue() + .submit(fetchJobFactory.create(projectNameKey, fromInput(input), metrics)); } private String getProjectRepositoryName(ProjectCreatedEvent projectCreatedEvent) {
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchActionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchActionTest.java index e2e4cd3..9dc736f 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchActionTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchActionTest.java
@@ -19,7 +19,6 @@ import static com.google.common.truth.Truth.assertThat; import static org.apache.http.HttpStatus.SC_OK; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.eq; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -27,7 +26,7 @@ import com.google.gerrit.extensions.restapi.Response; import com.google.gerrit.extensions.restapi.RestApiException; import com.google.gerrit.server.project.ProjectResource; -import java.util.List; +import java.util.Set; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -52,54 +51,49 @@ } @Test - public void shouldDelegateToFetchActionWithBatchInputForListOfFetchInput() - throws RestApiException { - FetchAction.Input first = createInput(master); - FetchAction.Input second = createInput(test); + public void shouldDelegateToFetchActionForEveryFetchInput() throws RestApiException { + FetchAction.BatchInput batchInput = createBatchInput(master, test); - batchFetchAction.apply(projectResource, List.of(first, second)); + batchFetchAction.apply(projectResource, batchInput); - verify(fetchAction).apply(eq(projectResource), any(FetchAction.BatchInput.class)); + verify(fetchAction).apply(projectResource, batchInput); } @Test public void shouldReturnOkResponseCodeWhenAllInputsAreProcessedSuccessfully() throws RestApiException { - FetchAction.Input first = createInput(master); - FetchAction.Input second = createInput(test); + FetchAction.BatchInput batchInput = createBatchInput(master, test); when(fetchAction.apply(any(ProjectResource.class), any(FetchAction.BatchInput.class))) .thenAnswer((Answer<Response<?>>) invocation -> Response.accepted("some-url")); - Response<?> response = batchFetchAction.apply(projectResource, List.of(first, second)); + Response<?> response = batchFetchAction.apply(projectResource, batchInput); assertThat(response.statusCode()).isEqualTo(SC_OK); } @Test - public void shouldReturnAResponsesOnSuccess() throws RestApiException { - FetchAction.Input first = createInput(master); - FetchAction.Input second = createInput(test); + public void shouldReturnAListWithAllResponsesOnSuccess() throws RestApiException { + FetchAction.BatchInput batchInput = createBatchInput(master, test); + String masterUrl = "master-url"; - String testUrl = "test-url"; - Response.Accepted batchResponse = Response.accepted(masterUrl); + Response.Accepted firstResponse = Response.accepted(masterUrl); - when(fetchAction.apply(eq(projectResource), any(FetchAction.BatchInput.class))) - .thenAnswer((Answer<Response<?>>) invocation -> batchResponse); - Response<?> response = batchFetchAction.apply(projectResource, List.of(first, second)); + when(fetchAction.apply(projectResource, batchInput)) + .thenAnswer((Answer<Response<?>>) invocation -> firstResponse); + Response<?> response = batchFetchAction.apply(projectResource, batchInput); - assertThat(response.value()).isEqualTo(batchResponse); + assertThat(response.value()).isEqualTo(firstResponse); } @Test(expected = RestApiException.class) public void shouldThrowRestApiExceptionWhenProcessingFailsForAnInput() throws RestApiException { - FetchAction.Input first = createInput(master); - FetchAction.Input second = createInput(test); + FetchAction.BatchInput batchInput = createBatchInput(master, test); String masterUrl = "master-url"; - when(fetchAction.apply(eq(projectResource), any(FetchAction.BatchInput.class))) + when(fetchAction.apply(projectResource, batchInput)) .thenThrow(new MergeConflictException("BOOM")); - batchFetchAction.apply(projectResource, List.of(first, second)); + batchFetchAction.apply(projectResource, batchInput); } private FetchAction.Input createInput(String refName) { @@ -108,4 +102,11 @@ input.refName = refName; return input; } + + private FetchAction.BatchInput createBatchInput(String... refNames) { + FetchAction.BatchInput batchInput = new FetchAction.BatchInput(); + batchInput.label = label; + batchInput.refsNames = Set.of(refNames); + return batchInput; + } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java index f8d12a9..ebf0076 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java
@@ -86,7 +86,7 @@ @Test public void shouldScheduleRefFetchWithDelay() throws Exception { - objectUnderTest.fetchAsync(projectName, label, REF_NAME_TO_FETCH, apiRequestMetrics); + objectUnderTest.fetchAsync(projectName, label, Set.of(REF_NAME_TO_FETCH), apiRequestMetrics); verify(source, times(1)) .schedule( @@ -101,7 +101,7 @@ @Test public void shouldMarkAllFetchTasksScheduled() throws Exception { - objectUnderTest.fetchAsync(projectName, label, REF_NAME_TO_FETCH, apiRequestMetrics); + objectUnderTest.fetchAsync(projectName, label, Set.of(REF_NAME_TO_FETCH), apiRequestMetrics); verify(source, times(1)) .schedule(projectName, REF_NAME_TO_FETCH, state, Optional.of(apiRequestMetrics));
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilterTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilterTest.java index bba638c..b193cd7 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilterTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilterTest.java
@@ -143,16 +143,11 @@ @Test public void shouldFilterBatchFetchAction() throws Exception { byte[] payloadBatchFetch = - ("[{" + ("{" + "\"label\":\"Replication\", " - + "\"ref_name\": \"refs/heads/master\", " + + "\"refs_names\": [ \"refs/heads/master\" , \"refs/heads/test\" ], " + "\"async\":false" - + "}," - + "{" - + "\"label\":\"Replication\", " - + "\"ref_name\": \"refs/heads/test\", " - + "\"async\":false" - + "}]") + + "}") .getBytes(StandardCharsets.UTF_8); defineBehaviours(payloadBatchFetch, BATCH_FETCH_URI);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java index e9e28ed..3aa5b5a 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java
@@ -41,6 +41,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import org.apache.http.Header; import org.apache.http.HttpHeaders; import org.apache.http.client.methods.HttpDelete; @@ -259,13 +260,13 @@ HttpPost httpPost = httpPostCaptor.getValue(); String expectedPayload = - "[{\"label\":\"Replication\", \"ref_name\": \"" + "{\"label\":\"Replication\", \"refs_names\": [ " + + '"' + refName - + "\", \"async\":true}," - + "{\"label\":\"Replication\", \"ref_name\": \"" - + refs.get(1) - + "\", \"async\":true}" - + "]"; + + "\",\"" + + testRef + + "\" ]" + + ", \"async\":true}"; assertThat(readPayload(httpPost)).isEqualTo(expectedPayload); } @@ -304,7 +305,7 @@ public void shouldCallSyncBatchFetchOnlyForMetaRef() throws Exception { String metaRefName = "refs/changes/01/101/meta"; String expectedMetaRefPayload = - "[{\"label\":\"Replication\", \"ref_name\": \"" + metaRefName + "\", \"async\":false}]"; + "{\"label\":\"Replication\", \"refs_names\": [ \"" + metaRefName + "\" ], \"async\":false}"; when(config.getStringList("replication", null, "syncRefs")) .thenReturn(new String[] {"^refs\\/changes\\/.*\\/meta"}); @@ -349,13 +350,12 @@ HttpPost httpPost = httpPostCaptor.getValue(); String expectedPayload = - "[{\"label\":\"Replication\", \"ref_name\": \"" + "{\"label\":\"Replication\", \"refs_names\": [ " + + '"' + refName - + "\", \"async\":false}," - + "{\"label\":\"Replication\", \"ref_name\": \"" + + "\",\"" + refs.get(1) - + "\", \"async\":false}" - + "]"; + + "\" ], \"async\":false}"; assertThat(readPayload(httpPost)).isEqualTo(expectedPayload); } @@ -383,14 +383,9 @@ HttpPost httpPosts = httpPostCaptor.getValue(); String expectedSyncPayload = - "[" - + "{\"label\":\"Replication\", \"ref_name\": \"" - + refName - + "\", \"async\":false}," - + "{\"label\":\"Replication\", \"ref_name\": \"" - + refs.get(1) - + "\", \"async\":false}" - + "]"; + "{\"label\":\"Replication\", \"refs_names\": [ " + + refs.stream().map(r -> '"' + r + '"').collect(Collectors.joining(",")) + + " ], \"async\":false}"; assertThat(readPayload(httpPosts)).isEqualTo(expectedSyncPayload); }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java index e91ce8d..0f4bd33 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java
@@ -37,7 +37,7 @@ import com.googlesource.gerrit.plugins.replication.pull.Source; import com.googlesource.gerrit.plugins.replication.pull.SourcesCollection; import com.googlesource.gerrit.plugins.replication.pull.api.DeleteRefCommand; -import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.Input; +import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction; import com.googlesource.gerrit.plugins.replication.pull.api.FetchJob; import com.googlesource.gerrit.plugins.replication.pull.api.ProjectInitializationAction; import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics; @@ -70,7 +70,7 @@ @Mock private FetchJob.Factory fetchJobFactory; @Mock private UpdateHeadCommand updateHeadCommand; @Mock private DeleteRefCommand deleteRefCommand; - @Captor ArgumentCaptor<Input> inputCaptor; + @Captor ArgumentCaptor<FetchAction.BatchInput> batchInputCaptor; @Mock private PullReplicationApiRequestMetrics metrics; @Mock private SourcesCollection sources; @Mock private Source source; @@ -181,11 +181,12 @@ objectUnderTest.onEvent(event); - verify(fetchJobFactory).create(eq(Project.nameKey(TEST_PROJECT)), inputCaptor.capture(), any()); + verify(fetchJobFactory) + .create(eq(Project.nameKey(TEST_PROJECT)), batchInputCaptor.capture(), any()); - Input input = inputCaptor.getValue(); - assertThat(input.label).isEqualTo(REMOTE_INSTANCE_ID); - assertThat(input.refName).isEqualTo(TEST_REF_NAME); + FetchAction.BatchInput batchInput = batchInputCaptor.getValue(); + assertThat(batchInput.label).isEqualTo(REMOTE_INSTANCE_ID); + assertThat(batchInput.refsNames).contains(TEST_REF_NAME); verify(executor).submit(any(FetchJob.class)); } @@ -253,11 +254,12 @@ objectUnderTest.onEvent(event); - verify(fetchJobFactory).create(eq(Project.nameKey(TEST_PROJECT)), inputCaptor.capture(), any()); + verify(fetchJobFactory) + .create(eq(Project.nameKey(TEST_PROJECT)), batchInputCaptor.capture(), any()); - Input input = inputCaptor.getValue(); + FetchAction.BatchInput input = batchInputCaptor.getValue(); assertThat(input.label).isEqualTo(REMOTE_INSTANCE_ID); - assertThat(input.refName).isEqualTo(FetchOne.ALL_REFS); + assertThat(input.refsNames).contains(FetchOne.ALL_REFS); verify(executor).submit(any(FetchJob.class)); }