Allow additional refs fallback to apply-objects Apply-objects REST-API that is capable of applying an entire chain of commits on the ref chain so that the Git fetch can be avoided altogether. Without this change only refs/changes/**/meta fallback to the apply-objects. Add `replication.fallbackToApplyObjectsRefs` parameter to allow additional refs fallback to the apply-objects when `MissingParentObject` exception is thrown by the receiver. Change-Id: If2b12dee0c552defd97d3119eadd47f3700dc710
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java index e99580a..fc77503 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
@@ -38,6 +38,7 @@ import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException; import com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient; import com.googlesource.gerrit.plugins.replication.pull.client.HttpResult; +import com.googlesource.gerrit.plugins.replication.pull.filter.ApplyObjectsRefsFilter; import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter; import java.io.IOException; import java.net.URISyntaxException; @@ -92,6 +93,7 @@ private final ApplyObjectMetrics applyObjectMetrics; private final FetchReplicationMetrics fetchMetrics; private final String instanceId; + private ApplyObjectsRefsFilter applyObjectsRefsFilter; @Inject ReplicationQueue( @@ -104,7 +106,8 @@ Provider<RevisionReader> revReaderProvider, ApplyObjectMetrics applyObjectMetrics, FetchReplicationMetrics fetchMetrics, - @GerritInstanceId String instanceId) { + @GerritInstanceId String instanceId, + ApplyObjectsRefsFilter applyObjectsRefsFilter) { workQueue = wq; dispatcher = dis; sources = rd; @@ -116,6 +119,7 @@ this.applyObjectMetrics = applyObjectMetrics; this.fetchMetrics = fetchMetrics; this.instanceId = instanceId; + this.applyObjectsRefsFilter = applyObjectsRefsFilter; } @Override @@ -370,7 +374,8 @@ if (!resultSuccessful) { if (result.isParentObjectMissing()) { - if (RefNames.isNoteDbMetaRef(refName) && revision.size() == 1) { + if ((RefNames.isNoteDbMetaRef(refName) || applyObjectsRefsFilter.match(refName)) + && revision.size() == 1) { List<RevisionData> allRevisions = fetchWholeMetaHistory(project, refName, revision.get(0)); repLog.info(
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/ApplyObjectsRefsFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/ApplyObjectsRefsFilter.java new file mode 100644 index 0000000..ff5899b --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/ApplyObjectsRefsFilter.java
@@ -0,0 +1,37 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.filter; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.googlesource.gerrit.plugins.replication.ReplicationConfig; +import java.util.List; +import org.eclipse.jgit.lib.Config; + +@Singleton +public class ApplyObjectsRefsFilter extends RefsFilter { + @Inject + public ApplyObjectsRefsFilter(ReplicationConfig replicationConfig) { + super(replicationConfig); + } + + @Override + protected List<String> getRefNamePatterns(Config cfg) { + String[] applyObjectsRefs = + cfg.getStringList("replication", null, "fallbackToApplyObjectsRefs"); + return ImmutableList.copyOf(applyObjectsRefs); + } +}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index 13b16ea..b85aeb6 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md
@@ -292,6 +292,33 @@ By default, set to '*' (all refs are replicated synchronously). +replication.fallbackToApplyObjectsRefs +: Specify for which refs will fallback to apply-objects REST-API that is capable of applying +an entire chain of commits on the ref chain so that the Git fetch can be avoided altogether. +It can be provided more than once, and supports three formats: regular expressions, +wildcard matching, and single ref matching. All three formats matches are case-sensitive. + + Values starting with a caret `^` are treated as regular + expressions. For the regular expressions details please follow + official [java documentation](https://docs.oracle.com/javase/tutorial/essential/regex/). + + Please note that regular expressions could also be used + with inverse match. + + Values that are not regular expressions and end in `*` are + treated as wildcard matches. Wildcards match refs whose + name agrees from the beginning until the trailing `*`. So + `foo/b*` would match the refs `foo/b`, `foo/bar`, and + `foo/baz`, but neither `foobar`, nor `bar/foo/baz`. + + Values that are neither regular expressions nor wildcards are + treated as single ref matches. So `foo/bar` matches only + the ref `foo/bar`, but no other refs. + + By default, set to 'refs/changes/**/meta'. + + Note that the refs/changes/**/meta always fallback to apply-objects REST-API + replication.maxApiPayloadSize : Maximum size in bytes of the ref to be sent as a REST Api call payload. For refs larger than threshold git fetch operation
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java index 0a48eaf..d65322f 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
@@ -52,6 +52,7 @@ import com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient; import com.googlesource.gerrit.plugins.replication.pull.client.FetchRestApiClient; import com.googlesource.gerrit.plugins.replication.pull.client.HttpResult; +import com.googlesource.gerrit.plugins.replication.pull.filter.ApplyObjectsRefsFilter; import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter; import java.io.IOException; import java.nio.file.Path; @@ -94,6 +95,7 @@ @Mock RevisionData revisionDataWithParents; List<ObjectId> revisionDataParentObjectIds; @Mock HttpResult httpResult; + @Mock ApplyObjectsRefsFilter applyObjectsRefsFilter; ApplyObjectMetrics applyObjectMetrics; FetchReplicationMetrics fetchMetrics; @@ -151,6 +153,7 @@ when(httpResult.isSuccessful()).thenReturn(true); when(fetchHttpResult.isSuccessful()).thenReturn(true); when(httpResult.isProjectMissing(any())).thenReturn(false); + when(applyObjectsRefsFilter.match(any())).thenReturn(false); applyObjectMetrics = new ApplyObjectMetrics("pull-replication", new DisabledMetricMaker()); fetchMetrics = new FetchReplicationMetrics("pull-replication", new DisabledMetricMaker()); @@ -166,7 +169,8 @@ () -> revReader, applyObjectMetrics, fetchMetrics, - LOCAL_INSTANCE_ID); + LOCAL_INSTANCE_ID, + applyObjectsRefsFilter); } @Test @@ -294,6 +298,34 @@ } @Test + public void shouldFallbackToApplyAllParentObjectsWhenParentObjectIsMissingOnAllowedRefs() + throws ClientProtocolException, IOException { + String refName = "refs/tags/test-tag"; + Event event = new TestEvent(refName); + objectUnderTest.start(); + + when(httpResult.isSuccessful()).thenReturn(false, true); + when(httpResult.isParentObjectMissing()).thenReturn(true, false); + when(fetchRestApiClient.callSendObjects(any(), anyString(), anyLong(), any(), any())) + .thenReturn(httpResult); + when(applyObjectsRefsFilter.match(refName)).thenReturn(true); + + objectUnderTest.onEvent(event); + + verify(fetchRestApiClient, times(2)) + .callSendObjects(any(), anyString(), anyLong(), revisionsDataCaptor.capture(), any()); + List<List<RevisionData>> revisionsDataValues = revisionsDataCaptor.getAllValues(); + assertThat(revisionsDataValues).hasSize(2); + + List<RevisionData> firstRevisionsValues = revisionsDataValues.get(0); + assertThat(firstRevisionsValues).hasSize(1); + assertThat(firstRevisionsValues).contains(revisionData); + + List<RevisionData> secondRevisionsValues = revisionsDataValues.get(1); + assertThat(secondRevisionsValues).hasSize(1 + revisionDataParentObjectIds.size()); + } + + @Test public void shouldSkipEventWhenMultiSiteVersionRef() throws IOException { FileBasedConfig fileConfig = new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED); @@ -313,7 +345,8 @@ () -> revReader, applyObjectMetrics, fetchMetrics, - LOCAL_INSTANCE_ID); + LOCAL_INSTANCE_ID, + applyObjectsRefsFilter); Event event = new TestEvent("refs/multi-site/version"); objectUnderTest.onEvent(event);