Allow additional refs fallback to apply-objects
Apply-objects REST-API that is capable of applying an entire chain
of commits on the ref chain so that the Git fetch can be avoided
altogether. Without this change only refs/changes/**/meta fallback
to the apply-objects. Add `replication.fallbackToApplyObjectsRefs`
parameter to allow additional refs fallback to the apply-objects
when `MissingParentObject` exception is thrown by the receiver.
Change-Id: If2b12dee0c552defd97d3119eadd47f3700dc710
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
index e99580a..fc77503 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
@@ -38,6 +38,7 @@
import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException;
import com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient;
import com.googlesource.gerrit.plugins.replication.pull.client.HttpResult;
+import com.googlesource.gerrit.plugins.replication.pull.filter.ApplyObjectsRefsFilter;
import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter;
import java.io.IOException;
import java.net.URISyntaxException;
@@ -92,6 +93,7 @@
private final ApplyObjectMetrics applyObjectMetrics;
private final FetchReplicationMetrics fetchMetrics;
private final String instanceId;
+ private ApplyObjectsRefsFilter applyObjectsRefsFilter;
@Inject
ReplicationQueue(
@@ -104,7 +106,8 @@
Provider<RevisionReader> revReaderProvider,
ApplyObjectMetrics applyObjectMetrics,
FetchReplicationMetrics fetchMetrics,
- @GerritInstanceId String instanceId) {
+ @GerritInstanceId String instanceId,
+ ApplyObjectsRefsFilter applyObjectsRefsFilter) {
workQueue = wq;
dispatcher = dis;
sources = rd;
@@ -116,6 +119,7 @@
this.applyObjectMetrics = applyObjectMetrics;
this.fetchMetrics = fetchMetrics;
this.instanceId = instanceId;
+ this.applyObjectsRefsFilter = applyObjectsRefsFilter;
}
@Override
@@ -370,7 +374,8 @@
if (!resultSuccessful) {
if (result.isParentObjectMissing()) {
- if (RefNames.isNoteDbMetaRef(refName) && revision.size() == 1) {
+ if ((RefNames.isNoteDbMetaRef(refName) || applyObjectsRefsFilter.match(refName))
+ && revision.size() == 1) {
List<RevisionData> allRevisions =
fetchWholeMetaHistory(project, refName, revision.get(0));
repLog.info(
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/ApplyObjectsRefsFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/ApplyObjectsRefsFilter.java
new file mode 100644
index 0000000..ff5899b
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/ApplyObjectsRefsFilter.java
@@ -0,0 +1,37 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull.filter;
+
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.replication.ReplicationConfig;
+import java.util.List;
+import org.eclipse.jgit.lib.Config;
+
+@Singleton
+public class ApplyObjectsRefsFilter extends RefsFilter {
+ @Inject
+ public ApplyObjectsRefsFilter(ReplicationConfig replicationConfig) {
+ super(replicationConfig);
+ }
+
+ @Override
+ protected List<String> getRefNamePatterns(Config cfg) {
+ String[] applyObjectsRefs =
+ cfg.getStringList("replication", null, "fallbackToApplyObjectsRefs");
+ return ImmutableList.copyOf(applyObjectsRefs);
+ }
+}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 13b16ea..b85aeb6 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -292,6 +292,33 @@
By default, set to '*' (all refs are replicated synchronously).
+replication.fallbackToApplyObjectsRefs
+: Specify for which refs will fallback to apply-objects REST-API that is capable of applying
+an entire chain of commits on the ref chain so that the Git fetch can be avoided altogether.
+It can be provided more than once, and supports three formats: regular expressions,
+wildcard matching, and single ref matching. All three formats matches are case-sensitive.
+
+ Values starting with a caret `^` are treated as regular
+ expressions. For the regular expressions details please follow
+ official [java documentation](https://docs.oracle.com/javase/tutorial/essential/regex/).
+
+ Please note that regular expressions could also be used
+ with inverse match.
+
+ Values that are not regular expressions and end in `*` are
+ treated as wildcard matches. Wildcards match refs whose
+ name agrees from the beginning until the trailing `*`. So
+ `foo/b*` would match the refs `foo/b`, `foo/bar`, and
+ `foo/baz`, but neither `foobar`, nor `bar/foo/baz`.
+
+ Values that are neither regular expressions nor wildcards are
+ treated as single ref matches. So `foo/bar` matches only
+ the ref `foo/bar`, but no other refs.
+
+ By default, set to 'refs/changes/**/meta'.
+
+ Note that the refs/changes/**/meta always fallback to apply-objects REST-API
+
replication.maxApiPayloadSize
: Maximum size in bytes of the ref to be sent as a REST Api call
payload. For refs larger than threshold git fetch operation
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
index 0a48eaf..d65322f 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
@@ -52,6 +52,7 @@
import com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient;
import com.googlesource.gerrit.plugins.replication.pull.client.FetchRestApiClient;
import com.googlesource.gerrit.plugins.replication.pull.client.HttpResult;
+import com.googlesource.gerrit.plugins.replication.pull.filter.ApplyObjectsRefsFilter;
import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter;
import java.io.IOException;
import java.nio.file.Path;
@@ -94,6 +95,7 @@
@Mock RevisionData revisionDataWithParents;
List<ObjectId> revisionDataParentObjectIds;
@Mock HttpResult httpResult;
+ @Mock ApplyObjectsRefsFilter applyObjectsRefsFilter;
ApplyObjectMetrics applyObjectMetrics;
FetchReplicationMetrics fetchMetrics;
@@ -151,6 +153,7 @@
when(httpResult.isSuccessful()).thenReturn(true);
when(fetchHttpResult.isSuccessful()).thenReturn(true);
when(httpResult.isProjectMissing(any())).thenReturn(false);
+ when(applyObjectsRefsFilter.match(any())).thenReturn(false);
applyObjectMetrics = new ApplyObjectMetrics("pull-replication", new DisabledMetricMaker());
fetchMetrics = new FetchReplicationMetrics("pull-replication", new DisabledMetricMaker());
@@ -166,7 +169,8 @@
() -> revReader,
applyObjectMetrics,
fetchMetrics,
- LOCAL_INSTANCE_ID);
+ LOCAL_INSTANCE_ID,
+ applyObjectsRefsFilter);
}
@Test
@@ -294,6 +298,34 @@
}
@Test
+ public void shouldFallbackToApplyAllParentObjectsWhenParentObjectIsMissingOnAllowedRefs()
+ throws ClientProtocolException, IOException {
+ String refName = "refs/tags/test-tag";
+ Event event = new TestEvent(refName);
+ objectUnderTest.start();
+
+ when(httpResult.isSuccessful()).thenReturn(false, true);
+ when(httpResult.isParentObjectMissing()).thenReturn(true, false);
+ when(fetchRestApiClient.callSendObjects(any(), anyString(), anyLong(), any(), any()))
+ .thenReturn(httpResult);
+ when(applyObjectsRefsFilter.match(refName)).thenReturn(true);
+
+ objectUnderTest.onEvent(event);
+
+ verify(fetchRestApiClient, times(2))
+ .callSendObjects(any(), anyString(), anyLong(), revisionsDataCaptor.capture(), any());
+ List<List<RevisionData>> revisionsDataValues = revisionsDataCaptor.getAllValues();
+ assertThat(revisionsDataValues).hasSize(2);
+
+ List<RevisionData> firstRevisionsValues = revisionsDataValues.get(0);
+ assertThat(firstRevisionsValues).hasSize(1);
+ assertThat(firstRevisionsValues).contains(revisionData);
+
+ List<RevisionData> secondRevisionsValues = revisionsDataValues.get(1);
+ assertThat(secondRevisionsValues).hasSize(1 + revisionDataParentObjectIds.size());
+ }
+
+ @Test
public void shouldSkipEventWhenMultiSiteVersionRef() throws IOException {
FileBasedConfig fileConfig =
new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED);
@@ -313,7 +345,8 @@
() -> revReader,
applyObjectMetrics,
fetchMetrics,
- LOCAL_INSTANCE_ID);
+ LOCAL_INSTANCE_ID,
+ applyObjectsRefsFilter);
Event event = new TestEvent("refs/multi-site/version");
objectUnderTest.onEvent(event);
