Merge branch 'stable-3.12' into stable-3.13 * stable-3.12: Fix --remote option to replication start Add RoundRobin URL selection per remote Improve the wording to explain url matching Release-Notes: skip Change-Id: I98294fefeca033b53b0d3bc7c12372656e1c9ec9
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java index f0b26f9..efac93d 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
@@ -158,6 +158,7 @@ private final DestinationConfiguration config; private final DynamicItem<EventDispatcher> eventDispatcher; private final Provider<ReplicationTasksStorage> replicationTasksStorage; + private final UrlDistributionStrategy.Instance urlDistributor; protected enum RetryReason { TRANSPORT_ERROR, @@ -200,6 +201,7 @@ this.replicationTasksStorage = rts; this.credentialsFactory = credentialsFactory; config = cfg; + urlDistributor = cfg.getUrlDistributionStrategy().newInstance(); ImmutableList<String> projects = cfg.getProjects(); int numStripes = projects.isEmpty() ? MAX_STRIPES : Math.min(projects.size(), MAX_STRIPES); @@ -800,6 +802,14 @@ return r; } + List<URIish> getDistributedUris(Project.NameKey project, String urlMatch) { + return getDistributedUris(getURIs(project, urlMatch)); + } + + List<URIish> getDistributedUris(List<URIish> candidates) { + return urlDistributor.select(candidates); + } + URIish getURI(URIish template, Project.NameKey project) throws URISyntaxException { return getURI(template, project, config.getRemoteNameStyle(), config.isSingleProjectMatch()); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java index b9e1be9..977b23a 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java
@@ -57,6 +57,7 @@ private final Supplier<Integer> pushBatchSize; private final ImmutableList<Pattern> excludedRefsPattern; private final boolean storeRefLog; + private final UrlDistributionStrategy urlDistributionStrategy; protected DestinationConfiguration(RemoteConfig remoteConfig, Config cfg) { this.remoteConfig = remoteConfig; @@ -124,6 +125,9 @@ }); excludedRefsPattern = getExcludedRefsPattern(cfg, name); storeRefLog = cfg.getBoolean("remote", name, "storeRefLog", false); + urlDistributionStrategy = + UrlDistributionStrategy.fromConfig( + cfg.getString("remote", name, "urlDistributionStrategy")); } @Override @@ -234,6 +238,11 @@ return storeRefLog; } + @Override + public UrlDistributionStrategy getUrlDistributionStrategy() { + return urlDistributionStrategy; + } + private ImmutableList<Pattern> getExcludedRefsPattern(Config cfg, String name) { List<Pattern> patterns = new ArrayList<>(); for (String regex : cfg.getStringList("remote", name, "excludedRefsPattern")) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationsCollection.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationsCollection.java index 471a408..82f33d7 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationsCollection.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationsCollection.java
@@ -99,6 +99,7 @@ } boolean adminURLUsed = false; + List<URIish> validUris = new ArrayList<>(); for (String url : config.getAdminUrls()) { if (Strings.isNullOrEmpty(url)) { @@ -128,15 +129,16 @@ continue; } } - uris.put(config, uri); + validUris.add(uri); adminURLUsed = true; } if (!adminURLUsed) { for (URIish uri : config.getURIs(projectName, "*")) { - uris.put(config, uri); + validUris.add(uri); } } + config.getDistributedUris(validUris).forEach(uri -> uris.put(config, uri)); } return uris; }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteConfiguration.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteConfiguration.java index c725d40..18dfca2 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteConfiguration.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteConfiguration.java
@@ -159,4 +159,9 @@ * @return true if new repositories should store ref-updates in their reflog */ boolean storeRefLog(); + + /** Returns the URL distribution strategy configured for this remote. */ + default UrlDistributionStrategy getUrlDistributionStrategy() { + return UrlDistributionStrategy.ALL; + } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java index 7b6079a..5d7fd19 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
@@ -231,7 +231,7 @@ } } if (!refNamesToPush.isEmpty()) { - for (URIish uri : cfg.getURIs(project, urlMatch)) { + for (URIish uri : cfg.getDistributedUris(project, urlMatch)) { replicationTasksStorage.create( ReplicateRefUpdate.create( project.get(), refNamesToPush, uri, cfg.getRemoteConfigName()));
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java index 507774f..92c619c 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java
@@ -39,11 +39,15 @@ @Option(name = "--all", usage = "push all known projects") private boolean all; - @Option(name = "--url", metaVar = "PATTERN", usage = "pattern to match URL on") + @Option(name = "--url", metaVar = "SUBSTRING", usage = "substring URL must match (or * to match everything)") private String urlMatch; + private final Set<String> remotesToConsider = new HashSet<>(); + @Option(name = "--remote", metaVar = "REMOTE", usage = "name of remote to replicate to") - private Set<String> remotesToConsider = new HashSet<>(); + void addRemote(String remote) { + remotesToConsider.add(remote); + } @Option(name = "--wait", usage = "wait for replication to finish before exiting") private boolean wait;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/UrlDistributionStrategy.java b/src/main/java/com/googlesource/gerrit/plugins/replication/UrlDistributionStrategy.java new file mode 100644 index 0000000..46183eb --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/UrlDistributionStrategy.java
@@ -0,0 +1,84 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.eclipse.jgit.transport.URIish; + +/** + * URL distribution strategy used when a remote has multiple configured URLs. + * + * <p>Each enum constant acts as a factory: call {@link #newInstance()} to obtain a stateful + * executor. Callers (e.g. {@link Destination}) hold the {@link Instance}, while the enum constant + * itself remains stateless and safe to use in equality checks. + * + * <p>Configured via {@code remote.NAME.urlDistribution} in {@code replication.config}. + */ +public enum UrlDistributionStrategy { + /** Push to all configured URLs. */ + ALL("all") { + @Override + public Instance newInstance() { + return candidates -> candidates; + } + }, + + /** + * Push to one URL at a time, rotating through the list on each push event. Particularly useful + * when multiple replica hosts share a single backend (likely via NFS): pushing to all URLs would + * cause redundant writes to the same underlying storage, while round-robin distributes load + * evenly and ensures each push is executed exactly once. + */ + ROUND_ROBIN("roundRobin") { + @Override + public Instance newInstance() { + final AtomicInteger index = new AtomicInteger(); + return candidates -> { + if (candidates.isEmpty()) { + return List.of(); + } + return List.of(candidates.get(Math.floorMod(index.getAndIncrement(), candidates.size()))); + }; + } + }; + + public final String configKey; + + UrlDistributionStrategy(String key) { + configKey = key; + } + + /** Creates a new stateful executor for this distribution strategy. */ + public abstract Instance newInstance(); + + /** + * Returns the distribution strategy for the given config value, or {@link #ALL} if the value is + * unrecognized or absent. + */ + public static UrlDistributionStrategy fromConfig(String value) { + return Arrays.stream(values()) + .filter(candidate -> candidate.configKey.equals(value)) + .findFirst() + .orElse(ALL); + } + + /** A stateful executor for a {@link UrlDistributionStrategy} strategy. */ + @FunctionalInterface + public interface Instance { + List<URIish> select(List<URIish> candidates); + } +}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index 150810e..3fa544b 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md
@@ -631,6 +631,24 @@ Do not exclude any refs pattern by default. +remote.NAME.urlDistributionStrategy +: URL distribution strategy to use when a remote has multiple configured URLs. + Applies to both push URLs (`url`) and admin URLs (`adminUrl`). + + `all` (default) + : Push to all configured URLs simultaneously. This is the standard + replication behaviour. + + `roundRobin` + : Push to one URL at a time, rotating through the list on each push event. + Particularly useful in high-availability setups where multiple replica + hosts share a single NFS backend. Pushing to all URLs simultaneously + would cause redundant writes to the same underlying storage; round-robin + distributes load evenly and ensures each push is written exactly once. + Has no effect if only one URL is configured. + + Defaults to `all`. + Directory `replication` -------------------- The optional directory `$site_path/etc/replication` contains Git-style
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/DestinationConfigurationTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/DestinationConfigurationTest.java index 646f915..c7b4174 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/DestinationConfigurationTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/DestinationConfigurationTest.java
@@ -98,4 +98,20 @@ // then assertThat(actual).isEqualTo(globalPushBatchSize); } + + @Test + public void shouldDefaultUrlDistributionToAll() { + assertThat(objectUnderTest.getUrlDistributionStrategy()).isEqualTo(UrlDistributionStrategy.ALL); + } + + @Test + public void shouldSetUrlDistributionToRoundRobinWhenConfigured() { + // given + when(cfgMock.getString("remote", REMOTE, "urlDistributionStrategy")).thenReturn("roundRobin"); + objectUnderTest = new DestinationConfiguration(remoteConfigMock, cfgMock); + + // when / then + assertThat(objectUnderTest.getUrlDistributionStrategy()) + .isEqualTo(UrlDistributionStrategy.ROUND_ROBIN); + } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationDaemon.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationDaemon.java index 11b7718..436fb1c 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationDaemon.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationDaemon.java
@@ -308,4 +308,10 @@ config.setBoolean("remote", remoteName, "replicateProjectDeletions", replicateProjectDeletion); config.save(); } + + protected void setUrlDistribution(String remoteName, UrlDistributionStrategy distribution) + throws IOException { + config.setString("remote", remoteName, "urlDistributionStrategy", distribution.configKey); + config.save(); + } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java index 5013014..8b9ea0e 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java
@@ -496,6 +496,102 @@ } @Test + public void shouldReplicateToAllUrlsByDefault() throws Exception { + Project.NameKey replica1Project = createTestProject(project + "replica1"); + Project.NameKey replica2Project = createTestProject(project + "replica2"); + + setReplicationDestination( + "foo", List.of("replica1", "replica2"), ALL_PROJECTS, TEST_REPLICATION_DELAY_SECONDS); + reloadConfig(); + + String newRef = "refs/heads/newForTest"; + createNewBranchWithoutPush("refs/heads/master", newRef); + + plugin + .getSysInjector() + .getInstance(ReplicationQueue.class) + .scheduleFullSync(project, null, new ReplicationState(NO_OP), true); + + // Wait for the push to land on both the refs + try (Repository r1 = repoManager.openRepository(replica1Project); + Repository r2 = repoManager.openRepository(replica2Project)) { + waitUntil(() -> checkedGetRef(r1, newRef) != null && checkedGetRef(r2, newRef) != null); + } + } + + @Test + public void shouldReplicateToOnlyOneUrlWhenRoundRobinEnabled() throws Exception { + Project.NameKey replica1Project = createTestProject(project + "replica1"); + Project.NameKey replica2Project = createTestProject(project + "replica2"); + + setReplicationDestination( + "foo", List.of("replica1", "replica2"), ALL_PROJECTS, TEST_REPLICATION_DELAY_SECONDS); + setUrlDistribution("foo", UrlDistributionStrategy.ROUND_ROBIN); + reloadConfig(); + + String newRef = "refs/heads/newForTest"; + createNewBranchWithoutPush("refs/heads/master", newRef); + + plugin + .getSysInjector() + .getInstance(ReplicationQueue.class) + .scheduleFullSync(project, null, new ReplicationState(NO_OP), true); + + // Wait for the push to land in at least one replica + try (Repository r1 = repoManager.openRepository(replica1Project); + Repository r2 = repoManager.openRepository(replica2Project)) { + waitUntil(() -> checkedGetRef(r1, newRef) != null || checkedGetRef(r2, newRef) != null); + + // Exactly one replica should have received the push + boolean r1HasRef = checkedGetRef(r1, newRef) != null; + boolean r2HasRef = checkedGetRef(r2, newRef) != null; + assertThat(r1HasRef ^ r2HasRef).isTrue(); + } + } + + @Test + public void shouldRotateAcrossUrlsOnConsecutivePushesWhenRoundRobinEnabled() throws Exception { + Project.NameKey replica1Project = createTestProject(project + "replica1"); + Project.NameKey replica2Project = createTestProject(project + "replica2"); + + setReplicationDestination( + "foo", List.of("replica1", "replica2"), ALL_PROJECTS, TEST_REPLICATION_DELAY_SECONDS); + setUrlDistribution("foo", UrlDistributionStrategy.ROUND_ROBIN); + reloadConfig(); + + String branch1 = "refs/heads/branch1"; + String branch2 = "refs/heads/branch2"; + createNewBranchWithoutPush("refs/heads/master", branch1); + + ReplicationQueue queue = plugin.getSysInjector().getInstance(ReplicationQueue.class); + + // First sync - goes to replica1 (index 0) + queue.scheduleFullSync(project, null, new ReplicationState(NO_OP), true); + + try (Repository r1 = repoManager.openRepository(replica1Project)) { + waitUntil(() -> checkedGetRef(r1, branch1) != null); + } + + // replica2 should not have received branch1 yet + try (Repository r2 = repoManager.openRepository(replica2Project)) { + assertThat(checkedGetRef(r2, branch1)).isNull(); + } + + // Second sync - goes to replica2 (index 1), includes branch1 and branch2 + createNewBranchWithoutPush("refs/heads/master", branch2); + queue.scheduleFullSync(project, null, new ReplicationState(NO_OP), true); + + try (Repository r2 = repoManager.openRepository(replica2Project)) { + waitUntil(() -> checkedGetRef(r2, branch1) != null && checkedGetRef(r2, branch2) != null); + } + + // replica1 should not have received branch2 (it was only in the second sync) + try (Repository r1 = repoManager.openRepository(replica1Project)) { + assertThat(checkedGetRef(r1, branch2)).isNull(); + } + } + + @Test public void shouldReplicateToMatchingRemote() throws Exception { Project.NameKey targetProject = createTestProject(project + "replica");