Merge "Auto-repair replication on missing necessary objects error"
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java index 0c77b16..e202729 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
@@ -23,6 +23,7 @@ import com.google.inject.Singleton; import com.googlesource.gerrit.plugins.replication.api.ReplicationConfig; import java.nio.file.Path; +import java.time.Duration; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -130,6 +131,21 @@ } @Override + public Duration getAutoRepairInterval() { + return currentConfig.getAutoRepairInterval(); + } + + @Override + public int getAutoRepairMaxAttempts() { + return currentConfig.getAutoRepairMaxAttempts(); + } + + @Override + public int getAutoRepairConcurrencyLimit() { + return currentConfig.getAutoRepairConcurrencyLimit(); + } + + @Override public Config getConfig() { return currentConfig.getConfig(); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoRepairHandler.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoRepairHandler.java new file mode 100644 index 0000000..1be8758 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoRepairHandler.java
@@ -0,0 +1,117 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog; + +import com.google.gerrit.entities.Project; +import com.google.gerrit.extensions.annotations.PluginName; +import com.google.gerrit.server.git.WorkQueue; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.googlesource.gerrit.plugins.replication.api.ReplicationConfig; +import com.googlesource.gerrit.plugins.replication.events.dispatcher.EventDispatcher; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import org.eclipse.jgit.transport.URIish; + +@Singleton +public class AutoRepairHandler { + static final String MISSING_NECESSARY_OBJECTS = "missing necessary objects"; + + private final AutoRepairTracker tracker; + private final ProjectRepairer projectRepairer; + private final ReplicationStarter replicationStarter; + private final EventDispatcher eventDispatcher; + private final ScheduledExecutorService executor; + + @Inject + AutoRepairHandler( + AutoRepairTracker tracker, + ProjectRepairer projectRepairer, + ReplicationStarter replicationStarter, + EventDispatcher eventDispatcher, + WorkQueue workQueue, + ReplicationConfig replicationConfig, + @PluginName String pluginName) { + this.tracker = tracker; + this.projectRepairer = projectRepairer; + this.replicationStarter = replicationStarter; + this.eventDispatcher = eventDispatcher; + this.executor = + workQueue.createQueue( + replicationConfig.getAutoRepairConcurrencyLimit(), pluginName + "_auto-repair"); + } + + public static boolean isMissingNecessaryObjectsError(String message) { + return message != null && message.contains(MISSING_NECESSARY_OBJECTS); + } + + public void handle( + Project.NameKey project, + URIish uri, + String remoteName, + UrlDistributionStrategy urlDistributionStrategy) { + if (!tracker.tryBeginRepair(project, uri, remoteName, urlDistributionStrategy)) { + return; + } + repLog.atInfo().log("Scheduling auto-repair for project %s to %s", project.get(), uri); + @SuppressWarnings("unused") + Future<?> possiblyIgnoredError = executor.submit(new AutoRepairTask(project, uri)); + } + + private class AutoRepairTask implements Runnable { + private final Project.NameKey project; + private final URIish uri; + + AutoRepairTask(Project.NameKey project, URIish uri) { + this.project = project; + this.uri = uri; + } + + @Override + public void run() { + ByteArrayOutputStream buf = new ByteArrayOutputStream(); + boolean isRepaired = projectRepairer.repair(project, uri, buf, true); + (isRepaired ? repLog.atInfo() : repLog.atWarning()) + .log( + "Auto-repair %s for project %s to %s:%s", + isRepaired ? "succeeded" : "failed", + project.get(), + uri, + buf.toString(StandardCharsets.UTF_8)); + if (isRepaired) { + replicationStarter.start( + uri.toString(), + Set.of(), + new ReplicationFilter(List.of(project.get()), Collections.emptyList()), + /* now= */ true, + /* wait= */ false, + new PushResultProcessing.GitUpdateProcessing(eventDispatcher)); + repLog.atInfo().log("Scheduled full replication of %s to %s", project.get(), uri); + } + } + + @Override + public String toString() { + return "auto-repair " + project.get() + " to " + uri; + } + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoRepairTracker.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoRepairTracker.java new file mode 100644 index 0000000..53d9c9e --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoRepairTracker.java
@@ -0,0 +1,118 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog; + +import com.google.gerrit.entities.Project; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.googlesource.gerrit.plugins.replication.api.ReplicationConfig; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.eclipse.jgit.transport.URIish; + +/** + * Tracks per-project, per-destination auto-repair attempts and enforces interval and attempt + * limits. + * + * <p>State is kept in memory and is reset whenever the plugin is reloaded or Gerrit restarts. + */ +@Singleton +public class AutoRepairTracker { + private final ReplicationConfig replicationConfig; + private final ConcurrentMap<DestinationKey, State> statesByDestination = + new ConcurrentHashMap<>(); + + @Inject + AutoRepairTracker(ReplicationConfig replicationConfig) { + this.replicationConfig = replicationConfig; + } + + public boolean isEnabled() { + return replicationConfig.getAutoRepairMaxAttempts() > 0; + } + + /** + * Returns whether a new auto-repair attempt is allowed for the project on the given destination. + * + * <p>If allowed, records the attempt before returning {@code true}. + */ + public boolean tryBeginRepair( + Project.NameKey project, + URIish uri, + String remoteName, + UrlDistributionStrategy urlDistributionStrategy) { + if (!isEnabled()) { + return false; + } + + if (!ProjectRepairer.canCopy(uri)) { + repLog.atWarning().log( + "Skipping auto-repair for %s to %s: only plain SSH destinations are supported", + project.get(), uri); + return false; + } + + DestinationKey key = DestinationKey.create(project, uri, remoteName, urlDistributionStrategy); + return statesByDestination.computeIfAbsent(key, k -> new State()).isRepairAllowed(project, uri); + } + + private record DestinationKey(Project.NameKey project, String destination) { + static DestinationKey create( + Project.NameKey project, + URIish uri, + String remoteName, + UrlDistributionStrategy urlDistributionStrategy) { + return new DestinationKey( + project, + urlDistributionStrategy == UrlDistributionStrategy.ROUND_ROBIN + ? remoteName + : uri.toString()); + } + } + + private class State { + private int repairAttemptCount; + private Instant lastAttempt = Instant.EPOCH; + + synchronized boolean isRepairAllowed(Project.NameKey project, URIish uri) { + int maxAttempts = replicationConfig.getAutoRepairMaxAttempts(); + if (repairAttemptCount >= maxAttempts) { + repLog.atWarning().log( + "Skipping auto-repair for %s to %s: reached max attempts (%d)", + project.get(), uri, maxAttempts); + return false; + } + Duration interval = replicationConfig.getAutoRepairInterval(); + Instant now = Instant.now(); + if (!interval.isZero()) { + Instant nextAllowed = lastAttempt.plus(interval); + if (now.isBefore(nextAllowed)) { + repLog.atInfo().log( + "Skipping auto-repair for %s to %s: interval not elapsed." + + " Next attempt allowed at %s", + project.get(), uri, nextAllowed); + return false; + } + } + lastAttempt = now; + repairAttemptCount++; + return true; + } + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java index 568bd0d..f83e1df 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
@@ -925,6 +925,10 @@ return config.getRemoteConfig().getName(); } + UrlDistributionStrategy getUrlDistributionStrategy() { + return config.getUrlDistributionStrategy(); + } + public int getMaxRetries() { return config.getMaxRetries(); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java index 600ce8c..caf3236 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
@@ -142,6 +142,7 @@ private final CreateProjectTask.Factory createProjectFactory; private final AtomicBoolean canceledWhileRunning; private final TransportFactory transportFactory; + private final AutoRepairHandler autoRepairHandler; private DynamicItem<ReplicationPushFilter> replicationPushFilter; @Inject @@ -159,6 +160,7 @@ ProjectCache pc, CreateProjectTask.Factory cpf, TransportFactory tf, + AutoRepairHandler autoRepairHandler, @Assisted Project.NameKey d, @Assisted URIish u) { gitManager = grm; @@ -181,6 +183,7 @@ canceledWhileRunning = new AtomicBoolean(false); maxRetries = p.getMaxRetries(); transportFactory = tf; + this.autoRepairHandler = autoRepairHandler; } @Inject(optional = true) @@ -832,6 +835,7 @@ throws UpdateRefFailureException { Set<String> doneRefs = new HashSet<>(); boolean anyRefFailed = false; + boolean autoRepairNeeded = false; RemoteRefUpdate.Status lastRefStatusError = RemoteRefUpdate.Status.OK; for (RemoteRefUpdate u : refUpdates) { @@ -877,6 +881,10 @@ || UPDATE_REF_FAILURE.equals(u.getMessage())) { throw new UpdateRefFailureException(uri, u.getMessage()); } else { + if (!autoRepairNeeded + && AutoRepairHandler.isMissingNecessaryObjectsError(u.getMessage())) { + autoRepairNeeded = true; + } stateLog.error( String.format( "Failed replicate of %s to %s, reason: %s", @@ -911,6 +919,10 @@ projectName.get(), entry.getKey(), uri, RefPushResult.NOT_ATTEMPTED, null); } } + if (autoRepairNeeded) { + autoRepairHandler.handle( + projectName, uri, pool.getRemoteConfigName(), pool.getUrlDistributionStrategy()); + } stateMap.clear(); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfigImpl.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfigImpl.java index f625232..72cd417 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfigImpl.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfigImpl.java
@@ -24,11 +24,15 @@ import com.google.inject.Inject; import com.googlesource.gerrit.plugins.replication.api.ReplicationConfig; import java.nio.file.Path; +import java.time.Duration; import org.eclipse.jgit.lib.Config; public class ReplicationConfigImpl implements ReplicationConfig { private static final int DEFAULT_SSH_CONNECTION_TIMEOUT_MS = 2 * 60 * 1000; // 2 minutes private static final String DEFAULT_RSYNC_PATH = "rsync"; + private static final Duration DEFAULT_AUTO_REPAIR_INTERVAL = Duration.ofDays(3); + private static final int DEFAULT_AUTO_REPAIR_MAX_ATTEMPTS = 0; + private static final int DEFAULT_AUTO_REPAIR_CONCURRENCY_LIMIT = 2; private final SitePaths site; private final MergedConfigResource configResource; @@ -39,6 +43,9 @@ private final int maxRefsToShow; private int sshCommandTimeout; private int sshConnectionTimeout; + private final Duration autoRepairInterval; + private final int autoRepairMaxAttempts; + private final int autoRepairConcurrencyLimit; private final Path pluginDataDir; private final Config config; @@ -63,6 +70,24 @@ "sshConnectionTimeout", DEFAULT_SSH_CONNECTION_TIMEOUT_MS, MILLISECONDS); + this.autoRepairInterval = + Duration.ofSeconds( + ConfigUtil.getTimeUnit( + config, + "replication", + null, + "autoRepairInterval", + DEFAULT_AUTO_REPAIR_INTERVAL.getSeconds(), + SECONDS)); + this.autoRepairMaxAttempts = + config.getInt("replication", "autoRepairMaxAttempts", DEFAULT_AUTO_REPAIR_MAX_ATTEMPTS); + this.autoRepairConcurrencyLimit = + Math.max( + 1, + config.getInt( + "replication", + "autoRepairConcurrencyLimit", + DEFAULT_AUTO_REPAIR_CONCURRENCY_LIMIT)); this.pluginDataDir = pluginDataDir; this.useLegacyCredentials = config.getBoolean("gerrit", "useLegacyCredentials", false); } @@ -152,4 +177,19 @@ String rsyncPath = getConfig().getString("replication", null, "rsyncPath"); return Strings.isNullOrEmpty(rsyncPath) ? DEFAULT_RSYNC_PATH : rsyncPath; } + + @Override + public Duration getAutoRepairInterval() { + return autoRepairInterval; + } + + @Override + public int getAutoRepairMaxAttempts() { + return autoRepairMaxAttempts; + } + + @Override + public int getAutoRepairConcurrencyLimit() { + return autoRepairConcurrencyLimit; + } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java index 60c0254..711228b 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
@@ -110,6 +110,8 @@ bind(ReplicationQueue.class).in(Scopes.SINGLETON); bind(ReplicationDestinations.class).to(DestinationsCollection.class); + bind(AutoRepairHandler.class).in(Scopes.SINGLETON); + bind(AutoRepairTracker.class).in(Scopes.SINGLETON); bind(ProjectRepairer.class).in(Scopes.SINGLETON); install(new FactoryModuleBuilder().build(Destination.Factory.class));
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStarter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStarter.java index 7202eab..e715747 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStarter.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStarter.java
@@ -42,8 +42,8 @@ ReplicationFilter filter, boolean now, boolean wait, - SshOutputCommand sink) { - ReplicationState state = new ReplicationState(new CommandProcessing(sink)); + PushResultProcessing processing) { + ReplicationState state = new ReplicationState(processing); Future<?> future = pushFactory @@ -68,11 +68,32 @@ try { state.waitForReplication(); } catch (InterruptedException e) { - sink.writeStdErrSync("We are interrupted while waiting replication to complete"); + processing.writeStdErr("We are interrupted while waiting replication to complete"); } } else { - sink.writeStdOutSync("Nothing to replicate"); + processing.writeStdOut("Nothing to replicate"); } } } + + void start( + @Nullable String urlMatch, + String refName, + Set<String> remotesToConsider, + ReplicationFilter filter, + boolean now, + boolean wait, + SshOutputCommand sink) { + start(urlMatch, refName, remotesToConsider, filter, now, wait, new CommandProcessing(sink)); + } + + void start( + @Nullable String urlMatch, + Set<String> remotesToConsider, + ReplicationFilter filter, + boolean now, + boolean wait, + PushResultProcessing processing) { + start(urlMatch, PushOne.ALL_REFS, remotesToConsider, filter, now, wait, processing); + } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/api/ReplicationConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/api/ReplicationConfig.java index 1714c1a..6860098 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/api/ReplicationConfig.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/api/ReplicationConfig.java
@@ -15,6 +15,7 @@ package com.googlesource.gerrit.plugins.replication.api; import java.nio.file.Path; +import java.time.Duration; import org.eclipse.jgit.lib.Config; /** Configuration of all the replication end points. */ @@ -97,6 +98,30 @@ String getRsyncPath(); /** + * Minimum interval between automatic repair attempts for the same project on the same + * destination. + * + * @return interval as a {@link Duration}, {@link Duration#ZERO} for no minimum interval between + * attempts. + */ + Duration getAutoRepairInterval(); + + /** + * Maximum number of automatic repair attempts per project on each destination. + * + * @return maximum attempts, zero to disable auto-repair. + */ + int getAutoRepairMaxAttempts(); + + /** + * Maximum number of automatic repair tasks allowed to run concurrently. Changing this value + * requires a plugin reload to take effect. + * + * @return concurrency limit, minimum 1. + */ + int getAutoRepairConcurrencyLimit(); + + /** * Current logical version string of the current configuration loaded in memory, depending on the * actual implementation of the configuration on the persistent storage. *
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index cffaa8c..305bb28 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md
@@ -282,6 +282,47 @@ Default: `rsync` (resolved via the Gerrit runtime user's `PATH`) +replication.autoRepairInterval +: Minimum interval between automatic repair attempts for the same + project on the same destination. Values are expressed with a time + unit suffix, e.g. `2h`, `30m`, `3d`. If no unit is given the value is + interpreted as seconds. + + Auto-repair is event-driven and each attempt is initiated by a failing + replication push and is not retried on its own. When a push fails with + a `missing necessary objects` error from `git-receive-pack`, the plugin + schedules a repair attempt for the affected SSH destination. Unlike + normal replication failures (which the plugin retries automatically), + repair attempts are not retried by themselves. If a repair completes + but the destination is still broken, the next repair runs only when + another replication push to the same destination fails with the + same missing objects error, and only after this interval has elapsed. + + Default: `3d` + +replication.autoRepairMaxAttempts +: Maximum number of automatic repair attempts per project on each + destination. After this limit is reached for a given project and + destination, further `missing necessary objects` errors for that pair + are logged but no additional automatic repairs are attempted. + + Auto-repair state is kept in memory and is reset whenever the plugin is + reloaded or Gerrit restarts. Enabling auto-repair in a clustered deployment + can lead to redundant repairs as the state is not shared between them. + + Default: `0` (auto-repair disabled) + +replication.autoRepairConcurrencyLimit +: Maximum number of automatic repair tasks allowed to run concurrently. + Each running repair occupies one slot for the duration of its repair + and the follow-up full replication, so this caps the load auto-repair + can put on the host and on destination SSH endpoints at any one time. + Increase it when many destinations are expected to need auto-repair + in parallel. The value is read once at plugin start, so changes only + take effect after a plugin reload. + + Minimum: `1`. Default: `2`. + remote.NAME.url : Address of the remote server to push to. Multiple URLs may be specified within a single remote block, listing different
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/AutoRepairTrackerTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/AutoRepairTrackerTest.java new file mode 100644 index 0000000..865ed23 --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/AutoRepairTrackerTest.java
@@ -0,0 +1,114 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.when; + +import com.google.gerrit.entities.Project; +import com.googlesource.gerrit.plugins.replication.api.ReplicationConfig; +import java.time.Duration; +import org.eclipse.jgit.transport.URIish; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +public class AutoRepairTrackerTest { + private static final Project.NameKey PROJECT = Project.nameKey("foo"); + private static final String REMOTE = "mirror"; + + @Mock private ReplicationConfig replicationConfig; + + private AutoCloseable mocks; + private AutoRepairTracker tracker; + private URIish uri1; + private URIish uri2; + + @Before + public void setUp() throws Exception { + mocks = MockitoAnnotations.openMocks(this); + when(replicationConfig.getAutoRepairInterval()).thenReturn(Duration.ofDays(3)); + when(replicationConfig.getAutoRepairMaxAttempts()).thenReturn(5); + + tracker = new AutoRepairTracker(replicationConfig); + uri1 = new URIish("ssh://mirror1.example.com/foo.git"); + uri2 = new URIish("ssh://mirror2.example.com/foo.git"); + } + + @After + public void tearDown() throws Exception { + mocks.close(); + } + + @Test + public void allowsRepairUpToMaxAttemptsPerDestination() { + when(replicationConfig.getAutoRepairInterval()).thenReturn(Duration.ZERO); + for (int i = 0; i < 5; i++) { + assertThat(tryRepair(uri1, UrlDistributionStrategy.ALL)).isTrue(); + } + assertThat(tryRepair(uri1, UrlDistributionStrategy.ALL)).isFalse(); + } + + @Test + public void tracksAttemptsSeparatelyPerDestination() { + when(replicationConfig.getAutoRepairInterval()).thenReturn(Duration.ZERO); + when(replicationConfig.getAutoRepairMaxAttempts()).thenReturn(1); + + assertThat(tryRepair(uri1, UrlDistributionStrategy.ALL)).isTrue(); + assertThat(tryRepair(uri1, UrlDistributionStrategy.ALL)).isFalse(); + assertThat(tryRepair(uri2, UrlDistributionStrategy.ALL)).isTrue(); + } + + @Test + public void roundRobinSharesAttemptsAcrossUrls() { + when(replicationConfig.getAutoRepairInterval()).thenReturn(Duration.ZERO); + when(replicationConfig.getAutoRepairMaxAttempts()).thenReturn(1); + + assertThat(tryRepair(uri1, UrlDistributionStrategy.ROUND_ROBIN)).isTrue(); + assertThat(tryRepair(uri2, UrlDistributionStrategy.ROUND_ROBIN)).isFalse(); + } + + @Test + public void allowsRepairWithoutIntervalWhenIntervalIsZero() { + when(replicationConfig.getAutoRepairInterval()).thenReturn(Duration.ZERO); + assertThat(tryRepair(uri1, UrlDistributionStrategy.ALL)).isTrue(); + assertThat(tryRepair(uri1, UrlDistributionStrategy.ALL)).isTrue(); + } + + @Test + public void disabledWhenMaxAttemptsIsZero() { + when(replicationConfig.getAutoRepairMaxAttempts()).thenReturn(0); + assertThat(tracker.isEnabled()).isFalse(); + assertThat(tryRepair(uri1, UrlDistributionStrategy.ALL)).isFalse(); + } + + @Test + public void enforcesIntervalBetweenAttempts() { + assertThat(tryRepair(uri1, UrlDistributionStrategy.ALL)).isTrue(); + assertThat(tryRepair(uri1, UrlDistributionStrategy.ALL)).isFalse(); + } + + @Test + public void rejectsNonCopyableDestination() throws Exception { + URIish httpUri = new URIish("http://mirror.example.com/foo.git"); + assertThat(tryRepair(httpUri, UrlDistributionStrategy.ALL)).isFalse(); + } + + private boolean tryRepair(URIish uri, UrlDistributionStrategy strategy) { + return tracker.tryBeginRepair(PROJECT, uri, REMOTE, strategy); + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java index 339123a..6def78f 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java
@@ -423,6 +423,7 @@ projectCacheMock, createProjectTaskFactoryMock, transportFactoryMock, + mock(AutoRepairHandler.class), projectNameKey, urish);