Merge branch 'stable-3.13' * origin/stable-3.13: Fix --remote option to replication start Add RoundRobin URL selection per remote Improve the wording to explain url matching Release-Notes: skip Change-Id: I7939ec71984a80496f7baf0bb9d86fb749316d7a
diff --git a/BUILD b/BUILD index 9c209ed..cc671d7 100644 --- a/BUILD +++ b/BUILD
@@ -1,6 +1,5 @@ load("@rules_java//java:defs.bzl", "java_library") -load("//tools/bzl:junit.bzl", "junit_tests") -load("//tools/bzl:plugin.bzl", "PLUGIN_DEPS", "PLUGIN_TEST_DEPS", "gerrit_plugin") +load("@com_googlesource_gerrit_bazlets//:gerrit_plugin.bzl", "gerrit_plugin", "gerrit_plugin_tests") gerrit_plugin( name = "replication", @@ -33,7 +32,7 @@ ], ) -junit_tests( +gerrit_plugin_tests( name = "replication_tests", timeout = "long", srcs = glob([ @@ -41,18 +40,19 @@ ]), tags = ["replication"], visibility = ["//visibility:public"], - deps = PLUGIN_TEST_DEPS + PLUGIN_DEPS + [ + deps = [ ":replication__plugin", ":replication_util", ], ) -[junit_tests( +[gerrit_plugin_tests( name = f[:f.index(".")].replace("/", "_"), + timeout = "long" if f.endswith("/ReplicationIT.java") else "moderate", srcs = [f], tags = ["replication"], visibility = ["//visibility:public"], - deps = PLUGIN_TEST_DEPS + PLUGIN_DEPS + [ + deps = [ ":replication__plugin", ":replication_util", ], @@ -65,7 +65,9 @@ ["src/test/java/**/*.java"], exclude = ["src/test/java/**/*Test.java"], ), - deps = PLUGIN_TEST_DEPS + PLUGIN_DEPS + [ + deps = [ ":replication__plugin", + "//java/com/google/gerrit/acceptance:lib", + "//plugins:plugin-lib", ], )
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java index 30e8245..25e9d18 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java
@@ -14,41 +14,54 @@ package com.googlesource.gerrit.plugins.replication; +import com.google.gerrit.common.Nullable; import com.google.inject.Inject; import com.google.inject.Singleton; +import com.googlesource.gerrit.plugins.replication.api.ReplicationConfig; import java.util.Optional; import org.eclipse.jgit.transport.URIish; /** Factory for creating an {@link AdminApi} instance for a remote URI. */ public interface AdminApiFactory { /** - * Create an {@link AdminApi} for the given remote URI. + * Create an {@link AdminApi} for the given remote URI with knowledge of which {@code + * remote.<remoteName>} section in {@code replication.config} the URI belongs to. * * @param uri the remote URI. + * @param remoteName the name of the {@code remote} section, or {@code null} when unknown. * @return An API for the given remote URI, or {@code Optional.empty} if there is no appropriate * API for the URI. */ - Optional<AdminApi> create(URIish uri); + Optional<AdminApi> create(URIish uri, @Nullable String remoteName); @Singleton static class DefaultAdminApiFactory implements AdminApiFactory { protected final SshHelper sshHelper; private final GerritRestApi.Factory gerritRestApiFactory; + private final ReplicationConfig replicationConfig; @Inject - public DefaultAdminApiFactory(SshHelper sshHelper, GerritRestApi.Factory gerritRestApiFactory) { + public DefaultAdminApiFactory( + SshHelper sshHelper, + GerritRestApi.Factory gerritRestApiFactory, + ReplicationConfig replicationConfig) { this.sshHelper = sshHelper; this.gerritRestApiFactory = gerritRestApiFactory; + this.replicationConfig = replicationConfig; } @Override - public Optional<AdminApi> create(URIish uri) { + public Optional<AdminApi> create(URIish uri, @Nullable String remoteName) { if (isGerrit(uri)) { return Optional.of(new GerritSshApi(sshHelper, uri)); } else if (!uri.isRemote()) { return Optional.of(new LocalFS(uri)); } else if (isSSH(uri)) { - return Optional.of(new RemoteSsh(sshHelper, uri)); + String gitPath = + remoteName == null + ? null + : replicationConfig.getConfig().getString("remote", remoteName, "gitPath"); + return Optional.of(new RemoteSsh(sshHelper, uri, gitPath)); } else if (isGerritHttp(uri)) { return Optional.of(gerritRestApiFactory.create(uri)); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java index 5e2c758..0c77b16 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
@@ -125,6 +125,11 @@ } @Override + public String getRsyncPath() { + return currentConfig.getRsyncPath(); + } + + @Override public Config getConfig() { return currentConfig.getConfig(); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java index 32903ab..04f9869 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java
@@ -61,7 +61,7 @@ private boolean createProject( URIish replicateURI, Project.NameKey projectName, String head, boolean storeRefLog) { - Optional<AdminApi> adminApi = adminApiFactory.get().create(replicateURI); + Optional<AdminApi> adminApi = adminApiFactory.get().create(replicateURI, config.getName()); if (adminApi.isPresent() && adminApi.get().createProject(projectName, head, storeRefLog)) { return true; }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java index 965ca94..f3fcafe 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java
@@ -24,6 +24,7 @@ import com.google.inject.assistedinject.Assisted; import com.googlesource.gerrit.plugins.replication.events.ProjectDeletionState; import java.util.Optional; +import org.eclipse.jgit.transport.RemoteConfig; import org.eclipse.jgit.transport.URIish; public class DeleteProjectTask implements Runnable { @@ -33,6 +34,7 @@ URIish replicateURI, Project.NameKey project, ProjectDeletionState state); } + private final RemoteConfig config; private final DynamicItem<AdminApiFactory> adminApiFactory; private final int id; private final URIish replicateURI; @@ -41,11 +43,13 @@ @Inject DeleteProjectTask( + RemoteConfig config, DynamicItem<AdminApiFactory> adminApiFactory, IdGenerator ig, @Assisted ProjectDeletionState state, @Assisted URIish replicateURI, @Assisted Project.NameKey project) { + this.config = config; this.adminApiFactory = adminApiFactory; this.id = ig.next(); this.replicateURI = replicateURI; @@ -55,7 +59,7 @@ @Override public void run() { - Optional<AdminApi> adminApi = adminApiFactory.get().create(replicateURI); + Optional<AdminApi> adminApi = adminApiFactory.get().create(replicateURI, config.getName()); if (adminApi.isPresent()) { if (adminApi.get().deleteProject(project)) { state.setSucceeded(replicateURI);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java index efac93d..54410f2 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
@@ -726,11 +726,12 @@ // by default push all projects ImmutableList<String> projects = config.getProjects(); - if (projects.isEmpty()) { + ImmutableList<String> excludeProjects = config.getExcludeProjects(); + if (projects.isEmpty() && excludeProjects.isEmpty()) { return true; } - boolean matches = new ReplicationFilter(projects).matches(project); + boolean matches = new ReplicationFilter(projects, excludeProjects).matches(project); if (!matches) { repLog.atFine().log( "Skipping replication of project %s; does not match filter", project.get()); @@ -908,7 +909,19 @@ return config.storeRefLog(); } - private static boolean matches(URIish uri, String urlMatch) { + String getUploadPack() { + return config.getUploadPack(); + } + + String getReceivePack() { + return config.getReceivePack(); + } + + String getGitPath() { + return config.getGitPath(); + } + + static boolean matches(URIish uri, String urlMatch) { if (urlMatch == null || urlMatch.equals("") || urlMatch.equals("*")) { return true; }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java index 977b23a..03ba914 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java
@@ -50,6 +50,7 @@ private final String remoteNameStyle; private final ImmutableList<String> urls; private final ImmutableList<String> projects; + private final ImmutableList<String> excludeProjects; private final ImmutableList<String> authGroupNames; private final RemoteConfig remoteConfig; private final int maxRetries; @@ -57,6 +58,9 @@ private final Supplier<Integer> pushBatchSize; private final ImmutableList<Pattern> excludedRefsPattern; private final boolean storeRefLog; + private final String uploadPack; + private final String receivePack; + private final String gitPath; private final UrlDistributionStrategy urlDistributionStrategy; protected DestinationConfiguration(RemoteConfig remoteConfig, Config cfg) { @@ -67,6 +71,7 @@ rescheduleDelay = Math.max(3, getInt(remoteConfig, cfg, "rescheduledelay", DEFAULT_RESCHEDULE_DELAY)); projects = ImmutableList.copyOf(cfg.getStringList("remote", name, "projects")); + excludeProjects = ImmutableList.copyOf(cfg.getStringList("remote", name, "excludeProjects")); adminUrls = ImmutableList.copyOf(cfg.getStringList("remote", name, "adminUrl")); retryDelay = Math.max(0, getInt(remoteConfig, cfg, "replicationretry", 1)); drainQueueAttempts = @@ -125,6 +130,9 @@ }); excludedRefsPattern = getExcludedRefsPattern(cfg, name); storeRefLog = cfg.getBoolean("remote", name, "storeRefLog", false); + uploadPack = cfg.getString("remote", name, "uploadpack"); + receivePack = cfg.getString("remote", name, "receivepack"); + gitPath = cfg.getString("remote", name, "gitPath"); urlDistributionStrategy = UrlDistributionStrategy.fromConfig( cfg.getString("remote", name, "urlDistributionStrategy")); @@ -173,6 +181,11 @@ } @Override + public ImmutableList<String> getExcludeProjects() { + return excludeProjects; + } + + @Override public ImmutableList<String> getAuthGroupNames() { return authGroupNames; } @@ -238,6 +251,18 @@ return storeRefLog; } + public String getUploadPack() { + return uploadPack; + } + + public String getReceivePack() { + return receivePack; + } + + public String getGitPath() { + return gitPath; + } + @Override public UrlDistributionStrategy getUrlDistributionStrategy() { return urlDistributionStrategy;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationsCollection.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationsCollection.java index 82f33d7..4896dcb 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationsCollection.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationsCollection.java
@@ -30,6 +30,7 @@ import com.google.common.eventbus.EventBus; import com.google.common.eventbus.Subscribe; import com.google.common.flogger.FluentLogger; +import com.google.gerrit.common.Nullable; import com.google.gerrit.entities.Project; import com.google.gerrit.server.git.WorkQueue; import com.google.inject.Inject; @@ -83,7 +84,10 @@ @Override public Multimap<Destination, URIish> getURIs( - Optional<String> remoteName, Project.NameKey projectName, FilterType filterType) { + Optional<String> remoteName, + Project.NameKey projectName, + FilterType filterType, + @Nullable String urlMatch) { if (getAll(filterType).isEmpty()) { return ImmutableMultimap.of(); } @@ -114,6 +118,7 @@ continue; } + boolean matchesConfigUrl = Destination.matches(uri, urlMatch); if (!isGerrit(uri) && !isGerritHttp(uri)) { String path = replaceName(uri.getPath(), projectName.get(), config.isSingleProjectMatch()); @@ -129,12 +134,14 @@ continue; } } - validUris.add(uri); - adminURLUsed = true; + if (matchesConfigUrl || Destination.matches(uri, urlMatch)) { + validUris.add(uri); + adminURLUsed = true; + } } if (!adminURLUsed) { - for (URIish uri : config.getURIs(projectName, "*")) { + for (URIish uri : config.getURIs(projectName, urlMatch)) { validUris.add(uri); } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ProjectRepairer.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ProjectRepairer.java new file mode 100644 index 0000000..5866569 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ProjectRepairer.java
@@ -0,0 +1,157 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog; + +import com.google.common.base.Strings; +import com.google.gerrit.entities.Project; +import com.google.gerrit.server.git.GitRepositoryManager; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.googlesource.gerrit.plugins.replication.api.ReplicationConfig; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.transport.URIish; +import org.eclipse.jgit.util.QuotedString; +import org.eclipse.jgit.util.io.StreamCopyThread; + +@Singleton +public class ProjectRepairer { + private final GitRepositoryManager gitManager; + private final ReplicationConfig replicationConfig; + + @Inject + ProjectRepairer(GitRepositoryManager gitManager, ReplicationConfig replicationConfig) { + this.gitManager = gitManager; + this.replicationConfig = replicationConfig; + } + + public boolean repair(Project.NameKey project, URIish uri, OutputStream out, boolean copyPacks) { + if (copyPacks && !copyPackTo(project, uri, out)) { + repLog.atSevere().log("Repair failed for %s on %s", project.get(), uri); + return false; + } + return true; + } + + public static boolean canCopy(URIish uri) { + return AdminApiFactory.isSSH(uri) && !AdminApiFactory.isGerrit(uri); + } + + private boolean copyPackTo(Project.NameKey project, URIish uri, OutputStream out) { + if (Strings.isNullOrEmpty(uri.getHost())) { + repLog.atSevere().log("Cannot repair %s: URI has no host: %s", project.get(), uri); + return false; + } + if (Strings.isNullOrEmpty(uri.getPath())) { + repLog.atSevere().log("Cannot repair %s: URI has no path: %s", project.get(), uri); + return false; + } + + Path packDir; + try (Repository repo = gitManager.openRepository(project)) { + packDir = repo.getDirectory().toPath().resolve("objects").resolve("pack"); + } catch (IOException e) { + repLog.atSevere().withCause(e).log("Cannot open repository %s for repair", project.get()); + return false; + } + + if (!Files.isDirectory(packDir)) { + repLog.atSevere().log("No objects/pack directory for project %s", project.get()); + return false; + } + + return copyInOrder(packDir, uri, out); + } + + private boolean copyInOrder(Path packDir, URIish uri, OutputStream out) { + try { + return copy(packDir, uri, out, "*.pack") == 0 + && copy(packDir, uri, out, "*.idx", "*.bitmap", "*.rev") == 0; + } catch (InterruptedException e) { + repLog.atWarning().withCause(e).log("Interrupted during copy to %s", uri); + return false; + } + } + + private int copy(Path src, URIish uri, OutputStream out, String... includes) + throws InterruptedException { + List<String> cmd = new ArrayList<>(); + cmd.add(replicationConfig.getRsyncPath()); + cmd.add("-avP"); + cmd.add("-e"); + cmd.add(buildSshTransport(uri)); + for (String inc : includes) { + cmd.add("--include=" + inc); + } + cmd.add("--exclude=*"); + cmd.add(src.toAbsolutePath().normalize() + "/"); + cmd.add(buildCopyDestination(uri)); + + repLog.atInfo().log("Running repair cmd: %s", String.join(" ", cmd)); + + ProcessBuilder pb = new ProcessBuilder(cmd); + pb.redirectErrorStream(true); + Process p; + try { + p = pb.start(); + } catch (IOException e) { + repLog.atWarning().withCause(e).log("Copy to %s failed", uri); + return -1; + } + + StreamCopyThread outStream = new StreamCopyThread(p.getInputStream(), out); + outStream.setName("copy-packs-output"); + outStream.start(); + try { + int code = p.waitFor(); + outStream.join(); + if (code != 0) { + repLog.atWarning().log("Copy to %s failed with exit code %d", uri, code); + } + return code; + } catch (InterruptedException e) { + p.destroyForcibly(); + outStream.halt(); + return -1; + } + } + + private static String buildCopyDestination(URIish uri) { + String host = uri.getHost(); + String path = uri.getPath(); + String remotePackPath = QuotedString.BOURNE.quote(path + "/objects/pack/"); + String user = uri.getUser(); + if (user != null && !user.isEmpty()) { + return user + "@" + host + ":" + remotePackPath; + } + return host + ":" + remotePackPath; + } + + private static String buildSshTransport(URIish uri) { + StringBuilder sb = new StringBuilder("ssh -o BatchMode=yes"); + int port = uri.getPort(); + if (port > 0) { + sb.append(" -p ").append(port); + } + return sb.toString(); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java index 030ba1c..0e04e1e 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
@@ -24,6 +24,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; +import com.google.common.base.Strings; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableSet; import com.google.common.collect.LinkedListMultimap; @@ -558,6 +559,7 @@ private PushResult pushVia(Repository git, Transport tn) throws IOException, PermissionBackendException { tn.applyConfig(config); + setUploadAndReceivePack(tn); tn.setCredentialsProvider(credentialsFactory.create(config.getName())); List<RemoteRefUpdate> todo = generateUpdates(git, tn); @@ -612,6 +614,24 @@ return result; } + private void setUploadAndReceivePack(Transport tn) { + String gitPath = pool.getGitPath(); + if (Strings.isNullOrEmpty(gitPath)) { + return; + } + int lastSlash = gitPath.lastIndexOf('/'); + if (lastSlash < 0) { + return; + } + String binDir = gitPath.substring(0, lastSlash + 1); + if (pool.getUploadPack() == null) { + tn.setOptionUploadPack(binDir + "git-upload-pack"); + } + if (pool.getReceivePack() == null) { + tn.setOptionReceivePack(binDir + "git-receive-pack"); + } + } + private static String refUpdatesForLogging(List<RemoteRefUpdate> refUpdates) { return refUpdates.stream().map(PushOne::refUpdateForLogging).collect(joining(", ")); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java index 6ed47d4..8d279d5 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java
@@ -90,11 +90,17 @@ return sb.toString(); } + public interface SshOutputCommand { + void writeStdOutSync(String message); + + void writeStdErrSync(String message); + } + public static class CommandProcessing implements PushResultProcessing { - private WeakReference<StartCommand> sshCommand; + private WeakReference<SshOutputCommand> sshCommand; private AtomicBoolean hasError = new AtomicBoolean(); - CommandProcessing(StartCommand sshCommand) { + CommandProcessing(SshOutputCommand sshCommand) { this.sshCommand = new WeakReference<>(sshCommand); } @@ -162,7 +168,7 @@ @Override public void writeStdOut(String message) { - StartCommand command = sshCommand.get(); + SshOutputCommand command = sshCommand.get(); if (command != null) { command.writeStdOutSync(message); } @@ -170,7 +176,7 @@ @Override public void writeStdErr(String message) { - StartCommand command = sshCommand.get(); + SshOutputCommand command = sshCommand.get(); if (command != null) { command.writeStdErrSync(message); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteConfiguration.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteConfiguration.java index 18dfca2..79bdbf3 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteConfiguration.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteConfiguration.java
@@ -66,6 +66,13 @@ ImmutableList<String> getProjects(); /** + * List of repositories that should be NOT replicated + * + * @return list of excluded project strings + */ + ImmutableList<String> getExcludeProjects(); + + /** * List of groups that should be used to access the repositories. * * @return list of group strings
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java index f96c157..2c9cb03 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java
@@ -16,6 +16,8 @@ import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog; +import com.google.common.base.Strings; +import com.google.gerrit.common.Nullable; import com.google.gerrit.entities.Project; import java.io.IOException; import java.io.OutputStream; @@ -26,18 +28,24 @@ private final SshHelper sshHelper; private URIish uri; + private final String git; RemoteSsh(SshHelper sshHelper, URIish uri) { + this(sshHelper, uri, null); + } + + RemoteSsh(SshHelper sshHelper, URIish uri, @Nullable String gitPath) { this.sshHelper = sshHelper; this.uri = uri; + this.git = Strings.isNullOrEmpty(gitPath) ? "git" : QuotedString.BOURNE.quote(gitPath); } @Override public boolean createProject(Project.NameKey project, String head) { String quotedPath = QuotedString.BOURNE.quote(uri.getPath()); - String cmd = "mkdir -p " + quotedPath + " && cd " + quotedPath + " && git init --bare"; + String cmd = "mkdir -p " + quotedPath + " && cd " + quotedPath + " && " + git + " init --bare"; if (head != null) { - cmd = cmd + " && git symbolic-ref HEAD " + QuotedString.BOURNE.quote(head); + cmd = cmd + " && " + git + " symbolic-ref HEAD " + QuotedString.BOURNE.quote(head); } OutputStream errStream = sshHelper.newErrorBufferStream(); try { @@ -79,7 +87,12 @@ public boolean updateHead(Project.NameKey project, String newHead) { String quotedPath = QuotedString.BOURNE.quote(uri.getPath()); String cmd = - "cd " + quotedPath + " && git symbolic-ref HEAD " + QuotedString.BOURNE.quote(newHead); + "cd " + + quotedPath + + " && " + + git + + " symbolic-ref HEAD " + + QuotedString.BOURNE.quote(newHead); OutputStream errStream = sshHelper.newErrorBufferStream(); try { sshHelper.executeRemoteSsh(uri, cmd, errStream);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RepairCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RepairCommand.java new file mode 100644 index 0000000..0371c9c --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RepairCommand.java
@@ -0,0 +1,156 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import com.google.gerrit.entities.Project; +import com.google.gerrit.exceptions.StorageException; +import com.google.gerrit.extensions.annotations.RequiresCapability; +import com.google.gerrit.server.project.ProjectCache; +import com.google.gerrit.sshd.CommandMetaData; +import com.google.gerrit.sshd.SshCommand; +import com.google.inject.Inject; +import com.googlesource.gerrit.plugins.replication.api.ReplicationConfig; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import org.eclipse.jgit.transport.URIish; +import org.kohsuke.args4j.Argument; +import org.kohsuke.args4j.Option; + +@RequiresCapability(StartReplicationCapability.START_REPLICATION) +@CommandMetaData(name = "repair", description = "Repair a project on replication destinations") +final class RepairCommand extends SshCommand implements PushResultProcessing.SshOutputCommand { + @Argument(index = 0, required = true, metaVar = "PROJECT", usage = "project name") + private String projectName; + + @Option( + name = "--url", + metaVar = "SUBSTRING", + usage = "substring URL must match (or * to match everything)") + private String urlMatch; + + @Option( + name = "--copy-packs", + usage = "rsync objects/pack files to SSH destinations before triggering replication") + private boolean copyPacks; + + @Option(name = "--full", usage = "run all supported repair actions (default)") + private boolean full; + + @Inject private ProjectCache projectCache; + @Inject private ReplicationDestinations destinations; + @Inject private ReplicationStarter replicationStarter; + @Inject private ProjectRepairer projectRepairer; + + private final Object outputLock = new Object(); + + @Override + protected void run() throws Failure { + Project.NameKey project = Project.nameKey(projectName); + try { + if (projectCache.get(project).isEmpty()) { + throw die("Project with name " + projectName + " not found."); + } + } catch (StorageException e) { + throw die(e); + } + + if (!copyPacks) { + full = true; + } + + Set<URIish> failedUris = repair(project); + if (!failedUris.isEmpty()) { + throw new UnloggedFailure(1, "Repair failed for " + failedUris.size() + " destination(s)"); + } + } + + private Set<URIish> repair(Project.NameKey project) throws Failure { + Set<URIish> copyTargets = new HashSet<>(); + Collection<URIish> destUris = + destinations + .getURIs(Optional.empty(), project, ReplicationConfig.FilterType.ALL, urlMatch) + .values(); + for (URIish uri : destUris) { + if (!ProjectRepairer.canCopy(uri)) { + writeStdErrSync( + "Warning: skipping " + uri + " as copy-packs only supports plain SSH destinations"); + continue; + } + copyTargets.add(uri); + } + + if (copyTargets.isEmpty()) { + throw die("No matching destinations found"); + } + + Set<URIish> failedUris = new HashSet<>(); + OutputStream out = getFlushingOutputStream(); + for (URIish uri : copyTargets) { + writeStdOutSync("\nRepairing " + uri + " ..."); + if (projectRepairer.repair(project, uri, out, full || copyPacks)) { + writeStdOutSync( + "\nRunning replication start for " + project.get() + " to " + uri.toString() + " ..."); + replicationStarter.start( + uri.toString(), + Set.of(), + new ReplicationFilter(List.of(project.get()), Collections.emptyList()), + /* now= */ true, + /* wait= */ true, + this); + } else { + failedUris.add(uri); + } + } + return failedUris; + } + + private OutputStream getFlushingOutputStream() { + return new OutputStream() { + @Override + public void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); + out.flush(); + } + + @Override + public void write(int b) throws IOException { + out.write(b); + out.flush(); + } + }; + } + + @Override + public void writeStdOutSync(String message) { + synchronized (outputLock) { + stdout.println(message); + stdout.flush(); + } + } + + @Override + public void writeStdErrSync(String message) { + synchronized (outputLock) { + stderr.println(message); + stderr.flush(); + } + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfigImpl.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfigImpl.java index 8f9b805..f625232 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfigImpl.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfigImpl.java
@@ -28,6 +28,7 @@ public class ReplicationConfigImpl implements ReplicationConfig { private static final int DEFAULT_SSH_CONNECTION_TIMEOUT_MS = 2 * 60 * 1000; // 2 minutes + private static final String DEFAULT_RSYNC_PATH = "rsync"; private final SitePaths site; private final MergedConfigResource configResource; @@ -145,4 +146,10 @@ public int getSshCommandTimeout() { return sshCommandTimeout; } + + @Override + public String getRsyncPath() { + String rsyncPath = getConfig().getString("replication", null, "rsyncPath"); + return Strings.isNullOrEmpty(rsyncPath) ? DEFAULT_RSYNC_PATH : rsyncPath; + } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationDestinations.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationDestinations.java index bcc07e5..91ced84 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationDestinations.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationDestinations.java
@@ -15,6 +15,7 @@ package com.googlesource.gerrit.plugins.replication; import com.google.common.collect.Multimap; +import com.google.gerrit.common.Nullable; import com.google.gerrit.entities.Project; import com.google.gerrit.server.git.WorkQueue; import com.googlesource.gerrit.plugins.replication.api.ReplicationConfig.FilterType; @@ -32,10 +33,19 @@ * @param remoteName name of the replication end or empty if selecting all ends. * @param projectName name of the project * @param filterType type of filter criteria for selecting projects + * @param urlMatch optional substring filter on configuration or expanded URLs; null matches all * @return the multi-map of destinations and the associated replication URIs */ Multimap<Destination, URIish> getURIs( - Optional<String> remoteName, Project.NameKey projectName, FilterType filterType); + Optional<String> remoteName, + Project.NameKey projectName, + FilterType filterType, + @Nullable String urlMatch); + + default Multimap<Destination, URIish> getURIs( + Optional<String> remoteName, Project.NameKey projectName, FilterType filterType) { + return getURIs(remoteName, projectName, filterType, null); + } /** * List of currently active replication destinations.
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFilter.java index 28f2fba..20192ed 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFilter.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFilter.java
@@ -18,6 +18,7 @@ import com.google.gerrit.entities.Project; import java.util.Collections; import java.util.List; +import java.util.Objects; public class ReplicationFilter { public enum PatternType { @@ -27,7 +28,7 @@ } public static ReplicationFilter all() { - return new ReplicationFilter(Collections.<String>emptyList()); + return new ReplicationFilter(null, null); } public static PatternType getPatternType(String pattern) { @@ -41,12 +42,18 @@ } private final List<String> projectPatterns; + private final List<String> excludePatterns; - public ReplicationFilter(List<String> patterns) { - projectPatterns = patterns; + public ReplicationFilter(List<String> includePatterns, List<String> excludePatterns) { + projectPatterns = Objects.requireNonNullElse(includePatterns, Collections.emptyList()); + this.excludePatterns = Objects.requireNonNullElse(excludePatterns, Collections.emptyList()); } public boolean matches(Project.NameKey name) { + return matchesProjectPatterns(name) && !matchesExcludePatterns(name); + } + + public boolean matchesProjectPatterns(Project.NameKey name) { if (projectPatterns.isEmpty()) { return true; } @@ -60,6 +67,20 @@ return false; } + public boolean matchesExcludePatterns(Project.NameKey name) { + if (excludePatterns.isEmpty()) { + return false; + } + String projectName = name.get(); + + for (String pattern : excludePatterns) { + if (matchesPattern(projectName, pattern)) { + return true; + } + } + return false; + } + private boolean matchesPattern(String projectName, String pattern) { boolean match = false; switch (getPatternType(pattern)) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java index e3e4021..bae633d 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
@@ -106,6 +106,8 @@ bind(ReplicationQueue.class).in(Scopes.SINGLETON); bind(ReplicationDestinations.class).to(DestinationsCollection.class); + bind(ProjectRepairer.class).in(Scopes.SINGLETON); + install(new FactoryModuleBuilder().build(Destination.Factory.class)); install(new FactoryModuleBuilder().build(ProjectDeletionState.Factory.class)); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStarter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStarter.java new file mode 100644 index 0000000..b7baab9 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStarter.java
@@ -0,0 +1,77 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import com.google.gerrit.common.Nullable; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.googlesource.gerrit.plugins.replication.PushResultProcessing.CommandProcessing; +import com.googlesource.gerrit.plugins.replication.PushResultProcessing.SshOutputCommand; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +@Singleton +class ReplicationStarter { + private final PushAll.Factory pushFactory; + private final ReplicationStateLogger stateLog; + + @Inject + ReplicationStarter(PushAll.Factory pushFactory, ReplicationStateLogger stateLog) { + this.pushFactory = pushFactory; + this.stateLog = stateLog; + } + + void start( + @Nullable String urlMatch, + Set<String> remotesToConsider, + ReplicationFilter filter, + boolean now, + boolean wait, + SshOutputCommand sink) { + ReplicationState state = new ReplicationState(new CommandProcessing(sink)); + + Future<?> future = + pushFactory + .create(urlMatch, remotesToConsider, filter, state, now) + .schedule(0, TimeUnit.SECONDS); + + if (wait) { + if (future != null) { + try { + future.get(); + } catch (InterruptedException e) { + stateLog.error( + "Thread was interrupted while waiting for PushAll operation to finish", e, state); + return; + } catch (ExecutionException e) { + stateLog.error("An exception was thrown in PushAll operation", e, state); + return; + } + } + + if (state.hasPushTask()) { + try { + state.waitForReplication(); + } catch (InterruptedException e) { + sink.writeStdErrSync("We are interrupted while waiting replication to complete"); + } + } else { + sink.writeStdOutSync("Nothing to replicate"); + } + } + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java index b60cc57..7fc4acb 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
@@ -226,12 +226,13 @@ } catch (NotDirectoryException e) { return Stream.of(path); } catch (Exception e) { - String message = "Error while walking directory %s"; + String message = "Error while walking directory"; if (isMultiPrimary() && e instanceof NoSuchFileException) { logger.atFine().log( - message + " (expected regularly with multi-primaries and distributor enabled)", path); + "%s %s (expected regularly with multi-primaries and distributor enabled)", + message, path); } else { - logger.atSevere().withCause(e).log(message, path); + logger.atSevere().withCause(e).log("%s %s", message, path); } return Stream.empty(); } @@ -401,15 +402,14 @@ logger.atFine().log("DELETE %s %s", running, updateLog()); Files.delete(running); } catch (IOException e) { - String message = "Error while deleting task %s"; + String message = "Error while deleting task"; if (isMultiPrimary() && e instanceof NoSuchFileException) { logger.atFine().log( - message - + " (expected after recovery from another node's startup with multi-primaries and" + "%s %s (expected after recovery from another node's startup with multi-primaries and" + " distributor enabled)", - taskKey); + message, taskKey); } else { - logger.atSevere().withCause(e).log(message, taskKey); + logger.atSevere().withCause(e).log("%s %s", message, taskKey); } } } @@ -421,13 +421,13 @@ Files.move(from, to, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); return true; } catch (IOException e) { - String message = "Error while renaming task %s"; + String message = "Error while renaming task"; if (isMultiPrimary() && e instanceof NoSuchFileException) { logger.atFine().log( - message + " (expected regularly with multi-primaries and distributor enabled)", - taskKey); + "%s %s (expected regularly with multi-primaries and distributor enabled)", + message, taskKey); } else { - logger.atSevere().withCause(e).log(message, taskKey); + logger.atSevere().withCause(e).log("%s %s", message, taskKey); } return false; }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/SshModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/SshModule.java index a66cce6..78410f8 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/SshModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/SshModule.java
@@ -27,6 +27,7 @@ @Override protected void configureCommands() { command(StartCommand.class); + command(RepairCommand.class); command(ListCommand.class); } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java index 92c619c..2c084b1 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java
@@ -18,14 +18,11 @@ import com.google.gerrit.sshd.CommandMetaData; import com.google.gerrit.sshd.SshCommand; import com.google.inject.Inject; -import com.googlesource.gerrit.plugins.replication.PushResultProcessing.CommandProcessing; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; import org.kohsuke.args4j.Argument; import org.kohsuke.args4j.Option; @@ -33,9 +30,7 @@ @CommandMetaData( name = "start", description = "Start replication for specific project or all projects") -final class StartCommand extends SshCommand { - @Inject private ReplicationStateLogger stateLog; - +final class StartCommand extends SshCommand implements PushResultProcessing.SshOutputCommand { @Option(name = "--all", usage = "push all known projects") private boolean all; @@ -58,7 +53,7 @@ @Argument(index = 0, multiValued = true, metaVar = "PATTERN", usage = "project name pattern") private List<String> projectPatterns = new ArrayList<>(2); - @Inject private PushAll.Factory pushFactory; + @Inject private ReplicationStarter replicationStarter; private final Object lock = new Object(); @@ -68,47 +63,15 @@ throw new UnloggedFailure(1, "error: cannot combine --all and PROJECT"); } - ReplicationState state = new ReplicationState(new CommandProcessing(this)); + ReplicationFilter projectFilter = + all + ? ReplicationFilter.all() + : new ReplicationFilter(projectPatterns, Collections.emptyList()); - ReplicationFilter projectFilter; - - if (all) { - projectFilter = ReplicationFilter.all(); - } else { - projectFilter = new ReplicationFilter(projectPatterns); - } - - Future<?> future = - pushFactory - .create(urlMatch, remotesToConsider, projectFilter, state, now) - .schedule(0, TimeUnit.SECONDS); - - if (wait) { - if (future != null) { - try { - future.get(); - } catch (InterruptedException e) { - stateLog.error( - "Thread was interrupted while waiting for PushAll operation to finish", e, state); - return; - } catch (ExecutionException e) { - stateLog.error("An exception was thrown in PushAll operation", e, state); - return; - } - } - - if (state.hasPushTask()) { - try { - state.waitForReplication(); - } catch (InterruptedException e) { - writeStdErrSync("We are interrupted while waiting replication to complete"); - } - } else { - writeStdOutSync("Nothing to replicate"); - } - } + replicationStarter.start(urlMatch, remotesToConsider, projectFilter, now, wait, this); } + @Override public void writeStdOutSync(String message) { if (wait) { synchronized (lock) { @@ -118,6 +81,7 @@ } } + @Override public void writeStdErrSync(String message) { if (wait) { synchronized (lock) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java index fdbd5e7..ce6370c 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java
@@ -23,10 +23,12 @@ import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; import java.util.Optional; +import org.eclipse.jgit.transport.RemoteConfig; import org.eclipse.jgit.transport.URIish; public class UpdateHeadTask implements Runnable { private final DynamicItem<AdminApiFactory> adminApiFactory; + private final RemoteConfig remoteConfig; private final int id; private final URIish replicateURI; private final Project.NameKey project; @@ -39,11 +41,13 @@ @Inject UpdateHeadTask( DynamicItem<AdminApiFactory> adminApiFactory, + RemoteConfig remoteConfig, IdGenerator ig, @Assisted URIish replicateURI, @Assisted Project.NameKey project, @Assisted String newHead) { this.adminApiFactory = adminApiFactory; + this.remoteConfig = remoteConfig; this.id = ig.next(); this.replicateURI = replicateURI; this.project = project; @@ -52,7 +56,8 @@ @Override public void run() { - Optional<AdminApi> adminApi = adminApiFactory.get().create(replicateURI); + Optional<AdminApi> adminApi = + adminApiFactory.get().create(replicateURI, remoteConfig.getName()); if (adminApi.isPresent()) { adminApi.get().updateHead(project, newHead); return;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/api/ReplicationConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/api/ReplicationConfig.java index d5ccb02..1714c1a 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/api/ReplicationConfig.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/api/ReplicationConfig.java
@@ -89,6 +89,14 @@ int getSshCommandTimeout(); /** + * Path of the {@code rsync} binary on the host running Gerrit, used by the {@code replication + * repair} command. + * + * @return the rsync binary path, or {@code "rsync"} to resolve via {@code PATH}. + */ + String getRsyncPath(); + + /** * Current logical version string of the current configuration loaded in memory, depending on the * actual implementation of the configuration on the persistent storage. *
diff --git a/src/main/resources/Documentation/cmd-repair.md b/src/main/resources/Documentation/cmd-repair.md new file mode 100644 index 0000000..3f95907 --- /dev/null +++ b/src/main/resources/Documentation/cmd-repair.md
@@ -0,0 +1,100 @@ +@PLUGIN@ repair +=============== + +NAME +---- +@PLUGIN@ repair - Repair a project on replication destinations + +SYNOPSIS +-------- + +```console +ssh -p @SSH_PORT@ @SSH_HOST@ @PLUGIN@ repair + [--url <PATTERN>] + [--full | --copy-packs] + <PROJECT> +``` + +DESCRIPTION +----------- +Repairs a project on its replication destinations, then runs a +`@PLUGIN@ start` for that project (with `--now --wait`) so +any refs that diverged during the repair are replicated. The command +blocks until replication finishes. + +If no repair action flag is supplied, `--full` is assumed. + +REQUIREMENTS +------------ +The Gerrit runtime user must have `ssh` on `PATH`, plus `rsync` either on +`PATH` or pointed at via [`replication.rsyncPath`](config.md#replication.rsyncPath). + +ACCESS +------ +Caller must be a member of the privileged 'Administrators' group, +or have been granted the 'Start Replication' plugin-owned capability. + +SCRIPTING +--------- +This command is intended to be used to repair repositories on the mirror. +Exit status is non-zero if the project is missing, or if the repair fails +for some reason. + +OPTIONS +------- + +`--url <PATTERN>` +: Restrict both the repair action(s) and the follow-up replication to +replication destinations whose configuration URL contains the substring +`PATTERN`, or whose expanded project URL contains `PATTERN`. + +`--full` +: Run every supported repair action. + +`--copy-packs` +: rsync regular files in `objects/pack/` whose names end with `.pack`, +`.idx`, `.bitmap`, or `.rev` to each matching destination. For each +remote, [remote.NAME.adminUrl](config.md#remote.NAME.adminUrl) is preferred +when set (same as repository creation); otherwise +[remote.NAME.url](config.md#remote.NAME.url) is used. Only plain SSH +destinations are eligible (for example `user@host:/path/to/repo.git`). +Destinations whose URL uses `gerrit+ssh`, HTTP(S), or a local path are +skipped. + +`PROJECT` +: Exact Gerrit project name. + +EXAMPLES +-------- +Run every supported repair action for `tools/gerrit` against every +eligible destination, then replicate refs (`--full` is implied since no +action flag is given): + +```console + $ ssh -p @SSH_PORT@ @SSH_HOST@ @PLUGIN@ repair tools/gerrit +``` + +Equivalent, with `--full` stated explicitly: + +```console + $ ssh -p @SSH_PORT@ @SSH_HOST@ @PLUGIN@ repair --full tools/gerrit +``` + +Only copy packs (no other repair actions, even if more are added later): + +```console + $ ssh -p @SSH_PORT@ @SSH_HOST@ @PLUGIN@ repair --copy-packs tools/gerrit +``` + +Repair only against destinations whose URL mentions `replica1`: + +```console + $ ssh -p @SSH_PORT@ @SSH_HOST@ @PLUGIN@ repair --url replica1 tools/gerrit +``` + +SEE ALSO +-------- + +* [@PLUGIN@ start](cmd-start.md) +* [Replication Configuration](config.md) +* [Access Control](../../../Documentation/access-control.html)
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index 3fa544b..fd3318a 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md
@@ -261,6 +261,14 @@ When not set, defaults to the plugin's data directory. +replication.rsyncPath +: Path to the `rsync` binary on the host running Gerrit, used by the + `@PLUGIN@ repair --copy-packs` command when transferring pack files + to SSH destinations. Set this when the Gerrit runtime user's `PATH` + does not contain `rsync`, or to pin a specific build. + + Default: `rsync` (resolved via the Gerrit runtime user's `PATH`) + remote.NAME.url : Address of the remote server to push to. Multiple URLs may be specified within a single remote block, listing different @@ -342,7 +350,9 @@ : Path of the `git-receive-pack` executable on the remote system, if using the SSH transport. - Defaults to `git-receive-pack`. + If not set and `remote.NAME.gitPath` is configured, defaults to + `git-receive-pack` in the directory of `gitPath`. Otherwise defaults + to `git-receive-pack`. remote.NAME.storeRefLog : `true` if the remote repositories should be enabled for storing @@ -359,7 +369,9 @@ : Path of the `git-upload-pack` executable on the remote system, if using the SSH transport. - Defaults to `git-upload-pack`. + If not set and `remote.NAME.gitPath` is configured, defaults to + `git-upload-pack` in the directory of `gitPath`. Otherwise defaults + to `git-upload-pack`. remote.NAME.push : Standard Git refspec denoting what should be replicated. @@ -500,6 +512,21 @@ By default, true, missing repositories are created. +remote.NAME.gitPath +: Absolute path to the `git` binary on this SSH destination, used when + creating a missing repository or updating its HEAD. Set this when the + non-interactive SSH session on the remote host does not have `git` + in its `PATH`. + + When set, the directory of `gitPath` is also used as the default + location for `git-upload-pack` and `git-receive-pack`, unless + `remote.NAME.uploadpack` or `remote.NAME.receivepack` are configured + explicitly. + + Only applies to SSH destinations. + + Default: `git` (resolved via the remote's `PATH`) + remote.NAME.replicatePermissions : If true, permissions-only projects and the refs/meta/config branch will also be replicated to the remote site. These @@ -593,6 +620,21 @@ By default, replicates without matching, i.e. replicates everything to all remotes. +remote.NAME.excludeProjects +: Specifies which repositories should NOT be replicated to the + remote. It can be provided more than once, and supports the same + formats as `projects`, and may be specified in combination with + `projects`. + + When both `projects` and `excludeProjects` are configured, a + project must match at least one `projects` pattern to be eligible + for replication, and must not match any `excludeProjects` pattern. + If a project matches both a `projects` pattern and an + `excludeProjects` pattern, it is excluded from replication. + + By default, replicates without matching, i.e. replicates + everything to all remotes. + <a name="remote.NAME.slowLatencyThreshold">remote.NAME.slowLatencyThreshold</a> : the time duration after which the replication of a project to this destination will be considered "slow". A slow project replication
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java index 8b9ea0e..d436f42 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java
@@ -283,7 +283,11 @@ .getSysInjector() .getInstance(PushAll.Factory.class) .create( - null, Set.of(), new ReplicationFilter(Arrays.asList(project.get())), state, false) + null, + Set.of(), + new ReplicationFilter(Arrays.asList(project.get()), null), + state, + false) .schedule(0, TimeUnit.SECONDS); future.get(); @@ -304,7 +308,11 @@ .getSysInjector() .getInstance(PushAll.Factory.class) .create( - null, Set.of(), new ReplicationFilter(Arrays.asList(project.get())), state, false) + null, + Set.of(), + new ReplicationFilter(Arrays.asList(project.get()), null), + state, + false) .schedule(0, TimeUnit.SECONDS); CountDownLatch latch = new CountDownLatch(1); @@ -616,6 +624,95 @@ } @Test + public void shouldReplicateWhenProjectNotExcluded() throws Exception { + Project.NameKey targetProject = createTestProject(project + "replica"); + + setReplicationDestination("foo", "replica", ALL_PROJECTS); + config.setString("remote", "foo", "excludeProjects", "excluded-prj"); + config.save(); + reloadConfig(); + + Result pushResult = createChange(); + RevCommit sourceCommit = pushResult.getCommit(); + String sourceRef = pushResult.getPatchSet().refName(); + + try (Repository repo = repoManager.openRepository(targetProject)) { + waitUntil(() -> checkedGetRef(repo, sourceRef) != null); + + Ref targetBranchRef = getRef(repo, sourceRef); + assertThat(targetBranchRef).isNotNull(); + assertThat(targetBranchRef.getObjectId()).isEqualTo(sourceCommit.getId()); + } + } + + @Test + public void shouldNotReplicateProjectMatchingExcludeProjects() throws Exception { + Project.NameKey prj1 = createTestProject("excludePrj1"); + Project.NameKey prj2 = createTestProject("excludePrj2"); + Project.NameKey prj3 = createTestProject("replicatePrj1"); + Project.NameKey prj4 = createTestProject("replicatePrj2"); + + Project.NameKey targetPrj1 = createTestProject(prj1.get() + "replica"); + Project.NameKey targetPrj2 = createTestProject(prj2.get() + "replica"); + Project.NameKey targetPrj3 = createTestProject(prj3.get() + "replica"); + Project.NameKey targetPrj4 = createTestProject(prj4.get() + "replica"); + + setReplicationDestination("foo", "replica", ALL_PROJECTS); + config.setString("remote", "foo", "excludeProjects", "^excludePrj.*"); + config.save(); + reloadConfig(); + + String newRef = "refs/heads/testBranch"; + createNewBranchWithoutPush(prj1, newRef); + createNewBranchWithoutPush(prj2, newRef); + ObjectId replicateTip1 = createNewBranchWithoutPush(prj3, newRef); + ObjectId replicateTip2 = createNewBranchWithoutPush(prj4, newRef); + + ReplicationQueue replicationQueue = plugin.getSysInjector().getInstance(ReplicationQueue.class); + ReplicationState state = new ReplicationState(NO_OP); + replicationQueue.scheduleFullSync(prj1, null, Set.of("foo"), state, true); + replicationQueue.scheduleFullSync(prj2, null, Set.of("foo"), state, true); + replicationQueue.scheduleFullSync(prj3, null, Set.of("foo"), state, true); + replicationQueue.scheduleFullSync(prj4, null, Set.of("foo"), state, true); + + try (Repository excludeRepo1 = repoManager.openRepository(targetPrj1); + Repository excludeRepo2 = repoManager.openRepository(targetPrj2); + Repository replicateRepo1 = repoManager.openRepository(targetPrj3); + Repository replicateRepo2 = repoManager.openRepository(targetPrj4)) { + waitUntil( + () -> + checkedGetRef(replicateRepo1, newRef) != null + && checkedGetRef(replicateRepo2, newRef) != null + && checkedGetRef(excludeRepo1, newRef) == null + && checkedGetRef(excludeRepo2, newRef) == null); + + assertThat(getRef(replicateRepo1, newRef).getObjectId()).isEqualTo(replicateTip1); + assertThat(getRef(replicateRepo2, newRef).getObjectId()).isEqualTo(replicateTip2); + assertThat(getRef(excludeRepo1, newRef)).isNull(); + assertThat(getRef(excludeRepo2, newRef)).isNull(); + } + } + + @Test + public void shouldNotReplicateProjectListedInProjectsAndExcludeProjects() throws Exception { + Project.NameKey targetProject = createTestProject(project + "replica"); + + setReplicationDestination("foo", "replica", Optional.of(project.get())); + config.setString("remote", "foo", "excludeProjects", project.get()); + config.save(); + reloadConfig(); + + Result pushResult = createChange(); + String sourceRef = pushResult.getPatchSet().refName(); + + try (Repository repo = repoManager.openRepository(targetProject)) { + assertThrows( + InterruptedException.class, + () -> waitUntil(() -> checkedGetRef(repo, sourceRef) != null)); + } + } + + @Test public void shouldNotReplicateToNonMatchingRemote() throws Exception { Project.NameKey targetProject = createTestProject(project + "replica"); @@ -671,4 +768,25 @@ return update.getNewObjectId(); } } + + private ObjectId createNewBranchWithoutPush(Project.NameKey projectKey, String newBranch) + throws Exception { + return createNewBranchWithoutPush(projectKey, "refs/heads/master", newBranch); + } + + private ObjectId createNewBranchWithoutPush( + Project.NameKey projectKey, String fromBranch, String newBranch) throws Exception { + try (Repository repo = repoManager.openRepository(projectKey); + RevWalk walk = new RevWalk(repo)) { + Ref ref = repo.exactRef(fromBranch); + RevCommit tip = null; + if (ref != null) { + tip = walk.parseCommit(ref.getObjectId()); + } + RefUpdate update = repo.updateRef(newBranch); + update.setNewObjectId(tip); + update.update(walk); + return update.getNewObjectId(); + } + } }