Merge branch 'stable-2.14' into stable-2.15
* stable-2.14:
Bazel: Add fixes for --incompatible_load_java_rules_from_bzl
Change-Id: I73e2b89362f0e29e2e61547e33c6c36778758168
diff --git a/BUILD b/BUILD
index f66fdd8..0a54c3e 100644
--- a/BUILD
+++ b/BUILD
@@ -1,6 +1,6 @@
load("@rules_java//java:defs.bzl", "java_library")
load("//tools/bzl:junit.bzl", "junit_tests")
-load("//tools/bzl:plugin.bzl", "gerrit_plugin")
+load("//tools/bzl:plugin.bzl", "PLUGIN_DEPS", "PLUGIN_TEST_DEPS", "gerrit_plugin")
gerrit_plugin(
name = "replication",
@@ -24,11 +24,9 @@
srcs = glob(["src/test/java/**/*Test.java"]),
tags = ["replication"],
visibility = ["//visibility:public"],
- deps = [
+ deps = PLUGIN_TEST_DEPS + PLUGIN_DEPS + [
":replication__plugin",
":replication_util",
- "//gerrit-acceptance-framework:lib",
- "//gerrit-plugin-api:lib",
],
)
@@ -39,9 +37,7 @@
["src/test/java/**/*.java"],
exclude = ["src/test/java/**/*Test.java"],
),
- deps = [
+ deps = PLUGIN_TEST_DEPS + PLUGIN_DEPS + [
":replication__plugin",
- "//gerrit-acceptance-framework:lib",
- "//gerrit-plugin-api:lib",
],
)
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
index 8b6b8fc..5d6e409 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
@@ -110,4 +110,14 @@
public synchronized void startup(WorkQueue workQueue) {
currentConfig.startup(workQueue);
}
+
+ @Override
+ public synchronized int getSshConnectionTimeout() {
+ return currentConfig.getSshConnectionTimeout();
+ }
+
+ @Override
+ public synchronized int getSshCommandTimeout() {
+ return currentConfig.getSshCommandTimeout();
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
index b2dd382..0a06093 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
@@ -15,6 +15,8 @@
package com.googlesource.gerrit.plugins.replication;
import static com.googlesource.gerrit.plugins.replication.PushResultProcessing.resolveNodeName;
+import static org.eclipse.jgit.transport.RemoteRefUpdate.Status.NON_EXISTING;
+import static org.eclipse.jgit.transport.RemoteRefUpdate.Status.REJECTED_OTHER_REASON;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
@@ -24,8 +26,10 @@
import com.google.common.collect.Lists;
import com.google.gerrit.common.EventDispatcher;
import com.google.gerrit.common.data.GroupReference;
+import com.google.gerrit.extensions.client.ProjectState;
import com.google.gerrit.extensions.config.FactoryModule;
import com.google.gerrit.extensions.registration.DynamicItem;
+import com.google.gerrit.extensions.restapi.AuthException;
import com.google.gerrit.reviewdb.client.AccountGroup;
import com.google.gerrit.reviewdb.client.Branch;
import com.google.gerrit.reviewdb.client.Project;
@@ -41,6 +45,10 @@
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.PerThreadRequestScope;
import com.google.gerrit.server.git.WorkQueue;
+import com.google.gerrit.server.permissions.PermissionBackend;
+import com.google.gerrit.server.permissions.PermissionBackendException;
+import com.google.gerrit.server.permissions.ProjectPermission;
+import com.google.gerrit.server.permissions.RefPermission;
import com.google.gerrit.server.project.NoSuchProjectException;
import com.google.gerrit.server.project.PerRequestProjectControlCache;
import com.google.gerrit.server.project.ProjectControl;
@@ -50,20 +58,26 @@
import com.google.inject.Provides;
import com.google.inject.assistedinject.FactoryModuleBuilder;
import com.google.inject.servlet.RequestScoped;
+import com.googlesource.gerrit.plugins.replication.ReplicationState.RefPushResult;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.Callable;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
import org.apache.commons.io.FilenameUtils;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.RefSpec;
import org.eclipse.jgit.transport.RemoteConfig;
+import org.eclipse.jgit.transport.RemoteRefUpdate;
import org.eclipse.jgit.transport.URIish;
import org.slf4j.Logger;
@@ -76,7 +90,8 @@
private final PushOne.Factory opFactory;
private final ProjectControl.Factory projectControlFactory;
private final GitRepositoryManager gitManager;
- private volatile WorkQueue.Executor pool;
+ private final PermissionBackend permissionBackend;
+ private volatile ScheduledExecutorService pool;
private final PerThreadRequestScope.Scoper threadScoper;
private final DestinationConfiguration config;
private final DynamicItem<EventDispatcher> eventDispatcher;
@@ -103,6 +118,7 @@
RemoteSiteUser.Factory replicationUserFactory,
PluginUser pluginUser,
GitRepositoryManager gitRepositoryManager,
+ PermissionBackend permissionBackend,
GroupBackend groupBackend,
ReplicationStateListener stateLog,
GroupIncludeCache groupIncludeCache,
@@ -110,9 +126,10 @@
config = cfg;
this.eventDispatcher = eventDispatcher;
gitManager = gitRepositoryManager;
+ this.permissionBackend = permissionBackend;
this.stateLog = stateLog;
- final CurrentUser remoteUser;
+ CurrentUser remoteUser;
if (!cfg.getAuthGroupNames().isEmpty()) {
ImmutableSet.Builder<AccountGroup.UUID> builder = ImmutableSet.builder();
for (String name : cfg.getAuthGroupNames()) {
@@ -200,33 +217,74 @@
public int shutdown() {
int cnt = 0;
if (pool != null) {
- for (Runnable r : pool.getQueue()) {
- repLog.warn("Cancelling replication event {}", r);
- }
+ repLog.warn("Cancelling replication events");
+
+ foreachPushOp(
+ pending,
+ push -> {
+ push.cancel();
+ return null;
+ });
+ pending.clear();
+ foreachPushOp(
+ inFlight,
+ push -> {
+ push.setCanceledWhileRunning();
+ return null;
+ });
+ inFlight.clear();
cnt = pool.shutdownNow().size();
- pool.unregisterWorkQueue();
pool = null;
}
return cnt;
}
- private boolean shouldReplicate(ProjectControl projectControl) {
- return projectControl.isReadable()
- && (!projectControl.isHidden() || config.replicateHiddenProjects());
+ private void foreachPushOp(Map<URIish, PushOne> opsMap, Function<PushOne, Void> pushOneFunction) {
+ for (PushOne pushOne : ImmutableList.copyOf(opsMap.values())) {
+ pushOneFunction.apply(pushOne);
+ }
+ }
+
+ private boolean shouldReplicate(ProjectControl ctl) throws PermissionBackendException {
+ if (!config.replicateHiddenProjects() && ctl.getProject().getState() == ProjectState.HIDDEN) {
+ return false;
+ }
+ try {
+ permissionBackend
+ .user(ctl.getUser())
+ .project(ctl.getProject().getNameKey())
+ .check(ProjectPermission.ACCESS);
+ return true;
+ } catch (AuthException e) {
+ return false;
+ }
}
private boolean shouldReplicate(
- final Project.NameKey project, final String ref, ReplicationState... states) {
+ final Project.NameKey project, String ref, ReplicationState... states) {
try {
return threadScoper
.scope(
new Callable<Boolean>() {
@Override
- public Boolean call() throws NoSuchProjectException {
+ public Boolean call() throws NoSuchProjectException, PermissionBackendException {
ProjectControl projectControl = controlFor(project);
- return shouldReplicate(projectControl)
- && (PushOne.ALL_REFS.equals(ref)
- || projectControl.controlForRef(ref).isVisible());
+ if (!shouldReplicate(projectControl)) {
+ return false;
+ }
+ if (PushOne.ALL_REFS.equals(ref)) {
+ return true;
+ }
+ try {
+ permissionBackend
+ .user(projectControl.getUser())
+ .project(project)
+ .ref(ref)
+ .check(RefPermission.READ);
+ } catch (AuthException e) {
+ return false;
+ }
+ return true;
}
})
.call();
@@ -239,13 +297,13 @@
return false;
}
- private boolean shouldReplicate(final Project.NameKey project, ReplicationState... states) {
+ private boolean shouldReplicate(Project.NameKey project, ReplicationState... states) {
try {
return threadScoper
.scope(
new Callable<Boolean>() {
@Override
- public Boolean call() throws NoSuchProjectException {
+ public Boolean call() throws NoSuchProjectException, PermissionBackendException {
return shouldReplicate(controlFor(project));
}
})
@@ -260,6 +318,11 @@
}
void schedule(Project.NameKey project, String ref, URIish uri, ReplicationState state) {
+ schedule(project, ref, uri, state, false);
+ }
+
+ void schedule(
+ Project.NameKey project, String ref, URIish uri, ReplicationState state, boolean now) {
repLog.info("scheduling replication {}:{} => {}", project, ref, uri);
if (!shouldReplicate(project, ref, state)) {
return;
@@ -268,7 +331,7 @@
if (!config.replicatePermissions()) {
PushOne e;
synchronized (stateLock) {
- e = pending.get(uri);
+ e = getPendingPush(uri);
}
if (e == null) {
try (Repository git = gitManager.openRepository(project)) {
@@ -291,12 +354,14 @@
}
synchronized (stateLock) {
- PushOne e = pending.get(uri);
+ PushOne e = getPendingPush(uri);
if (e == null) {
e = opFactory.create(project, uri);
addRef(e, ref);
e.addState(ref, state);
- pool.schedule(e, config.getDelay(), TimeUnit.SECONDS);
+ @SuppressWarnings("unused")
+ ScheduledFuture<?> ignored =
+ pool.schedule(e, now ? 0 : config.getDelay(), TimeUnit.SECONDS);
pending.put(uri, e);
} else if (!e.getRefs().contains(ref)) {
addRef(e, ref);
@@ -307,6 +372,14 @@
}
}
+ private PushOne getPendingPush(URIish uri) {
+ PushOne e = pending.get(uri);
+ if (e != null && !e.wasCanceled()) {
+ return e;
+ }
+ return null;
+ }
+
void pushWasCanceled(PushOne pushOp) {
synchronized (stateLock) {
URIish uri = pushOp.getURI();
@@ -316,7 +389,7 @@
private void addRef(PushOne e, String ref) {
e.addRef(ref);
- postEvent(e, ref);
+ postReplicationScheduledEvent(e, ref);
}
/**
@@ -343,7 +416,7 @@
void reschedule(PushOne pushOp, RetryReason reason) {
synchronized (stateLock) {
URIish uri = pushOp.getURI();
- PushOne pendingPushOp = pending.get(uri);
+ PushOne pendingPushOp = getPendingPush(uri);
if (pendingPushOp != null) {
// There is one PushOp instance already pending to same URI.
@@ -389,13 +462,23 @@
pending.put(uri, pushOp);
switch (reason) {
case COLLISION:
- pool.schedule(pushOp, config.getRescheduleDelay(), TimeUnit.SECONDS);
+ @SuppressWarnings("unused")
+ ScheduledFuture<?> ignored =
+ pool.schedule(pushOp, config.getRescheduleDelay(), TimeUnit.SECONDS);
break;
case TRANSPORT_ERROR:
case REPOSITORY_MISSING:
default:
+ RemoteRefUpdate.Status status =
+ RetryReason.REPOSITORY_MISSING.equals(reason)
+ ? NON_EXISTING
+ : REJECTED_OTHER_REASON;
+ postReplicationFailedEvent(pushOp, status);
if (pushOp.setToRetry()) {
- pool.schedule(pushOp, config.getRetryDelay(), TimeUnit.MINUTES);
+ postReplicationScheduledEvent(pushOp);
+ @SuppressWarnings("unused")
+ ScheduledFuture<?> ignored2 =
+ pool.schedule(pushOp, config.getRetryDelay(), TimeUnit.MINUTES);
} else {
pushOp.canceledByReplication();
pending.remove(uri);
@@ -413,18 +496,19 @@
return projectControlFactory.controlFor(project);
}
- boolean requestRunway(PushOne op) {
+ RunwayStatus requestRunway(PushOne op) {
synchronized (stateLock) {
if (op.wasCanceled()) {
- return false;
+ return RunwayStatus.canceled();
}
pending.remove(op.getURI());
- if (inFlight.containsKey(op.getURI())) {
- return false;
+ PushOne inFlightOp = inFlight.get(op.getURI());
+ if (inFlightOp != null) {
+ return RunwayStatus.denied(inFlightOp.getId());
}
inFlight.put(op.getURI(), op);
}
- return true;
+ return RunwayStatus.allowed();
}
void notifyFinished(PushOne op) {
@@ -572,10 +656,36 @@
return uri.toString().contains(urlMatch);
}
- private void postEvent(PushOne pushOp, String ref) {
+ private void postReplicationScheduledEvent(PushOne pushOp) {
+ postReplicationScheduledEvent(pushOp, null);
+ }
+
+ private void postReplicationScheduledEvent(PushOne pushOp, String inputRef) {
+ Set<String> refs = inputRef == null ? pushOp.getRefs() : ImmutableSet.of(inputRef);
Project.NameKey project = pushOp.getProjectNameKey();
String targetNode = resolveNodeName(pushOp.getURI());
- ReplicationScheduledEvent event = new ReplicationScheduledEvent(project.get(), ref, targetNode);
- eventDispatcher.get().postEvent(new Branch.NameKey(project, ref), event);
+ for (String ref : refs) {
+ ReplicationScheduledEvent event =
+ new ReplicationScheduledEvent(project.get(), ref, targetNode);
+ try {
+ eventDispatcher.get().postEvent(new Branch.NameKey(project, ref), event);
+ } catch (PermissionBackendException e) {
+ repLog.error("error posting event", e);
+ }
+ }
+ }
+
+ private void postReplicationFailedEvent(PushOne pushOp, RemoteRefUpdate.Status status) {
+ Project.NameKey project = pushOp.getProjectNameKey();
+ String targetNode = resolveNodeName(pushOp.getURI());
+ for (String ref : pushOp.getRefs()) {
+ RefReplicatedEvent event =
+ new RefReplicatedEvent(project.get(), ref, targetNode, RefPushResult.FAILED, status);
+ try {
+ eventDispatcher.get().postEvent(new Branch.NameKey(project, ref), event);
+ } catch (PermissionBackendException e) {
+ repLog.error("error posting event", e);
+ }
+ }
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java
index 856ffb1..b2d0de2 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java
@@ -19,7 +19,7 @@
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.transport.RemoteConfig;
-class DestinationConfiguration {
+public class DestinationConfiguration {
static final int DEFAULT_REPLICATION_DELAY = 15;
static final int DEFAULT_RESCHEDULE_DELAY = 3;
@@ -40,7 +40,7 @@
private final RemoteConfig remoteConfig;
private final int maxRetries;
- DestinationConfiguration(RemoteConfig remoteConfig, Config cfg) {
+ protected DestinationConfiguration(RemoteConfig remoteConfig, Config cfg) {
this.remoteConfig = remoteConfig;
String name = remoteConfig.getName();
urls = ImmutableList.copyOf(cfg.getStringList("remote", name, "url"));
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationFactory.java
index df886cb..83eab86 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationFactory.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationFactory.java
@@ -20,6 +20,7 @@
import com.google.gerrit.server.account.GroupBackend;
import com.google.gerrit.server.account.GroupIncludeCache;
import com.google.gerrit.server.git.GitRepositoryManager;
+import com.google.gerrit.server.permissions.PermissionBackend;
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.google.inject.Singleton;
@@ -30,6 +31,7 @@
private final RemoteSiteUser.Factory replicationUserFactory;
private final PluginUser pluginUser;
private final GitRepositoryManager gitRepositoryManager;
+ private final PermissionBackend permissionBackend;
private final GroupBackend groupBackend;
private final ReplicationStateListener stateLog;
private final GroupIncludeCache groupIncludeCache;
@@ -41,6 +43,7 @@
RemoteSiteUser.Factory replicationUserFactory,
PluginUser pluginUser,
GitRepositoryManager gitRepositoryManager,
+ PermissionBackend permissionBackend,
GroupBackend groupBackend,
ReplicationStateListener stateLog,
GroupIncludeCache groupIncludeCache,
@@ -49,6 +52,7 @@
this.replicationUserFactory = replicationUserFactory;
this.pluginUser = pluginUser;
this.gitRepositoryManager = gitRepositoryManager;
+ this.permissionBackend = permissionBackend;
this.groupBackend = groupBackend;
this.stateLog = stateLog;
this.groupIncludeCache = groupIncludeCache;
@@ -62,6 +66,7 @@
replicationUserFactory,
pluginUser,
gitRepositoryManager,
+ permissionBackend,
groupBackend,
stateLog,
groupIncludeCache,
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java
new file mode 100644
index 0000000..b46a0d9
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java
@@ -0,0 +1,132 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.server.ssh.SshAddressesModule;
+import com.google.inject.Inject;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URISyntaxException;
+import java.util.HashSet;
+import java.util.Set;
+import org.eclipse.jgit.transport.URIish;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class GerritSshApi {
+ static int SSH_COMMAND_FAILED = -1;
+ private static final Logger log = LoggerFactory.getLogger(GerritSshApi.class);
+ private static String GERRIT_ADMIN_PROTOCOL_PREFIX = "gerrit+";
+
+ private final SshHelper sshHelper;
+
+ private final Set<URIish> withoutDeleteProjectPlugin = new HashSet<>();
+
+ @Inject
+ protected GerritSshApi(SshHelper sshHelper) {
+ this.sshHelper = sshHelper;
+ }
+
+ protected boolean createProject(URIish uri, Project.NameKey projectName, String head) {
+ OutputStream errStream = sshHelper.newErrorBufferStream();
+ String cmd = "gerrit create-project --branch " + head + " " + projectName.get();
+ try {
+ execute(uri, cmd, errStream);
+ } catch (IOException e) {
+ logError("creating", uri, errStream, cmd, e);
+ return false;
+ }
+ return true;
+ }
+
+ protected boolean deleteProject(URIish uri, Project.NameKey projectName) {
+ if (!withoutDeleteProjectPlugin.contains(uri)) {
+ OutputStream errStream = sshHelper.newErrorBufferStream();
+ String cmd = "deleteproject delete --yes-really-delete --force " + projectName.get();
+ int exitCode = -1;
+ try {
+ exitCode = execute(uri, cmd, errStream);
+ } catch (IOException e) {
+ logError("deleting", uri, errStream, cmd, e);
+ return false;
+ }
+ if (exitCode == 1) {
+ log.info(
+ "DeleteProject plugin is not installed on {}; will not try to forward this operation to that host");
+ withoutDeleteProjectPlugin.add(uri);
+ return true;
+ }
+ }
+ return true;
+ }
+
+ protected boolean updateHead(URIish uri, Project.NameKey projectName, String newHead) {
+ OutputStream errStream = sshHelper.newErrorBufferStream();
+ String cmd = "gerrit set-head " + projectName.get() + " --new-head " + newHead;
+ try {
+ execute(uri, cmd, errStream);
+ } catch (IOException e) {
+ log.error(
+ "Error updating HEAD of remote repository at {} to {}:\n"
+ + " Exception: {}\n Command: {}\n Output: {}",
+ uri,
+ newHead,
+ e,
+ cmd,
+ errStream,
+ e);
+ return false;
+ }
+ return true;
+ }
+
+ private URIish toSshUri(URIish uri) throws URISyntaxException {
+ String uriStr = uri.toString();
+ if (uri.getHost() != null && uriStr.startsWith(GERRIT_ADMIN_PROTOCOL_PREFIX)) {
+ return new URIish(uriStr.substring(GERRIT_ADMIN_PROTOCOL_PREFIX.length()));
+ }
+ String rawPath = uri.getRawPath();
+ if (!rawPath.endsWith("/")) {
+ rawPath = rawPath + "/";
+ }
+ URIish sshUri = new URIish("ssh://" + rawPath);
+ if (sshUri.getPort() < 0) {
+ sshUri = sshUri.setPort(SshAddressesModule.DEFAULT_PORT);
+ }
+ return sshUri;
+ }
+
+ private int execute(URIish uri, String cmd, OutputStream errStream) throws IOException {
+ try {
+ URIish sshUri = toSshUri(uri);
+ return sshHelper.executeRemoteSsh(sshUri, cmd, errStream);
+ } catch (URISyntaxException e) {
+ log.error("Cannot convert {} to SSH uri", uri, e);
+ }
+ return SSH_COMMAND_FAILED;
+ }
+
+ public void logError(String msg, URIish uri, OutputStream errStream, String cmd, IOException e) {
+ log.error(
+ "Error {} remote repository at {}:\n Exception: {}\n Command: {}\n Output: {}",
+ msg,
+ uri,
+ e,
+ cmd,
+ errStream,
+ e);
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java b/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java
index a6b38c1..227804d 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java
@@ -56,7 +56,9 @@
&& config.isReplicateAllOnPluginStart()) {
ReplicationState state = new ReplicationState(new GitUpdateProcessing(eventDispatcher.get()));
pushAllFuture.set(
- pushAll.create(null, ReplicationFilter.all(), state).schedule(30, TimeUnit.SECONDS));
+ pushAll
+ .create(null, ReplicationFilter.all(), state, false)
+ .schedule(30, TimeUnit.SECONDS));
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java
index da32ecd..db067e2 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java
@@ -27,7 +27,7 @@
private final ReplicationStateListener stateLog;
public interface Factory {
- PushAll create(String urlMatch, ReplicationFilter filter, ReplicationState state);
+ PushAll create(String urlMatch, ReplicationFilter filter, ReplicationState state, boolean now);
}
private final WorkQueue workQueue;
@@ -36,6 +36,7 @@
private final String urlMatch;
private final ReplicationFilter filter;
private final ReplicationState state;
+ private final boolean now;
@Inject
protected PushAll(
@@ -45,7 +46,8 @@
ReplicationStateListener stateLog,
@Assisted @Nullable String urlMatch,
@Assisted ReplicationFilter filter,
- @Assisted ReplicationState state) {
+ @Assisted ReplicationState state,
+ @Assisted boolean now) {
this.workQueue = wq;
this.projectCache = projectCache;
this.replication = rq;
@@ -53,6 +55,7 @@
this.urlMatch = urlMatch;
this.filter = filter;
this.state = state;
+ this.now = now;
}
Future<?> schedule(long delay, TimeUnit unit) {
@@ -64,7 +67,7 @@
try {
for (Project.NameKey nameKey : projectCache.all()) {
if (filter.matches(nameKey)) {
- replication.scheduleFullSync(nameKey, urlMatch, state);
+ replication.scheduleFullSync(nameKey, urlMatch, state, now);
}
}
} catch (Exception e) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
index 64f5152..5f0c066 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
@@ -21,25 +21,22 @@
import com.google.common.collect.LinkedListMultimap;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Sets;
-import com.google.gerrit.common.Nullable;
import com.google.gerrit.extensions.events.GitReferenceUpdatedListener;
+import com.google.gerrit.extensions.restapi.AuthException;
import com.google.gerrit.metrics.Timer1;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.client.RefNames;
-import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.PerThreadRequestScope;
import com.google.gerrit.server.git.ProjectRunnable;
-import com.google.gerrit.server.git.SearchingChangeCacheImpl;
-import com.google.gerrit.server.git.TagCache;
import com.google.gerrit.server.git.VisibleRefFilter;
import com.google.gerrit.server.git.WorkQueue.CanceledWhileRunning;
-import com.google.gerrit.server.notedb.ChangeNotes;
+import com.google.gerrit.server.permissions.PermissionBackend;
+import com.google.gerrit.server.permissions.PermissionBackendException;
+import com.google.gerrit.server.permissions.ProjectPermission;
import com.google.gerrit.server.project.NoSuchProjectException;
import com.google.gerrit.server.project.ProjectControl;
import com.google.gerrit.server.util.IdGenerator;
-import com.google.gwtorm.server.OrmException;
-import com.google.gwtorm.server.SchemaFactory;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.googlesource.gerrit.plugins.replication.ReplicationState.RefPushResult;
@@ -90,14 +87,12 @@
}
private final GitRepositoryManager gitManager;
- private final SchemaFactory<ReviewDb> schema;
+ private final PermissionBackend permissionBackend;
private final Destination pool;
private final RemoteConfig config;
private final CredentialsProvider credentialsProvider;
- private final TagCache tagCache;
private final PerThreadRequestScope.Scoper threadScoper;
- private final ChangeNotes.Factory changeNotesFactory;
- private final SearchingChangeCacheImpl changeCache;
+ private final VisibleRefFilter.Factory refFilterFactory;
private final ReplicationQueue replicationQueue;
private final Project.NameKey projectName;
@@ -120,14 +115,12 @@
@Inject
PushOne(
GitRepositoryManager grm,
- SchemaFactory<ReviewDb> s,
+ PermissionBackend permissionBackend,
Destination p,
RemoteConfig c,
+ VisibleRefFilter.Factory rff,
CredentialsFactory cpFactory,
- TagCache tc,
PerThreadRequestScope.Scoper ts,
- ChangeNotes.Factory nf,
- @Nullable SearchingChangeCacheImpl cc,
ReplicationQueue rq,
IdGenerator ig,
ReplicationStateListener sl,
@@ -135,14 +128,12 @@
@Assisted Project.NameKey d,
@Assisted URIish u) {
gitManager = grm;
- schema = s;
+ this.permissionBackend = permissionBackend;
pool = p;
config = c;
+ refFilterFactory = rff;
credentialsProvider = cpFactory.create(c.getName());
- tagCache = tc;
threadScoper = ts;
- changeNotesFactory = nf;
- changeCache = cc;
replicationQueue = rq;
projectName = d;
uri = u;
@@ -158,14 +149,17 @@
@Override
public void cancel() {
- repLog.info("Replication {} was canceled", getURI());
+ repLog.info("Replication [{}] to {} was canceled", IdGenerator.format(id), getURI());
canceledByReplication();
pool.pushWasCanceled(this);
}
@Override
public void setCanceledWhileRunning() {
- repLog.info("Replication {} was canceled while being executed", getURI());
+ repLog.info(
+ "Replication [{}] to {} was canceled while being executed",
+ IdGenerator.format(id),
+ getURI());
canceledWhileRunning.set(true);
}
@@ -209,7 +203,7 @@
}
boolean wasCanceled() {
- return canceled;
+ return canceled || canceledWhileRunning.get();
}
URIish getURI() {
@@ -258,6 +252,10 @@
return states.toArray(new ReplicationState[states.size()]);
}
+ public int getId() {
+ return id;
+ }
+
void addStates(ListMultimap<String, ReplicationState> states) {
stateMap.putAll(states);
}
@@ -304,10 +302,15 @@
// created and scheduled for a future point in time.)
//
MDC.put(ID_MDC_KEY, IdGenerator.format(id));
- if (!pool.requestRunway(this)) {
- if (!canceled) {
+ RunwayStatus status = pool.requestRunway(this);
+ if (!status.isAllowed()) {
+ if (status.isCanceled()) {
+ repLog.info("PushOp for replication to {} was canceled and thus won't be rescheduled", uri);
+ } else {
repLog.info(
- "Rescheduling replication to {} to avoid collision with an in-flight push.", uri);
+ "Rescheduling replication to {} to avoid collision with the in-flight push [{}].",
+ uri,
+ IdGenerator.format(status.getInFlightPushId()));
pool.reschedule(this, Destination.RetryReason.COLLISION);
}
return;
@@ -338,7 +341,10 @@
// does not exist. In this case NoRemoteRepositoryException is not
// raised.
String msg = e.getMessage();
- if (msg.contains("access denied") || msg.contains("no such repository")) {
+ if (msg.contains("access denied")
+ || msg.contains("no such repository")
+ || msg.contains("Git repository not found")
+ || msg.contains("unavailable")) {
createRepository();
} else {
repLog.error("Cannot replicate {}; Remote repository error: {}", projectName, msg);
@@ -383,13 +389,13 @@
}
} catch (IOException e) {
stateLog.error("Cannot replicate to " + uri, e, getStatesAsArray());
- } catch (RuntimeException | Error e) {
+ } catch (PermissionBackendException | RuntimeException | Error e) {
stateLog.error("Unexpected error during replication to " + uri, e, getStatesAsArray());
} finally {
+ pool.notifyFinished(this);
if (git != null) {
git.close();
}
- pool.notifyFinished(this);
}
}
@@ -430,7 +436,7 @@
return target.getName();
}
- private void runImpl() throws IOException {
+ private void runImpl() throws IOException, PermissionBackendException {
PushResult res;
try (Transport tn = Transport.open(git, uri)) {
res = pushVia(tn);
@@ -439,7 +445,7 @@
}
private PushResult pushVia(Transport tn)
- throws IOException, NotSupportedException, TransportException {
+ throws IOException, NotSupportedException, TransportException, PermissionBackendException {
tn.applyConfig(config);
tn.setCredentialsProvider(credentialsProvider);
@@ -457,7 +463,8 @@
return tn.push(NullProgressMonitor.INSTANCE, todo);
}
- private List<RemoteRefUpdate> generateUpdates(Transport tn) throws IOException {
+ private List<RemoteRefUpdate> generateUpdates(Transport tn)
+ throws IOException, PermissionBackendException {
ProjectControl pc;
try {
pc = pool.controlFor(projectName);
@@ -466,7 +473,14 @@
}
Map<String, Ref> local = git.getAllRefs();
- if (!pc.allRefsAreVisible()) {
+ boolean filter;
+ try {
+ permissionBackend.user(pc.getUser()).project(projectName).check(ProjectPermission.READ);
+ filter = false;
+ } catch (AuthException e) {
+ filter = true;
+ }
+ if (filter) {
if (!pushAllRefs) {
// If we aren't mirroring, reduce the space we need to filter
// to only the references we will update during this operation.
@@ -480,16 +494,7 @@
}
local = n;
}
-
- try (ReviewDb db = schema.open()) {
- local =
- new VisibleRefFilter(tagCache, changeNotesFactory, changeCache, git, pc, db, true)
- .filter(local, true);
- } catch (OrmException e) {
- stateLog.error(
- "Cannot read database to replicate to " + projectName, e, getStatesAsArray());
- return Collections.emptyList();
- }
+ local = refFilterFactory.create(pc.getProjectState(), git).filter(local, true);
}
return pushAllRefs ? doPushAll(tn, local) : doPushDelta(local);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java
index 43d5c5e..c71a792 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java
@@ -16,6 +16,7 @@
import com.google.gerrit.common.EventDispatcher;
import com.google.gerrit.server.events.RefEvent;
+import com.google.gerrit.server.permissions.PermissionBackendException;
import com.google.gwtorm.server.OrmException;
import com.googlesource.gerrit.plugins.replication.ReplicationState.RefPushResult;
import java.lang.ref.WeakReference;
@@ -43,7 +44,7 @@
*
* @param message message text.
*/
- void writeStdOut(final String message) {
+ void writeStdOut(String message) {
// Default doing nothing
}
@@ -52,7 +53,7 @@
*
* @param message message text.
*/
- void writeStdErr(final String message) {
+ void writeStdErr(String message) {
// Default doing nothing
}
@@ -141,7 +142,7 @@
}
@Override
- void writeStdOut(final String message) {
+ void writeStdOut(String message) {
StartCommand command = sshCommand.get();
if (command != null) {
command.writeStdOutSync(message);
@@ -149,7 +150,7 @@
}
@Override
- void writeStdErr(final String message) {
+ void writeStdErr(String message) {
StartCommand command = sshCommand.get();
if (command != null) {
command.writeStdErrSync(message);
@@ -187,7 +188,7 @@
private void postEvent(RefEvent event) {
try {
dispatcher.postEvent(event);
- } catch (OrmException e) {
+ } catch (OrmException | PermissionBackendException e) {
log.error("Cannot post event", e);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicatedEvent.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicatedEvent.java
index 364f1b4..fccdb7b 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicatedEvent.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicatedEvent.java
@@ -21,13 +21,13 @@
import org.eclipse.jgit.transport.RemoteRefUpdate.Status;
public class RefReplicatedEvent extends RefEvent {
- static final String TYPE = "ref-replicated";
+ public static final String TYPE = "ref-replicated";
- final String project;
- final String ref;
- final String targetNode;
- final String status;
- final Status refStatus;
+ public final String project;
+ public final String ref;
+ public final String targetNode;
+ public final String status;
+ public final Status refStatus;
public RefReplicatedEvent(
String project,
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicationDoneEvent.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicationDoneEvent.java
index 90595b3..4789a96 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicationDoneEvent.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicationDoneEvent.java
@@ -18,7 +18,7 @@
import com.google.gerrit.server.events.RefEvent;
public class RefReplicationDoneEvent extends RefEvent {
- static final String TYPE = "ref-replication-done";
+ public static final String TYPE = "ref-replication-done";
final String project;
final String ref;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java
index f3dc04d..91fce7f 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java
@@ -15,7 +15,6 @@
package com.googlesource.gerrit.plugins.replication;
import com.google.gerrit.server.CurrentUser;
-import com.google.gerrit.server.account.CapabilityControl;
import com.google.gerrit.server.account.GroupMembership;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
@@ -28,9 +27,7 @@
private final GroupMembership effectiveGroups;
@Inject
- RemoteSiteUser(
- CapabilityControl.Factory capabilityControlFactory, @Assisted GroupMembership authGroups) {
- super(capabilityControlFactory);
+ RemoteSiteUser(@Assisted GroupMembership authGroups) {
effectiveGroups = authGroups;
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
index e94abbd..869a49b 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
@@ -35,4 +35,8 @@
int shutdown();
void startup(WorkQueue workQueue);
+
+ int getSshConnectionTimeout();
+
+ int getSshCommandTimeout();
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
index ee1f16d..db9f35d 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
@@ -13,10 +13,13 @@
// limitations under the License.
package com.googlesource.gerrit.plugins.replication;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toList;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import com.google.gerrit.server.config.ConfigUtil;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.git.WorkQueue;
import com.google.inject.Inject;
@@ -41,10 +44,14 @@
@Singleton
public class ReplicationFileBasedConfig implements ReplicationConfig {
private static final Logger log = LoggerFactory.getLogger(ReplicationFileBasedConfig.class);
+ private static final int DEFAULT_SSH_CONNECTION_TIMEOUT_MS = 2 * 60 * 1000; // 2 minutes
+
private List<Destination> destinations;
private Path cfgPath;
private boolean replicateAllOnPluginStart;
private boolean defaultForceUpdate;
+ private int sshCommandTimeout;
+ private int sshConnectionTimeout = DEFAULT_SSH_CONNECTION_TIMEOUT_MS;
private final FileBasedConfig config;
@Inject
@@ -104,6 +111,18 @@
defaultForceUpdate = config.getBoolean("gerrit", "defaultForceUpdate", false);
+ sshCommandTimeout =
+ (int) ConfigUtil.getTimeUnit(config, "gerrit", null, "sshCommandTimeout", 0, SECONDS);
+ sshConnectionTimeout =
+ (int)
+ ConfigUtil.getTimeUnit(
+ config,
+ "gerrit",
+ null,
+ "sshConnectionTimeout",
+ DEFAULT_SSH_CONNECTION_TIMEOUT_MS,
+ MILLISECONDS);
+
ImmutableList.Builder<Destination> dest = ImmutableList.builder();
for (RemoteConfig c : allRemotes(config)) {
if (c.getURIs().isEmpty()) {
@@ -203,4 +222,14 @@
cfg.start(workQueue);
}
}
+
+ @Override
+ public int getSshConnectionTimeout() {
+ return sshConnectionTimeout;
+ }
+
+ @Override
+ public int getSshCommandTimeout() {
+ return sshCommandTimeout;
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
index 57893d8..30aff44 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
@@ -25,7 +25,6 @@
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.git.WorkQueue;
import com.google.inject.Inject;
-import com.google.inject.Provider;
import com.googlesource.gerrit.plugins.replication.PushResultProcessing.GitUpdateProcessing;
import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType;
import java.io.File;
@@ -35,17 +34,12 @@
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
-import org.eclipse.jgit.errors.TransportException;
import org.eclipse.jgit.internal.storage.file.FileRepository;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.RefUpdate;
import org.eclipse.jgit.lib.Repository;
-import org.eclipse.jgit.transport.RemoteSession;
-import org.eclipse.jgit.transport.SshSessionFactory;
import org.eclipse.jgit.transport.URIish;
-import org.eclipse.jgit.util.FS;
import org.eclipse.jgit.util.QuotedString;
-import org.eclipse.jgit.util.io.StreamCopyThread;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -58,7 +52,6 @@
HeadUpdatedListener {
static final String REPLICATION_LOG_NAME = "replication_log";
static final Logger repLog = LoggerFactory.getLogger(REPLICATION_LOG_NAME);
- private static final int SSH_REMOTE_TIMEOUT = 120 * 1000;
private final ReplicationStateListener stateLog;
@@ -75,23 +68,26 @@
}
private final WorkQueue workQueue;
+ private final SshHelper sshHelper;
private final DynamicItem<EventDispatcher> dispatcher;
private final ReplicationConfig config;
- private final Provider<SshSessionFactory> sshSessionFactoryProvider;
+ private final GerritSshApi gerritAdmin;
private volatile boolean running;
@Inject
ReplicationQueue(
WorkQueue wq,
+ SshHelper sh,
+ GerritSshApi ga,
ReplicationConfig rc,
DynamicItem<EventDispatcher> dis,
- ReplicationStateListener sl,
- Provider<SshSessionFactory> sshSessionFactoryProvider) {
+ ReplicationStateListener sl) {
workQueue = wq;
+ sshHelper = sh;
dispatcher = dis;
config = rc;
stateLog = sl;
- this.sshSessionFactoryProvider = sshSessionFactoryProvider;
+ gerritAdmin = ga;
}
@Override
@@ -109,8 +105,12 @@
}
}
+ void scheduleFullSync(Project.NameKey project, String urlMatch, ReplicationState state) {
+ scheduleFullSync(project, urlMatch, state, false);
+ }
+
void scheduleFullSync(
- final Project.NameKey project, final String urlMatch, ReplicationState state) {
+ Project.NameKey project, String urlMatch, ReplicationState state, boolean now) {
if (!running) {
stateLog.warn("Replication plugin did not finish startup before event", state);
return;
@@ -119,7 +119,7 @@
for (Destination cfg : config.getDestinations(FilterType.ALL)) {
if (cfg.wouldPushProject(project)) {
for (URIish uri : cfg.getURIs(project, urlMatch)) {
- cfg.schedule(project, PushOne.ALL_REFS, uri, state);
+ cfg.schedule(project, PushOne.ALL_REFS, uri, state, now);
}
}
}
@@ -146,24 +146,25 @@
@Override
public void onNewProjectCreated(NewProjectCreatedListener.Event event) {
- for (URIish uri :
- getURIs(new Project.NameKey(event.getProjectName()), FilterType.PROJECT_CREATION)) {
- createProject(uri, event.getHeadName());
+ Project.NameKey projectName = new Project.NameKey(event.getProjectName());
+ for (URIish uri : getURIs(projectName, FilterType.PROJECT_CREATION)) {
+ createProject(uri, projectName, event.getHeadName());
}
}
@Override
public void onProjectDeleted(ProjectDeletedListener.Event event) {
- for (URIish uri :
- getURIs(new Project.NameKey(event.getProjectName()), FilterType.PROJECT_DELETION)) {
- deleteProject(uri);
+ Project.NameKey projectName = new Project.NameKey(event.getProjectName());
+ for (URIish uri : getURIs(projectName, FilterType.PROJECT_DELETION)) {
+ deleteProject(uri, projectName);
}
}
@Override
public void onHeadUpdated(HeadUpdatedListener.Event event) {
- for (URIish uri : getURIs(new Project.NameKey(event.getProjectName()), FilterType.ALL)) {
- updateHead(uri, event.getNewHeadName());
+ Project.NameKey project = new Project.NameKey(event.getProjectName());
+ for (URIish uri : getURIs(project, FilterType.ALL)) {
+ updateHead(uri, project, event.getNewHeadName());
}
}
@@ -197,18 +198,20 @@
continue;
}
- String path = replaceName(uri.getPath(), projectName.get(), config.isSingleProjectMatch());
- if (path == null) {
- repLog.warn("adminURL {} does not contain ${name}", uri);
- continue;
- }
+ if (!isGerrit(uri)) {
+ String path =
+ replaceName(uri.getPath(), projectName.get(), config.isSingleProjectMatch());
+ if (path == null) {
+ repLog.warn("adminURL {} does not contain ${name}", uri);
+ continue;
+ }
- uri = uri.setPath(path);
- if (!isSSH(uri)) {
- repLog.warn("adminURL '{}' is invalid: only SSH is supported", uri);
- continue;
+ uri = uri.setPath(path);
+ if (!isSSH(uri)) {
+ repLog.warn("adminURL '{}' is invalid: only SSH is supported", uri);
+ continue;
+ }
}
-
uris.add(uri);
adminURLUsed = true;
}
@@ -225,13 +228,15 @@
public boolean createProject(Project.NameKey project, String head) {
boolean success = true;
for (URIish uri : getURIs(project, FilterType.PROJECT_CREATION)) {
- success &= createProject(uri, head);
+ success &= createProject(uri, project, head);
}
return success;
}
- private boolean createProject(URIish replicateURI, String head) {
- if (!replicateURI.isRemote()) {
+ private boolean createProject(URIish replicateURI, Project.NameKey projectName, String head) {
+ if (isGerrit(replicateURI)) {
+ gerritAdmin.createProject(replicateURI, projectName, head);
+ } else if (!replicateURI.isRemote()) {
createLocally(replicateURI, head);
} else if (isSSH(replicateURI)) {
createRemoteSsh(replicateURI, head);
@@ -267,9 +272,9 @@
if (head != null) {
cmd = cmd + " && git symbolic-ref HEAD " + QuotedString.BOURNE.quote(head);
}
- OutputStream errStream = newErrorBufferStream();
+ OutputStream errStream = sshHelper.newErrorBufferStream();
try {
- executeRemoteSsh(uri, cmd, errStream);
+ sshHelper.executeRemoteSsh(uri, cmd, errStream);
repLog.info("Created remote repository: {}", uri);
} catch (IOException e) {
repLog.error(
@@ -285,8 +290,11 @@
}
}
- private void deleteProject(URIish replicateURI) {
- if (!replicateURI.isRemote()) {
+ private void deleteProject(URIish replicateURI, Project.NameKey projectName) {
+ if (isGerrit(replicateURI)) {
+ gerritAdmin.deleteProject(replicateURI, projectName);
+ repLog.info("Deleted remote repository: " + replicateURI);
+ } else if (!replicateURI.isRemote()) {
deleteLocally(replicateURI);
} else if (isSSH(replicateURI)) {
deleteRemoteSsh(replicateURI);
@@ -329,9 +337,9 @@
private void deleteRemoteSsh(URIish uri) {
String quotedPath = QuotedString.BOURNE.quote(uri.getPath());
String cmd = "rm -rf " + quotedPath;
- OutputStream errStream = newErrorBufferStream();
+ OutputStream errStream = sshHelper.newErrorBufferStream();
try {
- executeRemoteSsh(uri, cmd, errStream);
+ sshHelper.executeRemoteSsh(uri, cmd, errStream);
repLog.info("Deleted remote repository: {}", uri);
} catch (IOException e) {
repLog.error(
@@ -347,8 +355,10 @@
}
}
- private void updateHead(URIish replicateURI, String newHead) {
- if (!replicateURI.isRemote()) {
+ private void updateHead(URIish replicateURI, Project.NameKey projectName, String newHead) {
+ if (isGerrit(replicateURI)) {
+ gerritAdmin.updateHead(replicateURI, projectName, newHead);
+ } else if (!replicateURI.isRemote()) {
updateHeadLocally(replicateURI, newHead);
} else if (isSSH(replicateURI)) {
updateHeadRemoteSsh(replicateURI, newHead);
@@ -365,9 +375,9 @@
String quotedPath = QuotedString.BOURNE.quote(uri.getPath());
String cmd =
"cd " + quotedPath + " && git symbolic-ref HEAD " + QuotedString.BOURNE.quote(newHead);
- OutputStream errStream = newErrorBufferStream();
+ OutputStream errStream = sshHelper.newErrorBufferStream();
try {
- executeRemoteSsh(uri, cmd, errStream);
+ sshHelper.executeRemoteSsh(uri, cmd, errStream);
} catch (IOException e) {
repLog.error(
"Error updating HEAD of remote repository at {} to {}:\n"
@@ -394,57 +404,6 @@
}
}
- private void executeRemoteSsh(URIish uri, String cmd, OutputStream errStream) throws IOException {
- RemoteSession ssh = connect(uri);
- Process proc = ssh.exec(cmd, 0);
- proc.getOutputStream().close();
- StreamCopyThread out = new StreamCopyThread(proc.getInputStream(), errStream);
- StreamCopyThread err = new StreamCopyThread(proc.getErrorStream(), errStream);
- out.start();
- err.start();
- try {
- proc.waitFor();
- out.halt();
- err.halt();
- } catch (InterruptedException interrupted) {
- // Don't wait, drop out immediately.
- }
- ssh.disconnect();
- }
-
- private RemoteSession connect(URIish uri) throws TransportException {
- return sshSessionFactoryProvider.get().getSession(uri, null, FS.DETECTED, SSH_REMOTE_TIMEOUT);
- }
-
- private static OutputStream newErrorBufferStream() {
- return new OutputStream() {
- private final StringBuilder out = new StringBuilder();
- private final StringBuilder line = new StringBuilder();
-
- @Override
- public synchronized String toString() {
- while (out.length() > 0 && out.charAt(out.length() - 1) == '\n') {
- out.setLength(out.length() - 1);
- }
- return out.toString();
- }
-
- @Override
- public synchronized void write(final int b) {
- if (b == '\r') {
- return;
- }
-
- line.append((char) b);
-
- if (b == '\n') {
- out.append(line);
- line.setLength(0);
- }
- }
- };
- }
-
private static boolean isSSH(URIish uri) {
String scheme = uri.getScheme();
if (!uri.isRemote()) {
@@ -458,4 +417,9 @@
}
return false;
}
+
+ private static boolean isGerrit(URIish uri) {
+ String scheme = uri.getScheme();
+ return scheme != null && scheme.toLowerCase().equals("gerrit+ssh");
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationScheduledEvent.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationScheduledEvent.java
index 7268709..cd7a3cf 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationScheduledEvent.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationScheduledEvent.java
@@ -19,11 +19,11 @@
import com.google.gerrit.server.events.RefEvent;
public class ReplicationScheduledEvent extends RefEvent {
- static final String TYPE = "ref-replication-scheduled";
+ public static final String TYPE = "ref-replication-scheduled";
- final String project;
- final String ref;
- final String targetNode;
+ public final String project;
+ public final String ref;
+ public final String targetNode;
public ReplicationScheduledEvent(String project, String ref, String targetNode) {
super(TYPE);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationState.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationState.java
index 9a68c83..86557e2 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationState.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationState.java
@@ -150,11 +150,11 @@
allPushTasksFinished.await();
}
- public void writeStdOut(final String message) {
+ public void writeStdOut(String message) {
pushResultProcessing.writeStdOut(message);
}
- public void writeStdErr(final String message) {
+ public void writeStdErr(String message) {
pushResultProcessing.writeStdErr(message);
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStateLogger.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStateLogger.java
index cfa95dd..f2d55de 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStateLogger.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStateLogger.java
@@ -26,7 +26,7 @@
* state to the stderr console.
*/
@Singleton
-class ReplicationStateLogger implements ReplicationStateListener {
+public class ReplicationStateLogger implements ReplicationStateListener {
@Override
public void warn(String msg, ReplicationState... states) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RunwayStatus.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RunwayStatus.java
new file mode 100644
index 0000000..bcb1e2f
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RunwayStatus.java
@@ -0,0 +1,49 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+public class RunwayStatus {
+ public static RunwayStatus allowed() {
+ return new RunwayStatus(true, 0);
+ }
+
+ public static RunwayStatus canceled() {
+ return new RunwayStatus(false, 0);
+ }
+
+ public static RunwayStatus denied(int inFlightPushId) {
+ return new RunwayStatus(false, inFlightPushId);
+ }
+
+ private final boolean allowed;
+ private final int inFlightPushId;
+
+ private RunwayStatus(boolean allowed, int inFlightPushId) {
+ this.allowed = allowed;
+ this.inFlightPushId = inFlightPushId;
+ }
+
+ public boolean isAllowed() {
+ return allowed;
+ }
+
+ public boolean isCanceled() {
+ return !allowed && inFlightPushId == 0;
+ }
+
+ public int getInFlightPushId() {
+ return inFlightPushId;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/SshHelper.java b/src/main/java/com/googlesource/gerrit/plugins/replication/SshHelper.java
new file mode 100644
index 0000000..d73c101
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/SshHelper.java
@@ -0,0 +1,93 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import java.io.IOException;
+import java.io.OutputStream;
+import org.eclipse.jgit.errors.TransportException;
+import org.eclipse.jgit.transport.RemoteSession;
+import org.eclipse.jgit.transport.SshSessionFactory;
+import org.eclipse.jgit.transport.URIish;
+import org.eclipse.jgit.util.FS;
+import org.eclipse.jgit.util.io.StreamCopyThread;
+
+/** Utility class that provides SSH access to remote URIs. */
+public class SshHelper {
+ private final Provider<SshSessionFactory> sshSessionFactoryProvider;
+ private final int commandTimeout;
+ private final int connectionTimeout;
+
+ @Inject
+ protected SshHelper(
+ ReplicationConfig replicationConfig, Provider<SshSessionFactory> sshSessionFactoryProvider) {
+ this.sshSessionFactoryProvider = sshSessionFactoryProvider;
+ this.commandTimeout = replicationConfig.getSshCommandTimeout();
+ this.connectionTimeout = replicationConfig.getSshConnectionTimeout();
+ }
+
+ public int executeRemoteSsh(URIish uri, String cmd, OutputStream errStream) throws IOException {
+ RemoteSession ssh = connect(uri);
+ Process proc = ssh.exec(cmd, commandTimeout);
+ proc.getOutputStream().close();
+ StreamCopyThread out = new StreamCopyThread(proc.getInputStream(), errStream);
+ StreamCopyThread err = new StreamCopyThread(proc.getErrorStream(), errStream);
+ out.start();
+ err.start();
+ try {
+ proc.waitFor();
+ out.halt();
+ err.halt();
+ } catch (InterruptedException interrupted) {
+ // Don't wait, drop out immediately.
+ }
+ ssh.disconnect();
+ return proc.exitValue();
+ }
+
+ public OutputStream newErrorBufferStream() {
+ return new OutputStream() {
+ private final StringBuilder out = new StringBuilder();
+ private final StringBuilder line = new StringBuilder();
+
+ @Override
+ public synchronized String toString() {
+ while (out.length() > 0 && out.charAt(out.length() - 1) == '\n') {
+ out.setLength(out.length() - 1);
+ }
+ return out.toString();
+ }
+
+ @Override
+ public synchronized void write(int b) {
+ if (b == '\r') {
+ return;
+ }
+
+ line.append((char) b);
+
+ if (b == '\n') {
+ out.append(line);
+ line.setLength(0);
+ }
+ }
+ };
+ }
+
+ protected RemoteSession connect(URIish uri) throws TransportException {
+ return sshSessionFactoryProvider.get().getSession(uri, null, FS.DETECTED, connectionTimeout);
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java
index 9336466..7115d5b 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java
@@ -43,11 +43,16 @@
@Option(name = "--wait", usage = "wait for replication to finish before exiting")
private boolean wait;
+ @Option(name = "--now", usage = "start replication without waiting for replicationDelay")
+ private boolean now;
+
@Argument(index = 0, multiValued = true, metaVar = "PATTERN", usage = "project name pattern")
private List<String> projectPatterns = new ArrayList<>(2);
@Inject private PushAll.Factory pushFactory;
+ private final Object lock = new Object();
+
@Override
protected void run() throws Failure {
if (all && projectPatterns.size() > 0) {
@@ -65,7 +70,7 @@
projectFilter = new ReplicationFilter(projectPatterns);
}
- future = pushFactory.create(urlMatch, projectFilter, state).schedule(0, TimeUnit.SECONDS);
+ future = pushFactory.create(urlMatch, projectFilter, state, now).schedule(0, TimeUnit.SECONDS);
if (wait) {
if (future != null) {
@@ -93,18 +98,18 @@
}
}
- public void writeStdOutSync(final String message) {
+ public void writeStdOutSync(String message) {
if (wait) {
- synchronized (stdout) {
+ synchronized (lock) {
stdout.println(message);
stdout.flush();
}
}
}
- public void writeStdErrSync(final String message) {
+ public void writeStdErrSync(String message) {
if (wait) {
- synchronized (stderr) {
+ synchronized (lock) {
stderr.println(message);
stderr.flush();
}
diff --git a/src/main/resources/Documentation/about.md b/src/main/resources/Documentation/about.md
index dd9cea5..69a371b 100644
--- a/src/main/resources/Documentation/about.md
+++ b/src/main/resources/Documentation/about.md
@@ -16,3 +16,15 @@
local path as replication target. This makes e.g. sense if a network
share is mounted to which the repositories should be replicated.
+Replication of account data (NoteDb)
+------------------------------------
+
+To replicate the account data in NoteDb the following branches from the
+`All-Users` repository must be replicated:
+
+* `refs/users/*` (user branches)
+* `refs/meta/external-ids` (external IDs)
+* `refs/starred-changes/*` (star labels)
+* `refs/sequences/accounts` (account sequence numbers, not needed for Gerrit
+ slaves)
+
diff --git a/src/main/resources/Documentation/cmd-start.md b/src/main/resources/Documentation/cmd-start.md
index 59c3d1d..6af73af 100644
--- a/src/main/resources/Documentation/cmd-start.md
+++ b/src/main/resources/Documentation/cmd-start.md
@@ -9,6 +9,7 @@
--------
```
ssh -p @SSH_PORT@ @SSH_HOST@ @PLUGIN@ start
+ [--now]
[--wait]
[--url <PATTERN>]
{--all | <PROJECT PATTERN> ...}
@@ -85,6 +86,10 @@
OPTIONS
-------
+`--now`
+: Start replicating right away without waiting the per remote
+ replication delay.
+
`--wait`
: Wait for replication to finish before exiting.
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 099608d..e70094c 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -77,6 +77,14 @@
: If true, the default push refspec will be set to use forced
update to the remote when no refspec is given. By default, false.
+gerrit.sshCommandTimeout
+: Timeout for SSH command execution. If 0, there is no timeout and
+ the client waits indefinitely. By default, 0.
+
+gerrit.sshConnectionTimeout
+: Timeout for SSH connections. If 0, there is no timeout and
+ the client waits indefinitely. By default, 2 minutes.
+
replication.lockErrorMaxRetries
: Number of times to retry a replication operation if a lock
error is detected.
@@ -153,6 +161,18 @@
local environment. In that case, an alternative SSH url could
be specified to repository creation.
+ To enable replication to different Gerrit instance use `gerrit+ssh://`
+ as protocol name followed by hostname of another Gerrit server eg.
+
+ `gerrit+ssh://replica1.my.org/`
+
+ In this case replication will use Gerrit's SSH API to
+ create/remove projects and update repository HEAD references.
+
+ NOTE: In order to replicate project deletion, the
+ link:https://gerrit-review.googlesource.com/admin/projects/plugins/delete-project delete-project[delete-project]
+ plugin must be installed on the other Gerrit.
+
remote.NAME.receivepack
: Path of the `git-receive-pack` executable on the remote
system, if using the SSH transport.
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java
index 41829bc..337bd1d 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java
@@ -24,6 +24,7 @@
import com.google.gerrit.common.EventDispatcher;
import com.google.gerrit.reviewdb.server.ReviewDb;
+import com.google.gerrit.server.permissions.PermissionBackendException;
import com.google.gwtorm.client.KeyUtil;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.SchemaFactory;
@@ -58,7 +59,8 @@
}
@Test
- public void headRefReplicated() throws URISyntaxException, OrmException {
+ public void headRefReplicated()
+ throws URISyntaxException, OrmException, PermissionBackendException {
reset(dispatcherMock);
RefReplicatedEvent expectedEvent =
new RefReplicatedEvent(
@@ -81,7 +83,8 @@
}
@Test
- public void changeRefReplicated() throws URISyntaxException, OrmException {
+ public void changeRefReplicated()
+ throws URISyntaxException, OrmException, PermissionBackendException {
reset(dispatcherMock);
RefReplicatedEvent expectedEvent =
new RefReplicatedEvent(
@@ -104,7 +107,7 @@
}
@Test
- public void onAllNodesReplicated() throws OrmException {
+ public void onAllNodesReplicated() throws OrmException, PermissionBackendException {
reset(dispatcherMock);
RefReplicationDoneEvent expectedDoneEvent =
new RefReplicationDoneEvent("someProject", "refs/heads/master", 5);
