Merge branch 'stable-2.15'
* stable-2.15:
Bazel: Reformat BUILD file with buildifier 0.17.2
Change-Id: If66e7cd9e00199148aa5bf6517cc4d530e236dd5
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApi.java
new file mode 100644
index 0000000..ef7e353
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApi.java
@@ -0,0 +1,25 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import com.google.gerrit.reviewdb.client.Project;
+
+public interface AdminApi {
+ public void createProject(Project.NameKey project, String head);
+
+ public void deleteProject(Project.NameKey project);
+
+ public void updateHead(Project.NameKey project, String newHead);
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java
new file mode 100644
index 0000000..528aff2
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java
@@ -0,0 +1,61 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import java.util.Optional;
+import org.eclipse.jgit.transport.URIish;
+
+@Singleton
+public class AdminApiFactory {
+
+ private final SshHelper sshHelper;
+
+ @Inject
+ AdminApiFactory(SshHelper sshHelper) {
+ this.sshHelper = sshHelper;
+ }
+
+ public Optional<AdminApi> create(URIish uri) {
+ if (isGerrit(uri)) {
+ return Optional.of(new GerritSshApi(sshHelper, uri));
+ } else if (!uri.isRemote()) {
+ return Optional.of(new LocalFS(uri));
+ } else if (isSSH(uri)) {
+ return Optional.of(new RemoteSsh(sshHelper, uri));
+ }
+ return Optional.empty();
+ }
+
+ public static boolean isGerrit(URIish uri) {
+ String scheme = uri.getScheme();
+ return scheme != null && scheme.toLowerCase().equals("gerrit+ssh");
+ }
+
+ public static boolean isSSH(URIish uri) {
+ if (!uri.isRemote()) {
+ return false;
+ }
+ String scheme = uri.getScheme();
+ if (scheme != null && scheme.toLowerCase().contains("ssh")) {
+ return true;
+ }
+ if (scheme == null && uri.getHost() != null && uri.getPath() != null) {
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
index 8b6b8fc..6ed438a 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
@@ -13,33 +13,40 @@
// limitations under the License.
package com.googlesource.gerrit.plugins.replication;
+import com.google.common.flogger.FluentLogger;
import com.google.gerrit.common.FileUtil;
+import com.google.gerrit.extensions.annotations.PluginData;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.git.WorkQueue;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.io.IOException;
+import java.nio.file.Path;
import java.util.List;
import org.eclipse.jgit.errors.ConfigInvalidException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
@Singleton
public class AutoReloadConfigDecorator implements ReplicationConfig {
- private static final Logger log = LoggerFactory.getLogger(AutoReloadConfigDecorator.class);
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
private ReplicationFileBasedConfig currentConfig;
private long currentConfigTs;
private final SitePaths site;
private final WorkQueue workQueue;
private final DestinationFactory destinationFactory;
+ private final Path pluginDataDir;
@Inject
public AutoReloadConfigDecorator(
- SitePaths site, WorkQueue workQueue, DestinationFactory destinationFactory)
+ SitePaths site,
+ WorkQueue workQueue,
+ DestinationFactory destinationFactory,
+ @PluginData Path pluginDataDir)
throws ConfigInvalidException, IOException {
this.site = site;
this.destinationFactory = destinationFactory;
+ this.pluginDataDir = pluginDataDir;
this.currentConfig = loadConfig();
this.currentConfigTs = getLastModified(currentConfig);
this.workQueue = workQueue;
@@ -50,7 +57,7 @@
}
private ReplicationFileBasedConfig loadConfig() throws ConfigInvalidException, IOException {
- return new ReplicationFileBasedConfig(site, destinationFactory);
+ return new ReplicationFileBasedConfig(site, destinationFactory, pluginDataDir);
}
private synchronized boolean isAutoReload() {
@@ -74,14 +81,14 @@
this.currentConfig = newConfig;
this.currentConfigTs = lastModified;
- log.info(
- "Configuration reloaded: {} destinations, {} replication events discarded",
- currentConfig.getDestinations(FilterType.ALL).size(),
- discarded);
+ logger.atInfo().log(
+ "Configuration reloaded: %d destinations, %d replication events discarded",
+ currentConfig.getDestinations(FilterType.ALL).size(), discarded);
}
}
} catch (Exception e) {
- log.error("Cannot reload replication configuration: keeping existing settings", e);
+ logger.atSevere().withCause(e).log(
+ "Cannot reload replication configuration: keeping existing settings");
return;
}
}
@@ -102,6 +109,11 @@
}
@Override
+ public Path getEventsDirectory() {
+ return currentConfig.getEventsDirectory();
+ }
+
+ @Override
public synchronized int shutdown() {
return currentConfig.shutdown();
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
index f8737b6..29a7ee6 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
@@ -16,18 +16,16 @@
import static com.google.gerrit.common.FileUtil.lastModified;
+import com.google.common.flogger.FluentLogger;
import com.google.gerrit.server.config.SitePaths;
import com.google.inject.Inject;
import java.io.IOException;
import java.nio.file.Files;
import java.util.concurrent.atomic.AtomicReference;
import org.eclipse.jgit.errors.ConfigInvalidException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public class AutoReloadSecureCredentialsFactoryDecorator implements CredentialsFactory {
- private static final Logger log =
- LoggerFactory.getLogger(AutoReloadSecureCredentialsFactoryDecorator.class);
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private final AtomicReference<SecureCredentialsFactory> secureCredentialsFactory;
private volatile long secureCredentialsFactoryLoadTs;
@@ -58,13 +56,12 @@
secureCredentialsFactory.compareAndSet(
secureCredentialsFactory.get(), new SecureCredentialsFactory(site));
secureCredentialsFactoryLoadTs = getSecureConfigLastEditTs();
- log.info("secure.config reloaded as it was updated on the file system");
+ logger.atInfo().log("secure.config reloaded as it was updated on the file system");
}
} catch (Exception e) {
- log.error(
+ logger.atSevere().withCause(e).log(
"Unexpected error while trying to reload "
- + "secure.config: keeping existing credentials",
- e);
+ + "secure.config: keeping existing credentials");
}
return secureCredentialsFactory.get().create(remoteName);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
index 0cee37c..b69bab6 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
@@ -24,9 +24,7 @@
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSet.Builder;
import com.google.common.collect.Lists;
-import com.google.gerrit.common.EventDispatcher;
import com.google.gerrit.common.data.GroupReference;
-import com.google.gerrit.extensions.client.ProjectState;
import com.google.gerrit.extensions.config.FactoryModule;
import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.extensions.restapi.AuthException;
@@ -42,6 +40,7 @@
import com.google.gerrit.server.account.GroupIncludeCache;
import com.google.gerrit.server.account.ListGroupMembership;
import com.google.gerrit.server.config.RequestScopedReviewDbProvider;
+import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.PerThreadRequestScope;
import com.google.gerrit.server.git.WorkQueue;
@@ -50,8 +49,8 @@
import com.google.gerrit.server.permissions.ProjectPermission;
import com.google.gerrit.server.permissions.RefPermission;
import com.google.gerrit.server.project.NoSuchProjectException;
-import com.google.gerrit.server.project.PerRequestProjectControlCache;
-import com.google.gerrit.server.project.ProjectControl;
+import com.google.gerrit.server.project.ProjectCache;
+import com.google.gerrit.server.project.ProjectState;
import com.google.gerrit.server.util.RequestContext;
import com.google.inject.Injector;
import com.google.inject.Provider;
@@ -86,9 +85,10 @@
private final Map<URIish, PushOne> pending = new HashMap<>();
private final Map<URIish, PushOne> inFlight = new HashMap<>();
private final PushOne.Factory opFactory;
- private final ProjectControl.Factory projectControlFactory;
private final GitRepositoryManager gitManager;
private final PermissionBackend permissionBackend;
+ private final Provider<CurrentUser> userProvider;
+ private final ProjectCache projectCache;
private volatile ScheduledExecutorService pool;
private final PerThreadRequestScope.Scoper threadScoper;
private final DestinationConfiguration config;
@@ -117,6 +117,8 @@
PluginUser pluginUser,
GitRepositoryManager gitRepositoryManager,
PermissionBackend permissionBackend,
+ Provider<CurrentUser> userProvider,
+ ProjectCache projectCache,
GroupBackend groupBackend,
ReplicationStateListener stateLog,
GroupIncludeCache groupIncludeCache,
@@ -125,6 +127,8 @@
this.eventDispatcher = eventDispatcher;
gitManager = gitRepositoryManager;
this.permissionBackend = permissionBackend;
+ this.userProvider = userProvider;
+ this.projectCache = projectCache;
this.stateLog = stateLog;
CurrentUser remoteUser;
@@ -151,7 +155,6 @@
protected void configure() {
bindScope(RequestScoped.class, PerThreadRequestScope.REQUEST);
bind(PerThreadRequestScope.Propagator.class);
- bind(PerRequestProjectControlCache.class).in(RequestScoped.class);
bind(Destination.class).toInstance(Destination.this);
bind(RemoteConfig.class).toInstance(config.getRemoteConfig());
@@ -183,7 +186,6 @@
}
});
- projectControlFactory = child.getInstance(ProjectControl.Factory.class);
opFactory = child.getInstance(PushOne.Factory.class);
threadScoper = child.getInstance(PerThreadRequestScope.Scoper.class);
}
@@ -223,15 +225,21 @@
return cnt;
}
- private boolean shouldReplicate(ProjectControl ctl) throws PermissionBackendException {
- if (!config.replicateHiddenProjects() && ctl.getProject().getState() == ProjectState.HIDDEN) {
+ private boolean shouldReplicate(ProjectState state, CurrentUser user)
+ throws PermissionBackendException {
+ if (!config.replicateHiddenProjects()
+ && state.getProject().getState()
+ == com.google.gerrit.extensions.client.ProjectState.HIDDEN) {
return false;
}
+
+ // Hidden projects(permitsRead = false) should only be accessible by the project owners.
+ // READ_CONFIG is checked here because it's only allowed to project owners(ACCESS may also
+ // be allowed for other users).
+ ProjectPermission permissionToCheck =
+ state.statePermitsRead() ? ProjectPermission.ACCESS : ProjectPermission.READ_CONFIG;
try {
- permissionBackend
- .user(ctl.getUser())
- .project(ctl.getProject().getNameKey())
- .check(ProjectPermission.ACCESS);
+ permissionBackend.user(user).project(state.getNameKey()).check(permissionToCheck);
return true;
} catch (AuthException e) {
return false;
@@ -246,8 +254,19 @@
new Callable<Boolean>() {
@Override
public Boolean call() throws NoSuchProjectException, PermissionBackendException {
- ProjectControl projectControl = controlFor(project);
- if (!shouldReplicate(projectControl)) {
+ ProjectState projectState;
+ try {
+ projectState = projectCache.checkedGet(project);
+ } catch (IOException e) {
+ return false;
+ }
+ if (projectState == null) {
+ throw new NoSuchProjectException(project);
+ }
+ if (!projectState.statePermitsRead()) {
+ return false;
+ }
+ if (!shouldReplicate(projectState, userProvider.get())) {
return false;
}
if (PushOne.ALL_REFS.equals(ref)) {
@@ -255,7 +274,7 @@
}
try {
permissionBackend
- .user(projectControl.getUser())
+ .user(userProvider.get())
.project(project)
.ref(ref)
.check(RefPermission.READ);
@@ -282,7 +301,16 @@
new Callable<Boolean>() {
@Override
public Boolean call() throws NoSuchProjectException, PermissionBackendException {
- return shouldReplicate(controlFor(project));
+ ProjectState projectState;
+ try {
+ projectState = projectCache.checkedGet(project);
+ } catch (IOException e) {
+ return false;
+ }
+ if (projectState == null) {
+ throw new NoSuchProjectException(project);
+ }
+ return shouldReplicate(projectState, userProvider.get());
}
})
.call();
@@ -456,10 +484,6 @@
}
}
- ProjectControl controlFor(Project.NameKey project) throws NoSuchProjectException {
- return projectControlFactory.controlFor(project);
- }
-
boolean requestRunway(PushOne op) {
synchronized (stateLock) {
if (op.wasCanceled()) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationFactory.java
index 83eab86..5e41e12 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationFactory.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationFactory.java
@@ -14,15 +14,18 @@
package com.googlesource.gerrit.plugins.replication;
-import com.google.gerrit.common.EventDispatcher;
import com.google.gerrit.extensions.registration.DynamicItem;
+import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.PluginUser;
import com.google.gerrit.server.account.GroupBackend;
import com.google.gerrit.server.account.GroupIncludeCache;
+import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.permissions.PermissionBackend;
+import com.google.gerrit.server.project.ProjectCache;
import com.google.inject.Inject;
import com.google.inject.Injector;
+import com.google.inject.Provider;
import com.google.inject.Singleton;
@Singleton
@@ -32,6 +35,8 @@
private final PluginUser pluginUser;
private final GitRepositoryManager gitRepositoryManager;
private final PermissionBackend permissionBackend;
+ private final Provider<CurrentUser> userProvider;
+ private final ProjectCache projectCache;
private final GroupBackend groupBackend;
private final ReplicationStateListener stateLog;
private final GroupIncludeCache groupIncludeCache;
@@ -44,6 +49,8 @@
PluginUser pluginUser,
GitRepositoryManager gitRepositoryManager,
PermissionBackend permissionBackend,
+ Provider<CurrentUser> userProvider,
+ ProjectCache projectCache,
GroupBackend groupBackend,
ReplicationStateListener stateLog,
GroupIncludeCache groupIncludeCache,
@@ -53,6 +60,8 @@
this.pluginUser = pluginUser;
this.gitRepositoryManager = gitRepositoryManager;
this.permissionBackend = permissionBackend;
+ this.userProvider = userProvider;
+ this.projectCache = projectCache;
this.groupBackend = groupBackend;
this.stateLog = stateLog;
this.groupIncludeCache = groupIncludeCache;
@@ -67,6 +76,8 @@
pluginUser,
gitRepositoryManager,
permissionBackend,
+ userProvider,
+ projectCache,
groupBackend,
stateLog,
groupIncludeCache,
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/EventsStorage.java b/src/main/java/com/googlesource/gerrit/plugins/replication/EventsStorage.java
new file mode 100644
index 0000000..d6ae6b4
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/EventsStorage.java
@@ -0,0 +1,110 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.flogger.FluentLogger;
+import com.google.common.hash.Hashing;
+import com.google.gson.Gson;
+import com.google.inject.Inject;
+import com.google.inject.ProvisionException;
+import com.google.inject.Singleton;
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import org.eclipse.jgit.lib.ObjectId;
+
+@Singleton
+public class EventsStorage {
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+ public static class ReplicateRefUpdate {
+ public String project;
+ public String ref;
+ }
+
+ private static Gson GSON = new Gson();
+
+ private final Path refUpdates;
+
+ @Inject
+ EventsStorage(ReplicationConfig config) {
+ refUpdates = config.getEventsDirectory().resolve("ref-updates");
+ }
+
+ public String persist(String project, String ref) {
+ ReplicateRefUpdate r = new ReplicateRefUpdate();
+ r.project = project;
+ r.ref = ref;
+
+ String json = GSON.toJson(r) + "\n";
+ String eventKey = sha1(json).name();
+ Path file = refUpdates().resolve(eventKey);
+
+ if (Files.exists(file)) {
+ return eventKey;
+ }
+
+ try {
+ Files.write(file, json.getBytes(UTF_8));
+ } catch (IOException e) {
+ logger.atWarning().log("Couldn't persist event %s", json);
+ }
+ return eventKey;
+ }
+
+ public void delete(String eventKey) {
+ if (eventKey != null) {
+ try {
+ Files.delete(refUpdates().resolve(eventKey));
+ } catch (IOException e) {
+ logger.atSevere().withCause(e).log("Error while deleting event %s", eventKey);
+ }
+ }
+ }
+
+ public List<ReplicateRefUpdate> list() {
+ ArrayList<ReplicateRefUpdate> result = new ArrayList<>();
+ try (DirectoryStream<Path> events = Files.newDirectoryStream(refUpdates())) {
+ for (Path e : events) {
+ if (Files.isRegularFile(e)) {
+ String json = new String(Files.readAllBytes(e), UTF_8);
+ result.add(GSON.fromJson(json, ReplicateRefUpdate.class));
+ Files.delete(e);
+ }
+ }
+ } catch (IOException e) {
+ logger.atSevere().withCause(e).log("Error when firing pending events");
+ }
+ return result;
+ }
+
+ @SuppressWarnings("deprecation")
+ private ObjectId sha1(String s) {
+ return ObjectId.fromRaw(Hashing.sha1().hashString(s, UTF_8).asBytes());
+ }
+
+ private Path refUpdates() {
+ try {
+ return Files.createDirectories(refUpdates);
+ } catch (IOException e) {
+ throw new ProvisionException(String.format("Couldn't create %s", refUpdates), e);
+ }
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java
index b46a0d9..85b17d0 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java
@@ -14,45 +14,45 @@
package com.googlesource.gerrit.plugins.replication;
+import com.google.common.flogger.FluentLogger;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.ssh.SshAddressesModule;
-import com.google.inject.Inject;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URISyntaxException;
import java.util.HashSet;
import java.util.Set;
import org.eclipse.jgit.transport.URIish;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-public class GerritSshApi {
+public class GerritSshApi implements AdminApi {
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
static int SSH_COMMAND_FAILED = -1;
- private static final Logger log = LoggerFactory.getLogger(GerritSshApi.class);
private static String GERRIT_ADMIN_PROTOCOL_PREFIX = "gerrit+";
private final SshHelper sshHelper;
+ private final URIish uri;
private final Set<URIish> withoutDeleteProjectPlugin = new HashSet<>();
- @Inject
- protected GerritSshApi(SshHelper sshHelper) {
+ protected GerritSshApi(SshHelper sshHelper, URIish uri) {
this.sshHelper = sshHelper;
+ this.uri = uri;
}
- protected boolean createProject(URIish uri, Project.NameKey projectName, String head) {
+ @Override
+ public void createProject(Project.NameKey projectName, String head) {
OutputStream errStream = sshHelper.newErrorBufferStream();
String cmd = "gerrit create-project --branch " + head + " " + projectName.get();
try {
execute(uri, cmd, errStream);
} catch (IOException e) {
logError("creating", uri, errStream, cmd, e);
- return false;
}
- return true;
}
- protected boolean deleteProject(URIish uri, Project.NameKey projectName) {
+ @Override
+ public void deleteProject(Project.NameKey projectName) {
if (!withoutDeleteProjectPlugin.contains(uri)) {
OutputStream errStream = sshHelper.newErrorBufferStream();
String cmd = "deleteproject delete --yes-really-delete --force " + projectName.get();
@@ -61,36 +61,28 @@
exitCode = execute(uri, cmd, errStream);
} catch (IOException e) {
logError("deleting", uri, errStream, cmd, e);
- return false;
}
if (exitCode == 1) {
- log.info(
- "DeleteProject plugin is not installed on {}; will not try to forward this operation to that host");
+ logger.atInfo().log(
+ "DeleteProject plugin is not installed on %s;"
+ + " will not try to forward this operation to that host");
withoutDeleteProjectPlugin.add(uri);
- return true;
}
}
- return true;
}
- protected boolean updateHead(URIish uri, Project.NameKey projectName, String newHead) {
+ @Override
+ public void updateHead(Project.NameKey projectName, String newHead) {
OutputStream errStream = sshHelper.newErrorBufferStream();
String cmd = "gerrit set-head " + projectName.get() + " --new-head " + newHead;
try {
execute(uri, cmd, errStream);
} catch (IOException e) {
- log.error(
- "Error updating HEAD of remote repository at {} to {}:\n"
- + " Exception: {}\n Command: {}\n Output: {}",
- uri,
- newHead,
- e,
- cmd,
- errStream,
- e);
- return false;
+ logger.atSevere().withCause(e).log(
+ "Error updating HEAD of remote repository at %s to %s:\n"
+ + " Exception: %s\n Command: %s\n Output: %s",
+ uri, newHead, e, cmd, errStream);
}
- return true;
}
private URIish toSshUri(URIish uri) throws URISyntaxException {
@@ -114,19 +106,14 @@
URIish sshUri = toSshUri(uri);
return sshHelper.executeRemoteSsh(sshUri, cmd, errStream);
} catch (URISyntaxException e) {
- log.error("Cannot convert {} to SSH uri", uri, e);
+ logger.atSevere().withCause(e).log("Cannot convert %s to SSH uri", uri);
}
return SSH_COMMAND_FAILED;
}
public void logError(String msg, URIish uri, OutputStream errStream, String cmd, IOException e) {
- log.error(
- "Error {} remote repository at {}:\n Exception: {}\n Command: {}\n Output: {}",
- msg,
- uri,
- e,
- cmd,
- errStream,
- e);
+ logger.atSevere().withCause(e).log(
+ "Error %s remote repository at %s:\n Exception: %s\n Command: %s\n Output: %s",
+ msg, uri, e, cmd, errStream);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java b/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java
new file mode 100644
index 0000000..1012cd7
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java
@@ -0,0 +1,91 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+
+import com.google.gerrit.reviewdb.client.Project;
+import java.io.File;
+import java.io.IOException;
+import org.eclipse.jgit.internal.storage.file.FileRepository;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.RefUpdate;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.transport.URIish;
+
+public class LocalFS implements AdminApi {
+
+ private final URIish uri;
+
+ public LocalFS(URIish uri) {
+ this.uri = uri;
+ }
+
+ @Override
+ public void createProject(Project.NameKey project, String head) {
+ try (Repository repo = new FileRepository(uri.getPath())) {
+ repo.create(true /* bare */);
+
+ if (head != null && head.startsWith(Constants.R_REFS)) {
+ RefUpdate u = repo.updateRef(Constants.HEAD);
+ u.disableRefLog();
+ u.link(head);
+ }
+ repLog.info("Created local repository: {}", uri);
+ } catch (IOException e) {
+ repLog.error("Error creating local repository {}", uri.getPath(), e);
+ }
+ }
+
+ @Override
+ public void deleteProject(Project.NameKey project) {
+ try {
+ recursivelyDelete(new File(uri.getPath()));
+ repLog.info("Deleted local repository: {}", uri);
+ } catch (IOException e) {
+ repLog.error("Error deleting local repository {}:\n", uri.getPath(), e);
+ }
+ }
+
+ @Override
+ public void updateHead(Project.NameKey project, String newHead) {
+ try (Repository repo = new FileRepository(uri.getPath())) {
+ if (newHead != null) {
+ RefUpdate u = repo.updateRef(Constants.HEAD);
+ u.link(newHead);
+ }
+ } catch (IOException e) {
+ repLog.error("Failed to update HEAD of repository {} to {}", uri.getPath(), newHead, e);
+ }
+ }
+
+ private static void recursivelyDelete(File dir) throws IOException {
+ File[] contents = dir.listFiles();
+ if (contents != null) {
+ for (File d : contents) {
+ if (d.isDirectory()) {
+ recursivelyDelete(d);
+ } else {
+ if (!d.delete()) {
+ throw new IOException("Failed to delete: " + d.getAbsolutePath());
+ }
+ }
+ }
+ }
+ if (!dir.delete()) {
+ throw new IOException("Failed to delete: " + dir.getAbsolutePath());
+ }
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java b/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java
index 227804d..ac0262d 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java
@@ -15,12 +15,13 @@
package com.googlesource.gerrit.plugins.replication;
import com.google.common.util.concurrent.Atomics;
-import com.google.gerrit.common.EventDispatcher;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.extensions.systemstatus.ServerInformation;
+import com.google.gerrit.server.events.EventDispatcher;
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.replication.PushResultProcessing.GitUpdateProcessing;
+import com.googlesource.gerrit.plugins.replication.ReplicationState.Factory;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
@@ -29,32 +30,31 @@
private final AtomicReference<Future<?>> pushAllFuture;
private final ServerInformation srvInfo;
private final PushAll.Factory pushAll;
- private final ReplicationQueue queue;
private final ReplicationConfig config;
private final DynamicItem<EventDispatcher> eventDispatcher;
+ private final Factory replicationStateFactory;
@Inject
protected OnStartStop(
ServerInformation srvInfo,
PushAll.Factory pushAll,
- ReplicationQueue queue,
ReplicationConfig config,
- DynamicItem<EventDispatcher> eventDispatcher) {
+ DynamicItem<EventDispatcher> eventDispatcher,
+ ReplicationState.Factory replicationStateFactory) {
this.srvInfo = srvInfo;
this.pushAll = pushAll;
- this.queue = queue;
this.config = config;
this.eventDispatcher = eventDispatcher;
+ this.replicationStateFactory = replicationStateFactory;
this.pushAllFuture = Atomics.newReference();
}
@Override
public void start() {
- queue.start();
-
if (srvInfo.getState() == ServerInformation.State.STARTUP
&& config.isReplicateAllOnPluginStart()) {
- ReplicationState state = new ReplicationState(new GitUpdateProcessing(eventDispatcher.get()));
+ ReplicationState state =
+ replicationStateFactory.create(new GitUpdateProcessing(eventDispatcher.get()));
pushAllFuture.set(
pushAll
.create(null, ReplicationFilter.all(), state, false)
@@ -68,6 +68,5 @@
if (f != null) {
f.cancel(true);
}
- queue.stop();
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
index 1efad4f..3ce3a19 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
@@ -23,19 +23,21 @@
import com.google.common.collect.Sets;
import com.google.gerrit.extensions.events.GitReferenceUpdatedListener;
import com.google.gerrit.extensions.restapi.AuthException;
+import com.google.gerrit.extensions.restapi.ResourceConflictException;
import com.google.gerrit.metrics.Timer1;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.client.RefNames;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.PerThreadRequestScope;
import com.google.gerrit.server.git.ProjectRunnable;
-import com.google.gerrit.server.git.VisibleRefFilter;
import com.google.gerrit.server.git.WorkQueue.CanceledWhileRunning;
+import com.google.gerrit.server.ioutil.HexFormat;
import com.google.gerrit.server.permissions.PermissionBackend;
+import com.google.gerrit.server.permissions.PermissionBackend.RefFilterOptions;
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.google.gerrit.server.permissions.ProjectPermission;
-import com.google.gerrit.server.project.NoSuchProjectException;
-import com.google.gerrit.server.project.ProjectControl;
+import com.google.gerrit.server.project.ProjectCache;
+import com.google.gerrit.server.project.ProjectState;
import com.google.gerrit.server.util.IdGenerator;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
@@ -92,7 +94,6 @@
private final RemoteConfig config;
private final CredentialsProvider credentialsProvider;
private final PerThreadRequestScope.Scoper threadScoper;
- private final VisibleRefFilter.Factory refFilterFactory;
private final ReplicationQueue replicationQueue;
private final Project.NameKey projectName;
@@ -110,6 +111,7 @@
private final int id;
private final long createdAt;
private final ReplicationMetrics metrics;
+ private final ProjectCache projectCache;
private final AtomicBoolean canceledWhileRunning;
@Inject
@@ -118,20 +120,19 @@
PermissionBackend permissionBackend,
Destination p,
RemoteConfig c,
- VisibleRefFilter.Factory rff,
CredentialsFactory cpFactory,
PerThreadRequestScope.Scoper ts,
ReplicationQueue rq,
IdGenerator ig,
ReplicationStateListener sl,
ReplicationMetrics m,
+ ProjectCache pc,
@Assisted Project.NameKey d,
@Assisted URIish u) {
gitManager = grm;
this.permissionBackend = permissionBackend;
pool = p;
config = c;
- refFilterFactory = rff;
credentialsProvider = cpFactory.create(c.getName());
threadScoper = ts;
replicationQueue = rq;
@@ -143,6 +144,7 @@
stateLog = sl;
createdAt = System.nanoTime();
metrics = m;
+ projectCache = pc;
canceledWhileRunning = new AtomicBoolean(false);
maxRetries = p.getMaxRetries();
}
@@ -177,7 +179,7 @@
@Override
public String toString() {
- String print = "[" + IdGenerator.format(id) + "] push " + uri;
+ String print = "[" + HexFormat.fromInt(id) + "] push " + uri;
if (retryCount > 0) {
print = "(retry " + retryCount + ") " + print;
@@ -294,7 +296,7 @@
// we start replication (instead a new instance, with the same URI, is
// created and scheduled for a future point in time.)
//
- MDC.put(ID_MDC_KEY, IdGenerator.format(id));
+ MDC.put(ID_MDC_KEY, HexFormat.fromInt(id));
if (!pool.requestRunway(this)) {
if (!canceled) {
repLog.info(
@@ -452,19 +454,19 @@
private List<RemoteRefUpdate> generateUpdates(Transport tn)
throws IOException, PermissionBackendException {
- ProjectControl pc;
- try {
- pc = pool.controlFor(projectName);
- } catch (NoSuchProjectException e) {
+ ProjectState projectState = projectCache.checkedGet(projectName);
+ if (projectState == null) {
return Collections.emptyList();
}
Map<String, Ref> local = git.getAllRefs();
boolean filter;
+ PermissionBackend.ForProject forProject = permissionBackend.currentUser().project(projectName);
try {
- permissionBackend.user(pc.getUser()).project(projectName).check(ProjectPermission.READ);
+ projectState.checkStatePermitsRead();
+ forProject.check(ProjectPermission.READ);
filter = false;
- } catch (AuthException e) {
+ } catch (AuthException | ResourceConflictException e) {
filter = true;
}
if (filter) {
@@ -481,7 +483,7 @@
}
local = n;
}
- local = refFilterFactory.create(pc.getProjectState(), git).filter(local, true);
+ local = forProject.filter(local, git, RefFilterOptions.builder().setFilterMeta(true).build());
}
return pushAllRefs ? doPushAll(tn, local) : doPushDelta(local);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java
index c71a792..ae0662d 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java
@@ -14,7 +14,8 @@
package com.googlesource.gerrit.plugins.replication;
-import com.google.gerrit.common.EventDispatcher;
+import com.google.common.flogger.FluentLogger;
+import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.events.RefEvent;
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.google.gwtorm.server.OrmException;
@@ -23,8 +24,6 @@
import java.util.concurrent.atomic.AtomicBoolean;
import org.eclipse.jgit.transport.RemoteRefUpdate;
import org.eclipse.jgit.transport.URIish;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public abstract class PushResultProcessing {
@@ -159,7 +158,7 @@
}
public static class GitUpdateProcessing extends PushResultProcessing {
- private static final Logger log = LoggerFactory.getLogger(GitUpdateProcessing.class);
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private final EventDispatcher dispatcher;
@@ -189,7 +188,7 @@
try {
dispatcher.postEvent(event);
} catch (OrmException | PermissionBackendException e) {
- log.error("Cannot post event", e);
+ logger.atSevere().withCause(e).log("Cannot post event");
}
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java
index 91fce7f..31d10b6 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSiteUser.java
@@ -35,4 +35,10 @@
public GroupMembership getEffectiveGroups() {
return effectiveGroups;
}
+
+ @Override
+ public Object getCacheKey() {
+ // Never cache a remote user
+ return new Object();
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java
new file mode 100644
index 0000000..dad1b0b
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java
@@ -0,0 +1,104 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+
+import com.google.gerrit.reviewdb.client.Project;
+import java.io.IOException;
+import java.io.OutputStream;
+import org.eclipse.jgit.transport.URIish;
+import org.eclipse.jgit.util.QuotedString;
+
+public class RemoteSsh implements AdminApi {
+
+ private final SshHelper sshHelper;
+ private URIish uri;
+
+ RemoteSsh(SshHelper sshHelper, URIish uri) {
+ this.sshHelper = sshHelper;
+ this.uri = uri;
+ }
+
+ @Override
+ public void createProject(Project.NameKey project, String head) {
+ String quotedPath = QuotedString.BOURNE.quote(uri.getPath());
+ String cmd = "mkdir -p " + quotedPath + " && cd " + quotedPath + " && git init --bare";
+ if (head != null) {
+ cmd = cmd + " && git symbolic-ref HEAD " + QuotedString.BOURNE.quote(head);
+ }
+ OutputStream errStream = sshHelper.newErrorBufferStream();
+ try {
+ sshHelper.executeRemoteSsh(uri, cmd, errStream);
+ repLog.info("Created remote repository: {}", uri);
+ } catch (IOException e) {
+ repLog.error(
+ "Error creating remote repository at {}:\n"
+ + " Exception: {}\n"
+ + " Command: {}\n"
+ + " Output: {}",
+ uri,
+ e,
+ cmd,
+ errStream,
+ e);
+ }
+ }
+
+ @Override
+ public void deleteProject(Project.NameKey project) {
+ String quotedPath = QuotedString.BOURNE.quote(uri.getPath());
+ String cmd = "rm -rf " + quotedPath;
+ OutputStream errStream = sshHelper.newErrorBufferStream();
+ try {
+ sshHelper.executeRemoteSsh(uri, cmd, errStream);
+ repLog.info("Deleted remote repository: {}", uri);
+ } catch (IOException e) {
+ repLog.error(
+ "Error deleting remote repository at {}:\n"
+ + " Exception: {}\n"
+ + " Command: {}\n"
+ + " Output: {}",
+ uri,
+ e,
+ cmd,
+ errStream,
+ e);
+ }
+ }
+
+ @Override
+ public void updateHead(Project.NameKey project, String newHead) {
+ String quotedPath = QuotedString.BOURNE.quote(uri.getPath());
+ String cmd =
+ "cd " + quotedPath + " && git symbolic-ref HEAD " + QuotedString.BOURNE.quote(newHead);
+ OutputStream errStream = sshHelper.newErrorBufferStream();
+ try {
+ sshHelper.executeRemoteSsh(uri, cmd, errStream);
+ } catch (IOException e) {
+ repLog.error(
+ "Error updating HEAD of remote repository at {} to {}:\n"
+ + " Exception: {}\n"
+ + " Command: {}\n"
+ + " Output: {}",
+ uri,
+ newHead,
+ e,
+ cmd,
+ errStream,
+ e);
+ }
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
index e94abbd..9693e2d 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
@@ -14,6 +14,7 @@
package com.googlesource.gerrit.plugins.replication;
import com.google.gerrit.server.git.WorkQueue;
+import java.nio.file.Path;
import java.util.List;
public interface ReplicationConfig {
@@ -32,6 +33,8 @@
boolean isEmpty();
+ Path getEventsDirectory();
+
int shutdown();
void startup(WorkQueue workQueue);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
index ee1f16d..3e8781e 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
@@ -15,8 +15,11 @@
import static java.util.stream.Collectors.toList;
+import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import com.google.common.flogger.FluentLogger;
+import com.google.gerrit.extensions.annotations.PluginData;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.git.WorkQueue;
import com.google.inject.Inject;
@@ -35,24 +38,28 @@
import org.eclipse.jgit.transport.RemoteConfig;
import org.eclipse.jgit.transport.URIish;
import org.eclipse.jgit.util.FS;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
@Singleton
public class ReplicationFileBasedConfig implements ReplicationConfig {
- private static final Logger log = LoggerFactory.getLogger(ReplicationFileBasedConfig.class);
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
private List<Destination> destinations;
+ private final SitePaths site;
private Path cfgPath;
private boolean replicateAllOnPluginStart;
private boolean defaultForceUpdate;
private final FileBasedConfig config;
+ private final Path pluginDataDir;
@Inject
- public ReplicationFileBasedConfig(SitePaths site, DestinationFactory destinationFactory)
+ public ReplicationFileBasedConfig(
+ SitePaths site, DestinationFactory destinationFactory, @PluginData Path pluginDataDir)
throws ConfigInvalidException, IOException {
+ this.site = site;
this.cfgPath = site.etc_dir.resolve("replication.config");
this.config = new FileBasedConfig(cfgPath.toFile(), FS.DETECTED);
this.destinations = allDestinations(destinationFactory);
+ this.pluginDataDir = pluginDataDir;
}
/*
@@ -82,11 +89,11 @@
private List<Destination> allDestinations(DestinationFactory destinationFactory)
throws ConfigInvalidException, IOException {
if (!config.getFile().exists()) {
- log.warn("Config file {} does not exist; not replicating", config.getFile());
+ logger.atWarning().log("Config file %s does not exist; not replicating", config.getFile());
return Collections.emptyList();
}
if (config.getFile().length() == 0) {
- log.info("Config file {} is empty; not replicating", config.getFile());
+ logger.atInfo().log("Config file %s is empty; not replicating", config.getFile());
return Collections.emptyList();
}
@@ -100,7 +107,7 @@
String.format("Cannot read %s: %s", config.getFile(), e.getMessage()), e);
}
- replicateAllOnPluginStart = config.getBoolean("gerrit", "replicateOnStartup", true);
+ replicateAllOnPluginStart = config.getBoolean("gerrit", "replicateOnStartup", false);
defaultForceUpdate = config.getBoolean("gerrit", "defaultForceUpdate", false);
@@ -180,6 +187,15 @@
return destinations.isEmpty();
}
+ @Override
+ public Path getEventsDirectory() {
+ String eventsDirectory = config.getString("replication", null, "eventsDirectory");
+ if (!Strings.isNullOrEmpty(eventsDirectory)) {
+ return site.resolve(eventsDirectory);
+ }
+ return pluginDataDir;
+ }
+
Path getCfgPath() {
return cfgPath;
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFilter.java
index 7b3486b..05bbb03 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFilter.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFilter.java
@@ -15,7 +15,7 @@
package com.googlesource.gerrit.plugins.replication;
import com.google.gerrit.common.data.AccessSection;
-import com.google.gerrit.reviewdb.client.Project.NameKey;
+import com.google.gerrit.reviewdb.client.Project;
import java.util.Collections;
import java.util.List;
@@ -46,7 +46,7 @@
projectPatterns = patterns;
}
- public boolean matches(NameKey name) {
+ public boolean matches(Project.NameKey name) {
if (projectPatterns.isEmpty()) {
return true;
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
index f30e13d..61c2e69 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
@@ -36,6 +36,9 @@
protected void configure() {
bind(DestinationFactory.class).in(Scopes.SINGLETON);
bind(ReplicationQueue.class).in(Scopes.SINGLETON);
+ bind(LifecycleListener.class)
+ .annotatedWith(UniqueAnnotations.create())
+ .to(ReplicationQueue.class);
DynamicSet.bind(binder(), GitReferenceUpdatedListener.class).to(ReplicationQueue.class);
DynamicSet.bind(binder(), NewProjectCreatedListener.class).to(ReplicationQueue.class);
@@ -56,6 +59,7 @@
install(new FactoryModuleBuilder().build(PushAll.Factory.class));
install(new FactoryModuleBuilder().build(RemoteSiteUser.Factory.class));
+ install(new FactoryModuleBuilder().build(ReplicationState.Factory.class));
bind(ReplicationConfig.class).to(AutoReloadConfigDecorator.class);
bind(ReplicationStateListener.class).to(ReplicationStateLogger.class);
@@ -64,5 +68,7 @@
EventTypes.register(RefReplicationDoneEvent.TYPE, RefReplicationDoneEvent.class);
EventTypes.register(ReplicationScheduledEvent.TYPE, ReplicationScheduledEvent.class);
bind(SshSessionFactory.class).toProvider(ReplicationSshSessionFactoryProvider.class);
+
+ bind(AdminApiFactory.class);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
index 30aff44..ae06f43 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
@@ -14,8 +14,10 @@
package com.googlesource.gerrit.plugins.replication;
+import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isGerrit;
+import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isSSH;
+
import com.google.common.base.Strings;
-import com.google.gerrit.common.EventDispatcher;
import com.google.gerrit.extensions.events.GitReferenceUpdatedListener;
import com.google.gerrit.extensions.events.HeadUpdatedListener;
import com.google.gerrit.extensions.events.LifecycleListener;
@@ -23,23 +25,17 @@
import com.google.gerrit.extensions.events.ProjectDeletedListener;
import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.git.WorkQueue;
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.replication.PushResultProcessing.GitUpdateProcessing;
import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType;
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
import java.net.URISyntaxException;
import java.util.Collections;
import java.util.HashSet;
+import java.util.Optional;
import java.util.Set;
-import org.eclipse.jgit.internal.storage.file.FileRepository;
-import org.eclipse.jgit.lib.Constants;
-import org.eclipse.jgit.lib.RefUpdate;
-import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.URIish;
-import org.eclipse.jgit.util.QuotedString;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -68,32 +64,36 @@
}
private final WorkQueue workQueue;
- private final SshHelper sshHelper;
private final DynamicItem<EventDispatcher> dispatcher;
private final ReplicationConfig config;
- private final GerritSshApi gerritAdmin;
+ private final AdminApiFactory adminApiFactory;
+ private final ReplicationState.Factory replicationStateFactory;
+ private final EventsStorage eventsStorage;
private volatile boolean running;
@Inject
ReplicationQueue(
WorkQueue wq,
- SshHelper sh,
- GerritSshApi ga,
+ AdminApiFactory aaf,
ReplicationConfig rc,
DynamicItem<EventDispatcher> dis,
- ReplicationStateListener sl) {
+ ReplicationStateListener sl,
+ ReplicationState.Factory rsf,
+ EventsStorage es) {
workQueue = wq;
- sshHelper = sh;
dispatcher = dis;
config = rc;
stateLog = sl;
- gerritAdmin = ga;
+ adminApiFactory = aaf;
+ replicationStateFactory = rsf;
+ eventsStorage = es;
}
@Override
public void start() {
config.startup(workQueue);
running = true;
+ firePendingEvents();
}
@Override
@@ -127,23 +127,37 @@
@Override
public void onGitReferenceUpdated(GitReferenceUpdatedListener.Event event) {
- ReplicationState state = new ReplicationState(new GitUpdateProcessing(dispatcher.get()));
+ onGitReferenceUpdated(event.getProjectName(), event.getRefName());
+ }
+
+ private void onGitReferenceUpdated(String projectName, String refName) {
+ ReplicationState state =
+ replicationStateFactory.create(new GitUpdateProcessing(dispatcher.get()));
if (!running) {
stateLog.warn("Replication plugin did not finish startup before event", state);
return;
}
- Project.NameKey project = new Project.NameKey(event.getProjectName());
+ Project.NameKey project = new Project.NameKey(projectName);
for (Destination cfg : config.getDestinations(FilterType.ALL)) {
- if (cfg.wouldPushProject(project) && cfg.wouldPushRef(event.getRefName())) {
+ if (cfg.wouldPushProject(project) && cfg.wouldPushRef(refName)) {
+ String eventKey = eventsStorage.persist(projectName, refName);
+ state.setEventKey(eventKey);
for (URIish uri : cfg.getURIs(project, null)) {
- cfg.schedule(project, event.getRefName(), uri, state);
+ cfg.schedule(project, refName, uri, state);
}
}
}
state.markAllPushTasksScheduled();
}
+ private void firePendingEvents() {
+ for (EventsStorage.ReplicateRefUpdate e : eventsStorage.list()) {
+ repLog.info("Firing pending event {}", e);
+ onGitReferenceUpdated(e.project, e.ref);
+ }
+ }
+
@Override
public void onNewProjectCreated(NewProjectCreatedListener.Event event) {
Project.NameKey projectName = new Project.NameKey(event.getProjectName());
@@ -234,192 +248,41 @@
}
private boolean createProject(URIish replicateURI, Project.NameKey projectName, String head) {
- if (isGerrit(replicateURI)) {
- gerritAdmin.createProject(replicateURI, projectName, head);
- } else if (!replicateURI.isRemote()) {
- createLocally(replicateURI, head);
- } else if (isSSH(replicateURI)) {
- createRemoteSsh(replicateURI, head);
- } else {
- repLog.warn(
- "Cannot create new project on remote site {}."
- + " Only local paths and SSH URLs are supported"
- + " for remote repository creation",
- replicateURI);
- return false;
- }
- return true;
- }
-
- private static void createLocally(URIish uri, String head) {
- try (Repository repo = new FileRepository(uri.getPath())) {
- repo.create(true /* bare */);
-
- if (head != null && head.startsWith(Constants.R_REFS)) {
- RefUpdate u = repo.updateRef(Constants.HEAD);
- u.disableRefLog();
- u.link(head);
- }
- repLog.info("Created local repository: {}", uri);
- } catch (IOException e) {
- repLog.error("Error creating local repository {}:\n", uri.getPath(), e);
- }
- }
-
- private void createRemoteSsh(URIish uri, String head) {
- String quotedPath = QuotedString.BOURNE.quote(uri.getPath());
- String cmd = "mkdir -p " + quotedPath + " && cd " + quotedPath + " && git init --bare";
- if (head != null) {
- cmd = cmd + " && git symbolic-ref HEAD " + QuotedString.BOURNE.quote(head);
- }
- OutputStream errStream = sshHelper.newErrorBufferStream();
- try {
- sshHelper.executeRemoteSsh(uri, cmd, errStream);
- repLog.info("Created remote repository: {}", uri);
- } catch (IOException e) {
- repLog.error(
- "Error creating remote repository at {}:\n"
- + " Exception: {}\n"
- + " Command: {}\n"
- + " Output: {}",
- uri,
- e,
- cmd,
- errStream,
- e);
- }
- }
-
- private void deleteProject(URIish replicateURI, Project.NameKey projectName) {
- if (isGerrit(replicateURI)) {
- gerritAdmin.deleteProject(replicateURI, projectName);
- repLog.info("Deleted remote repository: " + replicateURI);
- } else if (!replicateURI.isRemote()) {
- deleteLocally(replicateURI);
- } else if (isSSH(replicateURI)) {
- deleteRemoteSsh(replicateURI);
- } else {
- repLog.warn(
- "Cannot delete project on remote site {}. "
- + "Only local paths and SSH URLs are supported"
- + " for remote repository deletion",
- replicateURI);
- }
- }
-
- private static void deleteLocally(URIish uri) {
- try {
- recursivelyDelete(new File(uri.getPath()));
- repLog.info("Deleted local repository: {}", uri);
- } catch (IOException e) {
- repLog.error("Error deleting local repository {}:\n", uri.getPath(), e);
- }
- }
-
- private static void recursivelyDelete(File dir) throws IOException {
- File[] contents = dir.listFiles();
- if (contents != null) {
- for (File d : contents) {
- if (d.isDirectory()) {
- recursivelyDelete(d);
- } else {
- if (!d.delete()) {
- throw new IOException("Failed to delete: " + d.getAbsolutePath());
- }
- }
- }
- }
- if (!dir.delete()) {
- throw new IOException("Failed to delete: " + dir.getAbsolutePath());
- }
- }
-
- private void deleteRemoteSsh(URIish uri) {
- String quotedPath = QuotedString.BOURNE.quote(uri.getPath());
- String cmd = "rm -rf " + quotedPath;
- OutputStream errStream = sshHelper.newErrorBufferStream();
- try {
- sshHelper.executeRemoteSsh(uri, cmd, errStream);
- repLog.info("Deleted remote repository: {}", uri);
- } catch (IOException e) {
- repLog.error(
- "Error deleting remote repository at {}:\n"
- + " Exception: {}\n"
- + " Command: {}\n"
- + " Output: {}",
- uri,
- e,
- cmd,
- errStream,
- e);
- }
- }
-
- private void updateHead(URIish replicateURI, Project.NameKey projectName, String newHead) {
- if (isGerrit(replicateURI)) {
- gerritAdmin.updateHead(replicateURI, projectName, newHead);
- } else if (!replicateURI.isRemote()) {
- updateHeadLocally(replicateURI, newHead);
- } else if (isSSH(replicateURI)) {
- updateHeadRemoteSsh(replicateURI, newHead);
- } else {
- repLog.warn(
- "Cannot update HEAD of project on remote site {}."
- + " Only local paths and SSH URLs are supported"
- + " for remote HEAD update.",
- replicateURI);
- }
- }
-
- private void updateHeadRemoteSsh(URIish uri, String newHead) {
- String quotedPath = QuotedString.BOURNE.quote(uri.getPath());
- String cmd =
- "cd " + quotedPath + " && git symbolic-ref HEAD " + QuotedString.BOURNE.quote(newHead);
- OutputStream errStream = sshHelper.newErrorBufferStream();
- try {
- sshHelper.executeRemoteSsh(uri, cmd, errStream);
- } catch (IOException e) {
- repLog.error(
- "Error updating HEAD of remote repository at {} to {}:\n"
- + " Exception: {}\n"
- + " Command: {}\n"
- + " Output: {}",
- uri,
- newHead,
- e,
- cmd,
- errStream,
- e);
- }
- }
-
- private static void updateHeadLocally(URIish uri, String newHead) {
- try (Repository repo = new FileRepository(uri.getPath())) {
- if (newHead != null) {
- RefUpdate u = repo.updateRef(Constants.HEAD);
- u.link(newHead);
- }
- } catch (IOException e) {
- repLog.error("Failed to update HEAD of repository {} to {}", uri.getPath(), newHead, e);
- }
- }
-
- private static boolean isSSH(URIish uri) {
- String scheme = uri.getScheme();
- if (!uri.isRemote()) {
- return false;
- }
- if (scheme != null && scheme.toLowerCase().contains("ssh")) {
+ Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI);
+ if (adminApi.isPresent()) {
+ adminApi.get().createProject(projectName, head);
return true;
}
- if (scheme == null && uri.getHost() != null && uri.getPath() != null) {
- return true;
- }
+
+ warnCannotPerform("create new project", replicateURI);
return false;
}
- private static boolean isGerrit(URIish uri) {
- String scheme = uri.getScheme();
- return scheme != null && scheme.toLowerCase().equals("gerrit+ssh");
+ private void deleteProject(URIish replicateURI, Project.NameKey projectName) {
+ Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI);
+ if (adminApi.isPresent()) {
+ adminApi.get().deleteProject(projectName);
+ return;
+ }
+
+ warnCannotPerform("delete project", replicateURI);
+ }
+
+ private void updateHead(URIish replicateURI, Project.NameKey projectName, String newHead) {
+ Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI);
+ if (adminApi.isPresent()) {
+ adminApi.get().updateHead(projectName, newHead);
+ return;
+ }
+
+ warnCannotPerform("update HEAD of project", replicateURI);
+ }
+
+ private void warnCannotPerform(String op, URIish uri) {
+ repLog.warn(
+ "Cannot {} on remote site {}."
+ + "Only local paths and SSH URLs are supported for this operation",
+ op,
+ uri);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationScheduledEvent.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationScheduledEvent.java
index 7268709..301219f 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationScheduledEvent.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationScheduledEvent.java
@@ -15,7 +15,6 @@
package com.googlesource.gerrit.plugins.replication;
import com.google.gerrit.reviewdb.client.Project;
-import com.google.gerrit.reviewdb.client.Project.NameKey;
import com.google.gerrit.server.events.RefEvent;
public class ReplicationScheduledEvent extends RefEvent {
@@ -38,7 +37,7 @@
}
@Override
- public NameKey getProjectNameKey() {
+ public Project.NameKey getProjectNameKey() {
return new Project.NameKey(project);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationState.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationState.java
index 86557e2..ec878db 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationState.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationState.java
@@ -16,6 +16,8 @@
import com.google.common.collect.HashBasedTable;
import com.google.common.collect.Table;
+import com.google.inject.assistedinject.Assisted;
+import com.google.inject.assistedinject.AssistedInject;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@@ -23,7 +25,13 @@
import org.eclipse.jgit.transport.URIish;
public class ReplicationState {
+
+ public interface Factory {
+ ReplicationState create(PushResultProcessing processing);
+ }
+
private boolean allScheduled;
+ private final EventsStorage eventsStorage;
private final PushResultProcessing pushResultProcessing;
private final Lock countingLock = new ReentrantLock();
@@ -49,7 +57,11 @@
private int totalPushTasksCount;
private int finishedPushTasksCount;
- public ReplicationState(PushResultProcessing processing) {
+ private String eventKey;
+
+ @AssistedInject
+ ReplicationState(EventsStorage storage, @Assisted PushResultProcessing processing) {
+ eventsStorage = storage;
pushResultProcessing = processing;
statusByProjectRef = HashBasedTable.create();
}
@@ -133,10 +145,17 @@
}
private void doRefPushTasksCompleted(RefReplicationStatus refStatus) {
+ deleteEvent();
pushResultProcessing.onRefReplicatedToAllNodes(
refStatus.project, refStatus.ref, refStatus.nodesToReplicateCount);
}
+ private void deleteEvent() {
+ if (eventKey != null) {
+ eventsStorage.delete(eventKey);
+ }
+ }
+
private RefReplicationStatus getRefStatus(String project, String ref) {
if (!statusByProjectRef.contains(project, ref)) {
RefReplicationStatus refStatus = new RefReplicationStatus(project, ref);
@@ -173,4 +192,8 @@
return name().toLowerCase().replace("_", "-");
}
}
+
+ public void setEventKey(String eventKey) {
+ this.eventKey = eventKey;
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java
index 5aa0861..77bc285 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java
@@ -51,13 +51,15 @@
@Inject private PushAll.Factory pushFactory;
+ @Inject private ReplicationState.Factory replicationStateFactory;
+
@Override
protected void run() throws Failure {
if (all && projectPatterns.size() > 0) {
throw new UnloggedFailure(1, "error: cannot combine --all and PROJECT");
}
- ReplicationState state = new ReplicationState(new CommandProcessing(this));
+ ReplicationState state = replicationStateFactory.create(new CommandProcessing(this));
Future<?> future = null;
ReplicationFilter projectFilter;
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index c066513..646d49c 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -65,7 +65,7 @@
gerrit.replicateOnStartup
: If true, replicates to all remotes on startup to ensure they
- are in-sync with this server. By default, true.
+ are in-sync with this server. By default, false.
gerrit.autoReload
: If true, automatically reloads replication destinations and settings
@@ -116,6 +116,17 @@
By default, pushes are retried indefinitely.
+replication.eventsDirectory
+: Directory where replication events are persisted
+
+ When scheduling a replication, the replication event is persisted
+ under this directory. When the replication is done, the event is deleted.
+ If plugin is stopped before all scheduled replications are done, the
+ persisted events will not be deleted. When the plugin is started again,
+ it will trigger all replications found under this directory.
+
+ When not set, defaults to the plugin's data directory.
+
remote.NAME.url
: Address of the remote server to push to. Multiple URLs may be
specified within a single remote block, listing different
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java
index 337bd1d..5fa7b98 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java
@@ -22,8 +22,8 @@
import static org.easymock.EasyMock.reset;
import static org.easymock.EasyMock.verify;
-import com.google.gerrit.common.EventDispatcher;
import com.google.gerrit.reviewdb.server.ReviewDb;
+import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.google.gwtorm.client.KeyUtil;
import com.google.gwtorm.server.OrmException;
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStateTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStateTest.java
index 193af1e..cf6715e 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStateTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStateTest.java
@@ -32,12 +32,15 @@
private ReplicationState replicationState;
private PushResultProcessing pushResultProcessingMock;
+ private EventsStorage eventsStorage;
@Before
public void setUp() throws Exception {
pushResultProcessingMock = createNiceMock(PushResultProcessing.class);
replay(pushResultProcessingMock);
- replicationState = new ReplicationState(pushResultProcessingMock);
+ eventsStorage = createNiceMock(EventsStorage.class);
+ replay(eventsStorage);
+ replicationState = new ReplicationState(eventsStorage, pushResultProcessingMock);
}
@Test