Merge branch 'stable-3.1'
* stable-3.1:
Clarify the limitations of gerrit+ssh in replication.config
PushOne: Don't log refs to push at ERROR level
PushOne: Remove redundant 'throws' declarations
Destination: Further improve the debug logs when not pushing
Improve logging of why a project or ref is not replicated
ReplicationQueue: Add handling of null ReplicationTasksStorage.ReplicateRefUpdate
Destination: Extract repeated string to a constant
Fix flaky ReplicationConfig tests
FanoutReplicationConfig: Make methods static where possible
Factor out and simplify the check if Path is a *.config file
Consistently use Files.list in FanoutReplicationConfig
Add multiple replication configuration file support
Use ReplicationConfig interface instead of ReplicationFileBasedConfig
Fix failing AutoReloadConfigDecorator tests
Move replication config parsing out of DestinationsCollection
Extract destinations logic into a new class
ReplicationQueue: Migrate to Flogger
Adapt existing replication logging code in master to Flogger,
to allow the merged code to build successfully on master.
Change-Id: Ice4e54ab5c1a1a53894432c3687137e4b3603c41
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
index c043baf..782ff4f 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
@@ -25,13 +25,14 @@
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
+import org.eclipse.jgit.lib.Config;
@Singleton
public class AutoReloadConfigDecorator implements ReplicationConfig, LifecycleListener {
private static final long RELOAD_DELAY = 120;
private static final long RELOAD_INTERVAL = 60;
- private volatile ReplicationFileBasedConfig currentConfig;
+ private volatile ReplicationConfig currentConfig;
private final ScheduledExecutorService autoReloadExecutor;
private ScheduledFuture<?> autoReloadRunnable;
@@ -41,7 +42,7 @@
public AutoReloadConfigDecorator(
@PluginName String pluginName,
WorkQueue workQueue,
- ReplicationFileBasedConfig replicationConfig,
+ @MainReplicationConfig ReplicationConfig replicationConfig,
AutoReloadRunnable reloadRunner,
EventBus eventBus) {
this.currentConfig = replicationConfig;
@@ -104,7 +105,7 @@
}
@Subscribe
- public void onReload(ReplicationFileBasedConfig newConfig) {
+ public void onReload(ReplicationConfig newConfig) {
currentConfig = newConfig;
}
@@ -117,4 +118,9 @@
public synchronized int getSshCommandTimeout() {
return currentConfig.getSshCommandTimeout();
}
+
+ @Override
+ public Config getConfig() {
+ return currentConfig.getConfig();
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadRunnable.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadRunnable.java
index a1084a8..71f7c67 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadRunnable.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadRunnable.java
@@ -16,42 +16,34 @@
import com.google.common.eventbus.EventBus;
import com.google.common.flogger.FluentLogger;
-import com.google.gerrit.extensions.annotations.PluginData;
-import com.google.gerrit.server.config.SitePaths;
import com.google.inject.Inject;
import com.google.inject.Provider;
-import java.nio.file.Path;
import java.util.List;
public class AutoReloadRunnable implements Runnable {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
- private final SitePaths site;
- private final Path pluginDataDir;
private final EventBus eventBus;
private final Provider<ObservableQueue> queueObserverProvider;
- private final ReplicationConfigValidator configValidator;
-
- private ReplicationFileBasedConfig loadedConfig;
+ private final ConfigParser configParser;
+ private ReplicationConfig loadedConfig;
+ private Provider<ReplicationConfig> replicationConfigProvider;
private String loadedConfigVersion;
private String lastFailedConfigVersion;
@Inject
public AutoReloadRunnable(
- ReplicationConfigValidator configValidator,
- ReplicationFileBasedConfig config,
- SitePaths site,
- @PluginData Path pluginDataDir,
+ ConfigParser configParser,
+ @MainReplicationConfig Provider<ReplicationConfig> replicationConfigProvider,
EventBus eventBus,
Provider<ObservableQueue> queueObserverProvider) {
- this.loadedConfig = config;
- this.loadedConfigVersion = config.getVersion();
+ this.replicationConfigProvider = replicationConfigProvider;
+ this.loadedConfig = replicationConfigProvider.get();
+ this.loadedConfigVersion = loadedConfig.getVersion();
this.lastFailedConfigVersion = "";
- this.site = site;
- this.pluginDataDir = pluginDataDir;
this.eventBus = eventBus;
this.queueObserverProvider = queueObserverProvider;
- this.configValidator = configValidator;
+ this.configParser = configParser;
}
@Override
@@ -71,9 +63,9 @@
synchronized void reload() {
String pendingConfigVersion = loadedConfig.getVersion();
try {
- ReplicationFileBasedConfig newConfig = new ReplicationFileBasedConfig(site, pluginDataDir);
+ ReplicationConfig newConfig = replicationConfigProvider.get();
final List<RemoteConfiguration> newValidDestinations =
- configValidator.validateConfig(newConfig);
+ configParser.parseRemotes(newConfig.getConfig());
loadedConfig = newConfig;
loadedConfigVersion = newConfig.getVersion();
lastFailedConfigVersion = "";
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
index 98f364d..5bae0af 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
@@ -31,11 +31,10 @@
private final AtomicReference<SecureCredentialsFactory> secureCredentialsFactory;
private volatile long secureCredentialsFactoryLoadTs;
private final SitePaths site;
- private ReplicationFileBasedConfig config;
+ private ReplicationConfig config;
@Inject
- public AutoReloadSecureCredentialsFactoryDecorator(
- SitePaths site, ReplicationFileBasedConfig config)
+ public AutoReloadSecureCredentialsFactoryDecorator(SitePaths site, ReplicationConfig config)
throws ConfigInvalidException, IOException {
this.site = site;
this.config = config;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ConfigParser.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ConfigParser.java
new file mode 100644
index 0000000..66251a5
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ConfigParser.java
@@ -0,0 +1,109 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.flogger.FluentLogger;
+import java.net.URISyntaxException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.transport.RefSpec;
+import org.eclipse.jgit.transport.RemoteConfig;
+import org.eclipse.jgit.transport.URIish;
+
+public class ConfigParser {
+
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+ /**
+ * parse the new replication config
+ *
+ * @param config new configuration to parse
+ * @return List of parsed {@link RemoteConfiguration}
+ * @throws ConfigInvalidException if the new configuration is not valid.
+ */
+ public List<RemoteConfiguration> parseRemotes(Config config) throws ConfigInvalidException {
+
+ if (config.getSections().isEmpty()) {
+ logger.atWarning().log("Replication config does not exist or it's empty; not replicating");
+ return Collections.emptyList();
+ }
+
+ boolean defaultForceUpdate = config.getBoolean("gerrit", "defaultForceUpdate", false);
+
+ ImmutableList.Builder<RemoteConfiguration> confs = ImmutableList.builder();
+ for (RemoteConfig c : allRemotes(config)) {
+ if (c.getURIs().isEmpty()) {
+ continue;
+ }
+
+ if (!c.getFetchRefSpecs().isEmpty()) {
+ repLog.atInfo().log("Ignore '%s' endpoint: not a 'push' target", c.getName());
+ continue;
+ }
+
+ // If destination for push is not set assume equal to source.
+ for (RefSpec ref : c.getPushRefSpecs()) {
+ if (ref.getDestination() == null) {
+ ref.setDestination(ref.getSource());
+ }
+ }
+
+ if (c.getPushRefSpecs().isEmpty()) {
+ c.addPushRefSpec(
+ new RefSpec()
+ .setSourceDestination("refs/*", "refs/*")
+ .setForceUpdate(defaultForceUpdate));
+ }
+
+ DestinationConfiguration destinationConfiguration = new DestinationConfiguration(c, config);
+
+ if (!destinationConfiguration.isSingleProjectMatch()) {
+ for (URIish u : c.getURIs()) {
+ if (u.getPath() == null || !u.getPath().contains("${name}")) {
+ throw new ConfigInvalidException(
+ String.format(
+ "remote.%s.url \"%s\" lacks ${name} placeholder in %s",
+ c.getName(), u, config));
+ }
+ }
+ }
+
+ confs.add(destinationConfiguration);
+ }
+
+ return confs.build();
+ }
+
+ private static List<RemoteConfig> allRemotes(Config cfg) throws ConfigInvalidException {
+ Set<String> names = cfg.getSubsections("remote");
+ List<RemoteConfig> result = Lists.newArrayListWithCapacity(names.size());
+ for (String name : names) {
+ try {
+ result.add(new RemoteConfig(cfg, name));
+ } catch (URISyntaxException e) {
+ throw new ConfigInvalidException(
+ String.format("remote %s has invalid URL in %s", name, cfg), e);
+ }
+ }
+ return result;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java
index fa26e82..424648e 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java
@@ -63,7 +63,8 @@
return true;
}
- repLog.warn("Cannot create new project {} on remote site {}.", projectName, replicateURI);
+ repLog.atWarning().log(
+ "Cannot create new project %s on remote site %s.", projectName, replicateURI);
return false;
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java
index 4617672..8ea7227 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java
@@ -55,7 +55,7 @@
return;
}
- repLog.warn("Cannot delete project {} on remote site {}.", project, replicateURI);
+ repLog.atWarning().log("Cannot delete project %s on remote site %s.", project, replicateURI);
}
@Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
index 7e4298b..4aa74c7 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
@@ -53,6 +53,7 @@
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectState;
import com.google.gerrit.server.util.RequestContext;
+import com.google.gerrit.util.logging.NamedFluentLogger;
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.google.inject.Provider;
@@ -84,10 +85,11 @@
import org.eclipse.jgit.transport.RemoteConfig;
import org.eclipse.jgit.transport.RemoteRefUpdate;
import org.eclipse.jgit.transport.URIish;
-import org.slf4j.Logger;
public class Destination {
- private static final Logger repLog = ReplicationQueue.repLog;
+ private static final NamedFluentLogger repLog = ReplicationQueue.repLog;
+
+ private static final String PROJECT_NOT_AVAILABLE = "source project %s not available";
public interface Factory {
Destination create(DestinationConfiguration config);
@@ -157,7 +159,7 @@
builder.add(g.getUUID());
addRecursiveParents(g.getUUID(), builder, groupIncludeCache);
} else {
- repLog.warn("Group \"{}\" not recognized, removing from authGroup", name);
+ repLog.atWarning().log("Group \"%s\" not recognized, removing from authGroup", name);
}
}
remoteUser = new RemoteSiteUser(new ListGroupMembership(builder.build()));
@@ -241,11 +243,9 @@
int numInFlight = inFlight.size();
if (numPending > 0 || numInFlight > 0) {
- repLog.warn(
- "Cancelling replication events (pending={}, inFlight={}) for destination {}",
- numPending,
- numInFlight,
- getRemoteConfigName());
+ repLog.atWarning().log(
+ "Cancelling replication events (pending=%d, inFlight=%d) for destination %s",
+ numPending, numInFlight, getRemoteConfigName());
foreachPushOp(
pending,
@@ -277,9 +277,12 @@
private boolean shouldReplicate(ProjectState state, CurrentUser user)
throws PermissionBackendException {
+ String name = state.getProject().getName();
if (!config.replicateHiddenProjects()
&& state.getProject().getState()
== com.google.gerrit.extensions.client.ProjectState.HIDDEN) {
+ repLog.atFine().log(
+ "Project %s is hidden and replication of hidden projects is disabled", name);
return false;
}
@@ -292,6 +295,9 @@
permissionBackend.user(user).project(state.getNameKey()).check(permissionToCheck);
return true;
} catch (AuthException e) {
+ repLog.atFine().log(
+ "Project %s is not visible to current user %s",
+ name, user.getUserName().orElse("unknown"));
return false;
}
}
@@ -306,12 +312,16 @@
try {
projectState = projectCache.get(project).orElseThrow(noSuchProject(project));
} catch (StorageException e) {
+ repLog.atWarning().withCause(e).log(
+ "Error reading project %s from cache", project);
return false;
}
if (!projectState.statePermitsRead()) {
+ repLog.atFine().log("Project %s does not permit read", project);
return false;
}
if (!shouldReplicate(projectState, userProvider.get())) {
+ repLog.atFine().log("Project %s should not be replicated", project);
return false;
}
if (PushOne.ALL_REFS.equals(ref)) {
@@ -324,13 +334,16 @@
.ref(ref)
.check(RefPermission.READ);
} catch (AuthException e) {
+ repLog.atFine().log(
+ "Ref %s on project %s is not visible to calling user",
+ ref, project, userProvider.get().getUserName().orElse("unknown"));
return false;
}
return true;
})
.call();
} catch (NoSuchProjectException err) {
- stateLog.error(String.format("source project %s not available", project), err, states);
+ stateLog.error(String.format(PROJECT_NOT_AVAILABLE, project), err, states);
} catch (Exception e) {
Throwables.throwIfUnchecked(e);
throw new RuntimeException(e);
@@ -353,7 +366,7 @@
})
.call();
} catch (NoSuchProjectException err) {
- stateLog.error(String.format("source project %s not available", project), err, states);
+ stateLog.error(String.format(PROJECT_NOT_AVAILABLE, project), err, states);
} catch (Exception e) {
Throwables.throwIfUnchecked(e);
throw new RuntimeException(e);
@@ -367,10 +380,11 @@
void schedule(
Project.NameKey project, String ref, URIish uri, ReplicationState state, boolean now) {
- repLog.info("scheduling replication {}:{} => {}", project, ref, uri);
if (!shouldReplicate(project, ref, state)) {
+ repLog.atFine().log("Not scheduling replication %s:%s => %s", project, ref, uri);
return;
}
+ repLog.atInfo().log("scheduling replication %s:%s => %s", project, ref, uri);
if (!config.replicatePermissions()) {
PushOne e;
@@ -391,7 +405,7 @@
return;
}
} catch (IOException err) {
- stateLog.error(String.format("source project %s not available", project), err, state);
+ stateLog.error(String.format(PROJECT_NOT_AVAILABLE, project), err, state);
return;
}
}
@@ -412,7 +426,8 @@
task.addState(ref, state);
}
state.increasePushTaskCount(project.get(), ref);
- repLog.info("scheduled {}:{} => {} to run after {}s", project, ref, task, config.getDelay());
+ repLog.atInfo().log(
+ "scheduled %s:%s => %s to run after %ds", project, ref, task, config.getDelay());
}
}
@@ -579,7 +594,7 @@
Set<String> names = new HashSet<>();
for (PushOne push : pending.values()) {
if (!replicationTasksStorage.get().isWaiting(push)) {
- repLog.debug("No longer isWaiting, can prune " + push.getURI());
+ repLog.atFine().log("No longer isWaiting, can prune %s", push.getURI());
names.add(push.toString());
}
}
@@ -588,6 +603,7 @@
boolean wouldPushProject(Project.NameKey project) {
if (!shouldReplicate(project)) {
+ repLog.atFine().log("Skipping replication of project %s", project.get());
return false;
}
@@ -597,7 +613,12 @@
return true;
}
- return (new ReplicationFilter(projects)).matches(project);
+ boolean matches = (new ReplicationFilter(projects)).matches(project);
+ if (!matches) {
+ repLog.atFine().log(
+ "Skipping replication of project %s; does not match filter", project.get());
+ }
+ return matches;
}
boolean isSingleProjectMatch() {
@@ -606,6 +627,7 @@
boolean wouldPushRef(String ref) {
if (!config.replicatePermissions() && RefNames.REFS_CONFIG.equals(ref)) {
+ repLog.atFine().log("Skipping push of ref %s; it is a meta ref", ref);
return false;
}
if (PushOne.ALL_REFS.equals(ref)) {
@@ -616,6 +638,7 @@
return true;
}
}
+ repLog.atFine().log("Skipping push of ref %s; it does not match push ref specs", ref);
return false;
}
@@ -647,7 +670,8 @@
} else if (remoteNameStyle.equals("basenameOnly")) {
name = FilenameUtils.getBaseName(name);
} else if (!remoteNameStyle.equals("slash")) {
- repLog.debug("Unknown remoteNameStyle: {}, falling back to slash", remoteNameStyle);
+ repLog.atFine().log(
+ "Unknown remoteNameStyle: %s, falling back to slash", remoteNameStyle);
}
String replacedPath = replaceName(uri.getPath(), name, config.isSingleProjectMatch());
if (replacedPath != null) {
@@ -739,7 +763,7 @@
try {
eventDispatcher.get().postEvent(BranchNameKey.create(project, ref), event);
} catch (PermissionBackendException e) {
- repLog.error("error posting event", e);
+ repLog.atSevere().withCause(e).log("error posting event");
}
}
}
@@ -753,7 +777,7 @@
try {
eventDispatcher.get().postEvent(BranchNameKey.create(project, ref), event);
} catch (PermissionBackendException e) {
- repLog.error("error posting event", e);
+ repLog.atSevere().withCause(e).log("error posting event");
}
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationsCollection.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationsCollection.java
index 0eeab48..d869e83 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationsCollection.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationsCollection.java
@@ -25,7 +25,6 @@
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMultimap;
-import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.common.collect.SetMultimap;
import com.google.common.eventbus.EventBus;
@@ -38,22 +37,16 @@
import com.google.inject.Singleton;
import com.googlesource.gerrit.plugins.replication.Destination.Factory;
import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType;
-import java.io.IOException;
import java.net.URISyntaxException;
-import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
-import java.util.Set;
import java.util.function.Predicate;
import org.eclipse.jgit.errors.ConfigInvalidException;
-import org.eclipse.jgit.storage.file.FileBasedConfig;
-import org.eclipse.jgit.transport.RefSpec;
-import org.eclipse.jgit.transport.RemoteConfig;
import org.eclipse.jgit.transport.URIish;
@Singleton
-public class DestinationsCollection implements ReplicationDestinations, ReplicationConfigValidator {
+public class DestinationsCollection implements ReplicationDestinations {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private final Factory destinationFactory;
@@ -73,12 +66,15 @@
public DestinationsCollection(
Destination.Factory destinationFactory,
Provider<ReplicationQueue> replicationQueue,
- ReplicationFileBasedConfig replicationConfig,
+ ReplicationConfig replicationConfig,
+ ConfigParser configParser,
EventBus eventBus)
throws ConfigInvalidException {
this.destinationFactory = destinationFactory;
this.replicationQueue = replicationQueue;
- this.destinations = allDestinations(destinationFactory, validateConfig(replicationConfig));
+ this.destinations =
+ allDestinations(
+ destinationFactory, configParser.parseRemotes(replicationConfig.getConfig()));
eventBus.register(this);
}
@@ -110,7 +106,7 @@
try {
uri = new URIish(url);
} catch (URISyntaxException e) {
- repLog.warn("adminURL '{}' is invalid: {}", url, e.getMessage());
+ repLog.atWarning().log("adminURL '%s' is invalid: %s", url, e.getMessage());
continue;
}
@@ -118,13 +114,14 @@
String path =
replaceName(uri.getPath(), projectName.get(), config.isSingleProjectMatch());
if (path == null) {
- repLog.warn("adminURL {} does not contain ${name}", uri);
+ repLog.atWarning().log("adminURL %s does not contain ${name}", uri);
continue;
}
uri = uri.setPath(path);
if (!isSSH(uri)) {
- repLog.warn("adminURL '{}' is invalid: only SSH and HTTP are supported", uri);
+ repLog.atWarning().log(
+ "adminURL '%s' is invalid: only SSH and HTTP are supported", uri);
continue;
}
}
@@ -247,84 +244,6 @@
}
}
- @Override
- public List<RemoteConfiguration> validateConfig(ReplicationFileBasedConfig replicationConfig)
- throws ConfigInvalidException {
- if (!replicationConfig.getConfig().getFile().exists()) {
- logger.atWarning().log(
- "Config file %s does not exist; not replicating",
- replicationConfig.getConfig().getFile());
- return Collections.emptyList();
- }
- if (replicationConfig.getConfig().getFile().length() == 0) {
- logger.atInfo().log(
- "Config file %s is empty; not replicating", replicationConfig.getConfig().getFile());
- return Collections.emptyList();
- }
-
- try {
- replicationConfig.getConfig().load();
- } catch (ConfigInvalidException e) {
- throw new ConfigInvalidException(
- String.format(
- "Config file %s is invalid: %s",
- replicationConfig.getConfig().getFile(), e.getMessage()),
- e);
- } catch (IOException e) {
- throw new ConfigInvalidException(
- String.format(
- "Cannot read %s: %s", replicationConfig.getConfig().getFile(), e.getMessage()),
- e);
- }
-
- boolean defaultForceUpdate =
- replicationConfig.getConfig().getBoolean("gerrit", "defaultForceUpdate", false);
-
- ImmutableList.Builder<RemoteConfiguration> confs = ImmutableList.builder();
- for (RemoteConfig c : allRemotes(replicationConfig.getConfig())) {
- if (c.getURIs().isEmpty()) {
- continue;
- }
-
- if (!c.getFetchRefSpecs().isEmpty()) {
- repLog.info("Ignore '{}' endpoint: not a 'push' target", c.getName());
- continue;
- }
-
- // If destination for push is not set assume equal to source.
- for (RefSpec ref : c.getPushRefSpecs()) {
- if (ref.getDestination() == null) {
- ref.setDestination(ref.getSource());
- }
- }
-
- if (c.getPushRefSpecs().isEmpty()) {
- c.addPushRefSpec(
- new RefSpec()
- .setSourceDestination("refs/*", "refs/*")
- .setForceUpdate(defaultForceUpdate));
- }
-
- DestinationConfiguration destinationConfiguration =
- new DestinationConfiguration(c, replicationConfig.getConfig());
-
- if (!destinationConfiguration.isSingleProjectMatch()) {
- for (URIish u : c.getURIs()) {
- if (u.getPath() == null || !u.getPath().contains("${name}")) {
- throw new ConfigInvalidException(
- String.format(
- "remote.%s.url \"%s\" lacks ${name} placeholder in %s",
- c.getName(), u, replicationConfig.getConfig().getFile()));
- }
- }
- }
-
- confs.add(destinationConfiguration);
- }
-
- return confs.build();
- }
-
private List<Destination> allDestinations(
Destination.Factory destinationFactory, List<RemoteConfiguration> remoteConfigurations) {
@@ -336,18 +255,4 @@
}
return dest.build();
}
-
- private static List<RemoteConfig> allRemotes(FileBasedConfig cfg) throws ConfigInvalidException {
- Set<String> names = cfg.getSubsections("remote");
- List<RemoteConfig> result = Lists.newArrayListWithCapacity(names.size());
- for (String name : names) {
- try {
- result.add(new RemoteConfig(cfg, name));
- } catch (URISyntaxException e) {
- throw new ConfigInvalidException(
- String.format("remote %s has invalid URL in %s", name, cfg.getFile()), e);
- }
- }
- return result;
- }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/FanoutReplicationConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/FanoutReplicationConfig.java
new file mode 100644
index 0000000..5b24bf5
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/FanoutReplicationConfig.java
@@ -0,0 +1,187 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.google.common.io.Files.getNameWithoutExtension;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.collect.Lists;
+import com.google.common.flogger.FluentLogger;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+import com.google.gerrit.extensions.annotations.PluginData;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.inject.Inject;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Stream;
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.internal.storage.file.FileSnapshot;
+import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
+
+public class FanoutReplicationConfig implements ReplicationConfig {
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+ private final ReplicationFileBasedConfig replicationConfig;
+ private final Config config;
+ private final Path remoteConfigsDirPath;
+
+ @Inject
+ public FanoutReplicationConfig(SitePaths site, @PluginData Path pluginDataDir)
+ throws IOException, ConfigInvalidException {
+
+ remoteConfigsDirPath = site.etc_dir.resolve("replication");
+ replicationConfig = new ReplicationFileBasedConfig(site, pluginDataDir);
+ config = replicationConfig.getConfig();
+ removeRemotes(config);
+
+ try (Stream<Path> files = Files.list(remoteConfigsDirPath)) {
+ files
+ .filter(Files::isRegularFile)
+ .filter(FanoutReplicationConfig::isConfig)
+ .map(FanoutReplicationConfig::loadConfig)
+ .filter(Optional::isPresent)
+ .map(Optional::get)
+ .filter(FanoutReplicationConfig::isValid)
+ .forEach(cfg -> addRemoteConfig(cfg, config));
+ } catch (IllegalStateException e) {
+ throw new ConfigInvalidException(e.getMessage());
+ }
+ }
+
+ private static void removeRemotes(Config config) {
+ Set<String> remoteNames = config.getSubsections("remote");
+ if (remoteNames.size() > 0) {
+ logger.atSevere().log(
+ "When replication directory is present replication.config file cannot contain remote configuration. Ignoring: %s",
+ String.join(",", remoteNames));
+
+ for (String name : remoteNames) {
+ config.unsetSection("remote", name);
+ }
+ }
+ }
+
+ private static void addRemoteConfig(FileBasedConfig source, Config destination) {
+ String remoteName = getNameWithoutExtension(source.getFile().getName());
+ for (String name : source.getNames("remote")) {
+ destination.setStringList(
+ "remote",
+ remoteName,
+ name,
+ Lists.newArrayList(source.getStringList("remote", null, name)));
+ }
+ }
+
+ private static boolean isValid(Config cfg) {
+ if (cfg.getSections().size() != 1 || !cfg.getSections().contains("remote")) {
+ logger.atSevere().log(
+ "Remote replication configuration file %s must contain only one remote section.", cfg);
+ return false;
+ }
+ if (cfg.getSubsections("remote").size() > 0) {
+ logger.atSevere().log(
+ "Remote replication configuration file %s cannot contain remote subsections.", cfg);
+ return false;
+ }
+
+ return true;
+ }
+
+ private static Optional<FileBasedConfig> loadConfig(Path path) {
+ FileBasedConfig cfg = new FileBasedConfig(path.toFile(), FS.DETECTED);
+ try {
+ cfg.load();
+ } catch (IOException | ConfigInvalidException e) {
+ logger.atSevere().withCause(e).log(
+ "Cannot load remote replication configuration file %s.", path);
+ return Optional.empty();
+ }
+ return Optional.of(cfg);
+ }
+
+ private static boolean isConfig(Path p) {
+ return p.toString().endsWith(".config");
+ }
+
+ @Override
+ public boolean isReplicateAllOnPluginStart() {
+ return replicationConfig.isReplicateAllOnPluginStart();
+ }
+
+ @Override
+ public boolean isDefaultForceUpdate() {
+ return replicationConfig.isDefaultForceUpdate();
+ }
+
+ @Override
+ public int getMaxRefsToLog() {
+ return replicationConfig.getMaxRefsToLog();
+ }
+
+ @Override
+ public Path getEventsDirectory() {
+ return replicationConfig.getEventsDirectory();
+ }
+
+ @Override
+ public int getSshConnectionTimeout() {
+ return replicationConfig.getSshConnectionTimeout();
+ }
+
+ @Override
+ public int getSshCommandTimeout() {
+ return replicationConfig.getSshCommandTimeout();
+ }
+
+ @Override
+ public int getDistributionInterval() {
+ return replicationConfig.getDistributionInterval();
+ }
+
+ @Override
+ public String getVersion() {
+ Hasher hasher = Hashing.murmur3_128().newHasher();
+ hasher.putString(replicationConfig.getVersion(), UTF_8);
+ try (Stream<Path> files = Files.list(remoteConfigsDirPath)) {
+ files
+ .filter(Files::isRegularFile)
+ .filter(FanoutReplicationConfig::isConfig)
+ .sorted()
+ .map(Path::toFile)
+ .map(FileSnapshot::save)
+ .forEach(
+ fileSnapshot ->
+ // hashCode is based on file size, file key and last modified time
+ hasher.putInt(fileSnapshot.hashCode()));
+ return hasher.hash().toString();
+ } catch (IOException e) {
+ logger.atSevere().withCause(e).log(
+ "Cannot list remote configuration files from %s. Returning replication.config file version",
+ remoteConfigsDirPath);
+ return replicationConfig.getVersion();
+ }
+ }
+
+ @Override
+ public Config getConfig() {
+ return config;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java
index f910a40..079087c 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java
@@ -62,34 +62,34 @@
@Override
public boolean createProject(Project.NameKey project, String head) {
- repLog.info("Creating project {} on {}", project, uri);
+ repLog.atInfo().log("Creating project %s on %s", project, uri);
String url = String.format("%s/a/projects/%s", toHttpUri(uri), Url.encode(project.get()));
try {
return httpClient
.execute(new HttpPut(url), new HttpResponseHandler(), getContext())
.isSuccessful();
} catch (IOException e) {
- repLog.error("Couldn't perform project creation on {}", uri, e);
+ repLog.atSevere().log("Couldn't perform project creation on %s", uri, e);
return false;
}
}
@Override
public boolean deleteProject(Project.NameKey project) {
- repLog.info("Deleting project {} on {}", project, uri);
+ repLog.atInfo().log("Deleting project %s on %s", project, uri);
String url = String.format("%s/a/projects/%s", toHttpUri(uri), Url.encode(project.get()));
try {
httpClient.execute(new HttpDelete(url), new HttpResponseHandler(), getContext());
return true;
} catch (IOException e) {
- repLog.error("Couldn't perform project deletion on {}", uri, e);
+ repLog.atSevere().log("Couldn't perform project deletion on %s", uri, e);
}
return false;
}
@Override
public boolean updateHead(Project.NameKey project, String newHead) {
- repLog.info("Updating head of {} on {}", project, uri);
+ repLog.atInfo().log("Updating head of %s on %s", project, uri);
String url = String.format("%s/a/projects/%s/HEAD", toHttpUri(uri), Url.encode(project.get()));
try {
HttpPut req = new HttpPut(url);
@@ -99,7 +99,7 @@
httpClient.execute(req, new HttpResponseHandler(), getContext());
return true;
} catch (IOException e) {
- repLog.error("Couldn't perform update head on {}", uri, e);
+ repLog.atSevere().log("Couldn't perform update head on %s", uri, e);
}
return false;
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java b/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java
index da960e6..b092363 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java
@@ -43,9 +43,9 @@
u.disableRefLog();
u.link(head);
}
- repLog.info("Created local repository: {}", uri);
+ repLog.atInfo().log("Created local repository: %s", uri);
} catch (IOException e) {
- repLog.error("Error creating local repository {}", uri.getPath(), e);
+ repLog.atSevere().withCause(e).log("Error creating local repository %s", uri.getPath());
return false;
}
return true;
@@ -55,9 +55,9 @@
public boolean deleteProject(Project.NameKey project) {
try {
recursivelyDelete(new File(uri.getPath()));
- repLog.info("Deleted local repository: {}", uri);
+ repLog.atInfo().log("Deleted local repository: %s", uri);
} catch (IOException e) {
- repLog.error("Error deleting local repository {}:\n", uri.getPath(), e);
+ repLog.atSevere().withCause(e).log("Error deleting local repository %s:\n", uri.getPath());
return false;
}
return true;
@@ -71,7 +71,8 @@
u.link(newHead);
}
} catch (IOException e) {
- repLog.error("Failed to update HEAD of repository {} to {}", uri.getPath(), newHead, e);
+ repLog.atSevere().withCause(e).log(
+ "Failed to update HEAD of repository %s to %s", uri.getPath(), newHead);
return false;
}
return true;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/MainReplicationConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/MainReplicationConfig.java
new file mode 100644
index 0000000..e8d95ec
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/MainReplicationConfig.java
@@ -0,0 +1,23 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import com.google.inject.BindingAnnotation;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+@BindingAnnotation
+@Retention(RetentionPolicy.RUNTIME)
+public @interface MainReplicationConfig {}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
index 8208264..36f52fa 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
@@ -166,17 +166,16 @@
@Override
public void cancel() {
- repLog.info("Replication [{}] to {} was canceled", HexFormat.fromInt(id), getURI());
+ repLog.atInfo().log("Replication [%s] to %s was canceled", HexFormat.fromInt(id), getURI());
canceledByReplication();
pool.pushWasCanceled(this);
}
@Override
public void setCanceledWhileRunning() {
- repLog.info(
- "Replication [{}] to {} was canceled while being executed",
- HexFormat.fromInt(id),
- getURI());
+ repLog.atInfo().log(
+ "Replication [%s] to %s was canceled while being executed",
+ HexFormat.fromInt(id), getURI());
canceledWhileRunning.set(true);
}
@@ -231,10 +230,10 @@
if (ALL_REFS.equals(ref)) {
delta.clear();
pushAllRefs = true;
- repLog.trace("Added all refs for replication to {}", uri);
+ repLog.atFinest().log("Added all refs for replication to %s", uri);
} else if (!pushAllRefs) {
delta.add(ref);
- repLog.trace("Added ref {} for replication to {}", ref, uri);
+ repLog.atFinest().log("Added ref %s for replication to %s", ref, uri);
}
}
@@ -319,20 +318,20 @@
RunwayStatus status = pool.requestRunway(this);
if (!status.isAllowed()) {
if (status.isCanceled()) {
- repLog.info("PushOp for replication to {} was canceled and thus won't be rescheduled", uri);
+ repLog.atInfo().log(
+ "PushOp for replication to %s was canceled and thus won't be rescheduled", uri);
} else if (status.isExternalInflight()) {
- repLog.info("PushOp for replication to {} was denied externally", uri);
+ repLog.atInfo().log("PushOp for replication to %s was denied externally", uri);
} else {
- repLog.info(
- "Rescheduling replication to {} to avoid collision with the in-flight push [{}].",
- uri,
- HexFormat.fromInt(status.getInFlightPushId()));
+ repLog.atInfo().log(
+ "Rescheduling replication to %s to avoid collision with the in-flight push [%s].",
+ uri, HexFormat.fromInt(status.getInFlightPushId()));
pool.reschedule(this, Destination.RetryReason.COLLISION);
}
return;
}
- repLog.info("Replication to {} started...", uri);
+ repLog.atInfo().log("Replication to %s started...", uri);
Timer1.Context<String> destinationContext = metrics.start(config.getName());
try {
long startedAt = destinationContext.getStartTime();
@@ -346,12 +345,9 @@
metrics.recordSlowProjectReplication(
config.getName(), projectName.get(), pool.getSlowLatencyThreshold(), elapsed);
}
- repLog.info(
- "Replication to {} completed in {}ms, {}ms delay, {} retries",
- uri,
- elapsed,
- delay,
- retryCount);
+ repLog.atInfo().log(
+ "Replication to %s completed in %dms, %dms delay, %d retries",
+ uri, elapsed, delay, retryCount);
} catch (RepositoryNotFoundException e) {
stateLog.error(
"Cannot replicate " + projectName + "; Local repository error: " + e.getMessage(),
@@ -368,7 +364,7 @@
|| msg.contains("unavailable")) {
createRepository();
} else {
- repLog.error("Cannot replicate {}; Remote repository error: {}", projectName, msg);
+ repLog.atSevere().log("Cannot replicate %s; Remote repository error: %s", projectName, msg);
}
} catch (NoRemoteRepositoryException e) {
@@ -378,12 +374,12 @@
} catch (TransportException e) {
Throwable cause = e.getCause();
if (cause instanceof JSchException && cause.getMessage().startsWith("UnknownHostKey:")) {
- repLog.error("Cannot replicate to {}: {}", uri, cause.getMessage());
+ repLog.atSevere().log("Cannot replicate to %s: %s", uri, cause.getMessage());
} else if (e instanceof LockFailureException) {
lockRetryCount++;
// The LockFailureException message contains both URI and reason
// for this failure.
- repLog.error("Cannot replicate to {}: {}", uri, e.getMessage());
+ repLog.atSevere().log("Cannot replicate to %s: %s", uri, e.getMessage());
// The remote push operation should be retried.
if (lockRetryCount <= maxLockRetries) {
@@ -393,17 +389,15 @@
pool.reschedule(this, Destination.RetryReason.TRANSPORT_ERROR);
}
} else {
- repLog.error(
- "Giving up after {} occurrences of this error: {} during replication to {}",
- lockRetryCount,
- e.getMessage(),
- uri);
+ repLog.atSevere().log(
+ "Giving up after %d occurrences of this error: %s during replication to %s",
+ lockRetryCount, e.getMessage(), uri);
}
} else {
if (canceledWhileRunning.get()) {
logCanceledWhileRunningException(e);
} else {
- repLog.error("Cannot replicate to {}", uri, e);
+ repLog.atSevere().withCause(e).log("Cannot replicate to %s", uri);
// The remote push operation should be retried.
pool.reschedule(this, Destination.RetryReason.TRANSPORT_ERROR);
}
@@ -421,7 +415,7 @@
}
private void logCanceledWhileRunningException(TransportException e) {
- repLog.info("Cannot replicate to {}. It was canceled while running", uri, e);
+ repLog.atInfo().withCause(e).log("Cannot replicate to %s. It was canceled while running", uri);
}
private void createRepository() {
@@ -429,10 +423,11 @@
try {
Ref head = git.exactRef(Constants.HEAD);
if (createProject(projectName, head != null ? getName(head) : null)) {
- repLog.warn("Missing repository created; retry replication to {}", uri);
+ repLog.atWarning().log("Missing repository created; retry replication to %s", uri);
pool.reschedule(this, Destination.RetryReason.REPOSITORY_MISSING);
} else {
- repLog.warn("Missing repository could not be created when replicating {}", uri);
+ repLog.atWarning().log(
+ "Missing repository could not be created when replicating %s", uri);
}
} catch (IOException ioe) {
stateLog.error(
@@ -465,8 +460,7 @@
updateStates(res.getRemoteUpdates());
}
- private PushResult pushVia(Transport tn)
- throws IOException, NotSupportedException, TransportException, PermissionBackendException {
+ private PushResult pushVia(Transport tn) throws IOException, PermissionBackendException {
tn.applyConfig(config);
tn.setCredentialsProvider(credentialsProvider);
@@ -480,10 +474,10 @@
}
if (replConfig.getMaxRefsToLog() == 0 || todo.size() <= replConfig.getMaxRefsToLog()) {
- repLog.info("Push to {} references: {}", uri, todo);
+ repLog.atInfo().log("Push to %s references: %s", uri, todo);
} else {
- repLog.info(
- "Push to {} references (first {} of {} listed): {}",
+ repLog.atInfo().log(
+ "Push to %s references (first %d of %d listed): %s",
uri,
replConfig.getMaxRefsToLog(),
todo.size(),
@@ -540,13 +534,13 @@
: replicationPushFilter.get().filter(projectName.get(), remoteUpdatesList);
}
- private List<RemoteRefUpdate> doPushAll(Transport tn, Map<String, Ref> local)
- throws NotSupportedException, TransportException, IOException {
+ private List<RemoteRefUpdate> doPushAll(Transport tn, Map<String, Ref> local) throws IOException {
List<RemoteRefUpdate> cmds = new ArrayList<>();
boolean noPerms = !pool.isReplicatePermissions();
Map<String, Ref> remote = listRemote(tn);
for (Ref src : local.values()) {
if (!canPushRef(src.getName(), noPerms)) {
+ repLog.atFine().log("Skipping push of ref %s", src.getName());
continue;
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java
index d4d979f..b8da2e4 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java
@@ -42,18 +42,14 @@
OutputStream errStream = sshHelper.newErrorBufferStream();
try {
sshHelper.executeRemoteSsh(uri, cmd, errStream);
- repLog.info("Created remote repository: {}", uri);
+ repLog.atInfo().log("Created remote repository: %s", uri);
} catch (IOException e) {
- repLog.error(
- "Error creating remote repository at {}:\n"
- + " Exception: {}\n"
- + " Command: {}\n"
- + " Output: {}",
- uri,
- e,
- cmd,
- errStream,
- e);
+ repLog.atSevere().withCause(e).log(
+ "Error creating remote repository at %s:\n"
+ + " Exception: %s\n"
+ + " Command: %s\n"
+ + " Output: %s",
+ uri, e, cmd, errStream);
return false;
}
return true;
@@ -66,18 +62,14 @@
OutputStream errStream = sshHelper.newErrorBufferStream();
try {
sshHelper.executeRemoteSsh(uri, cmd, errStream);
- repLog.info("Deleted remote repository: {}", uri);
+ repLog.atInfo().log("Deleted remote repository: %s", uri);
} catch (IOException e) {
- repLog.error(
- "Error deleting remote repository at {}:\n"
- + " Exception: {}\n"
- + " Command: {}\n"
- + " Output: {}",
- uri,
- e,
- cmd,
- errStream,
- e);
+ repLog.atSevere().withCause(e).log(
+ "Error deleting remote repository at %s}:\n"
+ + " Exception: %s\n"
+ + " Command: %s\n"
+ + " Output: %s",
+ uri, e, cmd, errStream);
return false;
}
return true;
@@ -92,17 +84,12 @@
try {
sshHelper.executeRemoteSsh(uri, cmd, errStream);
} catch (IOException e) {
- repLog.error(
- "Error updating HEAD of remote repository at {} to {}:\n"
- + " Exception: {}\n"
- + " Command: {}\n"
- + " Output: {}",
- uri,
- newHead,
- e,
- cmd,
- errStream,
- e);
+ repLog.atSevere().withCause(e).log(
+ "Error updating HEAD of remote repository at %s to %s:\n"
+ + " Exception: %s\n"
+ + " Command: %s\n"
+ + " Output: %s",
+ uri, newHead, e, cmd, errStream);
return false;
}
return true;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
index 68fe430..99e5ee4 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
@@ -15,6 +15,7 @@
package com.googlesource.gerrit.plugins.replication;
import java.nio.file.Path;
+import org.eclipse.jgit.lib.Config;
/** Configuration of all the replication end points. */
public interface ReplicationConfig {
@@ -77,4 +78,6 @@
* @return current logical version number.
*/
String getVersion();
+
+ Config getConfig();
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfigValidator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfigValidator.java
deleted file mode 100644
index 7883114..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfigValidator.java
+++ /dev/null
@@ -1,31 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.replication;
-
-import java.util.List;
-import org.eclipse.jgit.errors.ConfigInvalidException;
-
-public interface ReplicationConfigValidator {
-
- /**
- * validate the new replication.config
- *
- * @param newConfig new configuration detected
- * @return List of validated {@link RemoteConfiguration}
- * @throws ConfigInvalidException if the new configuration is not valid.
- */
- List<RemoteConfiguration> validateConfig(ReplicationFileBasedConfig newConfig)
- throws ConfigInvalidException;
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
index d714376..2631cbe 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
@@ -13,16 +13,19 @@
// limitations under the License.
package com.googlesource.gerrit.plugins.replication;
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+
import com.google.common.base.Strings;
import com.google.gerrit.extensions.annotations.PluginData;
import com.google.gerrit.server.config.SitePaths;
import com.google.inject.Inject;
-import com.google.inject.Singleton;
+import java.io.IOException;
import java.nio.file.Path;
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.util.FS;
-@Singleton
public class ReplicationFileBasedConfig implements ReplicationConfig {
private static final int DEFAULT_SSH_CONNECTION_TIMEOUT_MS = 2 * 60 * 1000; // 2 minutes
@@ -41,6 +44,13 @@
this.site = site;
this.cfgPath = site.etc_dir.resolve("replication.config");
this.config = new FileBasedConfig(cfgPath.toFile(), FS.DETECTED);
+ try {
+ config.load();
+ } catch (ConfigInvalidException e) {
+ repLog.atSevere().withCause(e).log("Config file %s is invalid: %s", cfgPath, e.getMessage());
+ } catch (IOException e) {
+ repLog.atSevere().withCause(e).log("Cannot read %s: %s", cfgPath, e.getMessage());
+ }
this.replicateAllOnPluginStart = config.getBoolean("gerrit", "replicateOnStartup", false);
this.defaultForceUpdate = config.getBoolean("gerrit", "defaultForceUpdate", false);
this.maxRefsToLog = config.getInt("gerrit", "maxRefsToLog", 0);
@@ -98,7 +108,8 @@
return cfgPath;
}
- public FileBasedConfig getConfig() {
+ @Override
+ public Config getConfig() {
return config;
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
index 979c8e3..ed1e348 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
@@ -34,6 +34,7 @@
import com.google.inject.internal.UniqueAnnotations;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Files;
import java.nio.file.Path;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.storage.file.FileBasedConfig;
@@ -41,10 +42,12 @@
import org.eclipse.jgit.util.FS;
class ReplicationModule extends AbstractModule {
+ private final SitePaths site;
private final Path cfgPath;
@Inject
public ReplicationModule(SitePaths site) {
+ this.site = site;
cfgPath = site.etc_dir.resolve("replication.config");
}
@@ -77,15 +80,18 @@
bind(EventBus.class).in(Scopes.SINGLETON);
bind(ReplicationDestinations.class).to(DestinationsCollection.class);
- bind(ReplicationConfigValidator.class).to(DestinationsCollection.class);
+ bind(ConfigParser.class).in(Scopes.SINGLETON);
if (getReplicationConfig().getBoolean("gerrit", "autoReload", false)) {
- bind(ReplicationConfig.class).to(AutoReloadConfigDecorator.class);
+ bind(ReplicationConfig.class)
+ .annotatedWith(MainReplicationConfig.class)
+ .to(getReplicationConfigClass());
+ bind(ReplicationConfig.class).to(AutoReloadConfigDecorator.class).in(Scopes.SINGLETON);
bind(LifecycleListener.class)
.annotatedWith(UniqueAnnotations.create())
.to(AutoReloadConfigDecorator.class);
} else {
- bind(ReplicationConfig.class).to(ReplicationFileBasedConfig.class);
+ bind(ReplicationConfig.class).to(getReplicationConfigClass()).in(Scopes.SINGLETON);
}
DynamicSet.setOf(binder(), ReplicationStateListener.class);
@@ -109,4 +115,11 @@
}
return config;
}
+
+ private Class<? extends ReplicationConfig> getReplicationConfigClass() {
+ if (Files.exists(site.etc_dir.resolve("replication"))) {
+ return FanoutReplicationConfig.class;
+ }
+ return ReplicationFileBasedConfig.class;
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
index 0dcfc95..a8ffeec 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
@@ -27,6 +27,7 @@
import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.git.WorkQueue;
+import com.google.gerrit.util.logging.NamedFluentLogger;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.googlesource.gerrit.plugins.replication.PushResultProcessing.GitUpdateProcessing;
@@ -39,8 +40,6 @@
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import org.eclipse.jgit.transport.URIish;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/** Manages automatic replication to remote repositories. */
public class ReplicationQueue
@@ -50,7 +49,7 @@
ProjectDeletedListener,
HeadUpdatedListener {
static final String REPLICATION_LOG_NAME = "replication_log";
- static final Logger repLog = LoggerFactory.getLogger(REPLICATION_LOG_NAME);
+ static final NamedFluentLogger repLog = NamedFluentLogger.forName(REPLICATION_LOG_NAME);
private final ReplicationStateListener stateLog;
@@ -99,7 +98,7 @@
distributor.stop();
int discarded = destinations.get().shutdown();
if (discarded > 0) {
- repLog.warn("Canceled {} replication events during shutdown", discarded);
+ repLog.atWarning().log("Canceled %d replication events during shutdown", discarded);
}
}
@@ -158,6 +157,8 @@
}
cfg.schedule(project, refName, uri, state, now);
}
+ } else {
+ repLog.atFine().log("Skipping ref %s on project %s", refName, project.get());
}
}
}
@@ -170,7 +171,7 @@
for (ReplicationTasksStorage.ReplicateRefUpdate t : replicationTasksStorage.listWaiting()) {
String eventKey = String.format("%s:%s", t.project, t.ref);
if (!eventsReplayed.contains(eventKey)) {
- repLog.info("Firing pending task {}", eventKey);
+ repLog.atInfo().log("Firing pending task %s", eventKey);
fire(t.project, t.ref, true);
eventsReplayed.add(eventKey);
}
@@ -196,7 +197,7 @@
if (state == WorkQueue.Task.State.SLEEPING || state == WorkQueue.Task.State.READY) {
if (task instanceof WorkQueue.ProjectTask) {
if (prunableTaskNames.contains(task.toString())) {
- repLog.debug("Pruning externally completed task:" + task);
+ repLog.atFine().log("Pruning externally completed task: %s", task);
task.cancel(false);
}
}
@@ -223,7 +224,7 @@
for (ReferenceUpdatedEvent event : beforeStartupEventsQueue) {
String eventKey = String.format("%s:%s", event.projectName(), event.refName());
if (!eventsReplayed.contains(eventKey)) {
- repLog.info("Firing pending task {}", event);
+ repLog.atInfo().log("Firing pending task %s", event);
fire(event.projectName(), event.refName(), false);
eventsReplayed.add(eventKey);
}
@@ -265,7 +266,7 @@
firePendingEvents();
pruneCompleted();
} catch (Exception e) {
- repLog.error("error distributing tasks", e);
+ repLog.atSevere().withCause(e).log("error distributing tasks");
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStateLogger.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStateLogger.java
index f2d55de..3e73033 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStateLogger.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStateLogger.java
@@ -31,19 +31,19 @@
@Override
public void warn(String msg, ReplicationState... states) {
stateWriteErr("Warning: " + msg, states);
- repLog.warn(msg);
+ repLog.atWarning().log(msg);
}
@Override
public void error(String msg, ReplicationState... states) {
stateWriteErr("Error: " + msg, states);
- repLog.error(msg);
+ repLog.atSevere().log(msg);
}
@Override
public void error(String msg, Throwable t, ReplicationState... states) {
stateWriteErr("Error: " + msg, states);
- repLog.error(msg, t);
+ repLog.atSevere().withCause(t).log(msg);
}
private void stateWriteErr(String msg, ReplicationState[] states) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java
index ffa6be1..fdbd5e7 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java
@@ -58,7 +58,8 @@
return;
}
- repLog.warn("Cannot update HEAD of project {} on remote site {}.", project, replicateURI);
+ repLog.atWarning().log(
+ "Cannot update HEAD of project %s on remote site %s.", project, replicateURI);
}
@Override
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 19a478d..fc3352d 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -200,23 +200,46 @@
local environment. In that case, an alternative SSH url could
be specified to repository creation.
- To enable replication to different Gerrit instance use `gerrit+ssh://`,
+ To enable replication to different Gerrit instance use
`gerrit+http://` or `gerrit+https://` as protocol name followed
by hostname of another Gerrit server eg.
- `gerrit+ssh://replica1.my.org/`
- <br>
`gerrit+http://replica2.my.org/`
<br>
`gerrit+https://replica3.my.org/`
- In this case replication will use Gerrit's SSH API or Gerrit's REST API
+ In this case replication will use Gerrit's REST API
to create/remove projects and update repository HEAD references.
NOTE: In order to replicate project deletion, the
link:https://gerrit-review.googlesource.com/admin/projects/plugins/delete-project delete-project[delete-project]
plugin must be installed on the other Gerrit.
+ *Backward compatibility notice*
+
+ Before Gerrit v2.13 it was possible to enable replication to different
+ Gerrit masters using `gerrit+ssh://`
+ as protocol name followed by hostname of another Gerrit server eg.
+
+ `gerrit+ssh://replica1.my.org/`
+
+ In that case replication would have used Gerrit's SSH API to
+ create/remove projects and update repository HEAD references.
+
+ The `gerrit+ssh` option is kept for backward compatibility, however
+ the use-case behind it is not valid anymore since the introduction of
+ Lucene indexes and the removal of ReviewDb, which would require
+ a lot more machinery to setup a master to master replication scenario.
+
+ The `gerrit+ssh` option is still possible but is limited to the
+ ability to replicate only regular Git repositories that do not
+ contain any code-review or NoteDb information.
+
+ Using `gerrit+ssh` for replicating all Gerrit repositories
+ would result in failures on the All-Users.git replication and
+ would not be able to replicate changes magic refs and indexes
+ across nodes.
+
remote.NAME.receivepack
: Path of the `git-receive-pack` executable on the remote
system, if using the SSH transport.
@@ -446,6 +469,79 @@
default: 15 minutes
+Directory `replication`
+--------------------
+The optional directory `$site_path/etc/replication` contains Git-style
+config files that controls the replication settings for the replication
+plugin. When present all `remote` sections from `replication.config` file are
+ignored.
+
+Files are composed of one `remote` section. Multiple `remote` sections or any
+other section makes the file invalid and skipped by the replication plugin.
+File name defines remote section name. Each section provides common configuration
+settings for one or more destination URLs. For more details how to setup `remote`
+sections please refer to the `replication.config` section.
+
+### Configuration example:
+
+Static configuration in `$site_path/etc/replication.config`:
+
+```
+[gerrit]
+ autoReload = true
+ replicateOnStartup = false
+[replication]
+ lockErrorMaxRetries = 5
+ maxRetries = 5
+```
+
+Remote sections in `$site_path/etc/replication` directory:
+
+* File `$site_path/etc/replication/host-one.config`
+
+ ```
+ [remote]
+ url = gerrit2@host-one.example.com:/some/path/${name}.git
+ ```
+
+
+* File `$site_path/etc/replication/pubmirror.config`
+
+ ```
+ [remote]
+ url = mirror1.us.some.org:/pub/git/${name}.git
+ url = mirror2.us.some.org:/pub/git/${name}.git
+ url = mirror3.us.some.org:/pub/git/${name}.git
+ push = +refs/heads/*:refs/heads/*
+ push = +refs/tags/*:refs/tags/*
+ threads = 3
+ authGroup = Public Mirror Group
+ authGroup = Second Public Mirror Group
+ ```
+
+Replication plugin resolves config files to the following configuration:
+
+```
+[gerrit]
+ autoReload = true
+ replicateOnStartup = false
+[replication]
+ lockErrorMaxRetries = 5
+ maxRetries = 5
+
+[remote "host-one"]
+ url = gerrit2@host-one.example.com:/some/path/${name}.git
+
+[remote "pubmirror"]
+ url = mirror1.us.some.org:/pub/git/${name}.git
+ url = mirror2.us.some.org:/pub/git/${name}.git
+ url = mirror3.us.some.org:/pub/git/${name}.git
+ push = +refs/heads/*:refs/heads/*
+ push = +refs/tags/*:refs/tags/*
+ threads = 3
+ authGroup = Public Mirror Group
+ authGroup = Second Public Mirror Group
+```
File `secure.config`
--------------------
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/AbstractConfigTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/AbstractConfigTest.java
index 3932fb3..c48bdbd 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/AbstractConfigTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/AbstractConfigTest.java
@@ -49,6 +49,7 @@
protected WorkQueue workQueueMock;
protected EventBus eventBus = new EventBus();
protected FakeExecutorService executorService = new FakeExecutorService();
+ protected ConfigParser configParser;
static class FakeDestination extends Destination {
public final DestinationConfiguration config;
@@ -71,6 +72,7 @@
sitePaths = new SitePaths(sitePath);
pluginDataPath = createTempPath("data");
destinationFactoryMock = mock(Destination.Factory.class);
+ configParser = new ConfigParser();
}
@Before
@@ -96,8 +98,12 @@
}
protected FileBasedConfig newReplicationConfig() {
+ return newReplicationConfig("replication.config");
+ }
+
+ protected FileBasedConfig newReplicationConfig(String path) {
FileBasedConfig replicationConfig =
- new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED);
+ new FileBasedConfig(sitePaths.etc_dir.resolve(path).toFile(), FS.DETECTED);
return replicationConfig;
}
@@ -122,12 +128,17 @@
assertThatIsDestination(matchingDestinations.get(0), remoteName, remoteUrls);
}
- protected DestinationsCollection newDestinationsCollections(
- ReplicationFileBasedConfig replicationFileBasedConfig) throws ConfigInvalidException {
+ protected DestinationsCollection newDestinationsCollections(ReplicationConfig replicationConfig)
+ throws ConfigInvalidException {
return new DestinationsCollection(
destinationFactoryMock,
Providers.of(replicationQueueMock),
- replicationFileBasedConfig,
+ replicationConfig,
+ configParser,
eventBus);
}
+
+ protected ReplicationConfig newReplicationFileBasedConfig() {
+ return new ReplicationFileBasedConfig(sitePaths, pluginDataPath);
+ }
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecoratorTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecoratorTest.java
index d85f622..b1b9453 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecoratorTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecoratorTest.java
@@ -16,44 +16,38 @@
import static com.google.common.truth.Truth.assertThat;
+import com.google.inject.Provider;
import com.google.inject.util.Providers;
import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.storage.file.FileBasedConfig;
-import org.junit.Before;
import org.junit.Test;
public class AutoReloadConfigDecoratorTest extends AbstractConfigTest {
- ReplicationFileBasedConfig replicationFileBasedConfig;
+ ReplicationConfig replicationConfig;
public AutoReloadConfigDecoratorTest() throws IOException {
super();
}
- @Override
- @Before
- public void setup() {
- super.setup();
-
- replicationFileBasedConfig = newReplicationFileBasedConfig();
- }
-
@Test
public void shouldAutoReloadReplicationConfig() throws Exception {
- FileBasedConfig replicationConfig = newReplicationConfig();
- replicationConfig.setBoolean("gerrit", null, "autoReload", true);
+ FileBasedConfig fileConfig = newReplicationConfig();
+ fileConfig.setBoolean("gerrit", null, "autoReload", true);
String remoteName1 = "foo";
String remoteUrl1 = "ssh://git@git.foo.com/${name}";
- replicationConfig.setString("remote", remoteName1, "url", remoteUrl1);
- replicationConfig.save();
+ fileConfig.setString("remote", remoteName1, "url", remoteUrl1);
+ fileConfig.save();
- newAutoReloadConfig().start();
+ replicationConfig = newReplicationFileBasedConfig();
- DestinationsCollection destinationsCollections =
- newDestinationsCollections(replicationFileBasedConfig);
+ newAutoReloadConfig(() -> newReplicationFileBasedConfig()).start();
+
+ DestinationsCollection destinationsCollections = newDestinationsCollections(replicationConfig);
destinationsCollections.startup(workQueueMock);
List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
assertThat(destinations).hasSize(1);
@@ -63,8 +57,8 @@
String remoteName2 = "bar";
String remoteUrl2 = "ssh://git@git.bar.com/${name}";
- replicationConfig.setString("remote", remoteName2, "url", remoteUrl2);
- replicationConfig.save();
+ fileConfig.setString("remote", remoteName2, "url", remoteUrl2);
+ fileConfig.save();
executorService.refreshCommand.run();
destinations = destinationsCollections.getAll(FilterType.ALL);
@@ -74,16 +68,151 @@
}
@Test
+ public void shouldAutoReloadFanoutReplicationConfigWhenConfigIsAdded() throws Exception {
+ String remoteName1 = "foo";
+ String remoteUrl1 = "ssh://git@git.foo.com/${name}";
+ FileBasedConfig remoteConfig = newReplicationConfig("replication/" + remoteName1 + ".config");
+ remoteConfig.setString("remote", null, "url", remoteUrl1);
+ remoteConfig.save();
+
+ replicationConfig = new FanoutReplicationConfig(sitePaths, pluginDataPath);
+
+ newAutoReloadConfig(
+ () -> {
+ try {
+ return new FanoutReplicationConfig(sitePaths, pluginDataPath);
+ } catch (IOException | ConfigInvalidException e) {
+ throw new RuntimeException(e);
+ }
+ })
+ .start();
+
+ DestinationsCollection destinationsCollections = newDestinationsCollections(replicationConfig);
+ destinationsCollections.startup(workQueueMock);
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(1);
+ assertThatContainsDestination(destinations, remoteName1, remoteUrl1);
+
+ TimeUnit.SECONDS.sleep(1); // Allow the filesystem to change the update TS
+
+ String remoteName2 = "foobar";
+ String remoteUrl2 = "ssh://git@git.foobar.com/${name}";
+ remoteConfig = newReplicationConfig("replication/" + remoteName2 + ".config");
+ remoteConfig.setString("remote", null, "url", remoteUrl2);
+ remoteConfig.save();
+ executorService.refreshCommand.run();
+
+ destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(2);
+ assertThatContainsDestination(destinations, remoteName1, remoteUrl1);
+ assertThatContainsDestination(destinations, remoteName2, remoteUrl2);
+ }
+
+ @Test
+ public void shouldAutoReloadFanoutReplicationConfigWhenConfigIsRemoved() throws Exception {
+ String remoteName1 = "foo";
+ String remoteUrl1 = "ssh://git@git.foo.com/${name}";
+ FileBasedConfig remoteConfig = newReplicationConfig("replication/" + remoteName1 + ".config");
+ remoteConfig.setString("remote", null, "url", remoteUrl1);
+ remoteConfig.save();
+
+ String remoteName2 = "foobar";
+ String remoteUrl2 = "ssh://git@git.foobar.com/${name}";
+ remoteConfig = newReplicationConfig("replication/" + remoteName2 + ".config");
+ remoteConfig.setString("remote", null, "url", remoteUrl2);
+ remoteConfig.save();
+
+ replicationConfig = new FanoutReplicationConfig(sitePaths, pluginDataPath);
+
+ newAutoReloadConfig(
+ () -> {
+ try {
+ return new FanoutReplicationConfig(sitePaths, pluginDataPath);
+ } catch (IOException | ConfigInvalidException e) {
+ throw new RuntimeException(e);
+ }
+ })
+ .start();
+
+ DestinationsCollection destinationsCollections = newDestinationsCollections(replicationConfig);
+ destinationsCollections.startup(workQueueMock);
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(2);
+ assertThatContainsDestination(destinations, remoteName1, remoteUrl1);
+ assertThatContainsDestination(destinations, remoteName2, remoteUrl2);
+
+ TimeUnit.SECONDS.sleep(1); // Allow the filesystem to change the update TS
+
+ assertThat(
+ sitePaths.etc_dir.resolve("replication/" + remoteName2 + ".config").toFile().delete())
+ .isTrue();
+
+ executorService.refreshCommand.run();
+
+ destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(1);
+ assertThatContainsDestination(destinations, remoteName1, remoteUrl1);
+ }
+
+ @Test
+ public void shouldAutoReloadFanoutReplicationConfigWhenConfigIsModified() throws Exception {
+ String remoteName1 = "foo";
+ String remoteUrl1 = "ssh://git@git.foo.com/${name}";
+ FileBasedConfig remoteConfig = newReplicationConfig("replication/" + remoteName1 + ".config");
+ remoteConfig.setString("remote", null, "url", remoteUrl1);
+ remoteConfig.save();
+
+ String remoteName2 = "bar";
+ String remoteUrl2 = "ssh://git@git.bar.com/${name}";
+ remoteConfig = newReplicationConfig("replication/" + remoteName2 + ".config");
+ remoteConfig.setString("remote", null, "url", remoteUrl2);
+ remoteConfig.save();
+
+ replicationConfig = new FanoutReplicationConfig(sitePaths, pluginDataPath);
+
+ newAutoReloadConfig(
+ () -> {
+ try {
+ return new FanoutReplicationConfig(sitePaths, pluginDataPath);
+ } catch (IOException | ConfigInvalidException e) {
+ throw new RuntimeException(e);
+ }
+ })
+ .start();
+
+ DestinationsCollection destinationsCollections = newDestinationsCollections(replicationConfig);
+ destinationsCollections.startup(workQueueMock);
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(2);
+ assertThatContainsDestination(destinations, remoteName1, remoteUrl1);
+ assertThatContainsDestination(destinations, remoteName2, remoteUrl2);
+
+ TimeUnit.SECONDS.sleep(1); // Allow the filesystem to change the update TS
+
+ String remoteUrl3 = "ssh://git@git.foobar.com/${name}";
+ remoteConfig.setString("remote", null, "url", remoteUrl3);
+ remoteConfig.save();
+
+ executorService.refreshCommand.run();
+
+ destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(2);
+ assertThatContainsDestination(destinations, remoteName1, remoteUrl1);
+ assertThatContainsDestination(destinations, remoteName2, remoteUrl3);
+ }
+
+ @Test
public void shouldNotAutoReloadReplicationConfigIfDisabled() throws Exception {
String remoteName1 = "foo";
String remoteUrl1 = "ssh://git@git.foo.com/${name}";
- FileBasedConfig replicationConfig = newReplicationConfig();
- replicationConfig.setBoolean("gerrit", null, "autoReload", false);
- replicationConfig.setString("remote", remoteName1, "url", remoteUrl1);
- replicationConfig.save();
+ FileBasedConfig fileConfig = newReplicationConfig();
+ fileConfig.setBoolean("gerrit", null, "autoReload", false);
+ fileConfig.setString("remote", remoteName1, "url", remoteUrl1);
+ fileConfig.save();
- DestinationsCollection destinationsCollections =
- newDestinationsCollections(replicationFileBasedConfig);
+ replicationConfig = newReplicationFileBasedConfig();
+
+ DestinationsCollection destinationsCollections = newDestinationsCollections(replicationConfig);
destinationsCollections.startup(workQueueMock);
List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
assertThat(destinations).hasSize(1);
@@ -91,27 +220,32 @@
TimeUnit.SECONDS.sleep(1); // Allow the filesystem to change the update TS
- replicationConfig.setString("remote", "bar", "url", "ssh://git@git.bar.com/${name}");
- replicationConfig.save();
+ fileConfig.setString("remote", "bar", "url", "ssh://git@git.bar.com/${name}");
+ fileConfig.save();
executorService.refreshCommand.run();
assertThat(destinationsCollections.getAll(FilterType.ALL)).isEqualTo(destinations);
}
- private AutoReloadConfigDecorator newAutoReloadConfig() throws ConfigInvalidException {
+ private AutoReloadConfigDecorator newAutoReloadConfig(
+ Supplier<ReplicationConfig> configSupplier) {
AutoReloadRunnable autoReloadRunnable =
new AutoReloadRunnable(
- newDestinationsCollections(replicationFileBasedConfig),
- replicationFileBasedConfig,
- sitePaths,
- pluginDataPath,
+ configParser,
+ new Provider<ReplicationConfig>() {
+
+ @Override
+ public ReplicationConfig get() {
+ return configSupplier.get();
+ }
+ },
eventBus,
Providers.of(replicationQueueMock));
return new AutoReloadConfigDecorator(
- "replication", workQueueMock, replicationFileBasedConfig, autoReloadRunnable, eventBus);
- }
-
- private ReplicationFileBasedConfig newReplicationFileBasedConfig() {
- return new ReplicationFileBasedConfig(sitePaths, pluginDataPath);
+ "replication",
+ workQueueMock,
+ newReplicationFileBasedConfig(),
+ autoReloadRunnable,
+ eventBus);
}
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadRunnableTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadRunnableTest.java
index b1aa7c8..93e8886 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadRunnableTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadRunnableTest.java
@@ -21,6 +21,7 @@
import com.google.common.eventbus.EventBus;
import com.google.common.eventbus.Subscribe;
import com.google.gerrit.server.config.SitePaths;
+import com.google.inject.Provider;
import com.google.inject.util.Providers;
import java.io.IOException;
import java.nio.file.Files;
@@ -28,6 +29,7 @@
import java.util.Collections;
import java.util.List;
import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.lib.Config;
import org.junit.Before;
import org.junit.Test;
@@ -54,41 +56,41 @@
}
@Test
- public void configurationIsReloadedWhenValidationSucceeds() {
- ReplicationConfigValidator validator = new TestValidConfigurationListener();
+ public void configurationIsReloadedWhenParsingSucceeds() {
+ ConfigParser parser = new TestValidConfigurationListener();
- attemptAutoReload(validator);
+ attemptAutoReload(parser);
assertThat(onReloadSubscriber.reloaded).isTrue();
}
@Test
- public void configurationIsNotReloadedWhenValidationFails() {
- ReplicationConfigValidator validator = new TestInvalidConfigurationListener();
+ public void configurationIsNotReloadedWhenParsingFails() {
+ ConfigParser parser = new TestInvalidConfigurationListener();
- attemptAutoReload(validator);
+ attemptAutoReload(parser);
assertThat(onReloadSubscriber.reloaded).isFalse();
}
- private void attemptAutoReload(ReplicationConfigValidator validator) {
+ private void attemptAutoReload(ConfigParser validator) {
final AutoReloadRunnable autoReloadRunnable =
new AutoReloadRunnable(
- validator,
- newVersionConfig(),
- sitePaths,
- sitePaths.data_dir,
- eventBus,
- Providers.of(replicationQueueMock));
+ validator, newVersionConfigProvider(), eventBus, Providers.of(replicationQueueMock));
autoReloadRunnable.run();
}
- private ReplicationFileBasedConfig newVersionConfig() {
- return new ReplicationFileBasedConfig(sitePaths, sitePaths.data_dir) {
+ private Provider<ReplicationConfig> newVersionConfigProvider() {
+ return new Provider<ReplicationConfig>() {
@Override
- public String getVersion() {
- return String.format("%s", System.nanoTime());
+ public ReplicationConfig get() {
+ return new ReplicationFileBasedConfig(sitePaths, sitePaths.data_dir) {
+ @Override
+ public String getVersion() {
+ return String.format("%s", System.nanoTime());
+ }
+ };
}
};
}
@@ -103,17 +105,17 @@
}
}
- private static class TestValidConfigurationListener implements ReplicationConfigValidator {
+ private static class TestValidConfigurationListener extends ConfigParser {
@Override
- public List<RemoteConfiguration> validateConfig(ReplicationFileBasedConfig newConfig) {
+ public List<RemoteConfiguration> parseRemotes(Config newConfig) {
return Collections.emptyList();
}
}
- private static class TestInvalidConfigurationListener implements ReplicationConfigValidator {
+ private static class TestInvalidConfigurationListener extends ConfigParser {
@Override
- public List<RemoteConfiguration> validateConfig(
- ReplicationFileBasedConfig configurationChangeEvent) throws ConfigInvalidException {
+ public List<RemoteConfiguration> parseRemotes(Config configurationChangeEvent)
+ throws ConfigInvalidException {
throw new ConfigInvalidException("expected test failure");
}
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/FanoutReplicationConfigTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/FanoutReplicationConfigTest.java
new file mode 100644
index 0000000..8cba4bb
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/FanoutReplicationConfigTest.java
@@ -0,0 +1,286 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE;
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.io.MoreFiles;
+import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType;
+import java.io.IOException;
+import java.util.List;
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
+import org.junit.Before;
+import org.junit.Test;
+
+public class FanoutReplicationConfigTest extends AbstractConfigTest {
+
+ public FanoutReplicationConfigTest() throws IOException {
+ super();
+ }
+
+ String remoteName1 = "foo";
+ String remoteUrl1 = "ssh://git@git.somewhere.com/${name}";
+ String remoteName2 = "bar";
+ String remoteUrl2 = "ssh://git@git.elsewhere.com/${name}";
+
+ @Before
+ public void setupTests() {
+ FileBasedConfig config = newReplicationConfig();
+ try {
+ config.save();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Test
+ public void shouldSkipRemoteConfigFromReplicationConfig() throws Exception {
+ String remoteName = "foo";
+ String remoteUrl = "ssh://git@git.somewhere.com/${name}";
+
+ FileBasedConfig config = newReplicationConfig();
+ config.setString("remote", remoteName, "url", remoteUrl);
+ config.save();
+
+ config = newRemoteConfig(remoteName2);
+ config.setString("remote", null, "url", remoteUrl2);
+ config.save();
+
+ DestinationsCollection destinationsCollections =
+ newDestinationsCollections(new FanoutReplicationConfig(sitePaths, pluginDataPath));
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(1);
+
+ assertThatIsDestination(destinations.get(0), remoteName2, remoteUrl2);
+ }
+
+ @Test
+ public void shouldLoadDestinationsFromMultipleFiles() throws Exception {
+
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", remoteUrl1);
+ config.save();
+
+ config = newRemoteConfig(remoteName2);
+ config.setString("remote", null, "url", remoteUrl2);
+ config.save();
+
+ DestinationsCollection destinationsCollections =
+ newDestinationsCollections(new FanoutReplicationConfig(sitePaths, pluginDataPath));
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(2);
+
+ assertThatContainsDestination(destinations, remoteName1, remoteUrl1);
+ assertThatContainsDestination(destinations, remoteName2, remoteUrl2);
+ }
+
+ @Test
+ public void shouldIgnoreDestinationsFromSubdirectories() throws Exception {
+
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", remoteUrl1);
+ config.save();
+
+ config = newRemoteConfig("subdirectory/" + remoteName2);
+ config.setString("remote", null, "url", remoteUrl2);
+ config.save();
+
+ DestinationsCollection destinationsCollections =
+ newDestinationsCollections(new FanoutReplicationConfig(sitePaths, pluginDataPath));
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(1);
+
+ assertThatIsDestination(destinations.get(0), remoteName1, remoteUrl1);
+ }
+
+ @Test
+ public void shouldIgnoreNonConfigFiles() throws Exception {
+
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", remoteUrl1);
+ config.save();
+
+ config =
+ new FileBasedConfig(
+ sitePaths.etc_dir.resolve("replication/" + remoteName2 + ".yaml").toFile(),
+ FS.DETECTED);
+ config.setString("remote", null, "url", remoteUrl2);
+ config.save();
+
+ DestinationsCollection destinationsCollections =
+ newDestinationsCollections(new FanoutReplicationConfig(sitePaths, pluginDataPath));
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(1);
+
+ assertThatIsDestination(destinations.get(0), remoteName1, remoteUrl1);
+ }
+
+ @Test(expected = ConfigInvalidException.class)
+ public void shouldThrowConfigInvalidExceptionWhenUrlIsMissingName() throws Exception {
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", "ssh://git@git.elsewhere.com/name");
+ config.save();
+
+ newDestinationsCollections(new FanoutReplicationConfig(sitePaths, pluginDataPath));
+ }
+
+ @Test
+ public void shouldIgnoreEmptyConfigFile() throws Exception {
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.save();
+
+ DestinationsCollection destinationsCollections =
+ newDestinationsCollections(new FanoutReplicationConfig(sitePaths, pluginDataPath));
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(0);
+ }
+
+ @Test
+ public void shouldIgnoreConfigWhenMoreThanOneRemoteInASingleFile() throws Exception {
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", remoteUrl1);
+ config.setString("remote", remoteName2, "url", remoteUrl2);
+ config.save();
+
+ DestinationsCollection destinationsCollections =
+ newDestinationsCollections(new FanoutReplicationConfig(sitePaths, pluginDataPath));
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(0);
+ }
+
+ @Test
+ public void shouldIgnoreConfigRemoteSection() throws Exception {
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("replication", null, "url", remoteUrl1);
+ config.save();
+
+ DestinationsCollection destinationsCollections =
+ newDestinationsCollections(new FanoutReplicationConfig(sitePaths, pluginDataPath));
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(0);
+ }
+
+ @Test
+ public void shouldReturnSameVersionWhenNoChanges() throws Exception {
+
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", remoteUrl1);
+ config.save();
+
+ config = newRemoteConfig(remoteName2);
+ config.setString("remote", null, "url", remoteUrl2);
+ config.save();
+
+ FanoutReplicationConfig objectUnderTest =
+ new FanoutReplicationConfig(sitePaths, pluginDataPath);
+
+ String version = objectUnderTest.getVersion();
+
+ objectUnderTest = new FanoutReplicationConfig(sitePaths, pluginDataPath);
+
+ assertThat(objectUnderTest.getVersion()).isEqualTo(version);
+ }
+
+ @Test
+ public void shouldReturnNewVersionWhenConfigFileAdded() throws Exception {
+
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", remoteUrl1);
+ config.save();
+
+ FanoutReplicationConfig objectUnderTest =
+ new FanoutReplicationConfig(sitePaths, pluginDataPath);
+
+ String version = objectUnderTest.getVersion();
+
+ config = newRemoteConfig(remoteName2);
+ config.setString("remote", null, "url", remoteUrl2);
+ config.save();
+
+ assertThat(objectUnderTest.getVersion()).isNotEqualTo(version);
+ }
+
+ @Test
+ public void shouldReturnNewVersionWhenConfigFileIsModified() throws Exception {
+
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", remoteUrl1);
+ config.save();
+
+ FanoutReplicationConfig objectUnderTest =
+ new FanoutReplicationConfig(sitePaths, pluginDataPath);
+
+ String version = objectUnderTest.getVersion();
+
+ config.setString("remote", null, "url", remoteUrl2);
+ config.save();
+
+ assertThat(objectUnderTest.getVersion()).isNotEqualTo(version);
+ }
+
+ @Test
+ public void shouldReturnNewVersionWhenConfigFileRemoved() throws Exception {
+
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", remoteUrl1);
+ config.save();
+
+ config = newRemoteConfig(remoteName2);
+ config.setString("remote", null, "url", remoteUrl2);
+ config.save();
+
+ FanoutReplicationConfig objectUnderTest =
+ new FanoutReplicationConfig(sitePaths, pluginDataPath);
+
+ String version = objectUnderTest.getVersion();
+ assertThat(
+ sitePaths.etc_dir.resolve("replication/" + remoteName2 + ".config").toFile().delete())
+ .isTrue();
+
+ assertThat(objectUnderTest.getVersion()).isNotEqualTo(version);
+ }
+
+ @Test
+ public void shouldReturnReplicationConfigVersionWhenReplicationConfigDirectoryRemoved()
+ throws Exception {
+
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", remoteUrl1);
+ config.save();
+
+ config = newRemoteConfig(remoteName2);
+ config.setString("remote", null, "url", remoteUrl2);
+ config.save();
+
+ FanoutReplicationConfig objectUnderTest =
+ new FanoutReplicationConfig(sitePaths, pluginDataPath);
+
+ String replicationConfigVersion =
+ new ReplicationFileBasedConfig(sitePaths, pluginDataPath).getVersion();
+
+ MoreFiles.deleteRecursively(sitePaths.etc_dir.resolve("replication"), ALLOW_INSECURE);
+
+ assertThat(objectUnderTest.getVersion()).isEqualTo(replicationConfigVersion);
+ }
+
+ protected FileBasedConfig newRemoteConfig(String configFileName) {
+ return new FileBasedConfig(
+ sitePaths.etc_dir.resolve("replication/" + configFileName + ".config").toFile(),
+ FS.DETECTED);
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFanoutIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFanoutIT.java
new file mode 100644
index 0000000..adb7fc8
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFanoutIT.java
@@ -0,0 +1,266 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.google.common.truth.Truth.assertThat;
+import static java.util.stream.Collectors.toList;
+
+import com.google.common.flogger.FluentLogger;
+import com.google.common.io.MoreFiles;
+import com.google.common.io.RecursiveDeleteOption;
+import com.google.gerrit.acceptance.LightweightPluginDaemonTest;
+import com.google.gerrit.acceptance.PushOneCommit.Result;
+import com.google.gerrit.acceptance.TestPlugin;
+import com.google.gerrit.acceptance.UseLocalDisk;
+import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
+import com.google.gerrit.entities.Project;
+import com.google.gerrit.extensions.annotations.PluginData;
+import com.google.gerrit.extensions.api.projects.BranchInput;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.inject.Inject;
+import com.google.inject.Key;
+import com.googlesource.gerrit.plugins.replication.ReplicationTasksStorage.ReplicateRefUpdate;
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Supplier;
+import java.util.regex.Pattern;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
+import org.junit.After;
+import org.junit.Test;
+
+@UseLocalDisk
+@TestPlugin(
+ name = "replication",
+ sysModule = "com.googlesource.gerrit.plugins.replication.ReplicationModule")
+public class ReplicationFanoutIT extends LightweightPluginDaemonTest {
+ private static final Optional<String> ALL_PROJECTS = Optional.empty();
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+ private static final int TEST_REPLICATION_DELAY = 1;
+ private static final Duration TEST_TIMEOUT = Duration.ofSeconds(TEST_REPLICATION_DELAY * 2);
+
+ @Inject private SitePaths sitePaths;
+ @Inject private ProjectOperations projectOperations;
+ private Path pluginDataDir;
+ private Path gitPath;
+ private Path storagePath;
+ private FileBasedConfig config;
+ private ReplicationTasksStorage tasksStorage;
+
+ @Override
+ public void setUpTestPlugin() throws Exception {
+ gitPath = sitePaths.site_path.resolve("git");
+
+ config =
+ new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED);
+ setAutoReload();
+ config.save();
+
+ setReplicationDestination("remote1", "suffix1", Optional.of("not-used-project"));
+
+ super.setUpTestPlugin();
+
+ pluginDataDir = plugin.getSysInjector().getInstance(Key.get(Path.class, PluginData.class));
+ storagePath = pluginDataDir.resolve("ref-updates");
+ tasksStorage = plugin.getSysInjector().getInstance(ReplicationTasksStorage.class);
+ cleanupReplicationTasks();
+ tasksStorage.disableDeleteForTesting(true);
+ }
+
+ @After
+ public void cleanUp() throws IOException {
+ if (Files.exists(sitePaths.etc_dir.resolve("replication"))) {
+ MoreFiles.deleteRecursively(
+ sitePaths.etc_dir.resolve("replication"), RecursiveDeleteOption.ALLOW_INSECURE);
+ }
+ }
+
+ @Test
+ public void shouldReplicateNewBranch() throws Exception {
+ setReplicationDestination("foo", "replica", ALL_PROJECTS);
+ reloadConfig();
+
+ Project.NameKey targetProject = createTestProject(project + "replica");
+ String newBranch = "refs/heads/mybranch";
+ String master = "refs/heads/master";
+ BranchInput input = new BranchInput();
+ input.revision = master;
+ gApi.projects().name(project.get()).branch(newBranch).create(input);
+
+ assertThat(listReplicationTasks("refs/heads/(mybranch|master)")).hasSize(2);
+
+ try (Repository repo = repoManager.openRepository(targetProject);
+ Repository sourceRepo = repoManager.openRepository(project)) {
+ waitUntil(() -> checkedGetRef(repo, newBranch) != null);
+
+ Ref masterRef = getRef(sourceRepo, master);
+ Ref targetBranchRef = getRef(repo, newBranch);
+ assertThat(targetBranchRef).isNotNull();
+ assertThat(targetBranchRef.getObjectId()).isEqualTo(masterRef.getObjectId());
+ }
+ }
+
+ @Test
+ public void shouldReplicateNewBranchToTwoRemotes() throws Exception {
+ Project.NameKey targetProject1 = createTestProject(project + "replica1");
+ Project.NameKey targetProject2 = createTestProject(project + "replica2");
+
+ setReplicationDestination("foo1", "replica1", ALL_PROJECTS);
+ setReplicationDestination("foo2", "replica2", ALL_PROJECTS);
+ reloadConfig();
+
+ Result pushResult = createChange();
+ RevCommit sourceCommit = pushResult.getCommit();
+ String sourceRef = pushResult.getPatchSet().refName();
+
+ assertThat(listReplicationTasks("refs/changes/\\d*/\\d*/\\d*")).hasSize(2);
+
+ try (Repository repo1 = repoManager.openRepository(targetProject1);
+ Repository repo2 = repoManager.openRepository(targetProject2)) {
+ waitUntil(
+ () ->
+ (checkedGetRef(repo1, sourceRef) != null && checkedGetRef(repo2, sourceRef) != null));
+
+ Ref targetBranchRef1 = getRef(repo1, sourceRef);
+ assertThat(targetBranchRef1).isNotNull();
+ assertThat(targetBranchRef1.getObjectId()).isEqualTo(sourceCommit.getId());
+
+ Ref targetBranchRef2 = getRef(repo2, sourceRef);
+ assertThat(targetBranchRef2).isNotNull();
+ assertThat(targetBranchRef2.getObjectId()).isEqualTo(sourceCommit.getId());
+ }
+ }
+
+ @Test
+ public void shouldCreateIndividualReplicationTasksForEveryRemoteUrlPair() throws Exception {
+ List<String> replicaSuffixes = Arrays.asList("replica1", "replica2");
+ createTestProject(project + "replica1");
+ createTestProject(project + "replica2");
+
+ setReplicationDestination("foo1", replicaSuffixes, ALL_PROJECTS);
+ setReplicationDestination("foo2", replicaSuffixes, ALL_PROJECTS);
+ config.setInt("remote", "foo1", "replicationDelay", TEST_REPLICATION_DELAY * 100);
+ config.setInt("remote", "foo2", "replicationDelay", TEST_REPLICATION_DELAY * 100);
+ reloadConfig();
+
+ createChange();
+
+ assertThat(listReplicationTasks("refs/changes/\\d*/\\d*/\\d*")).hasSize(4);
+
+ setReplicationDestination("foo1", replicaSuffixes, ALL_PROJECTS);
+ setReplicationDestination("foo2", replicaSuffixes, ALL_PROJECTS);
+ }
+
+ private Ref getRef(Repository repo, String branchName) throws IOException {
+ return repo.getRefDatabase().exactRef(branchName);
+ }
+
+ private Ref checkedGetRef(Repository repo, String branchName) {
+ try {
+ return repo.getRefDatabase().exactRef(branchName);
+ } catch (Exception e) {
+ logger.atSevere().withCause(e).log("failed to get ref %s in repo %s", branchName, repo);
+ return null;
+ }
+ }
+
+ private void setReplicationDestination(
+ String remoteName, String replicaSuffix, Optional<String> project) throws IOException {
+ setReplicationDestination(remoteName, Arrays.asList(replicaSuffix), project);
+ }
+
+ private void setReplicationDestination(
+ String remoteName, List<String> replicaSuffixes, Optional<String> allProjects)
+ throws IOException {
+ FileBasedConfig remoteConfig =
+ new FileBasedConfig(
+ sitePaths.etc_dir.resolve("replication/" + remoteName + ".config").toFile(),
+ FS.DETECTED);
+
+ setReplicationDestination(remoteConfig, replicaSuffixes, allProjects);
+ }
+
+ private void setAutoReload() throws IOException {
+ config.setBoolean("gerrit", null, "autoReload", true);
+ config.save();
+ }
+
+ private void setReplicationDestination(
+ FileBasedConfig config, List<String> replicaSuffixes, Optional<String> project)
+ throws IOException {
+
+ List<String> replicaUrls =
+ replicaSuffixes.stream()
+ .map(suffix -> gitPath.resolve("${name}" + suffix + ".git").toString())
+ .collect(toList());
+ config.setStringList("remote", null, "url", replicaUrls);
+ config.setInt("remote", null, "replicationDelay", TEST_REPLICATION_DELAY);
+ project.ifPresent(prj -> config.setString("remote", null, "projects", prj));
+
+ config.save();
+ }
+
+ private void waitUntil(Supplier<Boolean> waitCondition) throws InterruptedException {
+ WaitUtil.waitUntil(waitCondition, TEST_TIMEOUT);
+ }
+
+ private void reloadConfig() {
+ getAutoReloadConfigDecoratorInstance().reload();
+ }
+
+ private AutoReloadConfigDecorator getAutoReloadConfigDecoratorInstance() {
+ return getInstance(AutoReloadConfigDecorator.class);
+ }
+
+ private <T> T getInstance(Class<T> classObj) {
+ return plugin.getSysInjector().getInstance(classObj);
+ }
+
+ private Project.NameKey createTestProject(String name) throws Exception {
+ return projectOperations.newProject().name(name).create();
+ }
+
+ private List<ReplicateRefUpdate> listReplicationTasks(String refRegex) {
+ Pattern refmaskPattern = Pattern.compile(refRegex);
+ return tasksStorage.list().stream()
+ .filter(task -> refmaskPattern.matcher(task.ref).matches())
+ .collect(toList());
+ }
+
+ public void cleanupReplicationTasks() throws IOException {
+ cleanupReplicationTasks(storagePath);
+ }
+
+ private void cleanupReplicationTasks(Path basePath) throws IOException {
+ try (DirectoryStream<Path> files = Files.newDirectoryStream(basePath)) {
+ for (Path path : files) {
+ if (Files.isDirectory(path)) {
+ cleanupReplicationTasks(path);
+ } else {
+ path.toFile().delete();
+ }
+ }
+ }
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfigTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfigTest.java
index 50ae385..79b05cc 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfigTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfigTest.java
@@ -62,8 +62,8 @@
List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
assertThat(destinations).hasSize(2);
- assertThatIsDestination(destinations.get(0), remoteName1, remoteUrl1);
- assertThatIsDestination(destinations.get(1), remoteName2, remoteUrl2);
+ assertThatContainsDestination(destinations, remoteName1, remoteUrl1);
+ assertThatContainsDestination(destinations, remoteName2, remoteUrl2);
}
@Test
@@ -86,10 +86,4 @@
assertThatIsDestination(destinations.get(0), pushRemote, aRemoteURL);
}
-
- private ReplicationFileBasedConfig newReplicationFileBasedConfig() {
- ReplicationFileBasedConfig replicationConfig =
- new ReplicationFileBasedConfig(sitePaths, pluginDataPath);
- return replicationConfig;
- }
}
