Merge "Destination: Fix ConcurrentModificationException"
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
index c043baf..782ff4f 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
@@ -25,13 +25,14 @@
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
+import org.eclipse.jgit.lib.Config;
@Singleton
public class AutoReloadConfigDecorator implements ReplicationConfig, LifecycleListener {
private static final long RELOAD_DELAY = 120;
private static final long RELOAD_INTERVAL = 60;
- private volatile ReplicationFileBasedConfig currentConfig;
+ private volatile ReplicationConfig currentConfig;
private final ScheduledExecutorService autoReloadExecutor;
private ScheduledFuture<?> autoReloadRunnable;
@@ -41,7 +42,7 @@
public AutoReloadConfigDecorator(
@PluginName String pluginName,
WorkQueue workQueue,
- ReplicationFileBasedConfig replicationConfig,
+ @MainReplicationConfig ReplicationConfig replicationConfig,
AutoReloadRunnable reloadRunner,
EventBus eventBus) {
this.currentConfig = replicationConfig;
@@ -104,7 +105,7 @@
}
@Subscribe
- public void onReload(ReplicationFileBasedConfig newConfig) {
+ public void onReload(ReplicationConfig newConfig) {
currentConfig = newConfig;
}
@@ -117,4 +118,9 @@
public synchronized int getSshCommandTimeout() {
return currentConfig.getSshCommandTimeout();
}
+
+ @Override
+ public Config getConfig() {
+ return currentConfig.getConfig();
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadRunnable.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadRunnable.java
index a1084a8..71f7c67 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadRunnable.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadRunnable.java
@@ -16,42 +16,34 @@
import com.google.common.eventbus.EventBus;
import com.google.common.flogger.FluentLogger;
-import com.google.gerrit.extensions.annotations.PluginData;
-import com.google.gerrit.server.config.SitePaths;
import com.google.inject.Inject;
import com.google.inject.Provider;
-import java.nio.file.Path;
import java.util.List;
public class AutoReloadRunnable implements Runnable {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
- private final SitePaths site;
- private final Path pluginDataDir;
private final EventBus eventBus;
private final Provider<ObservableQueue> queueObserverProvider;
- private final ReplicationConfigValidator configValidator;
-
- private ReplicationFileBasedConfig loadedConfig;
+ private final ConfigParser configParser;
+ private ReplicationConfig loadedConfig;
+ private Provider<ReplicationConfig> replicationConfigProvider;
private String loadedConfigVersion;
private String lastFailedConfigVersion;
@Inject
public AutoReloadRunnable(
- ReplicationConfigValidator configValidator,
- ReplicationFileBasedConfig config,
- SitePaths site,
- @PluginData Path pluginDataDir,
+ ConfigParser configParser,
+ @MainReplicationConfig Provider<ReplicationConfig> replicationConfigProvider,
EventBus eventBus,
Provider<ObservableQueue> queueObserverProvider) {
- this.loadedConfig = config;
- this.loadedConfigVersion = config.getVersion();
+ this.replicationConfigProvider = replicationConfigProvider;
+ this.loadedConfig = replicationConfigProvider.get();
+ this.loadedConfigVersion = loadedConfig.getVersion();
this.lastFailedConfigVersion = "";
- this.site = site;
- this.pluginDataDir = pluginDataDir;
this.eventBus = eventBus;
this.queueObserverProvider = queueObserverProvider;
- this.configValidator = configValidator;
+ this.configParser = configParser;
}
@Override
@@ -71,9 +63,9 @@
synchronized void reload() {
String pendingConfigVersion = loadedConfig.getVersion();
try {
- ReplicationFileBasedConfig newConfig = new ReplicationFileBasedConfig(site, pluginDataDir);
+ ReplicationConfig newConfig = replicationConfigProvider.get();
final List<RemoteConfiguration> newValidDestinations =
- configValidator.validateConfig(newConfig);
+ configParser.parseRemotes(newConfig.getConfig());
loadedConfig = newConfig;
loadedConfigVersion = newConfig.getVersion();
lastFailedConfigVersion = "";
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
index 98f364d..5bae0af 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
@@ -31,11 +31,10 @@
private final AtomicReference<SecureCredentialsFactory> secureCredentialsFactory;
private volatile long secureCredentialsFactoryLoadTs;
private final SitePaths site;
- private ReplicationFileBasedConfig config;
+ private ReplicationConfig config;
@Inject
- public AutoReloadSecureCredentialsFactoryDecorator(
- SitePaths site, ReplicationFileBasedConfig config)
+ public AutoReloadSecureCredentialsFactoryDecorator(SitePaths site, ReplicationConfig config)
throws ConfigInvalidException, IOException {
this.site = site;
this.config = config;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ConfigParser.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ConfigParser.java
new file mode 100644
index 0000000..66251a5
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ConfigParser.java
@@ -0,0 +1,109 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.flogger.FluentLogger;
+import java.net.URISyntaxException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.transport.RefSpec;
+import org.eclipse.jgit.transport.RemoteConfig;
+import org.eclipse.jgit.transport.URIish;
+
+public class ConfigParser {
+
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+ /**
+ * parse the new replication config
+ *
+ * @param config new configuration to parse
+ * @return List of parsed {@link RemoteConfiguration}
+ * @throws ConfigInvalidException if the new configuration is not valid.
+ */
+ public List<RemoteConfiguration> parseRemotes(Config config) throws ConfigInvalidException {
+
+ if (config.getSections().isEmpty()) {
+ logger.atWarning().log("Replication config does not exist or it's empty; not replicating");
+ return Collections.emptyList();
+ }
+
+ boolean defaultForceUpdate = config.getBoolean("gerrit", "defaultForceUpdate", false);
+
+ ImmutableList.Builder<RemoteConfiguration> confs = ImmutableList.builder();
+ for (RemoteConfig c : allRemotes(config)) {
+ if (c.getURIs().isEmpty()) {
+ continue;
+ }
+
+ if (!c.getFetchRefSpecs().isEmpty()) {
+ repLog.atInfo().log("Ignore '%s' endpoint: not a 'push' target", c.getName());
+ continue;
+ }
+
+ // If destination for push is not set assume equal to source.
+ for (RefSpec ref : c.getPushRefSpecs()) {
+ if (ref.getDestination() == null) {
+ ref.setDestination(ref.getSource());
+ }
+ }
+
+ if (c.getPushRefSpecs().isEmpty()) {
+ c.addPushRefSpec(
+ new RefSpec()
+ .setSourceDestination("refs/*", "refs/*")
+ .setForceUpdate(defaultForceUpdate));
+ }
+
+ DestinationConfiguration destinationConfiguration = new DestinationConfiguration(c, config);
+
+ if (!destinationConfiguration.isSingleProjectMatch()) {
+ for (URIish u : c.getURIs()) {
+ if (u.getPath() == null || !u.getPath().contains("${name}")) {
+ throw new ConfigInvalidException(
+ String.format(
+ "remote.%s.url \"%s\" lacks ${name} placeholder in %s",
+ c.getName(), u, config));
+ }
+ }
+ }
+
+ confs.add(destinationConfiguration);
+ }
+
+ return confs.build();
+ }
+
+ private static List<RemoteConfig> allRemotes(Config cfg) throws ConfigInvalidException {
+ Set<String> names = cfg.getSubsections("remote");
+ List<RemoteConfig> result = Lists.newArrayListWithCapacity(names.size());
+ for (String name : names) {
+ try {
+ result.add(new RemoteConfig(cfg, name));
+ } catch (URISyntaxException e) {
+ throw new ConfigInvalidException(
+ String.format("remote %s has invalid URL in %s", name, cfg), e);
+ }
+ }
+ return result;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java
index fa26e82..424648e 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java
@@ -63,7 +63,8 @@
return true;
}
- repLog.warn("Cannot create new project {} on remote site {}.", projectName, replicateURI);
+ repLog.atWarning().log(
+ "Cannot create new project %s on remote site %s.", projectName, replicateURI);
return false;
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java
index 4617672..8ea7227 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java
@@ -55,7 +55,7 @@
return;
}
- repLog.warn("Cannot delete project {} on remote site {}.", project, replicateURI);
+ repLog.atWarning().log("Cannot delete project %s on remote site %s.", project, replicateURI);
}
@Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
index 444bb46..b2821e9 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
@@ -53,6 +53,7 @@
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectState;
import com.google.gerrit.server.util.RequestContext;
+import com.google.gerrit.util.logging.NamedFluentLogger;
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.google.inject.Provider;
@@ -86,10 +87,11 @@
import org.eclipse.jgit.transport.RemoteConfig;
import org.eclipse.jgit.transport.RemoteRefUpdate;
import org.eclipse.jgit.transport.URIish;
-import org.slf4j.Logger;
public class Destination {
- private static final Logger repLog = ReplicationQueue.repLog;
+ private static final NamedFluentLogger repLog = ReplicationQueue.repLog;
+
+ private static final String PROJECT_NOT_AVAILABLE = "source project %s not available";
public interface Factory {
Destination create(DestinationConfiguration config);
@@ -161,7 +163,7 @@
builder.add(g.getUUID());
addRecursiveParents(g.getUUID(), builder, groupIncludeCache);
} else {
- repLog.warn("Group \"{}\" not recognized, removing from authGroup", name);
+ repLog.atWarning().log("Group \"%s\" not recognized, removing from authGroup", name);
}
}
remoteUser = new RemoteSiteUser(new ListGroupMembership(builder.build()));
@@ -245,11 +247,9 @@
int numInFlight = inFlight.size();
if (numPending > 0 || numInFlight > 0) {
- repLog.warn(
- "Cancelling replication events (pending={}, inFlight={}) for destination {}",
- numPending,
- numInFlight,
- getRemoteConfigName());
+ repLog.atWarning().log(
+ "Cancelling replication events (pending=%d, inFlight=%d) for destination %s",
+ numPending, numInFlight, getRemoteConfigName());
foreachPushOp(
pending,
@@ -281,9 +281,12 @@
private boolean shouldReplicate(ProjectState state, CurrentUser user)
throws PermissionBackendException {
+ String name = state.getProject().getName();
if (!config.replicateHiddenProjects()
&& state.getProject().getState()
== com.google.gerrit.extensions.client.ProjectState.HIDDEN) {
+ repLog.atFine().log(
+ "Project %s is hidden and replication of hidden projects is disabled", name);
return false;
}
@@ -296,6 +299,9 @@
permissionBackend.user(user).project(state.getNameKey()).check(permissionToCheck);
return true;
} catch (AuthException e) {
+ repLog.atFine().log(
+ "Project %s is not visible to current user %s",
+ name, user.getUserName().orElse("unknown"));
return false;
}
}
@@ -310,12 +316,16 @@
try {
projectState = projectCache.get(project).orElseThrow(noSuchProject(project));
} catch (StorageException e) {
+ repLog.atWarning().withCause(e).log(
+ "Error reading project %s from cache", project);
return false;
}
if (!projectState.statePermitsRead()) {
+ repLog.atFine().log("Project %s does not permit read", project);
return false;
}
if (!shouldReplicate(projectState, userProvider.get())) {
+ repLog.atFine().log("Project %s should not be replicated", project);
return false;
}
if (PushOne.ALL_REFS.equals(ref)) {
@@ -328,13 +338,16 @@
.ref(ref)
.check(RefPermission.READ);
} catch (AuthException e) {
+ repLog.atFine().log(
+ "Ref %s on project %s is not visible to calling user",
+ ref, project, userProvider.get().getUserName().orElse("unknown"));
return false;
}
return true;
})
.call();
} catch (NoSuchProjectException err) {
- stateLog.error(String.format("source project %s not available", project), err, states);
+ stateLog.error(String.format(PROJECT_NOT_AVAILABLE, project), err, states);
} catch (Exception e) {
Throwables.throwIfUnchecked(e);
throw new RuntimeException(e);
@@ -357,7 +370,7 @@
})
.call();
} catch (NoSuchProjectException err) {
- stateLog.error(String.format("source project %s not available", project), err, states);
+ stateLog.error(String.format(PROJECT_NOT_AVAILABLE, project), err, states);
} catch (Exception e) {
Throwables.throwIfUnchecked(e);
throw new RuntimeException(e);
@@ -371,10 +384,11 @@
void schedule(
Project.NameKey project, String ref, URIish uri, ReplicationState state, boolean now) {
- repLog.info("scheduling replication {}:{} => {}", project, ref, uri);
if (!shouldReplicate(project, ref, state)) {
+ repLog.atFine().log("Not scheduling replication %s:%s => %s", project, ref, uri);
return;
}
+ repLog.atInfo().log("scheduling replication %s:%s => %s", project, ref, uri);
if (!config.replicatePermissions()) {
PushOne e;
@@ -395,7 +409,7 @@
return;
}
} catch (IOException err) {
- stateLog.error(String.format("source project %s not available", project), err, state);
+ stateLog.error(String.format(PROJECT_NOT_AVAILABLE, project), err, state);
return;
}
}
@@ -416,7 +430,8 @@
task.addState(ref, state);
}
state.increasePushTaskCount(project.get(), ref);
- repLog.info("scheduled {}:{} => {} to run after {}s", project, ref, task, config.getDelay());
+ repLog.atInfo().log(
+ "scheduled %s:%s => %s to run after %ds", project, ref, task, config.getDelay());
}
}
@@ -522,7 +537,6 @@
pending.put(uri, pushOp);
switch (reason) {
case COLLISION:
- replicationTasksStorage.get().reset(pushOp);
@SuppressWarnings("unused")
ScheduledFuture<?> ignored =
pool.schedule(pushOp, config.getRescheduleDelay(), TimeUnit.SECONDS);
@@ -574,7 +588,9 @@
void notifyFinished(PushOne op) {
synchronized (stateLock) {
- replicationTasksStorage.get().finish(op);
+ if (!op.isRetrying()) {
+ replicationTasksStorage.get().finish(op);
+ }
inFlight.remove(op.getURI());
}
}
@@ -583,7 +599,7 @@
Set<String> names = new HashSet<>();
for (PushOne push : pending.values()) {
if (!replicationTasksStorage.get().isWaiting(push)) {
- repLog.debug("No longer isWaiting, can prune " + push.getURI());
+ repLog.atFine().log("No longer isWaiting, can prune %s", push.getURI());
names.add(push.toString());
}
}
@@ -592,6 +608,7 @@
boolean wouldPushProject(Project.NameKey project) {
if (!shouldReplicate(project)) {
+ repLog.atFine().log("Skipping replication of project %s", project.get());
return false;
}
@@ -601,7 +618,12 @@
return true;
}
- return (new ReplicationFilter(projects)).matches(project);
+ boolean matches = (new ReplicationFilter(projects)).matches(project);
+ if (!matches) {
+ repLog.atFine().log(
+ "Skipping replication of project %s; does not match filter", project.get());
+ }
+ return matches;
}
boolean isSingleProjectMatch() {
@@ -610,6 +632,7 @@
boolean wouldPushRef(String ref) {
if (!config.replicatePermissions() && RefNames.REFS_CONFIG.equals(ref)) {
+ repLog.atFine().log("Skipping push of ref %s; it is a meta ref", ref);
return false;
}
if (PushOne.ALL_REFS.equals(ref)) {
@@ -620,6 +643,7 @@
return true;
}
}
+ repLog.atFine().log("Skipping push of ref %s; it does not match push ref specs", ref);
return false;
}
@@ -651,7 +675,8 @@
} else if (remoteNameStyle.equals("basenameOnly")) {
name = FilenameUtils.getBaseName(name);
} else if (!remoteNameStyle.equals("slash")) {
- repLog.debug("Unknown remoteNameStyle: {}, falling back to slash", remoteNameStyle);
+ repLog.atFine().log(
+ "Unknown remoteNameStyle: %s, falling back to slash", remoteNameStyle);
}
String replacedPath = replaceName(uri.getPath(), name, config.isSingleProjectMatch());
if (replacedPath != null) {
@@ -743,7 +768,7 @@
try {
eventDispatcher.get().postEvent(BranchNameKey.create(project, ref), event);
} catch (PermissionBackendException e) {
- repLog.error("error posting event", e);
+ repLog.atSevere().withCause(e).log("error posting event");
}
}
}
@@ -757,7 +782,7 @@
try {
eventDispatcher.get().postEvent(BranchNameKey.create(project, ref), event);
} catch (PermissionBackendException e) {
- repLog.error("error posting event", e);
+ repLog.atSevere().withCause(e).log("error posting event");
}
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationsCollection.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationsCollection.java
index 0eeab48..d869e83 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationsCollection.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationsCollection.java
@@ -25,7 +25,6 @@
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMultimap;
-import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.common.collect.SetMultimap;
import com.google.common.eventbus.EventBus;
@@ -38,22 +37,16 @@
import com.google.inject.Singleton;
import com.googlesource.gerrit.plugins.replication.Destination.Factory;
import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType;
-import java.io.IOException;
import java.net.URISyntaxException;
-import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
-import java.util.Set;
import java.util.function.Predicate;
import org.eclipse.jgit.errors.ConfigInvalidException;
-import org.eclipse.jgit.storage.file.FileBasedConfig;
-import org.eclipse.jgit.transport.RefSpec;
-import org.eclipse.jgit.transport.RemoteConfig;
import org.eclipse.jgit.transport.URIish;
@Singleton
-public class DestinationsCollection implements ReplicationDestinations, ReplicationConfigValidator {
+public class DestinationsCollection implements ReplicationDestinations {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private final Factory destinationFactory;
@@ -73,12 +66,15 @@
public DestinationsCollection(
Destination.Factory destinationFactory,
Provider<ReplicationQueue> replicationQueue,
- ReplicationFileBasedConfig replicationConfig,
+ ReplicationConfig replicationConfig,
+ ConfigParser configParser,
EventBus eventBus)
throws ConfigInvalidException {
this.destinationFactory = destinationFactory;
this.replicationQueue = replicationQueue;
- this.destinations = allDestinations(destinationFactory, validateConfig(replicationConfig));
+ this.destinations =
+ allDestinations(
+ destinationFactory, configParser.parseRemotes(replicationConfig.getConfig()));
eventBus.register(this);
}
@@ -110,7 +106,7 @@
try {
uri = new URIish(url);
} catch (URISyntaxException e) {
- repLog.warn("adminURL '{}' is invalid: {}", url, e.getMessage());
+ repLog.atWarning().log("adminURL '%s' is invalid: %s", url, e.getMessage());
continue;
}
@@ -118,13 +114,14 @@
String path =
replaceName(uri.getPath(), projectName.get(), config.isSingleProjectMatch());
if (path == null) {
- repLog.warn("adminURL {} does not contain ${name}", uri);
+ repLog.atWarning().log("adminURL %s does not contain ${name}", uri);
continue;
}
uri = uri.setPath(path);
if (!isSSH(uri)) {
- repLog.warn("adminURL '{}' is invalid: only SSH and HTTP are supported", uri);
+ repLog.atWarning().log(
+ "adminURL '%s' is invalid: only SSH and HTTP are supported", uri);
continue;
}
}
@@ -247,84 +244,6 @@
}
}
- @Override
- public List<RemoteConfiguration> validateConfig(ReplicationFileBasedConfig replicationConfig)
- throws ConfigInvalidException {
- if (!replicationConfig.getConfig().getFile().exists()) {
- logger.atWarning().log(
- "Config file %s does not exist; not replicating",
- replicationConfig.getConfig().getFile());
- return Collections.emptyList();
- }
- if (replicationConfig.getConfig().getFile().length() == 0) {
- logger.atInfo().log(
- "Config file %s is empty; not replicating", replicationConfig.getConfig().getFile());
- return Collections.emptyList();
- }
-
- try {
- replicationConfig.getConfig().load();
- } catch (ConfigInvalidException e) {
- throw new ConfigInvalidException(
- String.format(
- "Config file %s is invalid: %s",
- replicationConfig.getConfig().getFile(), e.getMessage()),
- e);
- } catch (IOException e) {
- throw new ConfigInvalidException(
- String.format(
- "Cannot read %s: %s", replicationConfig.getConfig().getFile(), e.getMessage()),
- e);
- }
-
- boolean defaultForceUpdate =
- replicationConfig.getConfig().getBoolean("gerrit", "defaultForceUpdate", false);
-
- ImmutableList.Builder<RemoteConfiguration> confs = ImmutableList.builder();
- for (RemoteConfig c : allRemotes(replicationConfig.getConfig())) {
- if (c.getURIs().isEmpty()) {
- continue;
- }
-
- if (!c.getFetchRefSpecs().isEmpty()) {
- repLog.info("Ignore '{}' endpoint: not a 'push' target", c.getName());
- continue;
- }
-
- // If destination for push is not set assume equal to source.
- for (RefSpec ref : c.getPushRefSpecs()) {
- if (ref.getDestination() == null) {
- ref.setDestination(ref.getSource());
- }
- }
-
- if (c.getPushRefSpecs().isEmpty()) {
- c.addPushRefSpec(
- new RefSpec()
- .setSourceDestination("refs/*", "refs/*")
- .setForceUpdate(defaultForceUpdate));
- }
-
- DestinationConfiguration destinationConfiguration =
- new DestinationConfiguration(c, replicationConfig.getConfig());
-
- if (!destinationConfiguration.isSingleProjectMatch()) {
- for (URIish u : c.getURIs()) {
- if (u.getPath() == null || !u.getPath().contains("${name}")) {
- throw new ConfigInvalidException(
- String.format(
- "remote.%s.url \"%s\" lacks ${name} placeholder in %s",
- c.getName(), u, replicationConfig.getConfig().getFile()));
- }
- }
- }
-
- confs.add(destinationConfiguration);
- }
-
- return confs.build();
- }
-
private List<Destination> allDestinations(
Destination.Factory destinationFactory, List<RemoteConfiguration> remoteConfigurations) {
@@ -336,18 +255,4 @@
}
return dest.build();
}
-
- private static List<RemoteConfig> allRemotes(FileBasedConfig cfg) throws ConfigInvalidException {
- Set<String> names = cfg.getSubsections("remote");
- List<RemoteConfig> result = Lists.newArrayListWithCapacity(names.size());
- for (String name : names) {
- try {
- result.add(new RemoteConfig(cfg, name));
- } catch (URISyntaxException e) {
- throw new ConfigInvalidException(
- String.format("remote %s has invalid URL in %s", name, cfg.getFile()), e);
- }
- }
- return result;
- }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/FanoutReplicationConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/FanoutReplicationConfig.java
new file mode 100644
index 0000000..5b24bf5
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/FanoutReplicationConfig.java
@@ -0,0 +1,187 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.google.common.io.Files.getNameWithoutExtension;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.collect.Lists;
+import com.google.common.flogger.FluentLogger;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+import com.google.gerrit.extensions.annotations.PluginData;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.inject.Inject;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Stream;
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.internal.storage.file.FileSnapshot;
+import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
+
+public class FanoutReplicationConfig implements ReplicationConfig {
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+ private final ReplicationFileBasedConfig replicationConfig;
+ private final Config config;
+ private final Path remoteConfigsDirPath;
+
+ @Inject
+ public FanoutReplicationConfig(SitePaths site, @PluginData Path pluginDataDir)
+ throws IOException, ConfigInvalidException {
+
+ remoteConfigsDirPath = site.etc_dir.resolve("replication");
+ replicationConfig = new ReplicationFileBasedConfig(site, pluginDataDir);
+ config = replicationConfig.getConfig();
+ removeRemotes(config);
+
+ try (Stream<Path> files = Files.list(remoteConfigsDirPath)) {
+ files
+ .filter(Files::isRegularFile)
+ .filter(FanoutReplicationConfig::isConfig)
+ .map(FanoutReplicationConfig::loadConfig)
+ .filter(Optional::isPresent)
+ .map(Optional::get)
+ .filter(FanoutReplicationConfig::isValid)
+ .forEach(cfg -> addRemoteConfig(cfg, config));
+ } catch (IllegalStateException e) {
+ throw new ConfigInvalidException(e.getMessage());
+ }
+ }
+
+ private static void removeRemotes(Config config) {
+ Set<String> remoteNames = config.getSubsections("remote");
+ if (remoteNames.size() > 0) {
+ logger.atSevere().log(
+ "When replication directory is present replication.config file cannot contain remote configuration. Ignoring: %s",
+ String.join(",", remoteNames));
+
+ for (String name : remoteNames) {
+ config.unsetSection("remote", name);
+ }
+ }
+ }
+
+ private static void addRemoteConfig(FileBasedConfig source, Config destination) {
+ String remoteName = getNameWithoutExtension(source.getFile().getName());
+ for (String name : source.getNames("remote")) {
+ destination.setStringList(
+ "remote",
+ remoteName,
+ name,
+ Lists.newArrayList(source.getStringList("remote", null, name)));
+ }
+ }
+
+ private static boolean isValid(Config cfg) {
+ if (cfg.getSections().size() != 1 || !cfg.getSections().contains("remote")) {
+ logger.atSevere().log(
+ "Remote replication configuration file %s must contain only one remote section.", cfg);
+ return false;
+ }
+ if (cfg.getSubsections("remote").size() > 0) {
+ logger.atSevere().log(
+ "Remote replication configuration file %s cannot contain remote subsections.", cfg);
+ return false;
+ }
+
+ return true;
+ }
+
+ private static Optional<FileBasedConfig> loadConfig(Path path) {
+ FileBasedConfig cfg = new FileBasedConfig(path.toFile(), FS.DETECTED);
+ try {
+ cfg.load();
+ } catch (IOException | ConfigInvalidException e) {
+ logger.atSevere().withCause(e).log(
+ "Cannot load remote replication configuration file %s.", path);
+ return Optional.empty();
+ }
+ return Optional.of(cfg);
+ }
+
+ private static boolean isConfig(Path p) {
+ return p.toString().endsWith(".config");
+ }
+
+ @Override
+ public boolean isReplicateAllOnPluginStart() {
+ return replicationConfig.isReplicateAllOnPluginStart();
+ }
+
+ @Override
+ public boolean isDefaultForceUpdate() {
+ return replicationConfig.isDefaultForceUpdate();
+ }
+
+ @Override
+ public int getMaxRefsToLog() {
+ return replicationConfig.getMaxRefsToLog();
+ }
+
+ @Override
+ public Path getEventsDirectory() {
+ return replicationConfig.getEventsDirectory();
+ }
+
+ @Override
+ public int getSshConnectionTimeout() {
+ return replicationConfig.getSshConnectionTimeout();
+ }
+
+ @Override
+ public int getSshCommandTimeout() {
+ return replicationConfig.getSshCommandTimeout();
+ }
+
+ @Override
+ public int getDistributionInterval() {
+ return replicationConfig.getDistributionInterval();
+ }
+
+ @Override
+ public String getVersion() {
+ Hasher hasher = Hashing.murmur3_128().newHasher();
+ hasher.putString(replicationConfig.getVersion(), UTF_8);
+ try (Stream<Path> files = Files.list(remoteConfigsDirPath)) {
+ files
+ .filter(Files::isRegularFile)
+ .filter(FanoutReplicationConfig::isConfig)
+ .sorted()
+ .map(Path::toFile)
+ .map(FileSnapshot::save)
+ .forEach(
+ fileSnapshot ->
+ // hashCode is based on file size, file key and last modified time
+ hasher.putInt(fileSnapshot.hashCode()));
+ return hasher.hash().toString();
+ } catch (IOException e) {
+ logger.atSevere().withCause(e).log(
+ "Cannot list remote configuration files from %s. Returning replication.config file version",
+ remoteConfigsDirPath);
+ return replicationConfig.getVersion();
+ }
+ }
+
+ @Override
+ public Config getConfig() {
+ return config;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java
index f910a40..079087c 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java
@@ -62,34 +62,34 @@
@Override
public boolean createProject(Project.NameKey project, String head) {
- repLog.info("Creating project {} on {}", project, uri);
+ repLog.atInfo().log("Creating project %s on %s", project, uri);
String url = String.format("%s/a/projects/%s", toHttpUri(uri), Url.encode(project.get()));
try {
return httpClient
.execute(new HttpPut(url), new HttpResponseHandler(), getContext())
.isSuccessful();
} catch (IOException e) {
- repLog.error("Couldn't perform project creation on {}", uri, e);
+ repLog.atSevere().log("Couldn't perform project creation on %s", uri, e);
return false;
}
}
@Override
public boolean deleteProject(Project.NameKey project) {
- repLog.info("Deleting project {} on {}", project, uri);
+ repLog.atInfo().log("Deleting project %s on %s", project, uri);
String url = String.format("%s/a/projects/%s", toHttpUri(uri), Url.encode(project.get()));
try {
httpClient.execute(new HttpDelete(url), new HttpResponseHandler(), getContext());
return true;
} catch (IOException e) {
- repLog.error("Couldn't perform project deletion on {}", uri, e);
+ repLog.atSevere().log("Couldn't perform project deletion on %s", uri, e);
}
return false;
}
@Override
public boolean updateHead(Project.NameKey project, String newHead) {
- repLog.info("Updating head of {} on {}", project, uri);
+ repLog.atInfo().log("Updating head of %s on %s", project, uri);
String url = String.format("%s/a/projects/%s/HEAD", toHttpUri(uri), Url.encode(project.get()));
try {
HttpPut req = new HttpPut(url);
@@ -99,7 +99,7 @@
httpClient.execute(req, new HttpResponseHandler(), getContext());
return true;
} catch (IOException e) {
- repLog.error("Couldn't perform update head on {}", uri, e);
+ repLog.atSevere().log("Couldn't perform update head on %s", uri, e);
}
return false;
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java b/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java
index da960e6..b092363 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java
@@ -43,9 +43,9 @@
u.disableRefLog();
u.link(head);
}
- repLog.info("Created local repository: {}", uri);
+ repLog.atInfo().log("Created local repository: %s", uri);
} catch (IOException e) {
- repLog.error("Error creating local repository {}", uri.getPath(), e);
+ repLog.atSevere().withCause(e).log("Error creating local repository %s", uri.getPath());
return false;
}
return true;
@@ -55,9 +55,9 @@
public boolean deleteProject(Project.NameKey project) {
try {
recursivelyDelete(new File(uri.getPath()));
- repLog.info("Deleted local repository: {}", uri);
+ repLog.atInfo().log("Deleted local repository: %s", uri);
} catch (IOException e) {
- repLog.error("Error deleting local repository {}:\n", uri.getPath(), e);
+ repLog.atSevere().withCause(e).log("Error deleting local repository %s:\n", uri.getPath());
return false;
}
return true;
@@ -71,7 +71,8 @@
u.link(newHead);
}
} catch (IOException e) {
- repLog.error("Failed to update HEAD of repository {} to {}", uri.getPath(), newHead, e);
+ repLog.atSevere().withCause(e).log(
+ "Failed to update HEAD of repository %s to %s", uri.getPath(), newHead);
return false;
}
return true;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/MainReplicationConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/MainReplicationConfig.java
new file mode 100644
index 0000000..e8d95ec
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/MainReplicationConfig.java
@@ -0,0 +1,23 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import com.google.inject.BindingAnnotation;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+@BindingAnnotation
+@Retention(RetentionPolicy.RUNTIME)
+public @interface MainReplicationConfig {}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Nfs.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Nfs.java
new file mode 100644
index 0000000..a347f3a
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Nfs.java
@@ -0,0 +1,55 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import java.io.IOException;
+import java.util.Locale;
+
+/** Some NFS utilities */
+public class Nfs {
+ /**
+ * Determine if a throwable or a cause in its causal chain is a Stale NFS File Handle
+ *
+ * @param throwable
+ * @return a boolean true if the throwable or a cause in its causal chain is a Stale NFS File
+ * Handle
+ */
+ public static boolean isStaleFileHandleInCausalChain(Throwable throwable) {
+ while (throwable != null) {
+ if (throwable instanceof IOException && isStaleFileHandle((IOException) throwable)) {
+ return true;
+ }
+ throwable = throwable.getCause();
+ }
+ return false;
+ }
+
+ /**
+ * Determine if an IOException is a Stale NFS File Handle
+ *
+ * @param ioe
+ * @return a boolean true if the IOException is a Stale NFS FIle Handle
+ */
+ public static boolean isStaleFileHandle(IOException ioe) {
+ String msg = ioe.getMessage();
+ return msg != null && msg.toLowerCase(Locale.ROOT).matches(".*stale .*file .*handle.*");
+ }
+
+ public static <T extends Throwable> void throwIfNotStaleFileHandle(T e) throws T {
+ if (!isStaleFileHandleInCausalChain(e)) {
+ throw e;
+ }
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
index 8208264..199acce 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
@@ -166,17 +166,16 @@
@Override
public void cancel() {
- repLog.info("Replication [{}] to {} was canceled", HexFormat.fromInt(id), getURI());
+ repLog.atInfo().log("Replication [%s] to %s was canceled", HexFormat.fromInt(id), getURI());
canceledByReplication();
pool.pushWasCanceled(this);
}
@Override
public void setCanceledWhileRunning() {
- repLog.info(
- "Replication [{}] to {} was canceled while being executed",
- HexFormat.fromInt(id),
- getURI());
+ repLog.atInfo().log(
+ "Replication [%s] to %s was canceled while being executed",
+ HexFormat.fromInt(id), getURI());
canceledWhileRunning.set(true);
}
@@ -231,10 +230,10 @@
if (ALL_REFS.equals(ref)) {
delta.clear();
pushAllRefs = true;
- repLog.trace("Added all refs for replication to {}", uri);
+ repLog.atFinest().log("Added all refs for replication to %s", uri);
} else if (!pushAllRefs) {
delta.add(ref);
- repLog.trace("Added ref {} for replication to {}", ref, uri);
+ repLog.atFinest().log("Added ref %s for replication to %s", ref, uri);
}
}
@@ -319,20 +318,20 @@
RunwayStatus status = pool.requestRunway(this);
if (!status.isAllowed()) {
if (status.isCanceled()) {
- repLog.info("PushOp for replication to {} was canceled and thus won't be rescheduled", uri);
+ repLog.atInfo().log(
+ "PushOp for replication to %s was canceled and thus won't be rescheduled", uri);
} else if (status.isExternalInflight()) {
- repLog.info("PushOp for replication to {} was denied externally", uri);
+ repLog.atInfo().log("PushOp for replication to %s was denied externally", uri);
} else {
- repLog.info(
- "Rescheduling replication to {} to avoid collision with the in-flight push [{}].",
- uri,
- HexFormat.fromInt(status.getInFlightPushId()));
+ repLog.atInfo().log(
+ "Rescheduling replication to %s to avoid collision with the in-flight push [%s].",
+ uri, HexFormat.fromInt(status.getInFlightPushId()));
pool.reschedule(this, Destination.RetryReason.COLLISION);
}
return;
}
- repLog.info("Replication to {} started...", uri);
+ repLog.atInfo().log("Replication to %s started...", uri);
Timer1.Context<String> destinationContext = metrics.start(config.getName());
try {
long startedAt = destinationContext.getStartTime();
@@ -346,12 +345,9 @@
metrics.recordSlowProjectReplication(
config.getName(), projectName.get(), pool.getSlowLatencyThreshold(), elapsed);
}
- repLog.info(
- "Replication to {} completed in {}ms, {}ms delay, {} retries",
- uri,
- elapsed,
- delay,
- retryCount);
+ repLog.atInfo().log(
+ "Replication to %s completed in %dms, %dms delay, %d retries",
+ uri, elapsed, delay, retryCount);
} catch (RepositoryNotFoundException e) {
stateLog.error(
"Cannot replicate " + projectName + "; Local repository error: " + e.getMessage(),
@@ -368,7 +364,7 @@
|| msg.contains("unavailable")) {
createRepository();
} else {
- repLog.error("Cannot replicate {}; Remote repository error: {}", projectName, msg);
+ repLog.atSevere().log("Cannot replicate %s; Remote repository error: %s", projectName, msg);
}
} catch (NoRemoteRepositoryException e) {
@@ -378,12 +374,10 @@
} catch (TransportException e) {
Throwable cause = e.getCause();
if (cause instanceof JSchException && cause.getMessage().startsWith("UnknownHostKey:")) {
- repLog.error("Cannot replicate to {}: {}", uri, cause.getMessage());
+ repLog.atSevere().log("Cannot replicate to %s: %s", uri, cause.getMessage());
} else if (e instanceof LockFailureException) {
lockRetryCount++;
- // The LockFailureException message contains both URI and reason
- // for this failure.
- repLog.error("Cannot replicate to {}: {}", uri, e.getMessage());
+ repLog.atSevere().log("Cannot replicate to %s due to lock failure", uri);
// The remote push operation should be retried.
if (lockRetryCount <= maxLockRetries) {
@@ -393,17 +387,14 @@
pool.reschedule(this, Destination.RetryReason.TRANSPORT_ERROR);
}
} else {
- repLog.error(
- "Giving up after {} occurrences of this error: {} during replication to {}",
- lockRetryCount,
- e.getMessage(),
- uri);
+ repLog.atSevere().log(
+ "Giving up after %d lock failures during replication to %s", lockRetryCount, uri);
}
} else {
if (canceledWhileRunning.get()) {
logCanceledWhileRunningException(e);
} else {
- repLog.error("Cannot replicate to {}", uri, e);
+ repLog.atSevere().withCause(e).log("Cannot replicate to %s", uri);
// The remote push operation should be retried.
pool.reschedule(this, Destination.RetryReason.TRANSPORT_ERROR);
}
@@ -421,7 +412,7 @@
}
private void logCanceledWhileRunningException(TransportException e) {
- repLog.info("Cannot replicate to {}. It was canceled while running", uri, e);
+ repLog.atInfo().withCause(e).log("Cannot replicate to %s. It was canceled while running", uri);
}
private void createRepository() {
@@ -429,10 +420,11 @@
try {
Ref head = git.exactRef(Constants.HEAD);
if (createProject(projectName, head != null ? getName(head) : null)) {
- repLog.warn("Missing repository created; retry replication to {}", uri);
+ repLog.atWarning().log("Missing repository created; retry replication to %s", uri);
pool.reschedule(this, Destination.RetryReason.REPOSITORY_MISSING);
} else {
- repLog.warn("Missing repository could not be created when replicating {}", uri);
+ repLog.atWarning().log(
+ "Missing repository could not be created when replicating %s", uri);
}
} catch (IOException ioe) {
stateLog.error(
@@ -465,8 +457,7 @@
updateStates(res.getRemoteUpdates());
}
- private PushResult pushVia(Transport tn)
- throws IOException, NotSupportedException, TransportException, PermissionBackendException {
+ private PushResult pushVia(Transport tn) throws IOException, PermissionBackendException {
tn.applyConfig(config);
tn.setCredentialsProvider(credentialsProvider);
@@ -480,10 +471,10 @@
}
if (replConfig.getMaxRefsToLog() == 0 || todo.size() <= replConfig.getMaxRefsToLog()) {
- repLog.info("Push to {} references: {}", uri, todo);
+ repLog.atInfo().log("Push to %s references: %s", uri, todo);
} else {
- repLog.info(
- "Push to {} references (first {} of {} listed): {}",
+ repLog.atInfo().log(
+ "Push to %s references (first %d of %d listed): %s",
uri,
replConfig.getMaxRefsToLog(),
todo.size(),
@@ -540,13 +531,13 @@
: replicationPushFilter.get().filter(projectName.get(), remoteUpdatesList);
}
- private List<RemoteRefUpdate> doPushAll(Transport tn, Map<String, Ref> local)
- throws NotSupportedException, TransportException, IOException {
+ private List<RemoteRefUpdate> doPushAll(Transport tn, Map<String, Ref> local) throws IOException {
List<RemoteRefUpdate> cmds = new ArrayList<>();
boolean noPerms = !pool.isReplicatePermissions();
Map<String, Ref> remote = listRemote(tn);
for (Ref src : local.values()) {
if (!canPushRef(src.getName(), noPerms)) {
+ repLog.atFine().log("Skipping push of ref %s", src.getName());
continue;
}
@@ -562,12 +553,14 @@
if (config.isMirror()) {
for (Ref ref : remote.values()) {
- if (!Constants.HEAD.equals(ref.getName())) {
- RefSpec spec = matchDst(ref.getName());
- if (spec != null && !local.containsKey(spec.getSource())) {
- // No longer on local side, request removal.
- delete(cmds, spec);
- }
+ if (Constants.HEAD.equals(ref.getName())) {
+ repLog.atFine().log("Skipping deletion of %s", ref.getName());
+ continue;
+ }
+ RefSpec spec = matchDst(ref.getName());
+ if (spec != null && !local.containsKey(spec.getSource())) {
+ // No longer on local side, request removal.
+ delete(cmds, spec);
}
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java
index d4d979f..b8da2e4 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java
@@ -42,18 +42,14 @@
OutputStream errStream = sshHelper.newErrorBufferStream();
try {
sshHelper.executeRemoteSsh(uri, cmd, errStream);
- repLog.info("Created remote repository: {}", uri);
+ repLog.atInfo().log("Created remote repository: %s", uri);
} catch (IOException e) {
- repLog.error(
- "Error creating remote repository at {}:\n"
- + " Exception: {}\n"
- + " Command: {}\n"
- + " Output: {}",
- uri,
- e,
- cmd,
- errStream,
- e);
+ repLog.atSevere().withCause(e).log(
+ "Error creating remote repository at %s:\n"
+ + " Exception: %s\n"
+ + " Command: %s\n"
+ + " Output: %s",
+ uri, e, cmd, errStream);
return false;
}
return true;
@@ -66,18 +62,14 @@
OutputStream errStream = sshHelper.newErrorBufferStream();
try {
sshHelper.executeRemoteSsh(uri, cmd, errStream);
- repLog.info("Deleted remote repository: {}", uri);
+ repLog.atInfo().log("Deleted remote repository: %s", uri);
} catch (IOException e) {
- repLog.error(
- "Error deleting remote repository at {}:\n"
- + " Exception: {}\n"
- + " Command: {}\n"
- + " Output: {}",
- uri,
- e,
- cmd,
- errStream,
- e);
+ repLog.atSevere().withCause(e).log(
+ "Error deleting remote repository at %s}:\n"
+ + " Exception: %s\n"
+ + " Command: %s\n"
+ + " Output: %s",
+ uri, e, cmd, errStream);
return false;
}
return true;
@@ -92,17 +84,12 @@
try {
sshHelper.executeRemoteSsh(uri, cmd, errStream);
} catch (IOException e) {
- repLog.error(
- "Error updating HEAD of remote repository at {} to {}:\n"
- + " Exception: {}\n"
- + " Command: {}\n"
- + " Output: {}",
- uri,
- newHead,
- e,
- cmd,
- errStream,
- e);
+ repLog.atSevere().withCause(e).log(
+ "Error updating HEAD of remote repository at %s to %s:\n"
+ + " Exception: %s\n"
+ + " Command: %s\n"
+ + " Output: %s",
+ uri, newHead, e, cmd, errStream);
return false;
}
return true;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
index 68fe430..99e5ee4 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
@@ -15,6 +15,7 @@
package com.googlesource.gerrit.plugins.replication;
import java.nio.file.Path;
+import org.eclipse.jgit.lib.Config;
/** Configuration of all the replication end points. */
public interface ReplicationConfig {
@@ -77,4 +78,6 @@
* @return current logical version number.
*/
String getVersion();
+
+ Config getConfig();
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfigValidator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfigValidator.java
deleted file mode 100644
index 7883114..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfigValidator.java
+++ /dev/null
@@ -1,31 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.replication;
-
-import java.util.List;
-import org.eclipse.jgit.errors.ConfigInvalidException;
-
-public interface ReplicationConfigValidator {
-
- /**
- * validate the new replication.config
- *
- * @param newConfig new configuration detected
- * @return List of validated {@link RemoteConfiguration}
- * @throws ConfigInvalidException if the new configuration is not valid.
- */
- List<RemoteConfiguration> validateConfig(ReplicationFileBasedConfig newConfig)
- throws ConfigInvalidException;
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
index d714376..2631cbe 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
@@ -13,16 +13,19 @@
// limitations under the License.
package com.googlesource.gerrit.plugins.replication;
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+
import com.google.common.base.Strings;
import com.google.gerrit.extensions.annotations.PluginData;
import com.google.gerrit.server.config.SitePaths;
import com.google.inject.Inject;
-import com.google.inject.Singleton;
+import java.io.IOException;
import java.nio.file.Path;
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.util.FS;
-@Singleton
public class ReplicationFileBasedConfig implements ReplicationConfig {
private static final int DEFAULT_SSH_CONNECTION_TIMEOUT_MS = 2 * 60 * 1000; // 2 minutes
@@ -41,6 +44,13 @@
this.site = site;
this.cfgPath = site.etc_dir.resolve("replication.config");
this.config = new FileBasedConfig(cfgPath.toFile(), FS.DETECTED);
+ try {
+ config.load();
+ } catch (ConfigInvalidException e) {
+ repLog.atSevere().withCause(e).log("Config file %s is invalid: %s", cfgPath, e.getMessage());
+ } catch (IOException e) {
+ repLog.atSevere().withCause(e).log("Cannot read %s: %s", cfgPath, e.getMessage());
+ }
this.replicateAllOnPluginStart = config.getBoolean("gerrit", "replicateOnStartup", false);
this.defaultForceUpdate = config.getBoolean("gerrit", "defaultForceUpdate", false);
this.maxRefsToLog = config.getInt("gerrit", "maxRefsToLog", 0);
@@ -98,7 +108,8 @@
return cfgPath;
}
- public FileBasedConfig getConfig() {
+ @Override
+ public Config getConfig() {
return config;
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
index 979c8e3..ed1e348 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
@@ -34,6 +34,7 @@
import com.google.inject.internal.UniqueAnnotations;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Files;
import java.nio.file.Path;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.storage.file.FileBasedConfig;
@@ -41,10 +42,12 @@
import org.eclipse.jgit.util.FS;
class ReplicationModule extends AbstractModule {
+ private final SitePaths site;
private final Path cfgPath;
@Inject
public ReplicationModule(SitePaths site) {
+ this.site = site;
cfgPath = site.etc_dir.resolve("replication.config");
}
@@ -77,15 +80,18 @@
bind(EventBus.class).in(Scopes.SINGLETON);
bind(ReplicationDestinations.class).to(DestinationsCollection.class);
- bind(ReplicationConfigValidator.class).to(DestinationsCollection.class);
+ bind(ConfigParser.class).in(Scopes.SINGLETON);
if (getReplicationConfig().getBoolean("gerrit", "autoReload", false)) {
- bind(ReplicationConfig.class).to(AutoReloadConfigDecorator.class);
+ bind(ReplicationConfig.class)
+ .annotatedWith(MainReplicationConfig.class)
+ .to(getReplicationConfigClass());
+ bind(ReplicationConfig.class).to(AutoReloadConfigDecorator.class).in(Scopes.SINGLETON);
bind(LifecycleListener.class)
.annotatedWith(UniqueAnnotations.create())
.to(AutoReloadConfigDecorator.class);
} else {
- bind(ReplicationConfig.class).to(ReplicationFileBasedConfig.class);
+ bind(ReplicationConfig.class).to(getReplicationConfigClass()).in(Scopes.SINGLETON);
}
DynamicSet.setOf(binder(), ReplicationStateListener.class);
@@ -109,4 +115,11 @@
}
return config;
}
+
+ private Class<? extends ReplicationConfig> getReplicationConfigClass() {
+ if (Files.exists(site.etc_dir.resolve("replication"))) {
+ return FanoutReplicationConfig.class;
+ }
+ return ReplicationFileBasedConfig.class;
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
index 0dcfc95..a8ffeec 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
@@ -27,6 +27,7 @@
import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.git.WorkQueue;
+import com.google.gerrit.util.logging.NamedFluentLogger;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.googlesource.gerrit.plugins.replication.PushResultProcessing.GitUpdateProcessing;
@@ -39,8 +40,6 @@
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import org.eclipse.jgit.transport.URIish;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/** Manages automatic replication to remote repositories. */
public class ReplicationQueue
@@ -50,7 +49,7 @@
ProjectDeletedListener,
HeadUpdatedListener {
static final String REPLICATION_LOG_NAME = "replication_log";
- static final Logger repLog = LoggerFactory.getLogger(REPLICATION_LOG_NAME);
+ static final NamedFluentLogger repLog = NamedFluentLogger.forName(REPLICATION_LOG_NAME);
private final ReplicationStateListener stateLog;
@@ -99,7 +98,7 @@
distributor.stop();
int discarded = destinations.get().shutdown();
if (discarded > 0) {
- repLog.warn("Canceled {} replication events during shutdown", discarded);
+ repLog.atWarning().log("Canceled %d replication events during shutdown", discarded);
}
}
@@ -158,6 +157,8 @@
}
cfg.schedule(project, refName, uri, state, now);
}
+ } else {
+ repLog.atFine().log("Skipping ref %s on project %s", refName, project.get());
}
}
}
@@ -170,7 +171,7 @@
for (ReplicationTasksStorage.ReplicateRefUpdate t : replicationTasksStorage.listWaiting()) {
String eventKey = String.format("%s:%s", t.project, t.ref);
if (!eventsReplayed.contains(eventKey)) {
- repLog.info("Firing pending task {}", eventKey);
+ repLog.atInfo().log("Firing pending task %s", eventKey);
fire(t.project, t.ref, true);
eventsReplayed.add(eventKey);
}
@@ -196,7 +197,7 @@
if (state == WorkQueue.Task.State.SLEEPING || state == WorkQueue.Task.State.READY) {
if (task instanceof WorkQueue.ProjectTask) {
if (prunableTaskNames.contains(task.toString())) {
- repLog.debug("Pruning externally completed task:" + task);
+ repLog.atFine().log("Pruning externally completed task: %s", task);
task.cancel(false);
}
}
@@ -223,7 +224,7 @@
for (ReferenceUpdatedEvent event : beforeStartupEventsQueue) {
String eventKey = String.format("%s:%s", event.projectName(), event.refName());
if (!eventsReplayed.contains(eventKey)) {
- repLog.info("Firing pending task {}", event);
+ repLog.atInfo().log("Firing pending task %s", event);
fire(event.projectName(), event.refName(), false);
eventsReplayed.add(eventKey);
}
@@ -265,7 +266,7 @@
firePendingEvents();
pruneCompleted();
} catch (Exception e) {
- repLog.error("error distributing tasks", e);
+ repLog.atSevere().withCause(e).log("error distributing tasks");
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStateLogger.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStateLogger.java
index f2d55de..3e73033 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStateLogger.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationStateLogger.java
@@ -31,19 +31,19 @@
@Override
public void warn(String msg, ReplicationState... states) {
stateWriteErr("Warning: " + msg, states);
- repLog.warn(msg);
+ repLog.atWarning().log(msg);
}
@Override
public void error(String msg, ReplicationState... states) {
stateWriteErr("Error: " + msg, states);
- repLog.error(msg);
+ repLog.atSevere().log(msg);
}
@Override
public void error(String msg, Throwable t, ReplicationState... states) {
stateWriteErr("Error: " + msg, states);
- repLog.error(msg, t);
+ repLog.atSevere().withCause(t).log(msg);
}
private void stateWriteErr(String msg, ReplicationState[] states) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
index 554f8bb..c764161 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
@@ -24,6 +24,7 @@
import com.google.inject.ProvisionException;
import com.google.inject.Singleton;
import java.io.IOException;
+import java.nio.file.DirectoryIteratorException;
import java.nio.file.DirectoryStream;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
@@ -221,7 +222,12 @@
String json = new String(Files.readAllBytes(e), UTF_8);
results.add(GSON.fromJson(json, ReplicateRefUpdate.class));
} else if (Files.isDirectory(e)) {
- results.addAll(list(e));
+ try {
+ results.addAll(list(e));
+ } catch (DirectoryIteratorException d) {
+ // iterating over the sub-directories is expected to have dirs disappear
+ Nfs.throwIfNotStaleFileHandle(d.getCause());
+ }
}
}
} catch (IOException e) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java
index ffa6be1..fdbd5e7 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java
@@ -58,7 +58,8 @@
return;
}
- repLog.warn("Cannot update HEAD of project {} on remote site {}.", project, replicateURI);
+ repLog.atWarning().log(
+ "Cannot update HEAD of project %s on remote site %s.", project, replicateURI);
}
@Override
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 19a478d..fc3352d 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -200,23 +200,46 @@
local environment. In that case, an alternative SSH url could
be specified to repository creation.
- To enable replication to different Gerrit instance use `gerrit+ssh://`,
+ To enable replication to different Gerrit instance use
`gerrit+http://` or `gerrit+https://` as protocol name followed
by hostname of another Gerrit server eg.
- `gerrit+ssh://replica1.my.org/`
- <br>
`gerrit+http://replica2.my.org/`
<br>
`gerrit+https://replica3.my.org/`
- In this case replication will use Gerrit's SSH API or Gerrit's REST API
+ In this case replication will use Gerrit's REST API
to create/remove projects and update repository HEAD references.
NOTE: In order to replicate project deletion, the
link:https://gerrit-review.googlesource.com/admin/projects/plugins/delete-project delete-project[delete-project]
plugin must be installed on the other Gerrit.
+ *Backward compatibility notice*
+
+ Before Gerrit v2.13 it was possible to enable replication to different
+ Gerrit masters using `gerrit+ssh://`
+ as protocol name followed by hostname of another Gerrit server eg.
+
+ `gerrit+ssh://replica1.my.org/`
+
+ In that case replication would have used Gerrit's SSH API to
+ create/remove projects and update repository HEAD references.
+
+ The `gerrit+ssh` option is kept for backward compatibility, however
+ the use-case behind it is not valid anymore since the introduction of
+ Lucene indexes and the removal of ReviewDb, which would require
+ a lot more machinery to setup a master to master replication scenario.
+
+ The `gerrit+ssh` option is still possible but is limited to the
+ ability to replicate only regular Git repositories that do not
+ contain any code-review or NoteDb information.
+
+ Using `gerrit+ssh` for replicating all Gerrit repositories
+ would result in failures on the All-Users.git replication and
+ would not be able to replicate changes magic refs and indexes
+ across nodes.
+
remote.NAME.receivepack
: Path of the `git-receive-pack` executable on the remote
system, if using the SSH transport.
@@ -446,6 +469,79 @@
default: 15 minutes
+Directory `replication`
+--------------------
+The optional directory `$site_path/etc/replication` contains Git-style
+config files that controls the replication settings for the replication
+plugin. When present all `remote` sections from `replication.config` file are
+ignored.
+
+Files are composed of one `remote` section. Multiple `remote` sections or any
+other section makes the file invalid and skipped by the replication plugin.
+File name defines remote section name. Each section provides common configuration
+settings for one or more destination URLs. For more details how to setup `remote`
+sections please refer to the `replication.config` section.
+
+### Configuration example:
+
+Static configuration in `$site_path/etc/replication.config`:
+
+```
+[gerrit]
+ autoReload = true
+ replicateOnStartup = false
+[replication]
+ lockErrorMaxRetries = 5
+ maxRetries = 5
+```
+
+Remote sections in `$site_path/etc/replication` directory:
+
+* File `$site_path/etc/replication/host-one.config`
+
+ ```
+ [remote]
+ url = gerrit2@host-one.example.com:/some/path/${name}.git
+ ```
+
+
+* File `$site_path/etc/replication/pubmirror.config`
+
+ ```
+ [remote]
+ url = mirror1.us.some.org:/pub/git/${name}.git
+ url = mirror2.us.some.org:/pub/git/${name}.git
+ url = mirror3.us.some.org:/pub/git/${name}.git
+ push = +refs/heads/*:refs/heads/*
+ push = +refs/tags/*:refs/tags/*
+ threads = 3
+ authGroup = Public Mirror Group
+ authGroup = Second Public Mirror Group
+ ```
+
+Replication plugin resolves config files to the following configuration:
+
+```
+[gerrit]
+ autoReload = true
+ replicateOnStartup = false
+[replication]
+ lockErrorMaxRetries = 5
+ maxRetries = 5
+
+[remote "host-one"]
+ url = gerrit2@host-one.example.com:/some/path/${name}.git
+
+[remote "pubmirror"]
+ url = mirror1.us.some.org:/pub/git/${name}.git
+ url = mirror2.us.some.org:/pub/git/${name}.git
+ url = mirror3.us.some.org:/pub/git/${name}.git
+ push = +refs/heads/*:refs/heads/*
+ push = +refs/tags/*:refs/tags/*
+ threads = 3
+ authGroup = Public Mirror Group
+ authGroup = Second Public Mirror Group
+```
File `secure.config`
--------------------
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/AbstractConfigTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/AbstractConfigTest.java
index 3932fb3..c48bdbd 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/AbstractConfigTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/AbstractConfigTest.java
@@ -49,6 +49,7 @@
protected WorkQueue workQueueMock;
protected EventBus eventBus = new EventBus();
protected FakeExecutorService executorService = new FakeExecutorService();
+ protected ConfigParser configParser;
static class FakeDestination extends Destination {
public final DestinationConfiguration config;
@@ -71,6 +72,7 @@
sitePaths = new SitePaths(sitePath);
pluginDataPath = createTempPath("data");
destinationFactoryMock = mock(Destination.Factory.class);
+ configParser = new ConfigParser();
}
@Before
@@ -96,8 +98,12 @@
}
protected FileBasedConfig newReplicationConfig() {
+ return newReplicationConfig("replication.config");
+ }
+
+ protected FileBasedConfig newReplicationConfig(String path) {
FileBasedConfig replicationConfig =
- new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED);
+ new FileBasedConfig(sitePaths.etc_dir.resolve(path).toFile(), FS.DETECTED);
return replicationConfig;
}
@@ -122,12 +128,17 @@
assertThatIsDestination(matchingDestinations.get(0), remoteName, remoteUrls);
}
- protected DestinationsCollection newDestinationsCollections(
- ReplicationFileBasedConfig replicationFileBasedConfig) throws ConfigInvalidException {
+ protected DestinationsCollection newDestinationsCollections(ReplicationConfig replicationConfig)
+ throws ConfigInvalidException {
return new DestinationsCollection(
destinationFactoryMock,
Providers.of(replicationQueueMock),
- replicationFileBasedConfig,
+ replicationConfig,
+ configParser,
eventBus);
}
+
+ protected ReplicationConfig newReplicationFileBasedConfig() {
+ return new ReplicationFileBasedConfig(sitePaths, pluginDataPath);
+ }
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecoratorTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecoratorTest.java
index d85f622..b1b9453 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecoratorTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecoratorTest.java
@@ -16,44 +16,38 @@
import static com.google.common.truth.Truth.assertThat;
+import com.google.inject.Provider;
import com.google.inject.util.Providers;
import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.storage.file.FileBasedConfig;
-import org.junit.Before;
import org.junit.Test;
public class AutoReloadConfigDecoratorTest extends AbstractConfigTest {
- ReplicationFileBasedConfig replicationFileBasedConfig;
+ ReplicationConfig replicationConfig;
public AutoReloadConfigDecoratorTest() throws IOException {
super();
}
- @Override
- @Before
- public void setup() {
- super.setup();
-
- replicationFileBasedConfig = newReplicationFileBasedConfig();
- }
-
@Test
public void shouldAutoReloadReplicationConfig() throws Exception {
- FileBasedConfig replicationConfig = newReplicationConfig();
- replicationConfig.setBoolean("gerrit", null, "autoReload", true);
+ FileBasedConfig fileConfig = newReplicationConfig();
+ fileConfig.setBoolean("gerrit", null, "autoReload", true);
String remoteName1 = "foo";
String remoteUrl1 = "ssh://git@git.foo.com/${name}";
- replicationConfig.setString("remote", remoteName1, "url", remoteUrl1);
- replicationConfig.save();
+ fileConfig.setString("remote", remoteName1, "url", remoteUrl1);
+ fileConfig.save();
- newAutoReloadConfig().start();
+ replicationConfig = newReplicationFileBasedConfig();
- DestinationsCollection destinationsCollections =
- newDestinationsCollections(replicationFileBasedConfig);
+ newAutoReloadConfig(() -> newReplicationFileBasedConfig()).start();
+
+ DestinationsCollection destinationsCollections = newDestinationsCollections(replicationConfig);
destinationsCollections.startup(workQueueMock);
List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
assertThat(destinations).hasSize(1);
@@ -63,8 +57,8 @@
String remoteName2 = "bar";
String remoteUrl2 = "ssh://git@git.bar.com/${name}";
- replicationConfig.setString("remote", remoteName2, "url", remoteUrl2);
- replicationConfig.save();
+ fileConfig.setString("remote", remoteName2, "url", remoteUrl2);
+ fileConfig.save();
executorService.refreshCommand.run();
destinations = destinationsCollections.getAll(FilterType.ALL);
@@ -74,16 +68,151 @@
}
@Test
+ public void shouldAutoReloadFanoutReplicationConfigWhenConfigIsAdded() throws Exception {
+ String remoteName1 = "foo";
+ String remoteUrl1 = "ssh://git@git.foo.com/${name}";
+ FileBasedConfig remoteConfig = newReplicationConfig("replication/" + remoteName1 + ".config");
+ remoteConfig.setString("remote", null, "url", remoteUrl1);
+ remoteConfig.save();
+
+ replicationConfig = new FanoutReplicationConfig(sitePaths, pluginDataPath);
+
+ newAutoReloadConfig(
+ () -> {
+ try {
+ return new FanoutReplicationConfig(sitePaths, pluginDataPath);
+ } catch (IOException | ConfigInvalidException e) {
+ throw new RuntimeException(e);
+ }
+ })
+ .start();
+
+ DestinationsCollection destinationsCollections = newDestinationsCollections(replicationConfig);
+ destinationsCollections.startup(workQueueMock);
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(1);
+ assertThatContainsDestination(destinations, remoteName1, remoteUrl1);
+
+ TimeUnit.SECONDS.sleep(1); // Allow the filesystem to change the update TS
+
+ String remoteName2 = "foobar";
+ String remoteUrl2 = "ssh://git@git.foobar.com/${name}";
+ remoteConfig = newReplicationConfig("replication/" + remoteName2 + ".config");
+ remoteConfig.setString("remote", null, "url", remoteUrl2);
+ remoteConfig.save();
+ executorService.refreshCommand.run();
+
+ destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(2);
+ assertThatContainsDestination(destinations, remoteName1, remoteUrl1);
+ assertThatContainsDestination(destinations, remoteName2, remoteUrl2);
+ }
+
+ @Test
+ public void shouldAutoReloadFanoutReplicationConfigWhenConfigIsRemoved() throws Exception {
+ String remoteName1 = "foo";
+ String remoteUrl1 = "ssh://git@git.foo.com/${name}";
+ FileBasedConfig remoteConfig = newReplicationConfig("replication/" + remoteName1 + ".config");
+ remoteConfig.setString("remote", null, "url", remoteUrl1);
+ remoteConfig.save();
+
+ String remoteName2 = "foobar";
+ String remoteUrl2 = "ssh://git@git.foobar.com/${name}";
+ remoteConfig = newReplicationConfig("replication/" + remoteName2 + ".config");
+ remoteConfig.setString("remote", null, "url", remoteUrl2);
+ remoteConfig.save();
+
+ replicationConfig = new FanoutReplicationConfig(sitePaths, pluginDataPath);
+
+ newAutoReloadConfig(
+ () -> {
+ try {
+ return new FanoutReplicationConfig(sitePaths, pluginDataPath);
+ } catch (IOException | ConfigInvalidException e) {
+ throw new RuntimeException(e);
+ }
+ })
+ .start();
+
+ DestinationsCollection destinationsCollections = newDestinationsCollections(replicationConfig);
+ destinationsCollections.startup(workQueueMock);
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(2);
+ assertThatContainsDestination(destinations, remoteName1, remoteUrl1);
+ assertThatContainsDestination(destinations, remoteName2, remoteUrl2);
+
+ TimeUnit.SECONDS.sleep(1); // Allow the filesystem to change the update TS
+
+ assertThat(
+ sitePaths.etc_dir.resolve("replication/" + remoteName2 + ".config").toFile().delete())
+ .isTrue();
+
+ executorService.refreshCommand.run();
+
+ destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(1);
+ assertThatContainsDestination(destinations, remoteName1, remoteUrl1);
+ }
+
+ @Test
+ public void shouldAutoReloadFanoutReplicationConfigWhenConfigIsModified() throws Exception {
+ String remoteName1 = "foo";
+ String remoteUrl1 = "ssh://git@git.foo.com/${name}";
+ FileBasedConfig remoteConfig = newReplicationConfig("replication/" + remoteName1 + ".config");
+ remoteConfig.setString("remote", null, "url", remoteUrl1);
+ remoteConfig.save();
+
+ String remoteName2 = "bar";
+ String remoteUrl2 = "ssh://git@git.bar.com/${name}";
+ remoteConfig = newReplicationConfig("replication/" + remoteName2 + ".config");
+ remoteConfig.setString("remote", null, "url", remoteUrl2);
+ remoteConfig.save();
+
+ replicationConfig = new FanoutReplicationConfig(sitePaths, pluginDataPath);
+
+ newAutoReloadConfig(
+ () -> {
+ try {
+ return new FanoutReplicationConfig(sitePaths, pluginDataPath);
+ } catch (IOException | ConfigInvalidException e) {
+ throw new RuntimeException(e);
+ }
+ })
+ .start();
+
+ DestinationsCollection destinationsCollections = newDestinationsCollections(replicationConfig);
+ destinationsCollections.startup(workQueueMock);
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(2);
+ assertThatContainsDestination(destinations, remoteName1, remoteUrl1);
+ assertThatContainsDestination(destinations, remoteName2, remoteUrl2);
+
+ TimeUnit.SECONDS.sleep(1); // Allow the filesystem to change the update TS
+
+ String remoteUrl3 = "ssh://git@git.foobar.com/${name}";
+ remoteConfig.setString("remote", null, "url", remoteUrl3);
+ remoteConfig.save();
+
+ executorService.refreshCommand.run();
+
+ destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(2);
+ assertThatContainsDestination(destinations, remoteName1, remoteUrl1);
+ assertThatContainsDestination(destinations, remoteName2, remoteUrl3);
+ }
+
+ @Test
public void shouldNotAutoReloadReplicationConfigIfDisabled() throws Exception {
String remoteName1 = "foo";
String remoteUrl1 = "ssh://git@git.foo.com/${name}";
- FileBasedConfig replicationConfig = newReplicationConfig();
- replicationConfig.setBoolean("gerrit", null, "autoReload", false);
- replicationConfig.setString("remote", remoteName1, "url", remoteUrl1);
- replicationConfig.save();
+ FileBasedConfig fileConfig = newReplicationConfig();
+ fileConfig.setBoolean("gerrit", null, "autoReload", false);
+ fileConfig.setString("remote", remoteName1, "url", remoteUrl1);
+ fileConfig.save();
- DestinationsCollection destinationsCollections =
- newDestinationsCollections(replicationFileBasedConfig);
+ replicationConfig = newReplicationFileBasedConfig();
+
+ DestinationsCollection destinationsCollections = newDestinationsCollections(replicationConfig);
destinationsCollections.startup(workQueueMock);
List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
assertThat(destinations).hasSize(1);
@@ -91,27 +220,32 @@
TimeUnit.SECONDS.sleep(1); // Allow the filesystem to change the update TS
- replicationConfig.setString("remote", "bar", "url", "ssh://git@git.bar.com/${name}");
- replicationConfig.save();
+ fileConfig.setString("remote", "bar", "url", "ssh://git@git.bar.com/${name}");
+ fileConfig.save();
executorService.refreshCommand.run();
assertThat(destinationsCollections.getAll(FilterType.ALL)).isEqualTo(destinations);
}
- private AutoReloadConfigDecorator newAutoReloadConfig() throws ConfigInvalidException {
+ private AutoReloadConfigDecorator newAutoReloadConfig(
+ Supplier<ReplicationConfig> configSupplier) {
AutoReloadRunnable autoReloadRunnable =
new AutoReloadRunnable(
- newDestinationsCollections(replicationFileBasedConfig),
- replicationFileBasedConfig,
- sitePaths,
- pluginDataPath,
+ configParser,
+ new Provider<ReplicationConfig>() {
+
+ @Override
+ public ReplicationConfig get() {
+ return configSupplier.get();
+ }
+ },
eventBus,
Providers.of(replicationQueueMock));
return new AutoReloadConfigDecorator(
- "replication", workQueueMock, replicationFileBasedConfig, autoReloadRunnable, eventBus);
- }
-
- private ReplicationFileBasedConfig newReplicationFileBasedConfig() {
- return new ReplicationFileBasedConfig(sitePaths, pluginDataPath);
+ "replication",
+ workQueueMock,
+ newReplicationFileBasedConfig(),
+ autoReloadRunnable,
+ eventBus);
}
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadRunnableTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadRunnableTest.java
index b1aa7c8..93e8886 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadRunnableTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadRunnableTest.java
@@ -21,6 +21,7 @@
import com.google.common.eventbus.EventBus;
import com.google.common.eventbus.Subscribe;
import com.google.gerrit.server.config.SitePaths;
+import com.google.inject.Provider;
import com.google.inject.util.Providers;
import java.io.IOException;
import java.nio.file.Files;
@@ -28,6 +29,7 @@
import java.util.Collections;
import java.util.List;
import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.lib.Config;
import org.junit.Before;
import org.junit.Test;
@@ -54,41 +56,41 @@
}
@Test
- public void configurationIsReloadedWhenValidationSucceeds() {
- ReplicationConfigValidator validator = new TestValidConfigurationListener();
+ public void configurationIsReloadedWhenParsingSucceeds() {
+ ConfigParser parser = new TestValidConfigurationListener();
- attemptAutoReload(validator);
+ attemptAutoReload(parser);
assertThat(onReloadSubscriber.reloaded).isTrue();
}
@Test
- public void configurationIsNotReloadedWhenValidationFails() {
- ReplicationConfigValidator validator = new TestInvalidConfigurationListener();
+ public void configurationIsNotReloadedWhenParsingFails() {
+ ConfigParser parser = new TestInvalidConfigurationListener();
- attemptAutoReload(validator);
+ attemptAutoReload(parser);
assertThat(onReloadSubscriber.reloaded).isFalse();
}
- private void attemptAutoReload(ReplicationConfigValidator validator) {
+ private void attemptAutoReload(ConfigParser validator) {
final AutoReloadRunnable autoReloadRunnable =
new AutoReloadRunnable(
- validator,
- newVersionConfig(),
- sitePaths,
- sitePaths.data_dir,
- eventBus,
- Providers.of(replicationQueueMock));
+ validator, newVersionConfigProvider(), eventBus, Providers.of(replicationQueueMock));
autoReloadRunnable.run();
}
- private ReplicationFileBasedConfig newVersionConfig() {
- return new ReplicationFileBasedConfig(sitePaths, sitePaths.data_dir) {
+ private Provider<ReplicationConfig> newVersionConfigProvider() {
+ return new Provider<ReplicationConfig>() {
@Override
- public String getVersion() {
- return String.format("%s", System.nanoTime());
+ public ReplicationConfig get() {
+ return new ReplicationFileBasedConfig(sitePaths, sitePaths.data_dir) {
+ @Override
+ public String getVersion() {
+ return String.format("%s", System.nanoTime());
+ }
+ };
}
};
}
@@ -103,17 +105,17 @@
}
}
- private static class TestValidConfigurationListener implements ReplicationConfigValidator {
+ private static class TestValidConfigurationListener extends ConfigParser {
@Override
- public List<RemoteConfiguration> validateConfig(ReplicationFileBasedConfig newConfig) {
+ public List<RemoteConfiguration> parseRemotes(Config newConfig) {
return Collections.emptyList();
}
}
- private static class TestInvalidConfigurationListener implements ReplicationConfigValidator {
+ private static class TestInvalidConfigurationListener extends ConfigParser {
@Override
- public List<RemoteConfiguration> validateConfig(
- ReplicationFileBasedConfig configurationChangeEvent) throws ConfigInvalidException {
+ public List<RemoteConfiguration> parseRemotes(Config configurationChangeEvent)
+ throws ConfigInvalidException {
throw new ConfigInvalidException("expected test failure");
}
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/FanoutReplicationConfigTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/FanoutReplicationConfigTest.java
new file mode 100644
index 0000000..8cba4bb
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/FanoutReplicationConfigTest.java
@@ -0,0 +1,286 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE;
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.io.MoreFiles;
+import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType;
+import java.io.IOException;
+import java.util.List;
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
+import org.junit.Before;
+import org.junit.Test;
+
+public class FanoutReplicationConfigTest extends AbstractConfigTest {
+
+ public FanoutReplicationConfigTest() throws IOException {
+ super();
+ }
+
+ String remoteName1 = "foo";
+ String remoteUrl1 = "ssh://git@git.somewhere.com/${name}";
+ String remoteName2 = "bar";
+ String remoteUrl2 = "ssh://git@git.elsewhere.com/${name}";
+
+ @Before
+ public void setupTests() {
+ FileBasedConfig config = newReplicationConfig();
+ try {
+ config.save();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Test
+ public void shouldSkipRemoteConfigFromReplicationConfig() throws Exception {
+ String remoteName = "foo";
+ String remoteUrl = "ssh://git@git.somewhere.com/${name}";
+
+ FileBasedConfig config = newReplicationConfig();
+ config.setString("remote", remoteName, "url", remoteUrl);
+ config.save();
+
+ config = newRemoteConfig(remoteName2);
+ config.setString("remote", null, "url", remoteUrl2);
+ config.save();
+
+ DestinationsCollection destinationsCollections =
+ newDestinationsCollections(new FanoutReplicationConfig(sitePaths, pluginDataPath));
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(1);
+
+ assertThatIsDestination(destinations.get(0), remoteName2, remoteUrl2);
+ }
+
+ @Test
+ public void shouldLoadDestinationsFromMultipleFiles() throws Exception {
+
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", remoteUrl1);
+ config.save();
+
+ config = newRemoteConfig(remoteName2);
+ config.setString("remote", null, "url", remoteUrl2);
+ config.save();
+
+ DestinationsCollection destinationsCollections =
+ newDestinationsCollections(new FanoutReplicationConfig(sitePaths, pluginDataPath));
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(2);
+
+ assertThatContainsDestination(destinations, remoteName1, remoteUrl1);
+ assertThatContainsDestination(destinations, remoteName2, remoteUrl2);
+ }
+
+ @Test
+ public void shouldIgnoreDestinationsFromSubdirectories() throws Exception {
+
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", remoteUrl1);
+ config.save();
+
+ config = newRemoteConfig("subdirectory/" + remoteName2);
+ config.setString("remote", null, "url", remoteUrl2);
+ config.save();
+
+ DestinationsCollection destinationsCollections =
+ newDestinationsCollections(new FanoutReplicationConfig(sitePaths, pluginDataPath));
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(1);
+
+ assertThatIsDestination(destinations.get(0), remoteName1, remoteUrl1);
+ }
+
+ @Test
+ public void shouldIgnoreNonConfigFiles() throws Exception {
+
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", remoteUrl1);
+ config.save();
+
+ config =
+ new FileBasedConfig(
+ sitePaths.etc_dir.resolve("replication/" + remoteName2 + ".yaml").toFile(),
+ FS.DETECTED);
+ config.setString("remote", null, "url", remoteUrl2);
+ config.save();
+
+ DestinationsCollection destinationsCollections =
+ newDestinationsCollections(new FanoutReplicationConfig(sitePaths, pluginDataPath));
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(1);
+
+ assertThatIsDestination(destinations.get(0), remoteName1, remoteUrl1);
+ }
+
+ @Test(expected = ConfigInvalidException.class)
+ public void shouldThrowConfigInvalidExceptionWhenUrlIsMissingName() throws Exception {
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", "ssh://git@git.elsewhere.com/name");
+ config.save();
+
+ newDestinationsCollections(new FanoutReplicationConfig(sitePaths, pluginDataPath));
+ }
+
+ @Test
+ public void shouldIgnoreEmptyConfigFile() throws Exception {
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.save();
+
+ DestinationsCollection destinationsCollections =
+ newDestinationsCollections(new FanoutReplicationConfig(sitePaths, pluginDataPath));
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(0);
+ }
+
+ @Test
+ public void shouldIgnoreConfigWhenMoreThanOneRemoteInASingleFile() throws Exception {
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", remoteUrl1);
+ config.setString("remote", remoteName2, "url", remoteUrl2);
+ config.save();
+
+ DestinationsCollection destinationsCollections =
+ newDestinationsCollections(new FanoutReplicationConfig(sitePaths, pluginDataPath));
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(0);
+ }
+
+ @Test
+ public void shouldIgnoreConfigRemoteSection() throws Exception {
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("replication", null, "url", remoteUrl1);
+ config.save();
+
+ DestinationsCollection destinationsCollections =
+ newDestinationsCollections(new FanoutReplicationConfig(sitePaths, pluginDataPath));
+ List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
+ assertThat(destinations).hasSize(0);
+ }
+
+ @Test
+ public void shouldReturnSameVersionWhenNoChanges() throws Exception {
+
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", remoteUrl1);
+ config.save();
+
+ config = newRemoteConfig(remoteName2);
+ config.setString("remote", null, "url", remoteUrl2);
+ config.save();
+
+ FanoutReplicationConfig objectUnderTest =
+ new FanoutReplicationConfig(sitePaths, pluginDataPath);
+
+ String version = objectUnderTest.getVersion();
+
+ objectUnderTest = new FanoutReplicationConfig(sitePaths, pluginDataPath);
+
+ assertThat(objectUnderTest.getVersion()).isEqualTo(version);
+ }
+
+ @Test
+ public void shouldReturnNewVersionWhenConfigFileAdded() throws Exception {
+
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", remoteUrl1);
+ config.save();
+
+ FanoutReplicationConfig objectUnderTest =
+ new FanoutReplicationConfig(sitePaths, pluginDataPath);
+
+ String version = objectUnderTest.getVersion();
+
+ config = newRemoteConfig(remoteName2);
+ config.setString("remote", null, "url", remoteUrl2);
+ config.save();
+
+ assertThat(objectUnderTest.getVersion()).isNotEqualTo(version);
+ }
+
+ @Test
+ public void shouldReturnNewVersionWhenConfigFileIsModified() throws Exception {
+
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", remoteUrl1);
+ config.save();
+
+ FanoutReplicationConfig objectUnderTest =
+ new FanoutReplicationConfig(sitePaths, pluginDataPath);
+
+ String version = objectUnderTest.getVersion();
+
+ config.setString("remote", null, "url", remoteUrl2);
+ config.save();
+
+ assertThat(objectUnderTest.getVersion()).isNotEqualTo(version);
+ }
+
+ @Test
+ public void shouldReturnNewVersionWhenConfigFileRemoved() throws Exception {
+
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", remoteUrl1);
+ config.save();
+
+ config = newRemoteConfig(remoteName2);
+ config.setString("remote", null, "url", remoteUrl2);
+ config.save();
+
+ FanoutReplicationConfig objectUnderTest =
+ new FanoutReplicationConfig(sitePaths, pluginDataPath);
+
+ String version = objectUnderTest.getVersion();
+ assertThat(
+ sitePaths.etc_dir.resolve("replication/" + remoteName2 + ".config").toFile().delete())
+ .isTrue();
+
+ assertThat(objectUnderTest.getVersion()).isNotEqualTo(version);
+ }
+
+ @Test
+ public void shouldReturnReplicationConfigVersionWhenReplicationConfigDirectoryRemoved()
+ throws Exception {
+
+ FileBasedConfig config = newRemoteConfig(remoteName1);
+ config.setString("remote", null, "url", remoteUrl1);
+ config.save();
+
+ config = newRemoteConfig(remoteName2);
+ config.setString("remote", null, "url", remoteUrl2);
+ config.save();
+
+ FanoutReplicationConfig objectUnderTest =
+ new FanoutReplicationConfig(sitePaths, pluginDataPath);
+
+ String replicationConfigVersion =
+ new ReplicationFileBasedConfig(sitePaths, pluginDataPath).getVersion();
+
+ MoreFiles.deleteRecursively(sitePaths.etc_dir.resolve("replication"), ALLOW_INSECURE);
+
+ assertThat(objectUnderTest.getVersion()).isEqualTo(replicationConfigVersion);
+ }
+
+ protected FileBasedConfig newRemoteConfig(String configFileName) {
+ return new FileBasedConfig(
+ sitePaths.etc_dir.resolve("replication/" + configFileName + ".config").toFile(),
+ FS.DETECTED);
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFanoutIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFanoutIT.java
new file mode 100644
index 0000000..adb7fc8
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFanoutIT.java
@@ -0,0 +1,266 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.google.common.truth.Truth.assertThat;
+import static java.util.stream.Collectors.toList;
+
+import com.google.common.flogger.FluentLogger;
+import com.google.common.io.MoreFiles;
+import com.google.common.io.RecursiveDeleteOption;
+import com.google.gerrit.acceptance.LightweightPluginDaemonTest;
+import com.google.gerrit.acceptance.PushOneCommit.Result;
+import com.google.gerrit.acceptance.TestPlugin;
+import com.google.gerrit.acceptance.UseLocalDisk;
+import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
+import com.google.gerrit.entities.Project;
+import com.google.gerrit.extensions.annotations.PluginData;
+import com.google.gerrit.extensions.api.projects.BranchInput;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.inject.Inject;
+import com.google.inject.Key;
+import com.googlesource.gerrit.plugins.replication.ReplicationTasksStorage.ReplicateRefUpdate;
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Supplier;
+import java.util.regex.Pattern;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
+import org.junit.After;
+import org.junit.Test;
+
+@UseLocalDisk
+@TestPlugin(
+ name = "replication",
+ sysModule = "com.googlesource.gerrit.plugins.replication.ReplicationModule")
+public class ReplicationFanoutIT extends LightweightPluginDaemonTest {
+ private static final Optional<String> ALL_PROJECTS = Optional.empty();
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+ private static final int TEST_REPLICATION_DELAY = 1;
+ private static final Duration TEST_TIMEOUT = Duration.ofSeconds(TEST_REPLICATION_DELAY * 2);
+
+ @Inject private SitePaths sitePaths;
+ @Inject private ProjectOperations projectOperations;
+ private Path pluginDataDir;
+ private Path gitPath;
+ private Path storagePath;
+ private FileBasedConfig config;
+ private ReplicationTasksStorage tasksStorage;
+
+ @Override
+ public void setUpTestPlugin() throws Exception {
+ gitPath = sitePaths.site_path.resolve("git");
+
+ config =
+ new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED);
+ setAutoReload();
+ config.save();
+
+ setReplicationDestination("remote1", "suffix1", Optional.of("not-used-project"));
+
+ super.setUpTestPlugin();
+
+ pluginDataDir = plugin.getSysInjector().getInstance(Key.get(Path.class, PluginData.class));
+ storagePath = pluginDataDir.resolve("ref-updates");
+ tasksStorage = plugin.getSysInjector().getInstance(ReplicationTasksStorage.class);
+ cleanupReplicationTasks();
+ tasksStorage.disableDeleteForTesting(true);
+ }
+
+ @After
+ public void cleanUp() throws IOException {
+ if (Files.exists(sitePaths.etc_dir.resolve("replication"))) {
+ MoreFiles.deleteRecursively(
+ sitePaths.etc_dir.resolve("replication"), RecursiveDeleteOption.ALLOW_INSECURE);
+ }
+ }
+
+ @Test
+ public void shouldReplicateNewBranch() throws Exception {
+ setReplicationDestination("foo", "replica", ALL_PROJECTS);
+ reloadConfig();
+
+ Project.NameKey targetProject = createTestProject(project + "replica");
+ String newBranch = "refs/heads/mybranch";
+ String master = "refs/heads/master";
+ BranchInput input = new BranchInput();
+ input.revision = master;
+ gApi.projects().name(project.get()).branch(newBranch).create(input);
+
+ assertThat(listReplicationTasks("refs/heads/(mybranch|master)")).hasSize(2);
+
+ try (Repository repo = repoManager.openRepository(targetProject);
+ Repository sourceRepo = repoManager.openRepository(project)) {
+ waitUntil(() -> checkedGetRef(repo, newBranch) != null);
+
+ Ref masterRef = getRef(sourceRepo, master);
+ Ref targetBranchRef = getRef(repo, newBranch);
+ assertThat(targetBranchRef).isNotNull();
+ assertThat(targetBranchRef.getObjectId()).isEqualTo(masterRef.getObjectId());
+ }
+ }
+
+ @Test
+ public void shouldReplicateNewBranchToTwoRemotes() throws Exception {
+ Project.NameKey targetProject1 = createTestProject(project + "replica1");
+ Project.NameKey targetProject2 = createTestProject(project + "replica2");
+
+ setReplicationDestination("foo1", "replica1", ALL_PROJECTS);
+ setReplicationDestination("foo2", "replica2", ALL_PROJECTS);
+ reloadConfig();
+
+ Result pushResult = createChange();
+ RevCommit sourceCommit = pushResult.getCommit();
+ String sourceRef = pushResult.getPatchSet().refName();
+
+ assertThat(listReplicationTasks("refs/changes/\\d*/\\d*/\\d*")).hasSize(2);
+
+ try (Repository repo1 = repoManager.openRepository(targetProject1);
+ Repository repo2 = repoManager.openRepository(targetProject2)) {
+ waitUntil(
+ () ->
+ (checkedGetRef(repo1, sourceRef) != null && checkedGetRef(repo2, sourceRef) != null));
+
+ Ref targetBranchRef1 = getRef(repo1, sourceRef);
+ assertThat(targetBranchRef1).isNotNull();
+ assertThat(targetBranchRef1.getObjectId()).isEqualTo(sourceCommit.getId());
+
+ Ref targetBranchRef2 = getRef(repo2, sourceRef);
+ assertThat(targetBranchRef2).isNotNull();
+ assertThat(targetBranchRef2.getObjectId()).isEqualTo(sourceCommit.getId());
+ }
+ }
+
+ @Test
+ public void shouldCreateIndividualReplicationTasksForEveryRemoteUrlPair() throws Exception {
+ List<String> replicaSuffixes = Arrays.asList("replica1", "replica2");
+ createTestProject(project + "replica1");
+ createTestProject(project + "replica2");
+
+ setReplicationDestination("foo1", replicaSuffixes, ALL_PROJECTS);
+ setReplicationDestination("foo2", replicaSuffixes, ALL_PROJECTS);
+ config.setInt("remote", "foo1", "replicationDelay", TEST_REPLICATION_DELAY * 100);
+ config.setInt("remote", "foo2", "replicationDelay", TEST_REPLICATION_DELAY * 100);
+ reloadConfig();
+
+ createChange();
+
+ assertThat(listReplicationTasks("refs/changes/\\d*/\\d*/\\d*")).hasSize(4);
+
+ setReplicationDestination("foo1", replicaSuffixes, ALL_PROJECTS);
+ setReplicationDestination("foo2", replicaSuffixes, ALL_PROJECTS);
+ }
+
+ private Ref getRef(Repository repo, String branchName) throws IOException {
+ return repo.getRefDatabase().exactRef(branchName);
+ }
+
+ private Ref checkedGetRef(Repository repo, String branchName) {
+ try {
+ return repo.getRefDatabase().exactRef(branchName);
+ } catch (Exception e) {
+ logger.atSevere().withCause(e).log("failed to get ref %s in repo %s", branchName, repo);
+ return null;
+ }
+ }
+
+ private void setReplicationDestination(
+ String remoteName, String replicaSuffix, Optional<String> project) throws IOException {
+ setReplicationDestination(remoteName, Arrays.asList(replicaSuffix), project);
+ }
+
+ private void setReplicationDestination(
+ String remoteName, List<String> replicaSuffixes, Optional<String> allProjects)
+ throws IOException {
+ FileBasedConfig remoteConfig =
+ new FileBasedConfig(
+ sitePaths.etc_dir.resolve("replication/" + remoteName + ".config").toFile(),
+ FS.DETECTED);
+
+ setReplicationDestination(remoteConfig, replicaSuffixes, allProjects);
+ }
+
+ private void setAutoReload() throws IOException {
+ config.setBoolean("gerrit", null, "autoReload", true);
+ config.save();
+ }
+
+ private void setReplicationDestination(
+ FileBasedConfig config, List<String> replicaSuffixes, Optional<String> project)
+ throws IOException {
+
+ List<String> replicaUrls =
+ replicaSuffixes.stream()
+ .map(suffix -> gitPath.resolve("${name}" + suffix + ".git").toString())
+ .collect(toList());
+ config.setStringList("remote", null, "url", replicaUrls);
+ config.setInt("remote", null, "replicationDelay", TEST_REPLICATION_DELAY);
+ project.ifPresent(prj -> config.setString("remote", null, "projects", prj));
+
+ config.save();
+ }
+
+ private void waitUntil(Supplier<Boolean> waitCondition) throws InterruptedException {
+ WaitUtil.waitUntil(waitCondition, TEST_TIMEOUT);
+ }
+
+ private void reloadConfig() {
+ getAutoReloadConfigDecoratorInstance().reload();
+ }
+
+ private AutoReloadConfigDecorator getAutoReloadConfigDecoratorInstance() {
+ return getInstance(AutoReloadConfigDecorator.class);
+ }
+
+ private <T> T getInstance(Class<T> classObj) {
+ return plugin.getSysInjector().getInstance(classObj);
+ }
+
+ private Project.NameKey createTestProject(String name) throws Exception {
+ return projectOperations.newProject().name(name).create();
+ }
+
+ private List<ReplicateRefUpdate> listReplicationTasks(String refRegex) {
+ Pattern refmaskPattern = Pattern.compile(refRegex);
+ return tasksStorage.list().stream()
+ .filter(task -> refmaskPattern.matcher(task.ref).matches())
+ .collect(toList());
+ }
+
+ public void cleanupReplicationTasks() throws IOException {
+ cleanupReplicationTasks(storagePath);
+ }
+
+ private void cleanupReplicationTasks(Path basePath) throws IOException {
+ try (DirectoryStream<Path> files = Files.newDirectoryStream(basePath)) {
+ for (Path path : files) {
+ if (Files.isDirectory(path)) {
+ cleanupReplicationTasks(path);
+ } else {
+ path.toFile().delete();
+ }
+ }
+ }
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfigTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfigTest.java
index 50ae385..79b05cc 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfigTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfigTest.java
@@ -62,8 +62,8 @@
List<Destination> destinations = destinationsCollections.getAll(FilterType.ALL);
assertThat(destinations).hasSize(2);
- assertThatIsDestination(destinations.get(0), remoteName1, remoteUrl1);
- assertThatIsDestination(destinations.get(1), remoteName2, remoteUrl2);
+ assertThatContainsDestination(destinations, remoteName1, remoteUrl1);
+ assertThatContainsDestination(destinations, remoteName2, remoteUrl2);
}
@Test
@@ -86,10 +86,4 @@
assertThatIsDestination(destinations.get(0), pushRemote, aRemoteURL);
}
-
- private ReplicationFileBasedConfig newReplicationFileBasedConfig() {
- ReplicationFileBasedConfig replicationConfig =
- new ReplicationFileBasedConfig(sitePaths, pluginDataPath);
- return replicationConfig;
- }
}
