Merge branch 'stable-2.10' into stable-2.11 * stable-2.10: Set connection timeout to 120 sec for SSH remote operations Change-Id: Icec15cad4912a5a8c38ab73694fd200fa7f64cd2
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java index 83b97c6..62cad2c 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
@@ -22,8 +22,6 @@ import com.google.inject.Injector; import com.google.inject.Singleton; -import com.googlesource.gerrit.plugins.replication.RemoteSiteUser; - import org.eclipse.jgit.errors.ConfigInvalidException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,9 +73,9 @@ } @Override - public synchronized List<Destination> getDestinations() { + public synchronized List<Destination> getDestinations(FilterType filterType) { reloadIfNeeded(); - return currentConfig.getDestinations(); + return currentConfig.getDestinations(filterType); } private void reloadIfNeeded() { @@ -91,7 +89,7 @@ this.currentConfig = newConfig; this.currentConfigTs = currentConfig.getCfgPath().lastModified(); log.info("Configuration reloaded: " - + currentConfig.getDestinations().size() + " destinations, " + + currentConfig.getDestinations(FilterType.ALL).size() + " destinations, " + discarded + " replication events discarded"); } catch (Exception e) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java index 9d6ce69..4017822 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
@@ -42,8 +42,7 @@ this.site = site; this.config = config; this.secureCredentialsFactory = - new AtomicReference<SecureCredentialsFactory>( - new SecureCredentialsFactory(site)); + new AtomicReference<>(new SecureCredentialsFactory(site)); this.secureCredentialsFactoryLoadTs = getSecureConfigLastEditTs(); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java index ef659f2..a175285 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
@@ -14,11 +14,10 @@ package com.googlesource.gerrit.plugins.replication; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; -import com.google.gerrit.common.data.AccessSection; import com.google.gerrit.common.data.GroupReference; import com.google.gerrit.reviewdb.client.AccountGroup; import com.google.gerrit.reviewdb.client.Project; @@ -57,6 +56,7 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -78,8 +78,8 @@ private final int retryDelay; private final Object stateLock = new Object(); private final int lockErrorMaxRetries; - private final Map<URIish, PushOne> pending = new HashMap<URIish, PushOne>(); - private final Map<URIish, PushOne> inFlight = new HashMap<URIish, PushOne>(); + private final Map<URIish, PushOne> pending = new HashMap<>(); + private final Map<URIish, PushOne> inFlight = new HashMap<>(); private final PushOne.Factory opFactory; private final ProjectControl.Factory projectControlFactory; private final GitRepositoryManager gitManager; @@ -116,7 +116,7 @@ cfg.getBoolean("remote", rc.getName(), "replicatePermissions", true); replicateProjectDeletions = cfg.getBoolean("remote", rc.getName(), "replicateProjectDeletions", false); - remoteNameStyle = Objects.firstNonNull( + remoteNameStyle = MoreObjects.firstNonNull( cfg.getString("remote", rc.getName(), "remoteNameStyle"), "slash"); projects = cfg.getStringList("remote", rc.getName(), "projects"); @@ -396,36 +396,15 @@ return true; } - String projectName = project.get(); - for (final String projectMatch : projects) { - if (isRE(projectMatch)) { - // projectMatch is a regular expression - if (projectName.matches(projectMatch)) { - return true; - } - } else if (isWildcard(projectMatch)) { - // projectMatch is a wildcard - if (projectName.startsWith( - projectMatch.substring(0, projectMatch.length() - 1))) { - return true; - } - } else { - // No special case, so we try to match directly - if (projectName.equals(projectMatch)) { - return true; - } - } - } - - // Nothing matched, so don't push the project - return false; + return (new ReplicationFilter(Arrays.asList(projects))).matches(project); } boolean isSingleProjectMatch() { boolean ret = (projects.length == 1); if (ret) { String projectMatch = projects[0]; - if (isRE(projectMatch) || isWildcard(projectMatch)) { + if (ReplicationFilter.getPatternType(projectMatch) + != ReplicationFilter.PatternType.EXACT_MATCH) { // projectMatch is either regular expression, or wild-card. // // Even though they might refer to a single project now, they need not @@ -437,14 +416,6 @@ return ret; } - private static boolean isRE(String str) { - return str.startsWith(AccessSection.REGEX_PREFIX); - } - - private static boolean isWildcard(String str) { - return str.endsWith("*"); - } - boolean wouldPushRef(String ref) { if (!replicatePermissions && RefNames.REFS_CONFIG.equals(ref)) { return false;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java b/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java index 37d47ca..5b737f1 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java
@@ -50,7 +50,8 @@ if (srvInfo.getState() == ServerInformation.State.STARTUP && config.isReplicateAllOnPluginStart()) { ReplicationState state = new ReplicationState(); - pushAllFuture.set(pushAll.create(null, state).schedule(30, TimeUnit.SECONDS)); + pushAllFuture.set(pushAll.create( + null, ReplicationFilter.all(), state).schedule(30, TimeUnit.SECONDS)); } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java index 720da54..c6ad873 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java
@@ -29,22 +29,30 @@ new ReplicationStateLogger(ReplicationQueue.repLog); interface Factory { - PushAll create(String urlMatch, ReplicationState state); + PushAll create(String urlMatch, + ReplicationFilter filter, + ReplicationState state); } private final WorkQueue workQueue; private final ProjectCache projectCache; private final ReplicationQueue replication; private final String urlMatch; + private final ReplicationFilter filter; private final ReplicationState state; @Inject - PushAll(WorkQueue wq, ProjectCache projectCache, ReplicationQueue rq, - @Assisted @Nullable String urlMatch, @Assisted ReplicationState state) { + PushAll(WorkQueue wq, + ProjectCache projectCache, + ReplicationQueue rq, + @Assisted @Nullable String urlMatch, + @Assisted ReplicationFilter filter, + @Assisted ReplicationState state) { this.workQueue = wq; this.projectCache = projectCache; this.replication = rq; this.urlMatch = urlMatch; + this.filter = filter; this.state = state; } @@ -56,7 +64,9 @@ public void run() { try { for (Project.NameKey nameKey : projectCache.all()) { - replication.scheduleFullSync(nameKey, urlMatch, state); + if (filter.matches(nameKey)) { + replication.scheduleFullSync(nameKey, urlMatch, state); + } } } catch (Exception e) { stateLog.error("Cannot enumerate known projects", e, state);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java index b699e0d..0614959 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
@@ -22,7 +22,6 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import com.google.gerrit.extensions.events.GitReferenceUpdatedListener; -import com.google.gerrit.extensions.events.NewProjectCreatedListener; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.reviewdb.client.RefNames; import com.google.gerrit.reviewdb.server.ReviewDb; @@ -221,7 +220,7 @@ } ReplicationState[] getStatesAsArray() { - Set<ReplicationState> statesSet = new HashSet<ReplicationState>(); + Set<ReplicationState> statesSet = new HashSet<>(); statesSet.addAll(stateMap.values()); return statesSet.toArray(new ReplicationState[statesSet.size()]); } @@ -297,7 +296,7 @@ // does not exist. In this case NoRemoteRepositoryException is not // raised. final String msg = e.getMessage(); - if (msg.contains("access denied")) { + if (msg.contains("access denied") || msg.contains("no such repository")) { createRepository(); } else { repLog.error("Cannot replicate " + projectName @@ -348,21 +347,14 @@ if (pool.isCreateMissingRepos()) { try { final Ref head = git.getRef(Constants.HEAD); - NewProjectCreatedListener.Event event = - new NewProjectCreatedListener.Event() { - @Override - public String getProjectName() { - return projectName.get(); - } - - @Override - public String getHeadName() { - return head != null ? head.getName() : null; - } - }; - replicationQueue.onNewProjectCreated(event); - repLog.warn("Missing repository created; retry replication to " + uri); - pool.reschedule(this, Destination.RetryReason.REPOSITORY_MISSING); + if (replicationQueue.createProject(projectName, head != null ? head.getName() : null)) { + repLog.warn("Missing repository created; retry replication to " + uri); + pool.reschedule(this, Destination.RetryReason.REPOSITORY_MISSING); + } else { + repLog.warn("Missing repository could not be created when replicating " + uri + + ". You can only create missing repositories locally, over SSH or when " + + "using adminUrl in replication.config. See documentation for more information."); + } } catch (IOException ioe) { stateLog.error("Cannot replicate to " + uri + "; failed to create missing repository", ioe, getStatesAsArray()); @@ -552,12 +544,12 @@ private void updateStates(Collection<RemoteRefUpdate> refUpdates) throws LockFailureException { - Set<String> doneRefs = new HashSet<String>(); + Set<String> doneRefs = new HashSet<>(); boolean anyRefFailed = false; for (RemoteRefUpdate u : refUpdates) { RefPushResult pushStatus = RefPushResult.SUCCEEDED; - Set<ReplicationState> logStates = new HashSet<ReplicationState>(); + Set<ReplicationState> logStates = new HashSet<>(); logStates.addAll(stateMap.get(u.getSrcRef())); logStates.addAll(stateMap.get(ALL_REFS)); @@ -625,5 +617,5 @@ public LockFailureException(URIish uri, String message) { super(uri, message); } - }; + } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java index 56c5d39..762f069 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java
@@ -14,13 +14,13 @@ package com.googlesource.gerrit.plugins.replication; -import com.google.gerrit.common.ChangeHooks; +import com.google.gerrit.common.EventDispatcher; import com.google.gerrit.reviewdb.client.Branch; import com.google.gerrit.reviewdb.client.Change; import com.google.gerrit.reviewdb.client.PatchSet; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.reviewdb.server.ReviewDb; -import com.google.gerrit.server.events.ChangeEvent; +import com.google.gerrit.server.events.RefEvent; import com.google.gwtorm.server.OrmException; import com.google.gwtorm.server.SchemaFactory; @@ -41,10 +41,20 @@ abstract void onAllRefsReplicatedToAllNodes(int totalPushTasksCount); + /** + * Write message to standard out. + * + * @param message message text. + */ void writeStdOut(final String message) { // Default doing nothing } + /** + * Write message to standard error. + * + * @param message message text. + */ void writeStdErr(final String message) { // Default doing nothing } @@ -68,7 +78,7 @@ private AtomicBoolean hasError = new AtomicBoolean(); CommandProcessing(StartCommand sshCommand) { - this.sshCommand = new WeakReference<StartCommand>(sshCommand); + this.sshCommand = new WeakReference<>(sshCommand); } @Override @@ -146,11 +156,12 @@ public static class GitUpdateProcessing extends PushResultProcessing { static final Logger log = LoggerFactory.getLogger(GitUpdateProcessing.class); - private final ChangeHooks hooks; + private final EventDispatcher dispatcher; private final SchemaFactory<ReviewDb> schema; - public GitUpdateProcessing(ChangeHooks hooks, SchemaFactory<ReviewDb> schema) { - this.hooks = hooks; + public GitUpdateProcessing(EventDispatcher dispatcher, + SchemaFactory<ReviewDb> schema) { + this.dispatcher = dispatcher; this.schema = schema; } @@ -173,14 +184,14 @@ void onAllRefsReplicatedToAllNodes(int totalPushTasksCount) { } - private void postEvent(String project, String ref, ChangeEvent event) { + private void postEvent(String project, String ref, RefEvent event) { if (PatchSet.isRef(ref)) { try { ReviewDb db = schema.open(); try { Change change = retrieveChange(ref, db); if (change != null) { - hooks.postEvent(change, event, db); + dispatcher.postEvent(change, event, db); } } finally { db.close(); @@ -190,7 +201,7 @@ } } else { Branch.NameKey branch = new Branch.NameKey(Project.NameKey.parse(project), ref); - hooks.postEvent(branch, event); + dispatcher.postEvent(branch, event); } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicatedEvent.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicatedEvent.java index 98435ac..664e819 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicatedEvent.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicatedEvent.java
@@ -14,12 +14,12 @@ package com.googlesource.gerrit.plugins.replication; -import com.google.gerrit.server.events.ChangeEvent; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.events.RefEvent; import com.googlesource.gerrit.plugins.replication.ReplicationState.RefPushResult; -public class RefReplicatedEvent extends ChangeEvent { - public final String type = "ref-replicated"; +public class RefReplicatedEvent extends RefEvent { public final String project; public final String ref; public final String targetNode; @@ -27,6 +27,7 @@ public RefReplicatedEvent(String project, String ref, String targetNode, RefPushResult status) { + super("ref-replicated"); this.project = project; this.ref = ref; this.targetNode = targetNode; @@ -36,4 +37,14 @@ private String toStatusString(RefPushResult status) { return status.name().toLowerCase().replace("_", "-"); } + + @Override + public Project.NameKey getProjectNameKey() { + return new Project.NameKey(project); + } + + @Override + public String getRefName() { + return ref; + } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicationDoneEvent.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicationDoneEvent.java index 7eaed66..fe92bc8 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicationDoneEvent.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RefReplicationDoneEvent.java
@@ -14,17 +14,28 @@ package com.googlesource.gerrit.plugins.replication; -import com.google.gerrit.server.events.ChangeEvent; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.events.RefEvent; -public class RefReplicationDoneEvent extends ChangeEvent { - public final String type = "ref-replication-done"; +public class RefReplicationDoneEvent extends RefEvent { public final String project; public final String ref; public final int nodesCount; public RefReplicationDoneEvent(String project, String ref, int nodesCount) { + super("ref-replication-done"); this.project = project; this.ref = ref; this.nodesCount = nodesCount; } + + @Override + public Project.NameKey getProjectNameKey() { + return new Project.NameKey(project); + } + + @Override + public String getRefName() { + return ref; + } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java index 5c18f75..241c881 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
@@ -19,7 +19,13 @@ public interface ReplicationConfig { - List<Destination> getDestinations(); + enum FilterType { + PROJECT_CREATION, + PROJECT_DELETION, + ALL + } + + List<Destination> getDestinations(FilterType filterType); boolean isReplicateAllOnPluginStart();
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java index 57d07b8..f56185d 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
@@ -13,6 +13,8 @@ // limitations under the License. package com.googlesource.gerrit.plugins.replication; +import com.google.common.base.Predicate; +import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.gerrit.server.PluginUser; @@ -69,14 +71,47 @@ this.destinations = allDestinations(); } - /* (non-Javadoc) - * @see com.googlesource.gerrit.plugins.replication.ReplicationConfig#getDestinations() + /* + * (non-Javadoc) + * @see + * com.googlesource.gerrit.plugins.replication.ReplicationConfig#getDestinations + * (com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType) */ @Override - public List<Destination> getDestinations() { - return destinations; - } + public List<Destination> getDestinations(FilterType filterType) { + Predicate<Destination> filter; + switch (filterType) { + case PROJECT_CREATION : + filter = new Predicate<Destination>() { + @Override + public boolean apply(Destination dest) { + if (dest == null || !dest.isCreateMissingRepos()) { + return false; + } + return true; + } + }; + break; + case PROJECT_DELETION : + filter = new Predicate<Destination>() { + + @Override + public boolean apply(Destination dest) { + if (dest == null || !dest.isReplicateProjectDeletions()) { + return false; + } + return true; + } + }; + break; + case ALL : + return destinations; + default : + return destinations; + } + return FluentIterable.from(destinations).filter(filter).toList(); + } private List<Destination> allDestinations() throws ConfigInvalidException, IOException { if (!config.getFile().exists()) { @@ -184,6 +219,7 @@ return cfgPath; } + @Override public int shutdown() { int discarded = 0; for (Destination cfg : destinations) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFilter.java new file mode 100644 index 0000000..6e7be66 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFilter.java
@@ -0,0 +1,78 @@ +// Copyright (C) 2014 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import com.google.gerrit.common.data.AccessSection; +import com.google.gerrit.reviewdb.client.Project.NameKey; + +import java.util.Collections; +import java.util.List; + +public class ReplicationFilter { + public enum PatternType { + REGEX, WILDCARD, EXACT_MATCH; + } + + public static ReplicationFilter all() { + return new ReplicationFilter(Collections.<String> emptyList()); + } + + public static PatternType getPatternType(String pattern) { + if (pattern.startsWith(AccessSection.REGEX_PREFIX)) { + return PatternType.REGEX; + } else if (pattern.endsWith("*")) { + return PatternType.WILDCARD; + } else { + return PatternType.EXACT_MATCH; + } + } + + private final List<String> projectPatterns; + + public ReplicationFilter(List<String> patterns) { + projectPatterns = patterns; + } + + public boolean matches(NameKey name) { + if (projectPatterns.isEmpty()) { + return true; + } else { + String projectName = name.get(); + + for (String pattern : projectPatterns) { + if (matchesPattern(projectName, pattern)) { + return true; + } + } + } + return false; + } + + private boolean matchesPattern(String projectName, String pattern) { + boolean match = false; + switch (getPatternType(pattern)) { + case REGEX: + match = projectName.matches(pattern); + break; + case WILDCARD: + match = + projectName.startsWith(pattern.substring(0, pattern.length() - 1)); + break; + case EXACT_MATCH: + match = projectName.equals(pattern); + } + return match; + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java index 6e2ffdc..b9575b9 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
@@ -14,6 +14,7 @@ package com.googlesource.gerrit.plugins.replication; +import static com.googlesource.gerrit.plugins.replication.ReplicationState.RefPushResult.SUCCEEDED; import static com.googlesource.gerrit.plugins.replication.StartReplicationCapability.START_REPLICATION; import com.google.gerrit.extensions.annotations.Exports; @@ -24,11 +25,13 @@ import com.google.gerrit.extensions.events.NewProjectCreatedListener; import com.google.gerrit.extensions.events.ProjectDeletedListener; import com.google.gerrit.extensions.registration.DynamicSet; +import com.google.gerrit.server.events.EventTypes; import com.google.inject.AbstractModule; import com.google.inject.Scopes; import com.google.inject.assistedinject.FactoryModuleBuilder; import com.google.inject.internal.UniqueAnnotations; + class ReplicationModule extends AbstractModule { @Override protected void configure() { @@ -59,5 +62,8 @@ install(new FactoryModuleBuilder().build(RemoteSiteUser.Factory.class)); bind(ReplicationConfig.class).to(AutoReloadConfigDecorator.class); + + EventTypes.registerClass(new RefReplicatedEvent(null, null, null, SUCCEEDED)); + EventTypes.registerClass(new RefReplicationDoneEvent(null, null, 0)); } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java index a34ac1c..617dc02 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
@@ -16,7 +16,7 @@ import com.google.common.base.Strings; import com.google.common.collect.Sets; -import com.google.gerrit.common.ChangeHooks; +import com.google.gerrit.common.EventDispatcher; import com.google.gerrit.extensions.events.GitReferenceUpdatedListener; import com.google.gerrit.extensions.events.HeadUpdatedListener; import com.google.gerrit.extensions.events.LifecycleListener; @@ -29,8 +29,8 @@ import com.google.inject.Inject; import com.googlesource.gerrit.plugins.replication.PushResultProcessing.GitUpdateProcessing; +import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType; -import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.internal.storage.file.FileRepository; import org.eclipse.jgit.lib.Constants; @@ -80,17 +80,16 @@ private final WorkQueue workQueue; private final SchemaFactory<ReviewDb> database; - private final ChangeHooks changeHooks; + private final EventDispatcher dispatcher; private final ReplicationConfig config; private volatile boolean running; @Inject ReplicationQueue(final WorkQueue wq, final ReplicationConfig rc, - final SchemaFactory<ReviewDb> db, final ChangeHooks ch) - throws ConfigInvalidException, IOException { + final SchemaFactory<ReviewDb> db, final EventDispatcher dis) { workQueue = wq; database = db; - changeHooks = ch; + dispatcher = dis; config = rc; } @@ -118,7 +117,7 @@ return; } - for (Destination cfg : config.getDestinations()) { + for (Destination cfg : config.getDestinations(FilterType.ALL)) { if (cfg.wouldPushProject(project)) { for (URIish uri : cfg.getURIs(project, urlMatch)) { cfg.schedule(project, PushOne.ALL_REFS, uri, state); @@ -129,14 +128,14 @@ @Override public void onGitReferenceUpdated(GitReferenceUpdatedListener.Event event) { - ReplicationState state = new ReplicationState(new GitUpdateProcessing(changeHooks, database)); + ReplicationState state = new ReplicationState(new GitUpdateProcessing(dispatcher, database)); if (!running) { stateLog.warn("Replication plugin did not finish startup before event", state); return; } Project.NameKey project = new Project.NameKey(event.getProjectName()); - for (Destination cfg : config.getDestinations()) { + for (Destination cfg : config.getDestinations(FilterType.ALL)) { if (cfg.wouldPushProject(project) && cfg.wouldPushRef(event.getRefName())) { for (URIish uri : cfg.getURIs(project, null)) { cfg.schedule(project, event.getRefName(), uri, state); @@ -148,28 +147,31 @@ @Override public void onNewProjectCreated(NewProjectCreatedListener.Event event) { - for (URIish uri : getURIs(new Project.NameKey(event.getProjectName()), false)) { + for (URIish uri : getURIs(new Project.NameKey(event.getProjectName()), + FilterType.PROJECT_CREATION)) { createProject(uri, event.getHeadName()); } } @Override public void onProjectDeleted(ProjectDeletedListener.Event event) { - for (URIish uri : getURIs(new Project.NameKey(event.getProjectName()), true)) { + for (URIish uri : getURIs(new Project.NameKey(event.getProjectName()), + FilterType.PROJECT_DELETION)) { deleteProject(uri); } } @Override public void onHeadUpdated(HeadUpdatedListener.Event event) { - for (URIish uri : getURIs(new Project.NameKey(event.getProjectName()), false)) { + for (URIish uri : getURIs(new Project.NameKey(event.getProjectName()), + FilterType.ALL)) { updateHead(uri, event.getNewHeadName()); } } private Set<URIish> getURIs(Project.NameKey projectName, - boolean forProjectDeletion) { - if (config.getDestinations().isEmpty()) { + FilterType filterType) { + if (config.getDestinations(filterType).isEmpty()) { return Collections.emptySet(); } if (!running) { @@ -178,13 +180,11 @@ } Set<URIish> uris = Sets.newHashSet(); - for (Destination config : this.config.getDestinations()) { + for (Destination config : this.config.getDestinations(filterType)) { if (!config.wouldPushProject(projectName)) { continue; } - if (forProjectDeletion && !config.isReplicateProjectDeletions()) { - continue; - } + List<URIish> uriList = config.getURIs(projectName, "*"); String[] adminUrls = config.getAdminUrls(); boolean adminURLUsed = false; @@ -231,7 +231,15 @@ return uris; } - private void createProject(URIish replicateURI, String head) { + public boolean createProject(Project.NameKey project, String head) { + boolean success = false; + for (URIish uri : getURIs(project, FilterType.PROJECT_CREATION)) { + success &= createProject(uri, head); + } + return success; + } + + private boolean createProject(URIish replicateURI, String head) { if (!replicateURI.isRemote()) { createLocally(replicateURI, head); repLog.info("Created local repository: " + replicateURI); @@ -242,7 +250,9 @@ repLog.warn(String.format("Cannot create new project on remote site %s." + " Only local paths and SSH URLs are supported" + " for remote repository creation", replicateURI)); + return false; } + return true; } private static void createLocally(URIish uri, String head) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java index 68433f1..32d3905 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java
@@ -51,6 +51,7 @@ return cfg; } + @Override public SecureCredentialsProvider create(String remoteName) { String user = config.getString("remote", remoteName, "username"); String pass = config.getString("remote", remoteName, "password");
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java index 44513c0..17a2df9 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java
@@ -15,8 +15,6 @@ package com.googlesource.gerrit.plugins.replication; import com.google.gerrit.extensions.annotations.RequiresCapability; -import com.google.gerrit.reviewdb.client.Project; -import com.google.gerrit.server.project.ProjectCache; import com.google.gerrit.sshd.CommandMetaData; import com.google.gerrit.sshd.SshCommand; import com.google.inject.Inject; @@ -49,40 +47,31 @@ usage = "wait for replication to finish before exiting") private boolean wait; - @Argument(index = 0, multiValued = true, metaVar = "PROJECT", usage = "project name") - private List<String> projectNames = new ArrayList<String>(2); + @Argument(index = 0, multiValued = true, metaVar = "PATTERN", usage = "project name pattern") + private List<String> projectPatterns = new ArrayList<>(2); @Inject - private PushAll.Factory pushAllFactory; - - @Inject - private ReplicationQueue replication; - - @Inject - private ProjectCache projectCache; + private PushAll.Factory pushFactory; @Override protected void run() throws Failure { - if (all && projectNames.size() > 0) { + if (all && projectPatterns.size() > 0) { throw new UnloggedFailure(1, "error: cannot combine --all and PROJECT"); } ReplicationState state = new ReplicationState(new CommandProcessing(this)); Future<?> future = null; + + ReplicationFilter projectFilter; + if (all) { - future = pushAllFactory.create(urlMatch, state).schedule(0, TimeUnit.SECONDS); + projectFilter = ReplicationFilter.all(); } else { - for (String name : projectNames) { - Project.NameKey key = new Project.NameKey(name); - if (projectCache.get(key) != null) { - replication.scheduleFullSync(key, urlMatch, state); - } else { - writeStdErrSync("error: '" + name + "': not a Gerrit project"); - } - } - state.markAllPushTasksScheduled(); + projectFilter = new ReplicationFilter(projectPatterns); } + future = pushFactory.create(urlMatch, projectFilter, state).schedule(0, TimeUnit.SECONDS); + if (wait) { if (future != null) { try {
diff --git a/src/main/resources/Documentation/cmd-start.md b/src/main/resources/Documentation/cmd-start.md index 32da642..49f8c94 100644 --- a/src/main/resources/Documentation/cmd-start.md +++ b/src/main/resources/Documentation/cmd-start.md
@@ -11,7 +11,7 @@ ssh -p @SSH_PORT@ @SSH_HOST@ @PLUGIN@ start [--wait] [--url <PATTERN>] - {--all | <PROJECT> ...} + {--all | <PROJECT PATTERN> ...} ``` DESCRIPTION @@ -60,6 +60,19 @@ pattern in command options, or the authGroup in the replication.config has no read access for the replicated projects. +If one or several project patterns are supplied, only those projects +conforming to both this/these pattern(s) and those defined in +replication.config for the target host(s) are queued for replication. + +The patterns follow the same format as those in replication.config, +where wildcard or regular expression patterns can be given. +Regular expression patterns must match a complete project name to be +considered a match. + +A regular expression pattern starts with `^` and a wildcard pattern ends +with a `*`. If the pattern starts with `^` and ends with `*`, it is +treated as a regular expression. + ACCESS ------ Caller must be a member of the privileged 'Administrators' group, @@ -106,6 +119,18 @@ $ ssh -p @SSH_PORT@ @SSH_HOST@ @PLUGIN@ start tools/gerrit ``` +Replicate only projects located in the `documentation` subdirectory: + +``` + $ ssh -p @SSH_PORT@ @SSH_HOST@ @PLUGIN@ start documentation/* +``` + +Replicate projects whose path includes a folder named `vendor` to host slave1: + +``` + $ ssh -p @SSH_PORT@ @SSH_HOST@ @PLUGIN@ start --url slave1 ^(|.*/)vendor(|/.*) +``` + SEE ALSO --------
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index 94b8473..bea2d92 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md
@@ -238,12 +238,13 @@ everything to all remotes. remote.NAME.createMissingRepositories -: If true, a repository is automatically created on the remote site - if during the replication of a ref it is found to be missing. - E.g. a repository can be missing on the remote site if the remote - site was not available at the moment when a new project was created - and hence the repository for the new project could not be created on - the remote site. +: If true, a repository is automatically created on the remote site. + If the remote site was not available at the moment when a new + project was created, it will be created if during the replication + of a ref it is found to be missing. + + If false, repositories are never created automatically on this + remote. By default, true, missing repositories are created.
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java index 93ea562..a400dca 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java
@@ -24,7 +24,7 @@ import static org.easymock.EasyMock.reset; import static org.easymock.EasyMock.verify; -import com.google.gerrit.common.ChangeHooks; +import com.google.gerrit.common.EventDispatcher; import com.google.gerrit.reviewdb.client.Branch; import com.google.gerrit.reviewdb.client.Change; import com.google.gerrit.reviewdb.server.ChangeAccess; @@ -49,14 +49,15 @@ KeyUtil.setEncoderImpl(new StandardKeyEncoder()); } - private ChangeHooks changeHooksMock; + private EventDispatcher dispatcherMock; private ChangeAccess changeAccessMock; private GitUpdateProcessing gitUpdateProcessing; + @Override protected void setUp() throws Exception { super.setUp(); - changeHooksMock = createMock(ChangeHooks.class); - replay(changeHooksMock); + dispatcherMock = createMock(EventDispatcher.class); + replay(dispatcherMock); changeAccessMock = createNiceMock(ChangeAccess.class); replay(changeAccessMock); ReviewDb reviewDbMock = createNiceMock(ReviewDb.class); @@ -65,22 +66,22 @@ SchemaFactory<ReviewDb> schemaMock = createMock(SchemaFactory.class); expect(schemaMock.open()).andReturn(reviewDbMock).anyTimes(); replay(schemaMock); - gitUpdateProcessing = new GitUpdateProcessing(changeHooksMock, schemaMock); + gitUpdateProcessing = new GitUpdateProcessing(dispatcherMock, schemaMock); } public void testHeadRefReplicated() throws URISyntaxException { - reset(changeHooksMock); + reset(dispatcherMock); RefReplicatedEvent expectedEvent = new RefReplicatedEvent("someProject", "refs/heads/master", "someHost", RefPushResult.SUCCEEDED); - changeHooksMock.postEvent(anyObject(Branch.NameKey.class), + dispatcherMock.postEvent(anyObject(Branch.NameKey.class), RefReplicatedEventEquals.eqEvent(expectedEvent)); expectLastCall().once(); - replay(changeHooksMock); + replay(dispatcherMock); gitUpdateProcessing.onRefReplicatedToOneNode("someProject", "refs/heads/master", new URIish("git://someHost/someProject.git"), RefPushResult.SUCCEEDED); - verify(changeHooksMock); + verify(dispatcherMock); } public void testChangeRefReplicated() throws URISyntaxException, OrmException { @@ -89,32 +90,32 @@ expect(changeAccessMock.get(anyObject(Change.Id.class))).andReturn(expectedChange); replay(changeAccessMock); - reset(changeHooksMock); + reset(dispatcherMock); RefReplicatedEvent expectedEvent = - new RefReplicatedEvent("someProject", "refs/changes/1/1/1", "someHost", + new RefReplicatedEvent("someProject", "refs/changes/01/1/1", "someHost", RefPushResult.FAILED); - changeHooksMock.postEvent(eq(expectedChange), + dispatcherMock.postEvent(eq(expectedChange), RefReplicatedEventEquals.eqEvent(expectedEvent), anyObject(ReviewDb.class)); expectLastCall().once(); - replay(changeHooksMock); + replay(dispatcherMock); gitUpdateProcessing.onRefReplicatedToOneNode("someProject", - "refs/changes/1/1/1", new URIish("git://someHost/someProject.git"), + "refs/changes/01/1/1", new URIish("git://someHost/someProject.git"), RefPushResult.FAILED); - verify(changeHooksMock); + verify(dispatcherMock); } - public void testOnAllNodesReplicated() throws URISyntaxException { - reset(changeHooksMock); + public void testOnAllNodesReplicated() { + reset(dispatcherMock); RefReplicationDoneEvent expectedDoneEvent = new RefReplicationDoneEvent("someProject", "refs/heads/master", 5); - changeHooksMock.postEvent(anyObject(Branch.NameKey.class), + dispatcherMock.postEvent(anyObject(Branch.NameKey.class), RefReplicationDoneEventEquals.eqEvent(expectedDoneEvent)); expectLastCall().once(); - replay(changeHooksMock); + replay(dispatcherMock); gitUpdateProcessing.onRefReplicatedToAllNodes("someProject", "refs/heads/master", 5); - verify(changeHooksMock); + verify(dispatcherMock); } }