Added regex and wildcard matching to ssh command
Patterns can now be used in the list of projects to replicate.
Actually queued projects are the intersection of patterns in
replication.config and the project list argument
This can be used e.g. when having large amounts of projects
where only those in a certain sub directory are to be replicated,
to avoid filling up the replication queue unnecessarily.
Change-Id: I44fa3fbfca5ee9067dd288e3ed45a256e1955d50
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
index 395d711..5e843ad 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
@@ -18,7 +18,6 @@
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
-import com.google.gerrit.common.data.AccessSection;
import com.google.gerrit.common.data.GroupReference;
import com.google.gerrit.reviewdb.client.AccountGroup;
import com.google.gerrit.reviewdb.client.Project;
@@ -57,6 +56,7 @@
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -396,36 +396,15 @@
return true;
}
- String projectName = project.get();
- for (final String projectMatch : projects) {
- if (isRE(projectMatch)) {
- // projectMatch is a regular expression
- if (projectName.matches(projectMatch)) {
- return true;
- }
- } else if (isWildcard(projectMatch)) {
- // projectMatch is a wildcard
- if (projectName.startsWith(
- projectMatch.substring(0, projectMatch.length() - 1))) {
- return true;
- }
- } else {
- // No special case, so we try to match directly
- if (projectName.equals(projectMatch)) {
- return true;
- }
- }
- }
-
- // Nothing matched, so don't push the project
- return false;
+ return (new ReplicationFilter(Arrays.asList(projects))).matches(project);
}
boolean isSingleProjectMatch() {
boolean ret = (projects.length == 1);
if (ret) {
String projectMatch = projects[0];
- if (isRE(projectMatch) || isWildcard(projectMatch)) {
+ if (ReplicationFilter.getPatternType(projectMatch)
+ != ReplicationFilter.PatternType.EXACT_MATCH) {
// projectMatch is either regular expression, or wild-card.
//
// Even though they might refer to a single project now, they need not
@@ -437,14 +416,6 @@
return ret;
}
- private static boolean isRE(String str) {
- return str.startsWith(AccessSection.REGEX_PREFIX);
- }
-
- private static boolean isWildcard(String str) {
- return str.endsWith("*");
- }
-
boolean wouldPushRef(String ref) {
if (!replicatePermissions && RefNames.REFS_CONFIG.equals(ref)) {
return false;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java b/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java
index 37d47ca..5b737f1 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/OnStartStop.java
@@ -50,7 +50,8 @@
if (srvInfo.getState() == ServerInformation.State.STARTUP
&& config.isReplicateAllOnPluginStart()) {
ReplicationState state = new ReplicationState();
- pushAllFuture.set(pushAll.create(null, state).schedule(30, TimeUnit.SECONDS));
+ pushAllFuture.set(pushAll.create(
+ null, ReplicationFilter.all(), state).schedule(30, TimeUnit.SECONDS));
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java
index 720da54..c6ad873 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushAll.java
@@ -29,22 +29,30 @@
new ReplicationStateLogger(ReplicationQueue.repLog);
interface Factory {
- PushAll create(String urlMatch, ReplicationState state);
+ PushAll create(String urlMatch,
+ ReplicationFilter filter,
+ ReplicationState state);
}
private final WorkQueue workQueue;
private final ProjectCache projectCache;
private final ReplicationQueue replication;
private final String urlMatch;
+ private final ReplicationFilter filter;
private final ReplicationState state;
@Inject
- PushAll(WorkQueue wq, ProjectCache projectCache, ReplicationQueue rq,
- @Assisted @Nullable String urlMatch, @Assisted ReplicationState state) {
+ PushAll(WorkQueue wq,
+ ProjectCache projectCache,
+ ReplicationQueue rq,
+ @Assisted @Nullable String urlMatch,
+ @Assisted ReplicationFilter filter,
+ @Assisted ReplicationState state) {
this.workQueue = wq;
this.projectCache = projectCache;
this.replication = rq;
this.urlMatch = urlMatch;
+ this.filter = filter;
this.state = state;
}
@@ -56,7 +64,9 @@
public void run() {
try {
for (Project.NameKey nameKey : projectCache.all()) {
- replication.scheduleFullSync(nameKey, urlMatch, state);
+ if (filter.matches(nameKey)) {
+ replication.scheduleFullSync(nameKey, urlMatch, state);
+ }
}
} catch (Exception e) {
stateLog.error("Cannot enumerate known projects", e, state);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFilter.java
new file mode 100644
index 0000000..6e7be66
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFilter.java
@@ -0,0 +1,78 @@
+// Copyright (C) 2014 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import com.google.gerrit.common.data.AccessSection;
+import com.google.gerrit.reviewdb.client.Project.NameKey;
+
+import java.util.Collections;
+import java.util.List;
+
+public class ReplicationFilter {
+ public enum PatternType {
+ REGEX, WILDCARD, EXACT_MATCH;
+ }
+
+ public static ReplicationFilter all() {
+ return new ReplicationFilter(Collections.<String> emptyList());
+ }
+
+ public static PatternType getPatternType(String pattern) {
+ if (pattern.startsWith(AccessSection.REGEX_PREFIX)) {
+ return PatternType.REGEX;
+ } else if (pattern.endsWith("*")) {
+ return PatternType.WILDCARD;
+ } else {
+ return PatternType.EXACT_MATCH;
+ }
+ }
+
+ private final List<String> projectPatterns;
+
+ public ReplicationFilter(List<String> patterns) {
+ projectPatterns = patterns;
+ }
+
+ public boolean matches(NameKey name) {
+ if (projectPatterns.isEmpty()) {
+ return true;
+ } else {
+ String projectName = name.get();
+
+ for (String pattern : projectPatterns) {
+ if (matchesPattern(projectName, pattern)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private boolean matchesPattern(String projectName, String pattern) {
+ boolean match = false;
+ switch (getPatternType(pattern)) {
+ case REGEX:
+ match = projectName.matches(pattern);
+ break;
+ case WILDCARD:
+ match =
+ projectName.startsWith(pattern.substring(0, pattern.length() - 1));
+ break;
+ case EXACT_MATCH:
+ match = projectName.equals(pattern);
+ }
+ return match;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java
index 44513c0..6cd41d6 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/StartCommand.java
@@ -15,8 +15,6 @@
package com.googlesource.gerrit.plugins.replication;
import com.google.gerrit.extensions.annotations.RequiresCapability;
-import com.google.gerrit.reviewdb.client.Project;
-import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.sshd.CommandMetaData;
import com.google.gerrit.sshd.SshCommand;
import com.google.inject.Inject;
@@ -49,40 +47,31 @@
usage = "wait for replication to finish before exiting")
private boolean wait;
- @Argument(index = 0, multiValued = true, metaVar = "PROJECT", usage = "project name")
- private List<String> projectNames = new ArrayList<String>(2);
+ @Argument(index = 0, multiValued = true, metaVar = "PATTERN", usage = "project name pattern")
+ private List<String> projectPatterns = new ArrayList<String>(2);
@Inject
- private PushAll.Factory pushAllFactory;
-
- @Inject
- private ReplicationQueue replication;
-
- @Inject
- private ProjectCache projectCache;
+ private PushAll.Factory pushFactory;
@Override
protected void run() throws Failure {
- if (all && projectNames.size() > 0) {
+ if (all && projectPatterns.size() > 0) {
throw new UnloggedFailure(1, "error: cannot combine --all and PROJECT");
}
ReplicationState state = new ReplicationState(new CommandProcessing(this));
Future<?> future = null;
+
+ ReplicationFilter projectFilter;
+
if (all) {
- future = pushAllFactory.create(urlMatch, state).schedule(0, TimeUnit.SECONDS);
+ projectFilter = ReplicationFilter.all();
} else {
- for (String name : projectNames) {
- Project.NameKey key = new Project.NameKey(name);
- if (projectCache.get(key) != null) {
- replication.scheduleFullSync(key, urlMatch, state);
- } else {
- writeStdErrSync("error: '" + name + "': not a Gerrit project");
- }
- }
- state.markAllPushTasksScheduled();
+ projectFilter = new ReplicationFilter(projectPatterns);
}
+ future = pushFactory.create(urlMatch, projectFilter, state).schedule(0, TimeUnit.SECONDS);
+
if (wait) {
if (future != null) {
try {
diff --git a/src/main/resources/Documentation/cmd-start.md b/src/main/resources/Documentation/cmd-start.md
index 32da642..49f8c94 100644
--- a/src/main/resources/Documentation/cmd-start.md
+++ b/src/main/resources/Documentation/cmd-start.md
@@ -11,7 +11,7 @@
ssh -p @SSH_PORT@ @SSH_HOST@ @PLUGIN@ start
[--wait]
[--url <PATTERN>]
- {--all | <PROJECT> ...}
+ {--all | <PROJECT PATTERN> ...}
```
DESCRIPTION
@@ -60,6 +60,19 @@
pattern in command options, or the authGroup in the replication.config
has no read access for the replicated projects.
+If one or several project patterns are supplied, only those projects
+conforming to both this/these pattern(s) and those defined in
+replication.config for the target host(s) are queued for replication.
+
+The patterns follow the same format as those in replication.config,
+where wildcard or regular expression patterns can be given.
+Regular expression patterns must match a complete project name to be
+considered a match.
+
+A regular expression pattern starts with `^` and a wildcard pattern ends
+with a `*`. If the pattern starts with `^` and ends with `*`, it is
+treated as a regular expression.
+
ACCESS
------
Caller must be a member of the privileged 'Administrators' group,
@@ -106,6 +119,18 @@
$ ssh -p @SSH_PORT@ @SSH_HOST@ @PLUGIN@ start tools/gerrit
```
+Replicate only projects located in the `documentation` subdirectory:
+
+```
+ $ ssh -p @SSH_PORT@ @SSH_HOST@ @PLUGIN@ start documentation/*
+```
+
+Replicate projects whose path includes a folder named `vendor` to host slave1:
+
+```
+ $ ssh -p @SSH_PORT@ @SSH_HOST@ @PLUGIN@ start --url slave1 ^(|.*/)vendor(|/.*)
+```
+
SEE ALSO
--------
