Add Batch Cleaner to the Batch Plugin
Cleans expired batches daily, by default it removes all batches
that are older than 3 days.
Change-Id: Ic0f10e5cb9f68152360e068706ab8db5fdb9ceb4
diff --git a/src/main/java/com/googlesource/gerrit/plugins/batch/BatchCleaner.java b/src/main/java/com/googlesource/gerrit/plugins/batch/BatchCleaner.java
new file mode 100644
index 0000000..fe5c795
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/batch/BatchCleaner.java
@@ -0,0 +1,154 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.batch;
+
+import static java.util.concurrent.TimeUnit.DAYS;
+import static java.util.concurrent.TimeUnit.MINUTES;
+
+import com.google.gerrit.extensions.annotations.PluginName;
+import com.google.gerrit.extensions.events.LifecycleListener;
+import com.google.gerrit.server.CurrentUser;
+import com.google.gerrit.server.config.ConfigUtil;
+import com.google.gerrit.server.config.PluginConfigFactory;
+import com.google.gerrit.server.git.WorkQueue;
+import com.google.gerrit.server.git.WorkQueue.CancelableRunnable;
+import com.google.gerrit.server.project.NoSuchProjectException;
+import com.google.gerrit.server.project.ProjectCache;
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import java.util.ArrayList;
+import java.util.concurrent.ScheduledFuture;
+import org.eclipse.jgit.lib.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Clean up expired batches daily. */
+public class BatchCleaner implements CancelableRunnable {
+ private static final Logger log = LoggerFactory.getLogger(BatchCleaner.class);
+
+ public static class Lifecycle implements LifecycleListener {
+ public static final long DEFAULT_START_MINUTES = MINUTES.convert(1, MINUTES);
+ public static final long DEFAULT_MINUTES = MINUTES.convert(1, DAYS);
+
+ protected final long startMinutes;
+ protected final long intervalMinutes;
+
+ protected final PluginConfigFactory cfgFactory;
+ protected final WorkQueue workQueue;
+ protected final BatchCleaner cleaner;
+ protected final ProjectCache projectCache;
+ protected final String pluginName;
+
+ @Inject
+ protected Lifecycle(
+ PluginConfigFactory cfgFactory,
+ WorkQueue workQueue,
+ BatchCleaner cleaner,
+ ProjectCache projectCache,
+ @PluginName String pluginName)
+ throws NoSuchProjectException {
+ this.cfgFactory = cfgFactory;
+ this.workQueue = workQueue;
+ this.cleaner = cleaner;
+ this.projectCache = projectCache;
+ this.pluginName = pluginName;
+ startMinutes = startDelay();
+ intervalMinutes = interval();
+ }
+
+ @Override
+ public void start() {
+ cleaner.future =
+ workQueue
+ .getDefaultQueue()
+ .scheduleWithFixedDelay(cleaner, startMinutes, intervalMinutes, MINUTES);
+ }
+
+ @Override
+ public void stop() {
+ cleaner.cancel();
+ }
+
+ protected long startDelay() throws NoSuchProjectException {
+ Config config = cfgFactory.getProjectPluginConfig(projectCache.getAllProjects(), pluginName);
+ return ConfigUtil.getTimeUnit(
+ config, "cleaner", null, "startDelay", DEFAULT_START_MINUTES, MINUTES);
+ }
+
+ protected long interval() throws NoSuchProjectException {
+ Config config = cfgFactory.getProjectPluginConfig(projectCache.getAllProjects(), pluginName);
+ String freq = config.getString("cleaner", null, "interval");
+ if (freq != null && ("disabled".equalsIgnoreCase(freq) || "off".equalsIgnoreCase(freq))) {
+ return Long.MAX_VALUE;
+ }
+ return ConfigUtil.getTimeUnit(config, "cleaner", null, "interval", DEFAULT_MINUTES, MINUTES);
+ }
+ }
+
+ protected final ListBatches list;
+ protected final BatchRemover remover;
+ protected final Provider<CurrentUser> userProvider;
+
+ protected ScheduledFuture<?> future;
+ protected volatile boolean canceled = false;
+
+ @Inject
+ protected BatchCleaner(
+ ListBatches list, BatchRemover remover, Provider<CurrentUser> userProvider) {
+ this.remover = remover;
+ this.list = list;
+
+ this.userProvider = userProvider;
+ list.query = new ArrayList<String>();
+ list.query.add("is:expired");
+ }
+
+ @Override
+ public void run() {
+ Iterable<Batch> batches = null;
+ try {
+ batches = list.getBatches();
+ } catch (Exception e) {
+ log.error("getting list of batches to clean.", e);
+ // Ignore errors and hope someone notices the log file and fixes before the next run
+ return;
+ }
+
+ for (Batch batch : batches) {
+ if (canceled) {
+ return;
+ }
+ try {
+ remover.remove(batch.id);
+ } catch (Exception e) {
+ log.error("cleaning batch: " + batch.id, e);
+ // Ignore errors and hope someone notices the log file and fixes before the next run
+ }
+ }
+ }
+
+ @Override
+ public void cancel() {
+ if (future != null) {
+ future.cancel(true);
+ canceled = true;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "Batch Cleaner";
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/batch/BatchRemover.java b/src/main/java/com/googlesource/gerrit/plugins/batch/BatchRemover.java
index ef37f86..ffb7b41 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/batch/BatchRemover.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/batch/BatchRemover.java
@@ -11,6 +11,7 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
+
package com.googlesource.gerrit.plugins.batch;
import com.google.gerrit.reviewdb.client.Branch;
@@ -20,6 +21,7 @@
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.batch.exception.NoSuchBatchException;
import java.io.IOException;
+import org.eclipse.jgit.errors.RepositoryNotFoundException;
public class BatchRemover {
protected final RefUpdater refUpdater;
@@ -32,12 +34,14 @@
}
public Batch remove(String id)
- throws IllegalStateException, IOException, NoSuchBatchException, NoSuchProjectException {
+ throws IllegalStateException, NoSuchBatchException, IOException, RepositoryNotFoundException,
+ NoSuchProjectException {
return remove(store.read(id));
}
public Batch remove(Batch batch)
- throws IOException, IllegalStateException, NoSuchProjectException {
+ throws IOException, IllegalStateException, RepositoryNotFoundException,
+ NoSuchProjectException {
if (batch.state == Batch.State.OPEN) {
throw new IllegalStateException(
"Invalid Operation for Batch(" + batch.id + "): " + batch.state.toString());
@@ -48,7 +52,8 @@
return batch;
}
- protected void removeDownloadRefs(Batch batch) throws IOException, NoSuchProjectException {
+ protected void removeDownloadRefs(Batch batch)
+ throws IOException, RepositoryNotFoundException, NoSuchProjectException {
for (Batch.Destination dest : batch.listDestinations()) {
Project.NameKey project = new Project.NameKey(dest.project);
Branch.NameKey branch = new Branch.NameKey(project, dest.downloadRef);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/batch/BatchStore.java b/src/main/java/com/googlesource/gerrit/plugins/batch/BatchStore.java
index 1e168b9..e2f89cd 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/batch/BatchStore.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/batch/BatchStore.java
@@ -59,11 +59,16 @@
}
/** Returns a list of batch objects */
- public List<Batch> find(boolean includeBatchInfo) throws IOException, NoSuchBatchException {
+ public List<Batch> find(boolean includeBatchInfo) throws IOException {
List<Batch> batches = new ArrayList<>();
try (Repository repo = repoManager.openRepository(project)) {
for (Map.Entry<String, Ref> entry : repo.getRefDatabase().getRefs(BATCHES_REF).entrySet()) {
- batches.add((includeBatchInfo == true) ? read(entry.getKey()) : new Batch(entry.getKey()));
+ try {
+ batches.add(
+ (includeBatchInfo == true) ? read(entry.getKey()) : new Batch(entry.getKey()));
+ } catch (NoSuchBatchException e) {
+ continue;
+ }
}
}
return batches;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/batch/ListBatches.java b/src/main/java/com/googlesource/gerrit/plugins/batch/ListBatches.java
index d183d98..778d72f 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/batch/ListBatches.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/batch/ListBatches.java
@@ -15,34 +15,51 @@
import static java.nio.charset.StandardCharsets.UTF_8;
+import com.google.common.base.Joiner;
import com.google.gerrit.common.data.GlobalCapability;
import com.google.gerrit.extensions.annotations.RequiresCapability;
+import com.google.gerrit.index.query.Predicate;
+import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.server.OutputFormat;
import com.google.gson.reflect.TypeToken;
+import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
-import com.googlesource.gerrit.plugins.batch.exception.NoSuchBatchException;
+import com.googlesource.gerrit.plugins.batch.query.BatchQueryBuilder;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
import java.util.List;
+import org.kohsuke.args4j.Argument;
import org.kohsuke.args4j.Option;
@RequiresCapability(GlobalCapability.ADMINISTRATE_SERVER)
public class ListBatches {
+ @Argument(
+ index = 0,
+ required = false,
+ multiValued = true,
+ metaVar = "QUERY",
+ usage = "Query to execute")
+ public List<String> query;
+
@Option(name = "--include-batch-info", usage = "include additional information for every batch")
protected boolean includeBatchInfo;
+ protected final BatchQueryBuilder queryBuilder;
protected final BatchStore store;
@Inject
- ListBatches(BatchStore store) {
+ ListBatches(BatchQueryBuilder queryBuilder, BatchStore store) {
+ this.queryBuilder = queryBuilder;
this.store = store;
}
- public void display(OutputStream displayOutputStream) throws IOException, NoSuchBatchException {
+ public void display(OutputStream displayOutputStream)
+ throws IOException, OrmException, QueryParseException {
try {
PrintWriter stdout =
new PrintWriter(new BufferedWriter(new OutputStreamWriter(displayOutputStream, UTF_8)));
@@ -59,7 +76,18 @@
}
}
- public List<Batch> getBatches() throws IOException, NoSuchBatchException {
- return store.find(includeBatchInfo);
+ public List<Batch> getBatches() throws IOException, OrmException, QueryParseException {
+ Predicate<Batch> pred = null;
+ if (query != null) {
+ pred = queryBuilder.parse(Joiner.on(" ").join(query));
+ includeBatchInfo = true;
+ }
+ List<Batch> batches = new ArrayList<Batch>();
+ for (Batch batch : store.find(includeBatchInfo)) {
+ if (pred == null || pred.asMatchable().match(batch)) {
+ batches.add(batch);
+ }
+ }
+ return batches;
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/batch/Module.java b/src/main/java/com/googlesource/gerrit/plugins/batch/Module.java
index cfb2c15..1ac2911 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/batch/Module.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/batch/Module.java
@@ -14,13 +14,21 @@
package com.googlesource.gerrit.plugins.batch;
import com.google.gerrit.extensions.config.FactoryModule;
+import com.google.gerrit.extensions.events.LifecycleListener;
+import com.google.gerrit.server.git.meta.GitFile;
+import com.google.inject.internal.UniqueAnnotations;
import com.googlesource.gerrit.plugins.batch.util.MergeBranch;
import com.googlesource.gerrit.plugins.batch.util.MergeBuilder;
public class Module extends FactoryModule {
@Override
protected void configure() {
+ factory(GitFile.Factory.class);
factory(MergeBranch.Factory.class);
factory(MergeBuilder.Factory.class);
+
+ bind(LifecycleListener.class)
+ .annotatedWith(UniqueAnnotations.create())
+ .to(BatchCleaner.Lifecycle.class);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/batch/query/BatchQueryBuilder.java b/src/main/java/com/googlesource/gerrit/plugins/batch/query/BatchQueryBuilder.java
new file mode 100644
index 0000000..53f1b6f
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/batch/query/BatchQueryBuilder.java
@@ -0,0 +1,99 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.batch.query;
+
+import com.google.gerrit.extensions.annotations.PluginName;
+import com.google.gerrit.index.query.Matchable;
+import com.google.gerrit.index.query.OperatorPredicate;
+import com.google.gerrit.index.query.Predicate;
+import com.google.gerrit.index.query.QueryBuilder;
+import com.google.gerrit.index.query.QueryParseException;
+import com.google.gerrit.server.config.ConfigUtil;
+import com.google.gerrit.server.config.PluginConfigFactory;
+import com.google.gerrit.server.project.NoSuchProjectException;
+import com.google.gerrit.server.project.ProjectCache;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.batch.Batch;
+import com.googlesource.gerrit.plugins.batch.BatchStore;
+import java.util.Date;
+import java.util.concurrent.TimeUnit;
+import org.eclipse.jgit.lib.Config;
+
+/** Parses a query string meant to be applied to batch objects. */
+public class BatchQueryBuilder extends QueryBuilder<Batch> {
+ public abstract static class SimplePredicate extends OperatorPredicate<Batch>
+ implements Matchable<Batch> {
+ public SimplePredicate(String op, String val) {
+ super(op, val);
+ }
+
+ @Override
+ public boolean match(Batch b) {
+ return false;
+ }
+
+ @Override
+ public int getCost() {
+ return 1;
+ }
+ }
+
+ protected static final QueryBuilder.Definition<Batch, BatchQueryBuilder> mydef =
+ new QueryBuilder.Definition<Batch, BatchQueryBuilder>(BatchQueryBuilder.class);
+
+ public static final long DEFAULT_SECONDS = TimeUnit.SECONDS.convert(3, TimeUnit.DAYS);
+
+ protected final ProjectCache projectCache;
+ protected final PluginConfigFactory cfgFactory;
+ protected final BatchStore store;
+ protected final String pluginName;
+
+ @Inject
+ public BatchQueryBuilder(
+ ProjectCache projectCache,
+ PluginConfigFactory cfgFactory,
+ BatchStore store,
+ @PluginName String pluginName) {
+ super(mydef);
+ this.projectCache = projectCache;
+ this.cfgFactory = cfgFactory;
+ this.store = store;
+ this.pluginName = pluginName;
+ }
+
+ public Date getExpiry() throws NoSuchProjectException {
+ Config config = cfgFactory.getProjectPluginConfig(projectCache.getAllProjects(), pluginName);
+ long seconds =
+ ConfigUtil.getTimeUnit(
+ config, "cleaner", null, "maxAge", DEFAULT_SECONDS, TimeUnit.SECONDS);
+ long ms = TimeUnit.MILLISECONDS.convert(seconds, TimeUnit.SECONDS);
+ return new Date(new Date().getTime() - ms);
+ }
+
+ @Operator
+ public Predicate<Batch> is(String value) throws NoSuchProjectException, QueryParseException {
+ if ("expired".equalsIgnoreCase(value)) {
+ return new SimplePredicate("is", value) {
+ Date expiry = getExpiry();
+
+ @Override
+ public boolean match(Batch b) {
+ return b.lastModified.before(expiry);
+ }
+ };
+ }
+ throw error("Invalid query");
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/batch/ssh/SshModule.java b/src/main/java/com/googlesource/gerrit/plugins/batch/ssh/SshModule.java
index 2fd7b81..b80cf29 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/batch/ssh/SshModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/batch/ssh/SshModule.java
@@ -13,15 +13,11 @@
// limitations under the License.
package com.googlesource.gerrit.plugins.batch.ssh;
-import com.google.gerrit.server.git.meta.GitFile;
import com.google.gerrit.sshd.PluginCommandModule;
-import com.google.inject.assistedinject.FactoryModuleBuilder;
public class SshModule extends PluginCommandModule {
@Override
protected void configureCommands() {
- install(new FactoryModuleBuilder().build(GitFile.Factory.class));
-
command(MergeChangeCommand.class);
command(DeleteCommand.class);
command(ListCommand.class);
diff --git a/src/main/resources/Documentation/about.md b/src/main/resources/Documentation/about.md
index 75bf414..8898d4c 100644
--- a/src/main/resources/Documentation/about.md
+++ b/src/main/resources/Documentation/about.md
@@ -90,3 +90,72 @@
the special refs/meta/batch/<batch_id> ref in the All-Projects project. This
is internal meta data to the batch plugin and these refs should not be
accessed or altered by users directly.
+
+Batch Cleanup
+-------------
+Batches are temporary proposed updates. They are meant to be
+created, tested, and then submitted to their destinations if
+they pass, or deleted if they fail. Since the output of good
+batches will likely persist on destination branches, and
+the results of bad batches are not typically desirable to keep
+around, the batch service has a background *cleaner* task
+which finds expired batches and deletes them automatically.
+This cleaning helps to ensure that resources are released when
+they are no longer needed.
+
+The cleaner task runs by default daily, and batches are expired
+by default after 3 days from their last modification. It is
+possible to configure expiration times, and the cleaner using
+the `All-Projects` `refs/meta/config` `@PLUGIN@.config` file.
+The `@PLUGIN@.config` file is a "git-config" style file
+and supports the following parameters:
+
+*`cleaner.maxAge`*
+
+: Age after which a batch is considered expired. Values should
+use common unit suffixes to express their setting:
+
+ * s, sec, second, seconds (default unit)
+ * m, min, minute, minutes
+ * h, hr, hour, hours
+ * d, day, days
+ * w, week, weeks (1 week is treated as 7 days)
+ * mon, month, months (1 month is treated as 30 days)
+ * y, year, years (1 year is treated as 365 days)
+
+: If a unit suffix is not specified, seconds is assumed. If 0 is
+supplied, the maximum age is infinite and items are never
+expired (they must be deleted manually). The default maxAge is
+3 days.
+
+*`cleaner.interval`*
+
+: Interval for periodic repetition of triggering the batch
+cleanups. The interval must be larger than zero. The following
+suffixes are supported to define the time unit for the interval:
+
+ * m, min, minute, minutes (default suffix)
+ * h, hr, hour, hours
+ * d, day, days
+ * w, week, weeks (1 week is treated as 7 days)
+ * mon, month, months (1 month is treated as 30 days)
+ * y, year, years (1 year is treated as 365 days)
+
+: If a unit suffix is not specified, minutes is assumed. The
+default interval is 1 day.
+
+*`cleaner.startDelay`*
+
+: One time delay to wait after plugin load before starting
+the periodic cleaner. The following suffixes are supported
+to define the time unit for the delay:
+
+ * m, min, minute, minutes (default suffix)
+ * h, hr, hour, hours
+ * d, day, days
+ * w, week, weeks (1 week is treated as 7 days)
+ * mon, month, months (1 month is treated as 30 days)
+ * y, year, years (1 year is treated as 365 days)
+
+: If a unit suffix is not specified, minutes is assumed. The
+default startDelay is 1 minute.
diff --git a/src/main/resources/Documentation/cmd-ls-batches.md b/src/main/resources/Documentation/cmd-ls-batches.md
index 76c0071..b70d0a7 100644
--- a/src/main/resources/Documentation/cmd-ls-batches.md
+++ b/src/main/resources/Documentation/cmd-ls-batches.md
@@ -10,6 +10,7 @@
```
ssh -p @SSH_PORT@ @SSH_HOST@ @PLUGIN@ ls-batches
[--include-batch-info]
+ <query>
```
OPTIONS
@@ -31,6 +32,21 @@
---------
This command is intended to be used in scripts.
+QUERIES
+-------
+Batches can be queried using a syntax that mimicks the change
+query syntax. Operators act as restrictions on the search. As
+more operators are added to the same query string, they further
+restrict the returned results.
+
+Operators
+---------
+<a name="is:expired"></a>
+*is:expired*
+
+: Batches which have expired ([see batch cleanup](about.md#cleanup))
+
+
EXAMPLES
--------
List visible batches:
