Scan index directory to determine Lucene versions
Mark index versions as "ready" when they are fully indexed and the
server is not running (i.e. from Reindex). By default, search from the
most recent ready index version, and write to both the most recent
ready version and (if different) the most recent known version.
At server startup, mark all versions except those we are about to
start writing to as not ready.
Change-Id: Icf42a3eb27b0445899300d60941cd701a8072d41
diff --git a/gerrit-lucene/BUCK b/gerrit-lucene/BUCK
index 5e9f376..b5e5d07 100644
--- a/gerrit-lucene/BUCK
+++ b/gerrit-lucene/BUCK
@@ -27,7 +27,9 @@
'//gerrit-server:server',
'//lib:guava',
'//lib:gwtorm',
+ '//lib:jsr305',
'//lib/guice:guice',
+ '//lib/guice:guice-assistedinject',
'//lib/jgit:jgit',
'//lib/log:api',
'//lib/lucene:analyzers-common',
diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/IndexVersionCheck.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/IndexVersionCheck.java
deleted file mode 100644
index 54591c9..0000000
--- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/IndexVersionCheck.java
+++ /dev/null
@@ -1,92 +0,0 @@
-// Copyright (C) 2013 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.gerrit.lucene;
-
-import static com.google.gerrit.lucene.LuceneChangeIndex.LUCENE_VERSION;
-import static org.apache.lucene.util.Version.LUCENE_CURRENT;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.gerrit.extensions.events.LifecycleListener;
-import com.google.gerrit.server.config.SitePaths;
-import com.google.gerrit.server.index.ChangeSchemas;
-import com.google.inject.Inject;
-import com.google.inject.ProvisionException;
-
-import org.apache.lucene.util.Version;
-import org.eclipse.jgit.errors.ConfigInvalidException;
-import org.eclipse.jgit.storage.file.FileBasedConfig;
-import org.eclipse.jgit.util.FS;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Map;
-
-public class IndexVersionCheck implements LifecycleListener {
- public static final Map<String, Integer> SCHEMA_VERSIONS = ImmutableMap.of(
- LuceneChangeIndex.CHANGES_OPEN, ChangeSchemas.getLatestRelease().getVersion(),
- LuceneChangeIndex.CHANGES_CLOSED, ChangeSchemas.getLatestRelease().getVersion());
-
- public static File gerritIndexConfig(SitePaths sitePaths) {
- return new File(sitePaths.index_dir, "gerrit_index.config");
- }
-
- private final SitePaths sitePaths;
-
- @Inject
- IndexVersionCheck(SitePaths sitePaths) {
- this.sitePaths = sitePaths;
- }
-
- @Override
- public void start() {
- File file = gerritIndexConfig(sitePaths);
- try {
- FileBasedConfig cfg = new FileBasedConfig(file, FS.detect());
- cfg.load();
- for (Map.Entry<String, Integer> e : SCHEMA_VERSIONS.entrySet()) {
- int schemaVersion = cfg.getInt("index", e.getKey(), "schemaVersion", 0);
- if (schemaVersion != e.getValue()) {
- throw new ProvisionException(String.format(
- "wrong index schema version for \"%s\": expected %d, found %d%s",
- e.getKey(), e.getValue(), schemaVersion, upgrade()));
- }
- }
- @SuppressWarnings("deprecation")
- Version luceneVersion =
- cfg.getEnum("lucene", null, "version", LUCENE_CURRENT);
- if (luceneVersion != LUCENE_VERSION) {
- throw new ProvisionException(String.format(
- "wrong Lucene version: expected %d, found %d%s",
- luceneVersion, LUCENE_VERSION, upgrade()));
-
- }
- } catch (IOException e) {
- throw new ProvisionException("unable to read " + file);
- } catch (ConfigInvalidException e) {
- throw new ProvisionException("invalid config file " + file);
- }
- }
-
- @Override
- public void stop() {
- // Do nothing.
- }
-
- private final String upgrade() {
- return "\nRun reindex to rebuild the index:\n"
- + "$ java -jar gerrit.war reindex -d "
- + sitePaths.site_path.getAbsolutePath();
- }
-}
diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
index ef2ef91..f87b328 100644
--- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
+++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
@@ -14,8 +14,6 @@
package com.google.gerrit.lucene;
-import static com.google.gerrit.lucene.IndexVersionCheck.SCHEMA_VERSIONS;
-import static com.google.gerrit.lucene.IndexVersionCheck.gerritIndexConfig;
import static com.google.gerrit.server.index.IndexRewriteImpl.CLOSED_STATUSES;
import static com.google.gerrit.server.index.IndexRewriteImpl.OPEN_STATUSES;
@@ -26,7 +24,6 @@
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningScheduledExecutorService;
-import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
@@ -35,7 +32,6 @@
import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.FieldType;
-import com.google.gerrit.server.index.IndexCollection;
import com.google.gerrit.server.index.IndexExecutor;
import com.google.gerrit.server.index.IndexRewriteImpl;
import com.google.gerrit.server.index.Schema;
@@ -45,6 +41,8 @@
import com.google.gerrit.server.query.change.ChangeDataSource;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
+import com.google.inject.assistedinject.Assisted;
+import com.google.inject.assistedinject.AssistedInject;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
@@ -69,7 +67,6 @@
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.file.FileBasedConfig;
-import org.eclipse.jgit.util.FS;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -79,11 +76,12 @@
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
+import javax.annotation.Nullable;
+
/**
* Secondary index implementation using Apache Lucene.
* <p>
@@ -92,7 +90,7 @@
* though there may be some lag between a committed write and it showing up to
* other threads' searchers.
*/
-public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
+public class LuceneChangeIndex implements ChangeIndex {
private static final Logger log =
LoggerFactory.getLogger(LuceneChangeIndex.class);
@@ -101,6 +99,10 @@
public static final String CHANGES_CLOSED = "closed";
private static final String ID_FIELD = ChangeField.LEGACY_ID.getName();
+ static interface Factory {
+ LuceneChangeIndex create(Schema<ChangeData> schema, String base);
+ }
+
private static IndexWriterConfig getIndexWriterConfig(Config cfg, String name) {
IndexWriterConfig writerConfig = new IndexWriterConfig(LUCENE_VERSION,
new StandardAnalyzer(LUCENE_VERSION));
@@ -115,28 +117,27 @@
private final SitePaths sitePaths;
private final FillArgs fillArgs;
- private final IndexCollection indexes;
private final ExecutorService executor;
+ private final File dir;
private final Schema<ChangeData> schema;
private final SubIndex openIndex;
private final SubIndex closedIndex;
- LuceneChangeIndex(@GerritServerConfig Config cfg,
+ @AssistedInject
+ LuceneChangeIndex(
+ @GerritServerConfig Config cfg,
SitePaths sitePaths,
- IndexCollection indexes,
@IndexExecutor ListeningScheduledExecutorService executor,
FillArgs fillArgs,
- Schema<ChangeData> schema,
- String base) throws IOException {
- this.indexes = indexes;
+ @Assisted Schema<ChangeData> schema,
+ @Assisted @Nullable String base) throws IOException {
this.sitePaths = sitePaths;
this.fillArgs = fillArgs;
this.executor = executor;
this.schema = schema;
- File dir;
if (base == null) {
- dir = new File(sitePaths.index_dir, "changes_" + schema.getVersion());
+ dir = LuceneVersionManager.getDir(sitePaths, schema);
} else {
dir = new File(base);
}
@@ -147,13 +148,7 @@
}
@Override
- public void start() {
- indexes.setSearchIndex(this);
- indexes.addWriteIndex(this);
- }
-
- @Override
- public void stop() {
+ public void close() {
List<Future<?>> closeFutures = Lists.newArrayListWithCapacity(2);
closeFutures.add(executor.submit(new Runnable() {
@Override
@@ -249,6 +244,18 @@
return new QuerySource(indexes, QueryBuilder.toQuery(p));
}
+ @Override
+ public void markReady() throws IOException {
+ try {
+ FileBasedConfig cfg = LuceneVersionManager.loadGerritIndexConfig(sitePaths);
+ cfg.setBoolean("index", Integer.toString(schema.getVersion()), "ready",
+ true);
+ cfg.save();
+ } catch (ConfigInvalidException e) {
+ throw new IOException(e);
+ }
+ }
+
private static class QuerySource implements ChangeDataSource {
// TODO(dborowitz): Push limit down from predicate tree.
private static final int LIMIT = 1000;
@@ -388,17 +395,4 @@
private static Field.Store store(FieldDef<?, ?> f) {
return f.isStored() ? Field.Store.YES : Field.Store.NO;
}
-
- @Override
- public void finishIndex() throws IOException,
- ConfigInvalidException {
- FileBasedConfig cfg =
- new FileBasedConfig(gerritIndexConfig(sitePaths), FS.detect());
-
- for (Map.Entry<String, Integer> e : SCHEMA_VERSIONS.entrySet()) {
- cfg.setInt("index", e.getKey(), "schemaVersion", e.getValue());
- }
- cfg.setEnum("lucene", null, "version", LUCENE_VERSION);
- cfg.save();
- }
}
diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java
index a302f6f..1e1dca7 100644
--- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java
+++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java
@@ -14,57 +14,89 @@
package com.google.gerrit.lucene;
-import com.google.common.util.concurrent.ListeningScheduledExecutorService;
+import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.lifecycle.LifecycleModule;
-import com.google.gerrit.server.config.GerritServerConfig;
+import com.google.gerrit.server.config.FactoryModule;
import com.google.gerrit.server.config.SitePaths;
-import com.google.gerrit.server.index.ChangeIndex;
import com.google.gerrit.server.index.ChangeSchemas;
-import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.IndexCollection;
-import com.google.gerrit.server.index.IndexExecutor;
import com.google.gerrit.server.index.IndexModule;
+import com.google.gerrit.server.index.Schema;
+import com.google.gerrit.server.query.change.ChangeData;
+import com.google.inject.Inject;
import com.google.inject.Provides;
import com.google.inject.Singleton;
-import org.eclipse.jgit.lib.Config;
-
-import java.io.IOException;
-
public class LuceneIndexModule extends LifecycleModule {
- private final boolean checkVersion;
+ private final Integer singleVersion;
private final int threads;
private final String base;
public LuceneIndexModule() {
- this(true, 0, null);
+ this(null, 0, null);
}
- public LuceneIndexModule(boolean checkVersion, int threads,
+ public LuceneIndexModule(Integer singleVersion, int threads,
String base) {
- this.checkVersion = checkVersion;
+ this.singleVersion = singleVersion;
this.threads = threads;
this.base = base;
}
@Override
protected void configure() {
+ install(new FactoryModule() {
+ @Override
+ public void configure() {
+ factory(LuceneChangeIndex.Factory.class);
+ }
+ });
install(new IndexModule(threads));
- bind(ChangeIndex.class).to(LuceneChangeIndex.class);
- listener().to(LuceneChangeIndex.class);
- if (checkVersion) {
- listener().to(IndexVersionCheck.class);
+ if (singleVersion == null && base == null) {
+ listener().to(LuceneVersionManager.class);
+ } else {
+ install(new SingleVersionModule());
}
}
- @Provides
+ private class SingleVersionModule extends LifecycleModule {
+ @Override
+ public void configure() {
+ listener().to(SingleVersionListener.class);
+ }
+
+ @Provides
+ @Singleton
+ LuceneChangeIndex getIndex(LuceneChangeIndex.Factory factory,
+ SitePaths sitePaths) {
+ Schema<ChangeData> schema = singleVersion != null
+ ? ChangeSchemas.get(singleVersion)
+ : ChangeSchemas.getLatest();
+ return factory.create(schema, base);
+ }
+ }
+
@Singleton
- public LuceneChangeIndex getChangeIndex(@GerritServerConfig Config cfg,
- SitePaths sitePaths,
- IndexCollection indexes,
- @IndexExecutor ListeningScheduledExecutorService executor,
- FillArgs fillArgs) throws IOException {
- return new LuceneChangeIndex(cfg, sitePaths, indexes, executor, fillArgs,
- ChangeSchemas.getLatestRelease(), base);
+ static class SingleVersionListener implements LifecycleListener {
+ private final IndexCollection indexes;
+ private final LuceneChangeIndex index;
+
+ @Inject
+ SingleVersionListener(IndexCollection indexes,
+ LuceneChangeIndex index) {
+ this.indexes = indexes;
+ this.index = index;
+ }
+
+ @Override
+ public void start() {
+ indexes.setSearchIndex(index);
+ indexes.addWriteIndex(index);
+ }
+
+ @Override
+ public void stop() {
+ index.close();
+ }
}
}
diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java
new file mode 100644
index 0000000..c63eb24
--- /dev/null
+++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java
@@ -0,0 +1,209 @@
+// Copyright (C) 2013 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.package com.google.gerrit.server.git;
+
+package com.google.gerrit.lucene;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.primitives.Ints;
+import com.google.gerrit.extensions.events.LifecycleListener;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.index.ChangeSchemas;
+import com.google.gerrit.server.index.IndexCollection;
+import com.google.gerrit.server.index.Schema;
+import com.google.gerrit.server.query.change.ChangeData;
+import com.google.inject.Inject;
+import com.google.inject.ProvisionException;
+import com.google.inject.Singleton;
+
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.TreeMap;
+
+@Singleton
+class LuceneVersionManager implements LifecycleListener {
+ private static final Logger log = LoggerFactory
+ .getLogger(LuceneVersionManager.class);
+
+ private static final String CHANGES_PREFIX = "changes_";
+
+ private static class Version {
+ private final Schema<ChangeData> schema;
+ private final int version;
+ private final boolean exists;
+ private final boolean ready;
+
+ private Version(Schema<ChangeData> schema, int version, boolean exists,
+ boolean ready) {
+ checkArgument(schema == null || schema.getVersion() == version);
+ this.schema = schema;
+ this.version = version;
+ this.exists = exists;
+ this.ready = ready;
+ }
+ }
+
+ static File getDir(SitePaths sitePaths, Schema<ChangeData> schema) {
+ return new File(sitePaths.index_dir, String.format("%s%04d",
+ CHANGES_PREFIX, schema.getVersion()));
+ }
+
+ static FileBasedConfig loadGerritIndexConfig(SitePaths sitePaths)
+ throws ConfigInvalidException, IOException {
+ FileBasedConfig cfg = new FileBasedConfig(
+ new File(sitePaths.index_dir, "gerrit_index.config"), FS.detect());
+ cfg.load();
+ return cfg;
+ }
+
+ private static boolean getReady(Config cfg, int version) {
+ return cfg.getBoolean("index", Integer.toString(version), "ready", false);
+ }
+
+ private static void setReady(Config cfg, int version, boolean ready) {
+ cfg.setBoolean("index", Integer.toString(version), "ready", ready);
+ }
+
+ private final SitePaths sitePaths;
+ private final LuceneChangeIndex.Factory indexFactory;
+ private final IndexCollection indexes;
+
+ @Inject
+ LuceneVersionManager(
+ SitePaths sitePaths,
+ LuceneChangeIndex.Factory indexFactory,
+ IndexCollection indexes) {
+ this.sitePaths = sitePaths;
+ this.indexFactory = indexFactory;
+ this.indexes = indexes;
+ }
+
+ @Override
+ public void start() {
+ FileBasedConfig cfg;
+ try {
+ cfg = loadGerritIndexConfig(sitePaths);
+ } catch (ConfigInvalidException e) {
+ throw fail(e);
+ } catch (IOException e) {
+ throw fail(e);
+ }
+
+ TreeMap<Integer, Version> versions = scanVersions(cfg);
+ // Search from the most recent ready version.
+ // Write to the most recent ready version and the most recent version.
+ Version search = null;
+ List<Version> write = Lists.newArrayListWithCapacity(2);
+ for (Version v : versions.descendingMap().values()) {
+ if (v.schema == null) {
+ continue;
+ }
+ if (write.isEmpty()) {
+ write.add(v);
+ }
+ if (v.ready) {
+ search = v;
+ if (!write.contains(v)) {
+ write.add(v);
+ }
+ break;
+ }
+ }
+ if (search == null) {
+ throw new ProvisionException("No index versions ready; run Reindex");
+ }
+
+ markNotReady(cfg, versions.values(), write);
+ LuceneChangeIndex searchIndex = indexFactory.create(search.schema, null);
+ indexes.setSearchIndex(searchIndex);
+ for (Version v : write) {
+ if (v.schema != null) {
+ if (v.version != search.version) {
+ indexes.addWriteIndex(indexFactory.create(v.schema, null));
+ } else {
+ indexes.addWriteIndex(searchIndex);
+ }
+ }
+ }
+ }
+
+ private TreeMap<Integer, Version> scanVersions(Config cfg) {
+ TreeMap<Integer, Version> versions = Maps.newTreeMap();
+ for (Schema<ChangeData> schema : ChangeSchemas.ALL.values()) {
+ File f = getDir(sitePaths, schema);
+ boolean exists = f.exists() && f.isDirectory();
+ if (exists && !f.isDirectory()) {
+ log.warn("Not a directory: %s", f.getAbsolutePath());
+ }
+ int v = schema.getVersion();
+ versions.put(v, new Version(schema, v, exists, getReady(cfg, v)));
+ }
+
+ for (File f : sitePaths.index_dir.listFiles()) {
+ if (!f.getName().startsWith(CHANGES_PREFIX)) {
+ continue;
+ }
+ String versionStr = f.getName().substring(CHANGES_PREFIX.length());
+ Integer v = Ints.tryParse(versionStr);
+ if (v == null || versionStr.length() != 4) {
+ log.warn("Unrecognized version in index directory: {}",
+ f.getAbsolutePath());
+ continue;
+ }
+ if (!versions.containsKey(v)) {
+ versions.put(v, new Version(null, v, true, getReady(cfg, v)));
+ }
+ }
+ return versions;
+ }
+
+ private void markNotReady(FileBasedConfig cfg, Iterable<Version> versions,
+ Collection<Version> inUse) {
+ boolean dirty = false;
+ for (Version v : versions) {
+ if (!inUse.contains(v) && v.exists) {
+ setReady(cfg, v.version, false);
+ dirty = true;
+ }
+ }
+ if (dirty) {
+ try {
+ cfg.save();
+ } catch (IOException e) {
+ throw fail(e);
+ }
+ }
+ }
+
+ private ProvisionException fail(Throwable t) {
+ ProvisionException e = new ProvisionException("Error scanning indexes");
+ e.initCause(t);
+ throw e;
+ }
+
+ @Override
+ public void stop() {
+ }
+}
diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java
index 1070a52..6d47124 100644
--- a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java
+++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java
@@ -43,6 +43,8 @@
import com.google.gerrit.server.git.MultiProgressMonitor.Task;
import com.google.gerrit.server.index.ChangeIndex;
import com.google.gerrit.server.index.ChangeIndexer;
+import com.google.gerrit.server.index.ChangeSchemas;
+import com.google.gerrit.server.index.IndexCollection;
import com.google.gerrit.server.index.IndexExecutor;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.IndexModule.IndexType;
@@ -63,7 +65,6 @@
import org.eclipse.jgit.diff.DiffEntry;
import org.eclipse.jgit.diff.DiffFormatter;
-import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectInserter;
@@ -98,14 +99,22 @@
@Option(name = "--threads", usage = "Number of threads to use for indexing")
private int threads = Runtime.getRuntime().availableProcessors();
+ @Option(name = "--schema-version",
+ usage = "Schema version to reindex; default is most recent version")
+ private Integer version;
+
@Option(name = "--output", usage = "Prefix for output; path for local disk index, or prefix for remote index")
private String outputBase;
@Option(name = "--verbose", usage = "Output debug information for each change")
private boolean verbose;
+ @Option(name = "--dry-run", usage = "Dry run: don't write anything to index")
+ private boolean dryRun;
+
private Injector dbInjector;
private Injector sysInjector;
+ private ChangeIndex index;
@Override
public int run() throws Exception {
@@ -114,22 +123,22 @@
if (IndexModule.getIndexType(dbInjector) == IndexType.SQL) {
throw die("index.type must be configured (or not SQL)");
}
-
+ if (version == null) {
+ version = ChangeSchemas.getLatest().getVersion();
+ }
LifecycleManager dbManager = new LifecycleManager();
dbManager.add(dbInjector);
dbManager.start();
sysInjector = createSysInjector();
-
- // Delete before any index may be created depending on this data.
- deleteAll();
-
LifecycleManager sysManager = new LifecycleManager();
sysManager.add(sysInjector);
sysManager.start();
+ index = sysInjector.getInstance(IndexCollection.class).getSearchIndex();
+ index.deleteAll();
int result = indexAll();
- writeVersion();
+ index.markReady();
sysManager.stop();
dbManager.stop();
@@ -142,7 +151,7 @@
AbstractModule changeIndexModule;
switch (IndexModule.getIndexType(dbInjector)) {
case LUCENE:
- changeIndexModule = new LuceneIndexModule(false, threads, outputBase);
+ changeIndexModule = new LuceneIndexModule(version, threads, outputBase);
break;
case SOLR:
changeIndexModule = new SolrIndexModule(false, threads, outputBase);
@@ -207,11 +216,6 @@
}
}
- private void deleteAll() throws IOException {
- ChangeIndex index = sysInjector.getInstance(ChangeIndex.class);
- index.deleteAll();
- }
-
private int indexAll() throws Exception {
ReviewDb db = sysInjector.getInstance(ReviewDb.class);
ListeningScheduledExecutorService executor = sysInjector.getInstance(
@@ -464,10 +468,4 @@
}
}
}
-
- private void writeVersion() throws IOException,
- ConfigInvalidException {
- ChangeIndex index = sysInjector.getInstance(ChangeIndex.class);
- index.finishIndex();
- }
}
diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java
index 7d7dcdb..1ffa6b5 100644
--- a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java
+++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java
@@ -21,8 +21,6 @@
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.ChangeDataSource;
-import org.eclipse.jgit.errors.ConfigInvalidException;
-
import java.io.IOException;
/**
@@ -64,20 +62,27 @@
}
@Override
- public ChangeDataSource getSource(Predicate<ChangeData> p)
- throws QueryParseException {
+ public ChangeDataSource getSource(Predicate<ChangeData> p) {
throw new UnsupportedOperationException();
}
@Override
- public void finishIndex() {
+ public void close() {
// Do nothing.
}
+
+ @Override
+ public void markReady() {
+ throw new UnsupportedOperationException();
+ }
};
/** @return the schema version used by this index. */
public Schema<ChangeData> getSchema();
+ /** Close this index. */
+ public void close();
+
/**
* Insert a change document into the index.
* <p>
@@ -137,11 +142,13 @@
throws QueryParseException;
/**
- * Mark completion of indexing.
+ * Mark this index as up-to-date and ready to serve reads.
+ * <p>
+ * Should only be called immediately after a reindex, either during an online
+ * schema upgrade while actively writing to this index, or during an offline
+ * reindex.
*
- * @throws ConfigInvalidException
* @throws IOException
*/
- public void finishIndex() throws IOException,
- ConfigInvalidException;
+ public void markReady() throws IOException;
}
diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeSchemas.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeSchemas.java
index 6094df1..87a4df1 100644
--- a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeSchemas.java
+++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeSchemas.java
@@ -15,9 +15,9 @@
package com.google.gerrit.server.index;
import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.gerrit.server.query.change.ChangeData;
@@ -58,23 +58,16 @@
return new Schema<ChangeData>(false, Arrays.asList(fields));
}
- private static final ImmutableMap<Integer, Schema<ChangeData>> ALL;
+ public static final ImmutableMap<Integer, Schema<ChangeData>> ALL;
- public Schema<ChangeData> get(int version) {
+ public static Schema<ChangeData> get(int version) {
Schema<ChangeData> schema = ALL.get(version);
checkArgument(schema != null, "Unrecognized schema version: %s", version);
return schema;
}
- public static Schema<ChangeData> getLatestRelease() {
- Schema<ChangeData> latest = null;
- for (Schema<ChangeData> schema : ALL.values()) {
- if (schema.isRelease()) {
- latest = schema;
- }
- }
- checkState(latest != null, "No released schema versions found");
- return latest;
+ public static Schema<ChangeData> getLatest() {
+ return Iterables.getLast(ALL.values());
}
static {
diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexCollection.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexCollection.java
index f07bf49..a6cd226 100644
--- a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexCollection.java
+++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexCollection.java
@@ -16,6 +16,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
+import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -28,7 +29,7 @@
/** Dynamic pointers to the index versions used for searching and writing. */
@Singleton
-public class IndexCollection {
+public class IndexCollection implements LifecycleListener {
private final CopyOnWriteArrayList<ChangeIndex> writeIndexes;
private final AtomicReference<ChangeIndex> searchIndex;
@@ -66,4 +67,21 @@
}
writeIndexes.add(index);
}
+
+ @Override
+ public void start() {
+ }
+
+ @Override
+ public void stop() {
+ ChangeIndex read = searchIndex.get();
+ if (read != null) {
+ read.close();
+ }
+ for (ChangeIndex write : writeIndexes) {
+ if (write != read) {
+ write.close();
+ }
+ }
+ }
}
diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java
index a04fe80..e96791a 100644
--- a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java
+++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java
@@ -16,11 +16,11 @@
import com.google.common.util.concurrent.ListeningScheduledExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
+import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.git.WorkQueue;
import com.google.gerrit.server.git.WorkQueue.Executor;
import com.google.gerrit.server.query.change.ChangeQueryRewriter;
-import com.google.inject.AbstractModule;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.Provides;
@@ -34,7 +34,7 @@
* This module should not be used directly except by specific secondary indexer
* implementations (e.g. Lucene).
*/
-public class IndexModule extends AbstractModule {
+public class IndexModule extends LifecycleModule {
public enum IndexType {
SQL, LUCENE, SOLR;
}
@@ -57,6 +57,8 @@
bind(ChangeIndexer.class).to(ChangeIndexerImpl.class);
bind(ChangeQueryRewriter.class).to(IndexRewriteImpl.class);
bind(IndexRewriteImpl.BasicRewritesImpl.class);
+ bind(IndexCollection.class);
+ listener().to(IndexCollection.class);
}
@Provides
diff --git a/gerrit-server/src/test/java/com/google/gerrit/server/index/IndexRewriteTest.java b/gerrit-server/src/test/java/com/google/gerrit/server/index/IndexRewriteTest.java
index 2bea308..9f4376b 100644
--- a/gerrit-server/src/test/java/com/google/gerrit/server/index/IndexRewriteTest.java
+++ b/gerrit-server/src/test/java/com/google/gerrit/server/index/IndexRewriteTest.java
@@ -90,7 +90,11 @@
}
@Override
- public void finishIndex() {
+ public void close() {
+ }
+
+ @Override
+ public void markReady() {
throw new UnsupportedOperationException();
}
}
diff --git a/gerrit-solr/src/main/java/com/google/gerrit/solr/IndexVersionCheck.java b/gerrit-solr/src/main/java/com/google/gerrit/solr/IndexVersionCheck.java
index ea33b05..7e32bac 100644
--- a/gerrit-solr/src/main/java/com/google/gerrit/solr/IndexVersionCheck.java
+++ b/gerrit-solr/src/main/java/com/google/gerrit/solr/IndexVersionCheck.java
@@ -31,8 +31,8 @@
class IndexVersionCheck implements LifecycleListener {
public static final Map<String, Integer> SCHEMA_VERSIONS = ImmutableMap.of(
- SolrChangeIndex.CHANGES_OPEN, ChangeSchemas.getLatestRelease().getVersion(),
- SolrChangeIndex.CHANGES_CLOSED, ChangeSchemas.getLatestRelease().getVersion());
+ SolrChangeIndex.CHANGES_OPEN, ChangeSchemas.getLatest().getVersion(),
+ SolrChangeIndex.CHANGES_CLOSED, ChangeSchemas.getLatest().getVersion());
public static File solrIndexConfig(SitePaths sitePaths) {
return new File(sitePaths.index_dir, "gerrit_index.config");
diff --git a/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java b/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java
index b338dff..b455c55 100644
--- a/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java
+++ b/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java
@@ -52,7 +52,6 @@
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
-import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.util.FS;
@@ -121,6 +120,11 @@
}
@Override
+ public void close() {
+ stop();
+ }
+
+ @Override
public ListenableFuture<Void> insert(ChangeData cd) throws IOException {
String id = cd.getId().toString();
SolrInputDocument doc = toDocument(cd);
@@ -321,8 +325,7 @@
}
@Override
- public void finishIndex() throws IOException,
- ConfigInvalidException {
+ public void markReady() throws IOException {
// TODO Move the schema version information to a special meta-document
FileBasedConfig cfg = new FileBasedConfig(
solrIndexConfig(sitePaths),
diff --git a/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrIndexModule.java b/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrIndexModule.java
index 4b892a8..8c614f7 100644
--- a/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrIndexModule.java
+++ b/gerrit-solr/src/main/java/com/google/gerrit/solr/SolrIndexModule.java
@@ -61,6 +61,6 @@
IndexCollection indexes,
FillArgs fillArgs) throws IOException {
return new SolrChangeIndex(cfg, fillArgs, sitePaths, indexes,
- ChangeSchemas.getLatestRelease(), base);
+ ChangeSchemas.getLatest(), base);
}
}
