Merge changes from topic 'lucene5'
* changes:
Update Lucene to 5.0.0
Add config option to disable online reindexing
diff --git a/Documentation/config-gerrit.txt b/Documentation/config-gerrit.txt
index c93a01b..9fd0db2 100644
--- a/Documentation/config-gerrit.txt
+++ b/Documentation/config-gerrit.txt
@@ -2176,6 +2176,18 @@
thread pool as interactive operations (unless
link:#changeMerge.threadPoolSize[changeMerge.threadPoolSize] is set).
+[[index.onlineUpgrade]]index.onlineUpgrade::
++
+Whether to upgrade to new index schema versions while the server is
+running. This is recommended as it prevents additional downtime during
+Gerrit version upgrades (avoiding the need for an offline reindex step
+using Reindex), but can add additional server load during the upgrade.
++
+If set to false, there is no way to upgrade the index schema to take
+advantage of new search features without restarting the server.
++
+Defaults to true.
+
==== Lucene configuration
Open and closed changes are indexed in separate indexes named
diff --git a/gerrit-lucene/BUCK b/gerrit-lucene/BUCK
index 2b45d2b..a146774 100644
--- a/gerrit-lucene/BUCK
+++ b/gerrit-lucene/BUCK
@@ -34,7 +34,9 @@
'//lib/jgit:jgit',
'//lib/log:api',
'//lib/lucene:analyzers-common',
+ '//lib/lucene:backward-codecs',
'//lib/lucene:core',
+ '//lib/lucene:misc',
],
visibility = ['PUBLIC'],
)
diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/AutoCommitWriter.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/AutoCommitWriter.java
index e0c13ae..27ded17 100644
--- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/AutoCommitWriter.java
+++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/AutoCommitWriter.java
@@ -14,7 +14,6 @@
package com.google.gerrit.lucene;
-import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -43,13 +42,6 @@
}
@Override
- public void addDocument(Iterable<? extends IndexableField> doc,
- Analyzer analyzer) throws IOException {
- super.addDocument(doc, analyzer);
- autoFlush();
- }
-
- @Override
public void addDocuments(
Iterable<? extends Iterable<? extends IndexableField>> docs)
throws IOException {
@@ -58,14 +50,6 @@
}
@Override
- public void addDocuments(
- Iterable<? extends Iterable<? extends IndexableField>> docs,
- Analyzer analyzer) throws IOException {
- super.addDocuments(docs, analyzer);
- autoFlush();
- }
-
- @Override
public void updateDocuments(Term delTerm,
Iterable<? extends Iterable<? extends IndexableField>> docs)
throws IOException {
@@ -74,14 +58,6 @@
}
@Override
- public void updateDocuments(Term delTerm,
- Iterable<? extends Iterable<? extends IndexableField>> docs,
- Analyzer analyzer) throws IOException {
- super.updateDocuments(delTerm, docs, analyzer);
- autoFlush();
- }
-
- @Override
public void deleteDocuments(Term... term) throws IOException {
super.deleteDocuments(term);
autoFlush();
@@ -111,13 +87,6 @@
}
@Override
- public void updateDocument(Term term, Iterable<? extends IndexableField> doc,
- Analyzer analyzer) throws IOException {
- super.updateDocument(term, doc, analyzer);
- autoFlush();
- }
-
- @Override
public void deleteAll() throws IOException {
super.deleteAll();
autoFlush();
diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
index 0084536..15b8fc3 100644
--- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
+++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
@@ -14,10 +14,12 @@
package com.google.gerrit.lucene;
-import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.Iterables.getOnlyElement;
import static com.google.gerrit.server.git.QueueProvider.QueueType.INTERACTIVE;
import static com.google.gerrit.server.index.IndexRewriteImpl.CLOSED_STATUSES;
import static com.google.gerrit.server.index.IndexRewriteImpl.OPEN_STATUSES;
+
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
@@ -39,7 +41,6 @@
import com.google.gerrit.server.index.ChangeField.ChangeProtoField;
import com.google.gerrit.server.index.ChangeField.PatchSetApprovalProtoField;
import com.google.gerrit.server.index.ChangeIndex;
-import com.google.gerrit.server.index.ChangeSchemas;
import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.FieldType;
@@ -64,9 +65,12 @@
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@@ -76,13 +80,14 @@
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.SearcherFactory;
import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.uninverting.UninvertingReader;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.Version;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.file.FileBasedConfig;
@@ -120,54 +125,19 @@
private static final String CHANGE_FIELD = ChangeField.CHANGE.getName();
private static final String DELETED_FIELD = ChangeField.DELETED.getName();
private static final String ID_FIELD = ChangeField.LEGACY_ID.getName();
+ private static final String ID_SORT_FIELD =
+ sortFieldName(ChangeField.LEGACY_ID);
private static final String MERGEABLE_FIELD = ChangeField.MERGEABLE.getName();
+ private static final String UPDATED_SORT_FIELD =
+ sortFieldName(ChangeField.UPDATED);
+
private static final ImmutableSet<String> FIELDS = ImmutableSet.of(
ADDED_FIELD, APPROVAL_FIELD, CHANGE_FIELD, DELETED_FIELD, ID_FIELD,
MERGEABLE_FIELD);
+
private static final Map<String, String> CUSTOM_CHAR_MAPPING = ImmutableMap.of(
"_", " ", ".", " ");
- private static final Map<Schema<ChangeData>, Version> LUCENE_VERSIONS;
- static {
- ImmutableMap.Builder<Schema<ChangeData>, Version> versions =
- ImmutableMap.builder();
- @SuppressWarnings("deprecation")
- Version lucene43 = Version.LUCENE_43;
- @SuppressWarnings("deprecation")
- Version lucene44 = Version.LUCENE_44;
- @SuppressWarnings("deprecation")
- Version lucene46 = Version.LUCENE_46;
- @SuppressWarnings("deprecation")
- Version lucene47 = Version.LUCENE_47;
- @SuppressWarnings("deprecation")
- Version lucene48 = Version.LUCENE_48;
- @SuppressWarnings("deprecation")
- Version lucene410 = Version.LUCENE_4_10_0;
- // We are using 4.10.2 but there is no difference in the index
- // format since 4.10.1, so we reuse the version here.
- @SuppressWarnings("deprecation")
- Version lucene4101 = Version.LUCENE_4_10_1;
- for (Map.Entry<Integer, Schema<ChangeData>> e
- : ChangeSchemas.ALL.entrySet()) {
- if (e.getKey() <= 3) {
- versions.put(e.getValue(), lucene43);
- } else if (e.getKey() <= 5) {
- versions.put(e.getValue(), lucene44);
- } else if (e.getKey() <= 8) {
- versions.put(e.getValue(), lucene46);
- } else if (e.getKey() <= 10) {
- versions.put(e.getValue(), lucene47);
- } else if (e.getKey() <= 11) {
- versions.put(e.getValue(), lucene48);
- } else if (e.getKey() <= 13) {
- versions.put(e.getValue(), lucene410);
- } else {
- versions.put(e.getValue(), lucene4101);
- }
- }
- LUCENE_VERSIONS = versions.build();
- }
-
public static void setReady(SitePaths sitePaths, int version, boolean ready)
throws IOException {
try {
@@ -180,6 +150,10 @@
}
}
+ private static String sortFieldName(FieldDef<?, ?> f) {
+ return f.getName() + "_SORT";
+ }
+
static interface Factory {
LuceneChangeIndex create(Schema<ChangeData> schema, String base);
}
@@ -188,12 +162,13 @@
private final IndexWriterConfig luceneConfig;
private long commitWithinMs;
- private GerritIndexWriterConfig(Version version, Config cfg, String name) {
+ private GerritIndexWriterConfig(Config cfg, String name) {
CustomMappingAnalyzer analyzer =
new CustomMappingAnalyzer(new StandardAnalyzer(
CharArraySet.EMPTY_SET), CUSTOM_CHAR_MAPPING);
- luceneConfig = new IndexWriterConfig(version, analyzer);
- luceneConfig.setOpenMode(OpenMode.CREATE_OR_APPEND);
+ luceneConfig = new IndexWriterConfig(analyzer)
+ .setOpenMode(OpenMode.CREATE_OR_APPEND)
+ .setCommitOnClose(true);
double m = 1 << 20;
luceneConfig.setRAMBufferSizeMB(cfg.getLong(
"index", name, "ramBufferSize",
@@ -229,6 +204,17 @@
private final SubIndex openIndex;
private final SubIndex closedIndex;
+ /**
+ * Whether to use DocValues for range/sorted numeric fields.
+ * <p>
+ * Lucene 5 removed support for sorting based on normal numeric fields, so we
+ * use the newer API for more strongly typed numeric fields in newer schema
+ * versions. These fields also are not stored, so we need to store auxiliary
+ * stored-only field for them as well.
+ */
+ // TODO(dborowitz): Delete when we delete support for pre-Lucene-5.0 schemas.
+ private final boolean useDocValuesForSorting;
+
@AssistedInject
LuceneChangeIndex(
@GerritServerConfig Config cfg,
@@ -245,10 +231,8 @@
this.db = db;
this.changeDataFactory = changeDataFactory;
this.schema = schema;
+ this.useDocValuesForSorting = schema.getVersion() >= 15;
- Version luceneVersion = checkNotNull(
- LUCENE_VERSIONS.get(schema),
- "unknown Lucene version for index schema: %s", schema);
CustomMappingAnalyzer analyzer =
new CustomMappingAnalyzer(new StandardAnalyzer(CharArraySet.EMPTY_SET),
CUSTOM_CHAR_MAPPING);
@@ -258,21 +242,44 @@
BooleanQuery.getMaxClauseCount()));
GerritIndexWriterConfig openConfig =
- new GerritIndexWriterConfig(luceneVersion, cfg, "changes_open");
+ new GerritIndexWriterConfig(cfg, "changes_open");
GerritIndexWriterConfig closedConfig =
- new GerritIndexWriterConfig(luceneVersion, cfg, "changes_closed");
+ new GerritIndexWriterConfig(cfg, "changes_closed");
+ SearcherFactory searcherFactory = newSearcherFactory();
if (cfg.getBoolean("index", "lucene", "testInmemory", false)) {
- openIndex = new SubIndex(new RAMDirectory(), "ramOpen", openConfig);
- closedIndex = new SubIndex(new RAMDirectory(), "ramClosed", closedConfig);
+ openIndex = new SubIndex(new RAMDirectory(), "ramOpen", openConfig,
+ searcherFactory);
+ closedIndex = new SubIndex(new RAMDirectory(), "ramClosed", closedConfig,
+ searcherFactory);
} else {
Path dir = base != null ? Paths.get(base)
: LuceneVersionManager.getDir(sitePaths, schema);
- openIndex = new SubIndex(dir.resolve(CHANGES_OPEN), openConfig);
- closedIndex = new SubIndex(dir.resolve(CHANGES_CLOSED), closedConfig);
+ openIndex = new SubIndex(dir.resolve(CHANGES_OPEN), openConfig,
+ searcherFactory);
+ closedIndex = new SubIndex(dir.resolve(CHANGES_CLOSED), closedConfig,
+ searcherFactory);
}
}
+ private SearcherFactory newSearcherFactory() {
+ if (useDocValuesForSorting) {
+ return new SearcherFactory();
+ }
+ final Map<String, UninvertingReader.Type> mapping = ImmutableMap.of(
+ ChangeField.LEGACY_ID.getName(), UninvertingReader.Type.INTEGER,
+ ChangeField.UPDATED.getName(), UninvertingReader.Type.LONG);
+ return new SearcherFactory() {
+ @Override
+ public IndexSearcher newSearcher(IndexReader reader) {
+ checkState(reader instanceof DirectoryReader,
+ "expected DirectoryReader, found %s", reader.getClass().getName());
+ return new IndexSearcher(
+ UninvertingReader.wrap((DirectoryReader) reader, mapping));
+ }
+ };
+ }
+
@Override
public void close() {
List<ListenableFuture<?>> closeFutures = Lists.newArrayListWithCapacity(2);
@@ -355,12 +362,18 @@
setReady(sitePaths, schema.getVersion(), ready);
}
- private static Sort getSort() {
- return new Sort(
- new SortField(
- ChangeField.UPDATED.getName(), SortField.Type.LONG, true),
- new SortField(
- ChangeField.LEGACY_ID.getName(), SortField.Type.INT, true));
+ private Sort getSort() {
+ if (useDocValuesForSorting) {
+ return new Sort(
+ new SortField(UPDATED_SORT_FIELD, SortField.Type.LONG, true),
+ new SortField(ID_SORT_FIELD, SortField.Type.LONG, true));
+ } else {
+ return new Sort(
+ new SortField(
+ ChangeField.UPDATED.getName(), SortField.Type.LONG, true),
+ new SortField(
+ ChangeField.LEGACY_ID.getName(), SortField.Type.INT, true));
+ }
}
private class QuerySource implements ChangeDataSource {
@@ -506,6 +519,16 @@
FieldType<?> type = values.getField().getType();
Store store = store(values.getField());
+ if (useDocValuesForSorting) {
+ if (values.getField() == ChangeField.LEGACY_ID) {
+ int v = (Integer) getOnlyElement(values.getValues());
+ doc.add(new NumericDocValuesField(ID_SORT_FIELD, v));
+ } else if (values.getField() == ChangeField.UPDATED) {
+ long t = ((Timestamp) getOnlyElement(values.getValues())).getTime();
+ doc.add(new NumericDocValuesField(UPDATED_SORT_FIELD, t));
+ }
+ }
+
if (type == FieldType.INTEGER || type == FieldType.INTEGER_RANGE) {
for (Object value : values.getValues()) {
doc.add(new IntField(name, (Integer) value, store));
diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java
index 3c38225..109525a 100644
--- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java
+++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java
@@ -20,6 +20,7 @@
import com.google.common.collect.Maps;
import com.google.common.primitives.Ints;
import com.google.gerrit.extensions.events.LifecycleListener;
+import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.ChangeSchemas;
import com.google.gerrit.server.index.IndexCollection;
@@ -93,9 +94,11 @@
private final LuceneChangeIndex.Factory indexFactory;
private final IndexCollection indexes;
private final OnlineReindexer.Factory reindexerFactory;
+ private final boolean onlineUpgrade;
@Inject
LuceneVersionManager(
+ @GerritServerConfig Config cfg,
SitePaths sitePaths,
LuceneChangeIndex.Factory indexFactory,
IndexCollection indexes,
@@ -104,6 +107,7 @@
this.indexFactory = indexFactory;
this.indexes = indexes;
this.reindexerFactory = reindexerFactory;
+ this.onlineUpgrade = cfg.getBoolean("index", null, "onlineUpgrade", true);
}
@Override
@@ -133,7 +137,7 @@
if (v.schema == null) {
continue;
}
- if (write.isEmpty()) {
+ if (write.isEmpty() && onlineUpgrade) {
write.add(v);
}
if (v.ready) {
@@ -162,7 +166,7 @@
}
int latest = write.get(0).version;
- if (latest != search.version) {
+ if (onlineUpgrade && latest != search.version) {
reindexerFactory.create(latest).start();
}
}
diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java
index f28bf05..5778008 100644
--- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java
+++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java
@@ -28,9 +28,9 @@
import org.apache.lucene.index.TrackingIndexWriter;
import org.apache.lucene.search.ControlledRealTimeReopenThread;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ReferenceManager;
import org.apache.lucene.search.ReferenceManager.RefreshListener;
import org.apache.lucene.search.SearcherFactory;
-import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@@ -52,17 +52,19 @@
private final Directory dir;
private final TrackingIndexWriter writer;
- private final SearcherManager searcherManager;
+ private final ReferenceManager<IndexSearcher> searcherManager;
private final ControlledRealTimeReopenThread<IndexSearcher> reopenThread;
private final Set<NrtFuture> notDoneNrtFutures;
- SubIndex(Path path, GerritIndexWriterConfig writerConfig) throws IOException {
- this(FSDirectory.open(path.toFile()), path.getFileName().toString(),
- writerConfig);
+ SubIndex(Path path, GerritIndexWriterConfig writerConfig,
+ SearcherFactory searcherFactory) throws IOException {
+ this(FSDirectory.open(path), path.getFileName().toString(), writerConfig,
+ searcherFactory);
}
SubIndex(Directory dir, final String dirName,
- GerritIndexWriterConfig writerConfig) throws IOException {
+ GerritIndexWriterConfig writerConfig,
+ SearcherFactory searcherFactory) throws IOException {
this.dir = dir;
IndexWriter delegateWriter;
long commitPeriod = writerConfig.getCommitWithinMs();
@@ -104,8 +106,8 @@
}, commitPeriod, commitPeriod, MILLISECONDS);
}
writer = new TrackingIndexWriter(delegateWriter);
- searcherManager = new SearcherManager(
- writer.getIndexWriter(), true, new SearcherFactory());
+ searcherManager = new WrappableSearcherManager(
+ writer.getIndexWriter(), true, searcherFactory);
notDoneNrtFutures = Sets.newConcurrentHashSet();
@@ -125,6 +127,8 @@
// searching generation being up to date when calling
// reopenThread.waitForGeneration(gen, 0), therefore the reopen thread's
// internal listener needs to be called first.
+ // TODO(dborowitz): This may have been fixed by
+ // http://issues.apache.org/jira/browse/LUCENE-5461
searcherManager.addListener(new RefreshListener() {
@Override
public void beforeRefresh() throws IOException {
@@ -158,12 +162,9 @@
}
try {
- writer.getIndexWriter().commit();
- try {
- writer.getIndexWriter().close();
- } catch (AlreadyClosedException e) {
- // Ignore.
- }
+ writer.getIndexWriter().close();
+ } catch (AlreadyClosedException e) {
+ // Ignore.
} catch (IOException e) {
log.warn("error closing Lucene writer", e);
}
diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/WrappableSearcherManager.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/WrappableSearcherManager.java
new file mode 100644
index 0000000..fe45f1d
--- /dev/null
+++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/WrappableSearcherManager.java
@@ -0,0 +1,217 @@
+package com.google.gerrit.lucene;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FilterDirectoryReader;
+import org.apache.lucene.index.FilterLeafReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ReferenceManager;
+import org.apache.lucene.search.SearcherFactory;
+import org.apache.lucene.store.Directory;
+
+/**
+ * Utility class to safely share {@link IndexSearcher} instances across multiple
+ * threads, while periodically reopening. This class ensures each searcher is
+ * closed only once all threads have finished using it.
+ *
+ * <p>
+ * Use {@link #acquire} to obtain the current searcher, and {@link #release} to
+ * release it, like this:
+ *
+ * <pre class="prettyprint">
+ * IndexSearcher s = manager.acquire();
+ * try {
+ * // Do searching, doc retrieval, etc. with s
+ * } finally {
+ * manager.release(s);
+ * }
+ * // Do not use s after this!
+ * s = null;
+ * </pre>
+ *
+ * <p>
+ * In addition you should periodically call {@link #maybeRefresh}. While it's
+ * possible to call this just before running each query, this is discouraged
+ * since it penalizes the unlucky queries that need to refresh. It's better to use
+ * a separate background thread, that periodically calls {@link #maybeRefresh}. Finally,
+ * be sure to call {@link #close} once you are done.
+ *
+ * @see SearcherFactory
+ *
+ * @lucene.experimental
+ */
+// This file was copied from:
+// https://github.com/apache/lucene-solr/blob/lucene_solr_5_0/lucene/core/src/java/org/apache/lucene/search/SearcherManager.java
+// The only change (other than class name and import fixes)
+// is to skip the check in getSearcher that searcherFactory.newSearcher wraps
+// the provided searcher exactly.
+final class WrappableSearcherManager extends ReferenceManager<IndexSearcher> {
+
+ private final SearcherFactory searcherFactory;
+
+ /**
+ * Creates and returns a new SearcherManager from the given
+ * {@link IndexWriter}.
+ *
+ * @param writer
+ * the IndexWriter to open the IndexReader from.
+ * @param applyAllDeletes
+ * If <code>true</code>, all buffered deletes will be applied (made
+ * visible) in the {@link IndexSearcher} / {@link DirectoryReader}.
+ * If <code>false</code>, the deletes may or may not be applied, but
+ * remain buffered (in IndexWriter) so that they will be applied in
+ * the future. Applying deletes can be costly, so if your app can
+ * tolerate deleted documents being returned you might gain some
+ * performance by passing <code>false</code>. See
+ * {@link DirectoryReader#openIfChanged(DirectoryReader, IndexWriter, boolean)}.
+ * @param searcherFactory
+ * An optional {@link SearcherFactory}. Pass <code>null</code> if you
+ * don't require the searcher to be warmed before going live or other
+ * custom behavior.
+ *
+ * @throws IOException if there is a low-level I/O error
+ */
+ public WrappableSearcherManager(IndexWriter writer, boolean applyAllDeletes, SearcherFactory searcherFactory) throws IOException {
+ if (searcherFactory == null) {
+ searcherFactory = new SearcherFactory();
+ }
+ this.searcherFactory = searcherFactory;
+ current = getSearcher(searcherFactory, DirectoryReader.open(writer, applyAllDeletes));
+ }
+
+ /**
+ * Creates and returns a new SearcherManager from the given {@link Directory}.
+ * @param dir the directory to open the DirectoryReader on.
+ * @param searcherFactory An optional {@link SearcherFactory}. Pass
+ * <code>null</code> if you don't require the searcher to be warmed
+ * before going live or other custom behavior.
+ *
+ * @throws IOException if there is a low-level I/O error
+ */
+ public WrappableSearcherManager(Directory dir, SearcherFactory searcherFactory) throws IOException {
+ if (searcherFactory == null) {
+ searcherFactory = new SearcherFactory();
+ }
+ this.searcherFactory = searcherFactory;
+ current = getSearcher(searcherFactory, DirectoryReader.open(dir));
+ }
+
+ /**
+ * Creates and returns a new SearcherManager from an existing {@link DirectoryReader}. Note that
+ * this steals the incoming reference.
+ *
+ * @param reader the DirectoryReader.
+ * @param searcherFactory An optional {@link SearcherFactory}. Pass
+ * <code>null</code> if you don't require the searcher to be warmed
+ * before going live or other custom behavior.
+ *
+ * @throws IOException if there is a low-level I/O error
+ */
+ public WrappableSearcherManager(DirectoryReader reader, SearcherFactory searcherFactory) throws IOException {
+ if (searcherFactory == null) {
+ searcherFactory = new SearcherFactory();
+ }
+ this.searcherFactory = searcherFactory;
+ this.current = getSearcher(searcherFactory, reader);
+ }
+
+ @Override
+ protected void decRef(IndexSearcher reference) throws IOException {
+ reference.getIndexReader().decRef();
+ }
+
+ @Override
+ protected IndexSearcher refreshIfNeeded(IndexSearcher referenceToRefresh) throws IOException {
+ final IndexReader r = referenceToRefresh.getIndexReader();
+ assert r instanceof DirectoryReader: "searcher's IndexReader should be a DirectoryReader, but got " + r;
+ final IndexReader newReader = DirectoryReader.openIfChanged((DirectoryReader) r);
+ if (newReader == null) {
+ return null;
+ } else {
+ return getSearcher(searcherFactory, newReader);
+ }
+ }
+
+ @Override
+ protected boolean tryIncRef(IndexSearcher reference) {
+ return reference.getIndexReader().tryIncRef();
+ }
+
+ @Override
+ protected int getRefCount(IndexSearcher reference) {
+ return reference.getIndexReader().getRefCount();
+ }
+
+ /**
+ * Returns <code>true</code> if no changes have occured since this searcher
+ * ie. reader was opened, otherwise <code>false</code>.
+ * @see DirectoryReader#isCurrent()
+ */
+ public boolean isSearcherCurrent() throws IOException {
+ final IndexSearcher searcher = acquire();
+ try {
+ final IndexReader r = searcher.getIndexReader();
+ assert r instanceof DirectoryReader: "searcher's IndexReader should be a DirectoryReader, but got " + r;
+ return ((DirectoryReader) r).isCurrent();
+ } finally {
+ release(searcher);
+ }
+ }
+
+ /** Expert: creates a searcher from the provided {@link
+ * IndexReader} using the provided {@link
+ * SearcherFactory}. NOTE: this decRefs incoming reader
+ * on throwing an exception. */
+ @SuppressWarnings("resource")
+ public static IndexSearcher getSearcher(SearcherFactory searcherFactory, IndexReader reader) throws IOException {
+ boolean success = false;
+ final IndexSearcher searcher;
+ try {
+ searcher = searcherFactory.newSearcher(reader);
+ // Modification for Gerrit: Allow searcherFactory to transitively wrap the
+ // provided reader.
+ IndexReader unwrapped = searcher.getIndexReader();
+ while (true) {
+ if (unwrapped == reader) {
+ break;
+ } else if (unwrapped instanceof FilterDirectoryReader) {
+ unwrapped = ((FilterDirectoryReader) unwrapped).getDelegate();
+ } else if (unwrapped instanceof FilterLeafReader) {
+ unwrapped = ((FilterLeafReader) unwrapped).getDelegate();
+ } else {
+ break;
+ }
+ }
+
+ if (unwrapped != reader) {
+ throw new IllegalStateException("SearcherFactory must wrap the provided reader (got " + searcher.getIndexReader() + " but expected " + reader + ")");
+ }
+ success = true;
+ } finally {
+ if (!success) {
+ reader.decRef();
+ }
+ }
+ return searcher;
+ }
+}
diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java
index 821343f..0040762 100644
--- a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java
+++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java
@@ -55,6 +55,10 @@
* {@link ChangeQueryBuilder} for querying that field, and a method on
* {@link ChangeData} used for populating the corresponding document fields in
* the secondary index.
+ * <p>
+ * Field names are all lowercase alphanumeric plus underscore; index
+ * implementations may create unambiguous derived field names containing other
+ * characters.
*/
public class ChangeField {
/** Legacy change ID. */
diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/FieldDef.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/FieldDef.java
index 557faeb..cf3fd09 100644
--- a/gerrit-server/src/main/java/com/google/gerrit/server/index/FieldDef.java
+++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/FieldDef.java
@@ -14,6 +14,9 @@
package com.google.gerrit.server.index;
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.base.CharMatcher;
import com.google.common.base.Preconditions;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.TrackingFooters;
@@ -78,11 +81,17 @@
private final boolean stored;
private FieldDef(String name, FieldType<?> type, boolean stored) {
- this.name = name;
+ this.name = checkName(name);
this.type = type;
this.stored = stored;
}
+ private static String checkName(String name) {
+ CharMatcher m = CharMatcher.anyOf("abcdefghijklmnopqrstuvwxyz0123456789_");
+ checkArgument(m.matchesAllOf(name), "illegal field name: %s", name);
+ return name;
+ }
+
/** @return name of the field. */
public final String getName() {
return name;
diff --git a/gerrit-server/src/test/java/com/google/gerrit/server/query/change/LuceneQueryChangesV14Test.java b/gerrit-server/src/test/java/com/google/gerrit/server/query/change/LuceneQueryChangesV14Test.java
new file mode 100644
index 0000000..42f5072
--- /dev/null
+++ b/gerrit-server/src/test/java/com/google/gerrit/server/query/change/LuceneQueryChangesV14Test.java
@@ -0,0 +1,49 @@
+// Copyright (C) 2015 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.query.change;
+
+import com.google.gerrit.testutil.InMemoryModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+
+import org.eclipse.jgit.lib.Config;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class LuceneQueryChangesV14Test extends LuceneQueryChangesTest {
+
+ @Override
+ protected Injector createInjector() {
+ Config luceneConfig = new Config(config);
+ InMemoryModule.setDefaults(luceneConfig);
+ // Latest version with a Lucene 4 index.
+ luceneConfig.setInt("index", "lucene", "testVersion", 14);
+ return Guice.createInjector(new InMemoryModule(luceneConfig));
+ }
+
+ @Override
+ @Ignore
+ @Test
+ public void byCommentBy() {
+ // Ignore.
+ }
+
+ @Override
+ @Ignore
+ @Test
+ public void byFrom() {
+ // Ignore.
+ }
+}
diff --git a/lib/asciidoctor/java/DocIndexer.java b/lib/asciidoctor/java/DocIndexer.java
index 7eb70c1..f06c662 100644
--- a/lib/asciidoctor/java/DocIndexer.java
+++ b/lib/asciidoctor/java/DocIndexer.java
@@ -25,7 +25,6 @@
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.Version;
import org.kohsuke.args4j.Argument;
import org.kohsuke.args4j.CmdLineException;
import org.kohsuke.args4j.CmdLineParser;
@@ -51,8 +50,6 @@
import java.util.zip.ZipOutputStream;
public class DocIndexer {
- @SuppressWarnings("deprecation")
- private static final Version LUCENE_VERSION = Version.LUCENE_4_10_1;
private static final Pattern SECTION_HEADER = Pattern.compile("^=+ (.*)");
@Option(name = "-o", usage = "output JAR file")
@@ -99,9 +96,9 @@
UnsupportedEncodingException, FileNotFoundException {
RAMDirectory directory = new RAMDirectory();
IndexWriterConfig config = new IndexWriterConfig(
- LUCENE_VERSION,
new StandardAnalyzer(CharArraySet.EMPTY_SET));
config.setOpenMode(OpenMode.CREATE);
+ config.setCommitOnClose(true);
IndexWriter iwriter = new IndexWriter(directory, config);
for (String inputFile : inputFiles) {
File file = new File(inputFile);
diff --git a/lib/lucene/BUCK b/lib/lucene/BUCK
index 9026f79..275f0bb 100644
--- a/lib/lucene/BUCK
+++ b/lib/lucene/BUCK
@@ -1,11 +1,11 @@
include_defs('//lib/maven.defs')
-VERSION = '4.10.2'
+VERSION = '5.0.0'
maven_jar(
name = 'core',
id = 'org.apache.lucene:lucene-core:' + VERSION,
- sha1 = 'c01e3d675d277e0a93e7890d03cc3246b2cdecaa',
+ sha1 = '4395e5ea987af804c4a9b96131e2ee75db061fdf',
license = 'Apache2.0',
exclude = [
'META-INF/LICENSE.txt',
@@ -16,8 +16,33 @@
maven_jar(
name = 'analyzers-common',
id = 'org.apache.lucene:lucene-analyzers-common:' + VERSION,
- sha1 = 'f977f8c443e8f4e9d1fd7fdfda80a6cf60b3e7c2',
+ sha1 = '6159cbc5c9631ef75e1f0e97b358ecdd8f1447a9',
license = 'Apache2.0',
+ deps = [':core'],
+ exclude = [
+ 'META-INF/LICENSE.txt',
+ 'META-INF/NOTICE.txt',
+ ],
+)
+
+maven_jar(
+ name = 'backward-codecs',
+ id = 'org.apache.lucene:lucene-backward-codecs:' + VERSION,
+ sha1 = '5cd11fc1be436ff96b63f0f76f299a9d25543b0b',
+ license = 'Apache2.0',
+ deps = [':core'],
+ exclude = [
+ 'META-INF/LICENSE.txt',
+ 'META-INF/NOTICE.txt',
+ ],
+)
+
+maven_jar(
+ name = 'misc',
+ id = 'org.apache.lucene:lucene-misc:' + VERSION,
+ sha1 = '06bd7cb030e598da81a8228f5c58630e5ce7b84a',
+ license = 'Apache2.0',
+ deps = [':core'],
exclude = [
'META-INF/LICENSE.txt',
'META-INF/NOTICE.txt',
@@ -27,6 +52,11 @@
maven_jar(
name = 'query-parser',
id = 'org.apache.lucene:lucene-queryparser:' + VERSION,
- sha1 = 'd70f54e1060d553ba7aeb4d49a71fd0c068499e8',
+ sha1 = 'f459326c0b58bb837612bfeb37f6015c1a8962db',
license = 'Apache2.0',
+ deps = [':core'],
+ exclude = [
+ 'META-INF/LICENSE.txt',
+ 'META-INF/NOTICE.txt',
+ ],
)
