Merge changes from topic 'api-tests'
* changes:
BranchApi: Implement get and delete
AbstractDaemonTest: Simplify helpers for getting changes
Delete StarredChangesIT
IdentifiedUser: Clear starred changes before rereading from API
Implement list branches extension API
Convert HashtagsIT to extension API
Convert most of CreateProjectIT to extension API
Convert GetAccountIT to extension API
Convert CreateChangeIT to extension API
Convert CommentsIT to extension API
diff --git a/Documentation/config-gerrit.txt b/Documentation/config-gerrit.txt
index c93a01b..9fd0db2 100644
--- a/Documentation/config-gerrit.txt
+++ b/Documentation/config-gerrit.txt
@@ -2176,6 +2176,18 @@
thread pool as interactive operations (unless
link:#changeMerge.threadPoolSize[changeMerge.threadPoolSize] is set).
+[[index.onlineUpgrade]]index.onlineUpgrade::
++
+Whether to upgrade to new index schema versions while the server is
+running. This is recommended as it prevents additional downtime during
+Gerrit version upgrades (avoiding the need for an offline reindex step
+using Reindex), but can add additional server load during the upgrade.
++
+If set to false, there is no way to upgrade the index schema to take
+advantage of new search features without restarting the server.
++
+Defaults to true.
+
==== Lucene configuration
Open and closed changes are indexed in separate indexes named
diff --git a/gerrit-lucene/BUCK b/gerrit-lucene/BUCK
index 2b45d2b..a146774 100644
--- a/gerrit-lucene/BUCK
+++ b/gerrit-lucene/BUCK
@@ -34,7 +34,9 @@
'//lib/jgit:jgit',
'//lib/log:api',
'//lib/lucene:analyzers-common',
+ '//lib/lucene:backward-codecs',
'//lib/lucene:core',
+ '//lib/lucene:misc',
],
visibility = ['PUBLIC'],
)
diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/AutoCommitWriter.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/AutoCommitWriter.java
index e0c13ae..27ded17 100644
--- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/AutoCommitWriter.java
+++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/AutoCommitWriter.java
@@ -14,7 +14,6 @@
package com.google.gerrit.lucene;
-import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -43,13 +42,6 @@
}
@Override
- public void addDocument(Iterable<? extends IndexableField> doc,
- Analyzer analyzer) throws IOException {
- super.addDocument(doc, analyzer);
- autoFlush();
- }
-
- @Override
public void addDocuments(
Iterable<? extends Iterable<? extends IndexableField>> docs)
throws IOException {
@@ -58,14 +50,6 @@
}
@Override
- public void addDocuments(
- Iterable<? extends Iterable<? extends IndexableField>> docs,
- Analyzer analyzer) throws IOException {
- super.addDocuments(docs, analyzer);
- autoFlush();
- }
-
- @Override
public void updateDocuments(Term delTerm,
Iterable<? extends Iterable<? extends IndexableField>> docs)
throws IOException {
@@ -74,14 +58,6 @@
}
@Override
- public void updateDocuments(Term delTerm,
- Iterable<? extends Iterable<? extends IndexableField>> docs,
- Analyzer analyzer) throws IOException {
- super.updateDocuments(delTerm, docs, analyzer);
- autoFlush();
- }
-
- @Override
public void deleteDocuments(Term... term) throws IOException {
super.deleteDocuments(term);
autoFlush();
@@ -111,13 +87,6 @@
}
@Override
- public void updateDocument(Term term, Iterable<? extends IndexableField> doc,
- Analyzer analyzer) throws IOException {
- super.updateDocument(term, doc, analyzer);
- autoFlush();
- }
-
- @Override
public void deleteAll() throws IOException {
super.deleteAll();
autoFlush();
diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
index 0084536..15b8fc3 100644
--- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
+++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
@@ -14,10 +14,12 @@
package com.google.gerrit.lucene;
-import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.Iterables.getOnlyElement;
import static com.google.gerrit.server.git.QueueProvider.QueueType.INTERACTIVE;
import static com.google.gerrit.server.index.IndexRewriteImpl.CLOSED_STATUSES;
import static com.google.gerrit.server.index.IndexRewriteImpl.OPEN_STATUSES;
+
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
@@ -39,7 +41,6 @@
import com.google.gerrit.server.index.ChangeField.ChangeProtoField;
import com.google.gerrit.server.index.ChangeField.PatchSetApprovalProtoField;
import com.google.gerrit.server.index.ChangeIndex;
-import com.google.gerrit.server.index.ChangeSchemas;
import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.FieldType;
@@ -64,9 +65,12 @@
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@@ -76,13 +80,14 @@
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.SearcherFactory;
import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.uninverting.UninvertingReader;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.Version;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.file.FileBasedConfig;
@@ -120,54 +125,19 @@
private static final String CHANGE_FIELD = ChangeField.CHANGE.getName();
private static final String DELETED_FIELD = ChangeField.DELETED.getName();
private static final String ID_FIELD = ChangeField.LEGACY_ID.getName();
+ private static final String ID_SORT_FIELD =
+ sortFieldName(ChangeField.LEGACY_ID);
private static final String MERGEABLE_FIELD = ChangeField.MERGEABLE.getName();
+ private static final String UPDATED_SORT_FIELD =
+ sortFieldName(ChangeField.UPDATED);
+
private static final ImmutableSet<String> FIELDS = ImmutableSet.of(
ADDED_FIELD, APPROVAL_FIELD, CHANGE_FIELD, DELETED_FIELD, ID_FIELD,
MERGEABLE_FIELD);
+
private static final Map<String, String> CUSTOM_CHAR_MAPPING = ImmutableMap.of(
"_", " ", ".", " ");
- private static final Map<Schema<ChangeData>, Version> LUCENE_VERSIONS;
- static {
- ImmutableMap.Builder<Schema<ChangeData>, Version> versions =
- ImmutableMap.builder();
- @SuppressWarnings("deprecation")
- Version lucene43 = Version.LUCENE_43;
- @SuppressWarnings("deprecation")
- Version lucene44 = Version.LUCENE_44;
- @SuppressWarnings("deprecation")
- Version lucene46 = Version.LUCENE_46;
- @SuppressWarnings("deprecation")
- Version lucene47 = Version.LUCENE_47;
- @SuppressWarnings("deprecation")
- Version lucene48 = Version.LUCENE_48;
- @SuppressWarnings("deprecation")
- Version lucene410 = Version.LUCENE_4_10_0;
- // We are using 4.10.2 but there is no difference in the index
- // format since 4.10.1, so we reuse the version here.
- @SuppressWarnings("deprecation")
- Version lucene4101 = Version.LUCENE_4_10_1;
- for (Map.Entry<Integer, Schema<ChangeData>> e
- : ChangeSchemas.ALL.entrySet()) {
- if (e.getKey() <= 3) {
- versions.put(e.getValue(), lucene43);
- } else if (e.getKey() <= 5) {
- versions.put(e.getValue(), lucene44);
- } else if (e.getKey() <= 8) {
- versions.put(e.getValue(), lucene46);
- } else if (e.getKey() <= 10) {
- versions.put(e.getValue(), lucene47);
- } else if (e.getKey() <= 11) {
- versions.put(e.getValue(), lucene48);
- } else if (e.getKey() <= 13) {
- versions.put(e.getValue(), lucene410);
- } else {
- versions.put(e.getValue(), lucene4101);
- }
- }
- LUCENE_VERSIONS = versions.build();
- }
-
public static void setReady(SitePaths sitePaths, int version, boolean ready)
throws IOException {
try {
@@ -180,6 +150,10 @@
}
}
+ private static String sortFieldName(FieldDef<?, ?> f) {
+ return f.getName() + "_SORT";
+ }
+
static interface Factory {
LuceneChangeIndex create(Schema<ChangeData> schema, String base);
}
@@ -188,12 +162,13 @@
private final IndexWriterConfig luceneConfig;
private long commitWithinMs;
- private GerritIndexWriterConfig(Version version, Config cfg, String name) {
+ private GerritIndexWriterConfig(Config cfg, String name) {
CustomMappingAnalyzer analyzer =
new CustomMappingAnalyzer(new StandardAnalyzer(
CharArraySet.EMPTY_SET), CUSTOM_CHAR_MAPPING);
- luceneConfig = new IndexWriterConfig(version, analyzer);
- luceneConfig.setOpenMode(OpenMode.CREATE_OR_APPEND);
+ luceneConfig = new IndexWriterConfig(analyzer)
+ .setOpenMode(OpenMode.CREATE_OR_APPEND)
+ .setCommitOnClose(true);
double m = 1 << 20;
luceneConfig.setRAMBufferSizeMB(cfg.getLong(
"index", name, "ramBufferSize",
@@ -229,6 +204,17 @@
private final SubIndex openIndex;
private final SubIndex closedIndex;
+ /**
+ * Whether to use DocValues for range/sorted numeric fields.
+ * <p>
+ * Lucene 5 removed support for sorting based on normal numeric fields, so we
+ * use the newer API for more strongly typed numeric fields in newer schema
+ * versions. These fields also are not stored, so we need to store auxiliary
+ * stored-only field for them as well.
+ */
+ // TODO(dborowitz): Delete when we delete support for pre-Lucene-5.0 schemas.
+ private final boolean useDocValuesForSorting;
+
@AssistedInject
LuceneChangeIndex(
@GerritServerConfig Config cfg,
@@ -245,10 +231,8 @@
this.db = db;
this.changeDataFactory = changeDataFactory;
this.schema = schema;
+ this.useDocValuesForSorting = schema.getVersion() >= 15;
- Version luceneVersion = checkNotNull(
- LUCENE_VERSIONS.get(schema),
- "unknown Lucene version for index schema: %s", schema);
CustomMappingAnalyzer analyzer =
new CustomMappingAnalyzer(new StandardAnalyzer(CharArraySet.EMPTY_SET),
CUSTOM_CHAR_MAPPING);
@@ -258,21 +242,44 @@
BooleanQuery.getMaxClauseCount()));
GerritIndexWriterConfig openConfig =
- new GerritIndexWriterConfig(luceneVersion, cfg, "changes_open");
+ new GerritIndexWriterConfig(cfg, "changes_open");
GerritIndexWriterConfig closedConfig =
- new GerritIndexWriterConfig(luceneVersion, cfg, "changes_closed");
+ new GerritIndexWriterConfig(cfg, "changes_closed");
+ SearcherFactory searcherFactory = newSearcherFactory();
if (cfg.getBoolean("index", "lucene", "testInmemory", false)) {
- openIndex = new SubIndex(new RAMDirectory(), "ramOpen", openConfig);
- closedIndex = new SubIndex(new RAMDirectory(), "ramClosed", closedConfig);
+ openIndex = new SubIndex(new RAMDirectory(), "ramOpen", openConfig,
+ searcherFactory);
+ closedIndex = new SubIndex(new RAMDirectory(), "ramClosed", closedConfig,
+ searcherFactory);
} else {
Path dir = base != null ? Paths.get(base)
: LuceneVersionManager.getDir(sitePaths, schema);
- openIndex = new SubIndex(dir.resolve(CHANGES_OPEN), openConfig);
- closedIndex = new SubIndex(dir.resolve(CHANGES_CLOSED), closedConfig);
+ openIndex = new SubIndex(dir.resolve(CHANGES_OPEN), openConfig,
+ searcherFactory);
+ closedIndex = new SubIndex(dir.resolve(CHANGES_CLOSED), closedConfig,
+ searcherFactory);
}
}
+ private SearcherFactory newSearcherFactory() {
+ if (useDocValuesForSorting) {
+ return new SearcherFactory();
+ }
+ final Map<String, UninvertingReader.Type> mapping = ImmutableMap.of(
+ ChangeField.LEGACY_ID.getName(), UninvertingReader.Type.INTEGER,
+ ChangeField.UPDATED.getName(), UninvertingReader.Type.LONG);
+ return new SearcherFactory() {
+ @Override
+ public IndexSearcher newSearcher(IndexReader reader) {
+ checkState(reader instanceof DirectoryReader,
+ "expected DirectoryReader, found %s", reader.getClass().getName());
+ return new IndexSearcher(
+ UninvertingReader.wrap((DirectoryReader) reader, mapping));
+ }
+ };
+ }
+
@Override
public void close() {
List<ListenableFuture<?>> closeFutures = Lists.newArrayListWithCapacity(2);
@@ -355,12 +362,18 @@
setReady(sitePaths, schema.getVersion(), ready);
}
- private static Sort getSort() {
- return new Sort(
- new SortField(
- ChangeField.UPDATED.getName(), SortField.Type.LONG, true),
- new SortField(
- ChangeField.LEGACY_ID.getName(), SortField.Type.INT, true));
+ private Sort getSort() {
+ if (useDocValuesForSorting) {
+ return new Sort(
+ new SortField(UPDATED_SORT_FIELD, SortField.Type.LONG, true),
+ new SortField(ID_SORT_FIELD, SortField.Type.LONG, true));
+ } else {
+ return new Sort(
+ new SortField(
+ ChangeField.UPDATED.getName(), SortField.Type.LONG, true),
+ new SortField(
+ ChangeField.LEGACY_ID.getName(), SortField.Type.INT, true));
+ }
}
private class QuerySource implements ChangeDataSource {
@@ -506,6 +519,16 @@
FieldType<?> type = values.getField().getType();
Store store = store(values.getField());
+ if (useDocValuesForSorting) {
+ if (values.getField() == ChangeField.LEGACY_ID) {
+ int v = (Integer) getOnlyElement(values.getValues());
+ doc.add(new NumericDocValuesField(ID_SORT_FIELD, v));
+ } else if (values.getField() == ChangeField.UPDATED) {
+ long t = ((Timestamp) getOnlyElement(values.getValues())).getTime();
+ doc.add(new NumericDocValuesField(UPDATED_SORT_FIELD, t));
+ }
+ }
+
if (type == FieldType.INTEGER || type == FieldType.INTEGER_RANGE) {
for (Object value : values.getValues()) {
doc.add(new IntField(name, (Integer) value, store));
diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java
index 3c38225..109525a 100644
--- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java
+++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java
@@ -20,6 +20,7 @@
import com.google.common.collect.Maps;
import com.google.common.primitives.Ints;
import com.google.gerrit.extensions.events.LifecycleListener;
+import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.ChangeSchemas;
import com.google.gerrit.server.index.IndexCollection;
@@ -93,9 +94,11 @@
private final LuceneChangeIndex.Factory indexFactory;
private final IndexCollection indexes;
private final OnlineReindexer.Factory reindexerFactory;
+ private final boolean onlineUpgrade;
@Inject
LuceneVersionManager(
+ @GerritServerConfig Config cfg,
SitePaths sitePaths,
LuceneChangeIndex.Factory indexFactory,
IndexCollection indexes,
@@ -104,6 +107,7 @@
this.indexFactory = indexFactory;
this.indexes = indexes;
this.reindexerFactory = reindexerFactory;
+ this.onlineUpgrade = cfg.getBoolean("index", null, "onlineUpgrade", true);
}
@Override
@@ -133,7 +137,7 @@
if (v.schema == null) {
continue;
}
- if (write.isEmpty()) {
+ if (write.isEmpty() && onlineUpgrade) {
write.add(v);
}
if (v.ready) {
@@ -162,7 +166,7 @@
}
int latest = write.get(0).version;
- if (latest != search.version) {
+ if (onlineUpgrade && latest != search.version) {
reindexerFactory.create(latest).start();
}
}
diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java
index f28bf05..5778008 100644
--- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java
+++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java
@@ -28,9 +28,9 @@
import org.apache.lucene.index.TrackingIndexWriter;
import org.apache.lucene.search.ControlledRealTimeReopenThread;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ReferenceManager;
import org.apache.lucene.search.ReferenceManager.RefreshListener;
import org.apache.lucene.search.SearcherFactory;
-import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@@ -52,17 +52,19 @@
private final Directory dir;
private final TrackingIndexWriter writer;
- private final SearcherManager searcherManager;
+ private final ReferenceManager<IndexSearcher> searcherManager;
private final ControlledRealTimeReopenThread<IndexSearcher> reopenThread;
private final Set<NrtFuture> notDoneNrtFutures;
- SubIndex(Path path, GerritIndexWriterConfig writerConfig) throws IOException {
- this(FSDirectory.open(path.toFile()), path.getFileName().toString(),
- writerConfig);
+ SubIndex(Path path, GerritIndexWriterConfig writerConfig,
+ SearcherFactory searcherFactory) throws IOException {
+ this(FSDirectory.open(path), path.getFileName().toString(), writerConfig,
+ searcherFactory);
}
SubIndex(Directory dir, final String dirName,
- GerritIndexWriterConfig writerConfig) throws IOException {
+ GerritIndexWriterConfig writerConfig,
+ SearcherFactory searcherFactory) throws IOException {
this.dir = dir;
IndexWriter delegateWriter;
long commitPeriod = writerConfig.getCommitWithinMs();
@@ -104,8 +106,8 @@
}, commitPeriod, commitPeriod, MILLISECONDS);
}
writer = new TrackingIndexWriter(delegateWriter);
- searcherManager = new SearcherManager(
- writer.getIndexWriter(), true, new SearcherFactory());
+ searcherManager = new WrappableSearcherManager(
+ writer.getIndexWriter(), true, searcherFactory);
notDoneNrtFutures = Sets.newConcurrentHashSet();
@@ -125,6 +127,8 @@
// searching generation being up to date when calling
// reopenThread.waitForGeneration(gen, 0), therefore the reopen thread's
// internal listener needs to be called first.
+ // TODO(dborowitz): This may have been fixed by
+ // http://issues.apache.org/jira/browse/LUCENE-5461
searcherManager.addListener(new RefreshListener() {
@Override
public void beforeRefresh() throws IOException {
@@ -158,12 +162,9 @@
}
try {
- writer.getIndexWriter().commit();
- try {
- writer.getIndexWriter().close();
- } catch (AlreadyClosedException e) {
- // Ignore.
- }
+ writer.getIndexWriter().close();
+ } catch (AlreadyClosedException e) {
+ // Ignore.
} catch (IOException e) {
log.warn("error closing Lucene writer", e);
}
diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/WrappableSearcherManager.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/WrappableSearcherManager.java
new file mode 100644
index 0000000..fe45f1d
--- /dev/null
+++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/WrappableSearcherManager.java
@@ -0,0 +1,217 @@
+package com.google.gerrit.lucene;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FilterDirectoryReader;
+import org.apache.lucene.index.FilterLeafReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ReferenceManager;
+import org.apache.lucene.search.SearcherFactory;
+import org.apache.lucene.store.Directory;
+
+/**
+ * Utility class to safely share {@link IndexSearcher} instances across multiple
+ * threads, while periodically reopening. This class ensures each searcher is
+ * closed only once all threads have finished using it.
+ *
+ * <p>
+ * Use {@link #acquire} to obtain the current searcher, and {@link #release} to
+ * release it, like this:
+ *
+ * <pre class="prettyprint">
+ * IndexSearcher s = manager.acquire();
+ * try {
+ * // Do searching, doc retrieval, etc. with s
+ * } finally {
+ * manager.release(s);
+ * }
+ * // Do not use s after this!
+ * s = null;
+ * </pre>
+ *
+ * <p>
+ * In addition you should periodically call {@link #maybeRefresh}. While it's
+ * possible to call this just before running each query, this is discouraged
+ * since it penalizes the unlucky queries that need to refresh. It's better to use
+ * a separate background thread, that periodically calls {@link #maybeRefresh}. Finally,
+ * be sure to call {@link #close} once you are done.
+ *
+ * @see SearcherFactory
+ *
+ * @lucene.experimental
+ */
+// This file was copied from:
+// https://github.com/apache/lucene-solr/blob/lucene_solr_5_0/lucene/core/src/java/org/apache/lucene/search/SearcherManager.java
+// The only change (other than class name and import fixes)
+// is to skip the check in getSearcher that searcherFactory.newSearcher wraps
+// the provided searcher exactly.
+final class WrappableSearcherManager extends ReferenceManager<IndexSearcher> {
+
+ private final SearcherFactory searcherFactory;
+
+ /**
+ * Creates and returns a new SearcherManager from the given
+ * {@link IndexWriter}.
+ *
+ * @param writer
+ * the IndexWriter to open the IndexReader from.
+ * @param applyAllDeletes
+ * If <code>true</code>, all buffered deletes will be applied (made
+ * visible) in the {@link IndexSearcher} / {@link DirectoryReader}.
+ * If <code>false</code>, the deletes may or may not be applied, but
+ * remain buffered (in IndexWriter) so that they will be applied in
+ * the future. Applying deletes can be costly, so if your app can
+ * tolerate deleted documents being returned you might gain some
+ * performance by passing <code>false</code>. See
+ * {@link DirectoryReader#openIfChanged(DirectoryReader, IndexWriter, boolean)}.
+ * @param searcherFactory
+ * An optional {@link SearcherFactory}. Pass <code>null</code> if you
+ * don't require the searcher to be warmed before going live or other
+ * custom behavior.
+ *
+ * @throws IOException if there is a low-level I/O error
+ */
+ public WrappableSearcherManager(IndexWriter writer, boolean applyAllDeletes, SearcherFactory searcherFactory) throws IOException {
+ if (searcherFactory == null) {
+ searcherFactory = new SearcherFactory();
+ }
+ this.searcherFactory = searcherFactory;
+ current = getSearcher(searcherFactory, DirectoryReader.open(writer, applyAllDeletes));
+ }
+
+ /**
+ * Creates and returns a new SearcherManager from the given {@link Directory}.
+ * @param dir the directory to open the DirectoryReader on.
+ * @param searcherFactory An optional {@link SearcherFactory}. Pass
+ * <code>null</code> if you don't require the searcher to be warmed
+ * before going live or other custom behavior.
+ *
+ * @throws IOException if there is a low-level I/O error
+ */
+ public WrappableSearcherManager(Directory dir, SearcherFactory searcherFactory) throws IOException {
+ if (searcherFactory == null) {
+ searcherFactory = new SearcherFactory();
+ }
+ this.searcherFactory = searcherFactory;
+ current = getSearcher(searcherFactory, DirectoryReader.open(dir));
+ }
+
+ /**
+ * Creates and returns a new SearcherManager from an existing {@link DirectoryReader}. Note that
+ * this steals the incoming reference.
+ *
+ * @param reader the DirectoryReader.
+ * @param searcherFactory An optional {@link SearcherFactory}. Pass
+ * <code>null</code> if you don't require the searcher to be warmed
+ * before going live or other custom behavior.
+ *
+ * @throws IOException if there is a low-level I/O error
+ */
+ public WrappableSearcherManager(DirectoryReader reader, SearcherFactory searcherFactory) throws IOException {
+ if (searcherFactory == null) {
+ searcherFactory = new SearcherFactory();
+ }
+ this.searcherFactory = searcherFactory;
+ this.current = getSearcher(searcherFactory, reader);
+ }
+
+ @Override
+ protected void decRef(IndexSearcher reference) throws IOException {
+ reference.getIndexReader().decRef();
+ }
+
+ @Override
+ protected IndexSearcher refreshIfNeeded(IndexSearcher referenceToRefresh) throws IOException {
+ final IndexReader r = referenceToRefresh.getIndexReader();
+ assert r instanceof DirectoryReader: "searcher's IndexReader should be a DirectoryReader, but got " + r;
+ final IndexReader newReader = DirectoryReader.openIfChanged((DirectoryReader) r);
+ if (newReader == null) {
+ return null;
+ } else {
+ return getSearcher(searcherFactory, newReader);
+ }
+ }
+
+ @Override
+ protected boolean tryIncRef(IndexSearcher reference) {
+ return reference.getIndexReader().tryIncRef();
+ }
+
+ @Override
+ protected int getRefCount(IndexSearcher reference) {
+ return reference.getIndexReader().getRefCount();
+ }
+
+ /**
+ * Returns <code>true</code> if no changes have occured since this searcher
+ * ie. reader was opened, otherwise <code>false</code>.
+ * @see DirectoryReader#isCurrent()
+ */
+ public boolean isSearcherCurrent() throws IOException {
+ final IndexSearcher searcher = acquire();
+ try {
+ final IndexReader r = searcher.getIndexReader();
+ assert r instanceof DirectoryReader: "searcher's IndexReader should be a DirectoryReader, but got " + r;
+ return ((DirectoryReader) r).isCurrent();
+ } finally {
+ release(searcher);
+ }
+ }
+
+ /** Expert: creates a searcher from the provided {@link
+ * IndexReader} using the provided {@link
+ * SearcherFactory}. NOTE: this decRefs incoming reader
+ * on throwing an exception. */
+ @SuppressWarnings("resource")
+ public static IndexSearcher getSearcher(SearcherFactory searcherFactory, IndexReader reader) throws IOException {
+ boolean success = false;
+ final IndexSearcher searcher;
+ try {
+ searcher = searcherFactory.newSearcher(reader);
+ // Modification for Gerrit: Allow searcherFactory to transitively wrap the
+ // provided reader.
+ IndexReader unwrapped = searcher.getIndexReader();
+ while (true) {
+ if (unwrapped == reader) {
+ break;
+ } else if (unwrapped instanceof FilterDirectoryReader) {
+ unwrapped = ((FilterDirectoryReader) unwrapped).getDelegate();
+ } else if (unwrapped instanceof FilterLeafReader) {
+ unwrapped = ((FilterLeafReader) unwrapped).getDelegate();
+ } else {
+ break;
+ }
+ }
+
+ if (unwrapped != reader) {
+ throw new IllegalStateException("SearcherFactory must wrap the provided reader (got " + searcher.getIndexReader() + " but expected " + reader + ")");
+ }
+ success = true;
+ } finally {
+ if (!success) {
+ reader.decRef();
+ }
+ }
+ return searcher;
+ }
+}
diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java
index 821343f..0040762 100644
--- a/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java
+++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java
@@ -55,6 +55,10 @@
* {@link ChangeQueryBuilder} for querying that field, and a method on
* {@link ChangeData} used for populating the corresponding document fields in
* the secondary index.
+ * <p>
+ * Field names are all lowercase alphanumeric plus underscore; index
+ * implementations may create unambiguous derived field names containing other
+ * characters.
*/
public class ChangeField {
/** Legacy change ID. */
diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/FieldDef.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/FieldDef.java
index 557faeb..cf3fd09 100644
--- a/gerrit-server/src/main/java/com/google/gerrit/server/index/FieldDef.java
+++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/FieldDef.java
@@ -14,6 +14,9 @@
package com.google.gerrit.server.index;
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.base.CharMatcher;
import com.google.common.base.Preconditions;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.TrackingFooters;
@@ -78,11 +81,17 @@
private final boolean stored;
private FieldDef(String name, FieldType<?> type, boolean stored) {
- this.name = name;
+ this.name = checkName(name);
this.type = type;
this.stored = stored;
}
+ private static String checkName(String name) {
+ CharMatcher m = CharMatcher.anyOf("abcdefghijklmnopqrstuvwxyz0123456789_");
+ checkArgument(m.matchesAllOf(name), "illegal field name: %s", name);
+ return name;
+ }
+
/** @return name of the field. */
public final String getName() {
return name;
diff --git a/gerrit-server/src/test/java/com/google/gerrit/server/query/change/LuceneQueryChangesV14Test.java b/gerrit-server/src/test/java/com/google/gerrit/server/query/change/LuceneQueryChangesV14Test.java
new file mode 100644
index 0000000..42f5072
--- /dev/null
+++ b/gerrit-server/src/test/java/com/google/gerrit/server/query/change/LuceneQueryChangesV14Test.java
@@ -0,0 +1,49 @@
+// Copyright (C) 2015 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.query.change;
+
+import com.google.gerrit.testutil.InMemoryModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+
+import org.eclipse.jgit.lib.Config;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class LuceneQueryChangesV14Test extends LuceneQueryChangesTest {
+
+ @Override
+ protected Injector createInjector() {
+ Config luceneConfig = new Config(config);
+ InMemoryModule.setDefaults(luceneConfig);
+ // Latest version with a Lucene 4 index.
+ luceneConfig.setInt("index", "lucene", "testVersion", 14);
+ return Guice.createInjector(new InMemoryModule(luceneConfig));
+ }
+
+ @Override
+ @Ignore
+ @Test
+ public void byCommentBy() {
+ // Ignore.
+ }
+
+ @Override
+ @Ignore
+ @Test
+ public void byFrom() {
+ // Ignore.
+ }
+}
diff --git a/lib/asciidoctor/java/DocIndexer.java b/lib/asciidoctor/java/DocIndexer.java
index 7eb70c1..f06c662 100644
--- a/lib/asciidoctor/java/DocIndexer.java
+++ b/lib/asciidoctor/java/DocIndexer.java
@@ -25,7 +25,6 @@
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.Version;
import org.kohsuke.args4j.Argument;
import org.kohsuke.args4j.CmdLineException;
import org.kohsuke.args4j.CmdLineParser;
@@ -51,8 +50,6 @@
import java.util.zip.ZipOutputStream;
public class DocIndexer {
- @SuppressWarnings("deprecation")
- private static final Version LUCENE_VERSION = Version.LUCENE_4_10_1;
private static final Pattern SECTION_HEADER = Pattern.compile("^=+ (.*)");
@Option(name = "-o", usage = "output JAR file")
@@ -99,9 +96,9 @@
UnsupportedEncodingException, FileNotFoundException {
RAMDirectory directory = new RAMDirectory();
IndexWriterConfig config = new IndexWriterConfig(
- LUCENE_VERSION,
new StandardAnalyzer(CharArraySet.EMPTY_SET));
config.setOpenMode(OpenMode.CREATE);
+ config.setCommitOnClose(true);
IndexWriter iwriter = new IndexWriter(directory, config);
for (String inputFile : inputFiles) {
File file = new File(inputFile);
diff --git a/lib/lucene/BUCK b/lib/lucene/BUCK
index 9026f79..275f0bb 100644
--- a/lib/lucene/BUCK
+++ b/lib/lucene/BUCK
@@ -1,11 +1,11 @@
include_defs('//lib/maven.defs')
-VERSION = '4.10.2'
+VERSION = '5.0.0'
maven_jar(
name = 'core',
id = 'org.apache.lucene:lucene-core:' + VERSION,
- sha1 = 'c01e3d675d277e0a93e7890d03cc3246b2cdecaa',
+ sha1 = '4395e5ea987af804c4a9b96131e2ee75db061fdf',
license = 'Apache2.0',
exclude = [
'META-INF/LICENSE.txt',
@@ -16,8 +16,33 @@
maven_jar(
name = 'analyzers-common',
id = 'org.apache.lucene:lucene-analyzers-common:' + VERSION,
- sha1 = 'f977f8c443e8f4e9d1fd7fdfda80a6cf60b3e7c2',
+ sha1 = '6159cbc5c9631ef75e1f0e97b358ecdd8f1447a9',
license = 'Apache2.0',
+ deps = [':core'],
+ exclude = [
+ 'META-INF/LICENSE.txt',
+ 'META-INF/NOTICE.txt',
+ ],
+)
+
+maven_jar(
+ name = 'backward-codecs',
+ id = 'org.apache.lucene:lucene-backward-codecs:' + VERSION,
+ sha1 = '5cd11fc1be436ff96b63f0f76f299a9d25543b0b',
+ license = 'Apache2.0',
+ deps = [':core'],
+ exclude = [
+ 'META-INF/LICENSE.txt',
+ 'META-INF/NOTICE.txt',
+ ],
+)
+
+maven_jar(
+ name = 'misc',
+ id = 'org.apache.lucene:lucene-misc:' + VERSION,
+ sha1 = '06bd7cb030e598da81a8228f5c58630e5ce7b84a',
+ license = 'Apache2.0',
+ deps = [':core'],
exclude = [
'META-INF/LICENSE.txt',
'META-INF/NOTICE.txt',
@@ -27,6 +52,11 @@
maven_jar(
name = 'query-parser',
id = 'org.apache.lucene:lucene-queryparser:' + VERSION,
- sha1 = 'd70f54e1060d553ba7aeb4d49a71fd0c068499e8',
+ sha1 = 'f459326c0b58bb837612bfeb37f6015c1a8962db',
license = 'Apache2.0',
+ deps = [':core'],
+ exclude = [
+ 'META-INF/LICENSE.txt',
+ 'META-INF/NOTICE.txt',
+ ],
)
