Initial implementation of group index in Lucene
Same as for the account index there is no open/closed split.
Change-Id: Iff98ecab6e8eb07a49b915c4b97bab77f9809d84
Signed-off-by: Edwin Kempin <ekempin@google.com>
diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneGroupIndex.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneGroupIndex.java
new file mode 100644
index 0000000..436ee70
--- /dev/null
+++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneGroupIndex.java
@@ -0,0 +1,198 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.lucene;
+
+import static com.google.gerrit.server.index.group.GroupField.UUID;
+
+import com.google.gerrit.reviewdb.client.AccountGroup;
+import com.google.gerrit.server.account.GroupCache;
+import com.google.gerrit.server.config.GerritServerConfig;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.index.IndexUtils;
+import com.google.gerrit.server.index.QueryOptions;
+import com.google.gerrit.server.index.Schema;
+import com.google.gerrit.server.index.group.GroupIndex;
+import com.google.gerrit.server.query.DataSource;
+import com.google.gerrit.server.query.Predicate;
+import com.google.gerrit.server.query.QueryParseException;
+import com.google.gwtorm.server.OrmException;
+import com.google.gwtorm.server.ResultSet;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.SearcherFactory;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TopFieldDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.RAMDirectory;
+import org.eclipse.jgit.lib.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
+public class LuceneGroupIndex extends
+ AbstractLuceneIndex<AccountGroup.UUID, AccountGroup> implements GroupIndex {
+ private static final Logger log =
+ LoggerFactory.getLogger(LuceneGroupIndex.class);
+
+ private static final String GROUPS = "groups";
+
+ private static final String UUID_SORT_FIELD = sortFieldName(UUID);
+
+ private static Term idTerm(AccountGroup group) {
+ return idTerm(group.getGroupUUID());
+ }
+
+ private static Term idTerm(AccountGroup.UUID uuid) {
+ return QueryBuilder.stringTerm(UUID.getName(), uuid.get());
+ }
+
+ private final GerritIndexWriterConfig indexWriterConfig;
+ private final QueryBuilder<AccountGroup> queryBuilder;
+ private final GroupCache groupCache;
+
+ private static Directory dir(Schema<AccountGroup> schema, Config cfg,
+ SitePaths sitePaths) throws IOException {
+ if (LuceneIndexModule.isInMemoryTest(cfg)) {
+ return new RAMDirectory();
+ }
+ Path indexDir =
+ LuceneVersionManager.getDir(sitePaths, GROUPS + "_", schema);
+ return FSDirectory.open(indexDir);
+ }
+
+ @Inject
+ LuceneGroupIndex(
+ @GerritServerConfig Config cfg,
+ SitePaths sitePaths,
+ GroupCache groupCache,
+ @Assisted Schema<AccountGroup> schema) throws IOException {
+ super(schema, sitePaths, dir(schema, cfg, sitePaths), GROUPS, null,
+ new GerritIndexWriterConfig(cfg, GROUPS), new SearcherFactory());
+ this.groupCache = groupCache;
+
+ indexWriterConfig =
+ new GerritIndexWriterConfig(cfg, GROUPS);
+ queryBuilder = new QueryBuilder<>(schema, indexWriterConfig.getAnalyzer());
+ }
+
+ @Override
+ public void replace(AccountGroup group) throws IOException {
+ try {
+ // No parts of FillArgs are currently required, just use null.
+ replace(idTerm(group), toDocument(group, null)).get();
+ } catch (ExecutionException | InterruptedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public void delete(AccountGroup.UUID key) throws IOException {
+ try {
+ delete(idTerm(key)).get();
+ } catch (ExecutionException | InterruptedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public DataSource<AccountGroup> getSource(Predicate<AccountGroup> p,
+ QueryOptions opts) throws QueryParseException {
+ return new QuerySource(opts, queryBuilder.toQuery(p),
+ new Sort(new SortField(UUID_SORT_FIELD, SortField.Type.STRING, false)));
+ }
+
+ private class QuerySource implements DataSource<AccountGroup> {
+ private final QueryOptions opts;
+ private final Query query;
+ private final Sort sort;
+
+ private QuerySource(QueryOptions opts, Query query, Sort sort) {
+ this.opts = opts;
+ this.query = query;
+ this.sort = sort;
+ }
+
+ @Override
+ public int getCardinality() {
+ return 10;
+ }
+
+ @Override
+ public ResultSet<AccountGroup> read() throws OrmException {
+ IndexSearcher searcher = null;
+ try {
+ searcher = acquire();
+ int realLimit = opts.start() + opts.limit();
+ TopFieldDocs docs = searcher.search(query, realLimit, sort);
+ List<AccountGroup> result = new ArrayList<>(docs.scoreDocs.length);
+ for (int i = opts.start(); i < docs.scoreDocs.length; i++) {
+ ScoreDoc sd = docs.scoreDocs[i];
+ Document doc = searcher.doc(sd.doc, IndexUtils.groupFields(opts));
+ result.add(toAccountGroup(doc));
+ }
+ final List<AccountGroup> r = Collections.unmodifiableList(result);
+ return new ResultSet<AccountGroup>() {
+ @Override
+ public Iterator<AccountGroup> iterator() {
+ return r.iterator();
+ }
+
+ @Override
+ public List<AccountGroup> toList() {
+ return r;
+ }
+
+ @Override
+ public void close() {
+ // Do nothing.
+ }
+ };
+ } catch (IOException e) {
+ throw new OrmException(e);
+ } finally {
+ if (searcher != null) {
+ try {
+ release(searcher);
+ } catch (IOException e) {
+ log.warn("cannot release Lucene searcher", e);
+ }
+ }
+ }
+ }
+ }
+
+ private AccountGroup toAccountGroup(Document doc) {
+ AccountGroup.UUID uuid =
+ new AccountGroup.UUID(doc.getField(UUID.getName()).stringValue());
+ // Use the GroupCache rather than depending on any stored fields in the
+ // document (of which there shouldn't be any).
+ return groupCache.get(uuid);
+ }
+}
diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java
index d23c5bb..1d1ca95 100644
--- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java
+++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java
@@ -22,6 +22,7 @@
import com.google.gerrit.server.index.SingleVersionModule;
import com.google.gerrit.server.index.account.AccountIndex;
import com.google.gerrit.server.index.change.ChangeIndex;
+import com.google.gerrit.server.index.group.GroupIndex;
import com.google.inject.Provides;
import com.google.inject.Singleton;
import com.google.inject.assistedinject.FactoryModuleBuilder;
@@ -61,12 +62,16 @@
protected void configure() {
install(
new FactoryModuleBuilder()
+ .implement(AccountIndex.class, LuceneAccountIndex.class)
+ .build(AccountIndex.Factory.class));
+ install(
+ new FactoryModuleBuilder()
.implement(ChangeIndex.class, LuceneChangeIndex.class)
.build(ChangeIndex.Factory.class));
install(
new FactoryModuleBuilder()
- .implement(AccountIndex.class, LuceneAccountIndex.class)
- .build(AccountIndex.Factory.class));
+ .implement(GroupIndex.class, LuceneGroupIndex.class)
+ .build(GroupIndex.Factory.class));
install(new IndexModule(threads));
if (singleVersions == null) {
diff --git a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/QueryBuilder.java b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/QueryBuilder.java
index a993b49..cfec648 100644
--- a/gerrit-lucene/src/main/java/com/google/gerrit/lucene/QueryBuilder.java
+++ b/gerrit-lucene/src/main/java/com/google/gerrit/lucene/QueryBuilder.java
@@ -15,6 +15,7 @@
package com.google.gerrit.lucene;
import static com.google.common.base.Preconditions.checkArgument;
+import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.lucene.search.BooleanClause.Occur.MUST;
import static org.apache.lucene.search.BooleanClause.Occur.MUST_NOT;
import static org.apache.lucene.search.BooleanClause.Occur.SHOULD;
@@ -54,6 +55,12 @@
return new Term(name, builder.get());
}
+ static Term stringTerm(String name, String value) {
+ BytesRefBuilder builder = new BytesRefBuilder();
+ builder.append(value.getBytes(UTF_8), 0, value.length());
+ return new Term(name, builder.get());
+ }
+
private final Schema<V> schema;
private final org.apache.lucene.util.QueryBuilder queryBuilder;
diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/DummyIndexModule.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/DummyIndexModule.java
index ed196c1..1706761 100644
--- a/gerrit-server/src/main/java/com/google/gerrit/server/index/DummyIndexModule.java
+++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/DummyIndexModule.java
@@ -14,10 +14,12 @@
package com.google.gerrit.server.index;
+import com.google.gerrit.reviewdb.client.AccountGroup;
import com.google.gerrit.server.account.AccountState;
import com.google.gerrit.server.index.account.AccountIndex;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.change.DummyChangeIndex;
+import com.google.gerrit.server.index.group.GroupIndex;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.inject.AbstractModule;
@@ -36,6 +38,13 @@
}
}
+ private static class DummyGroupIndexFactory implements GroupIndex.Factory {
+ @Override
+ public GroupIndex create(Schema<AccountGroup> schema) {
+ throw new UnsupportedOperationException();
+ }
+ }
+
@Override
protected void configure() {
install(new IndexModule(1));
@@ -43,5 +52,6 @@
bind(Index.class).toInstance(new DummyChangeIndex());
bind(AccountIndex.Factory.class).toInstance(new DummyAccountIndexFactory());
bind(ChangeIndex.Factory.class).toInstance(new DummyChangeIndexFactory());
+ bind(GroupIndex.Factory.class).toInstance(new DummyGroupIndexFactory());
}
}
diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java
index 9e0be86..13e7b85 100644
--- a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java
+++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java
@@ -36,6 +36,8 @@
import com.google.gerrit.server.index.change.ChangeIndexRewriter;
import com.google.gerrit.server.index.change.ChangeIndexer;
import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
+import com.google.gerrit.server.index.group.GroupIndexDefinition;
+import com.google.gerrit.server.index.group.GroupSchemaDefinitions;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.Provides;
@@ -61,7 +63,8 @@
public static final ImmutableCollection<SchemaDefinitions<?>> ALL_SCHEMA_DEFS =
ImmutableList.<SchemaDefinitions<?>> of(
AccountSchemaDefinitions.INSTANCE,
- ChangeSchemaDefinitions.INSTANCE);
+ ChangeSchemaDefinitions.INSTANCE,
+ GroupSchemaDefinitions.INSTANCE);
/** Type of secondary index. */
public static IndexType getIndexType(Injector injector) {
@@ -103,10 +106,12 @@
@Provides
Collection<IndexDefinition<?, ?, ?>> getIndexDefinitions(
AccountIndexDefinition accounts,
- ChangeIndexDefinition changes) {
+ ChangeIndexDefinition changes,
+ GroupIndexDefinition groups) {
Collection<IndexDefinition<?, ?, ?>> result =
ImmutableList.<IndexDefinition<?, ?, ?>> of(
accounts,
+ groups,
changes);
Set<String> expected = FluentIterable.from(ALL_SCHEMA_DEFS)
.transform(SchemaDefinitions::getName)
diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexUtils.java b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexUtils.java
index bcc7f7b..eceec59 100644
--- a/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexUtils.java
+++ b/gerrit-server/src/main/java/com/google/gerrit/server/index/IndexUtils.java
@@ -14,7 +14,6 @@
package com.google.gerrit.server.index;
-import static com.google.gerrit.server.index.account.AccountField.ID;
import static com.google.gerrit.server.index.change.ChangeField.CHANGE;
import static com.google.gerrit.server.index.change.ChangeField.LEGACY_ID;
import static com.google.gerrit.server.index.change.ChangeField.PROJECT;
@@ -23,6 +22,8 @@
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.index.account.AccountField;
+import com.google.gerrit.server.index.group.GroupField;
import org.eclipse.jgit.errors.ConfigInvalidException;
@@ -47,9 +48,9 @@
public static Set<String> accountFields(QueryOptions opts) {
Set<String> fs = opts.fields();
- return fs.contains(ID.getName())
+ return fs.contains(AccountField.ID.getName())
? fs
- : Sets.union(fs, ImmutableSet.of(ID.getName()));
+ : Sets.union(fs, ImmutableSet.of(AccountField.ID.getName()));
}
public static Set<String> changeFields(QueryOptions opts) {
@@ -68,6 +69,13 @@
ImmutableSet.of(LEGACY_ID.getName(), PROJECT.getName()));
}
+ public static Set<String> groupFields(QueryOptions opts) {
+ Set<String> fs = opts.fields();
+ return fs.contains(GroupField.UUID.getName())
+ ? fs
+ : Sets.union(fs, ImmutableSet.of(GroupField.UUID.getName()));
+ }
+
private IndexUtils() {
// hide default constructor
}
