Merge "Fix buggy Index-Interactive Executor" into stable-2.16
diff --git a/Documentation/config-gerrit.txt b/Documentation/config-gerrit.txt
index 749d4c2..9885e71 100644
--- a/Documentation/config-gerrit.txt
+++ b/Documentation/config-gerrit.txt
@@ -3130,6 +3130,11 @@
+
Defaults to true (throttling enabled).
+During offline reindexing, setting ramBufferSize greater than the size
+of index (size of specific index folder under <site_dir>/index) and
+maxBufferedDocs as -1 avoids unnecessary flushes and triggers only a
+single flush at the end of the process.
+
Sample Lucene index configuration:
----
[index]
diff --git a/java/com/google/gerrit/acceptance/GerritServer.java b/java/com/google/gerrit/acceptance/GerritServer.java
index a397923..b18df5b 100644
--- a/java/com/google/gerrit/acceptance/GerritServer.java
+++ b/java/com/google/gerrit/acceptance/GerritServer.java
@@ -35,6 +35,7 @@
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePath;
import com.google.gerrit.server.git.receive.AsyncReceiveCommits;
+import com.google.gerrit.server.index.AutoFlush;
import com.google.gerrit.server.ssh.NoSshModule;
import com.google.gerrit.server.util.ManualRequestContext;
import com.google.gerrit.server.util.OneOffRequestContext;
@@ -353,7 +354,8 @@
cfg.setBoolean("index", null, "onlineUpgrade", false);
cfg.setString("gitweb", null, "cgi", "");
daemon.setEnableHttpd(desc.httpd());
- daemon.setLuceneModule(LuceneIndexModule.singleVersionAllLatest(0, isSlave(baseConfig)));
+ daemon.setLuceneModule(
+ LuceneIndexModule.singleVersionAllLatest(0, isSlave(baseConfig), AutoFlush.ENABLED));
daemon.setDatabaseForTesting(
ImmutableList.<Module>of(
new InMemoryTestingDatabaseModule(
diff --git a/java/com/google/gerrit/httpd/init/WebAppInitializer.java b/java/com/google/gerrit/httpd/init/WebAppInitializer.java
index c2d4a14..9f2ecb4 100644
--- a/java/com/google/gerrit/httpd/init/WebAppInitializer.java
+++ b/java/com/google/gerrit/httpd/init/WebAppInitializer.java
@@ -74,6 +74,7 @@
import com.google.gerrit.server.git.GitRepositoryManagerModule;
import com.google.gerrit.server.git.SearchingChangeCacheImpl;
import com.google.gerrit.server.git.WorkQueue;
+import com.google.gerrit.server.index.AutoFlush;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.IndexModule.IndexType;
import com.google.gerrit.server.index.OnlineUpgrader;
@@ -387,7 +388,7 @@
private Module createIndexModule() {
switch (indexType) {
case LUCENE:
- return LuceneIndexModule.latestVersion(false);
+ return LuceneIndexModule.latestVersion(false, AutoFlush.ENABLED);
case ELASTICSEARCH:
return ElasticIndexModule.latestVersion(false);
default:
diff --git a/java/com/google/gerrit/lucene/AbstractLuceneIndex.java b/java/com/google/gerrit/lucene/AbstractLuceneIndex.java
index cb26488..ef42fe0 100644
--- a/java/com/google/gerrit/lucene/AbstractLuceneIndex.java
+++ b/java/com/google/gerrit/lucene/AbstractLuceneIndex.java
@@ -39,6 +39,7 @@
import com.google.gerrit.index.query.DataSource;
import com.google.gerrit.index.query.FieldBundle;
import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.index.AutoFlush;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.logging.LoggingContextAwareExecutorService;
import com.google.gerrit.server.logging.LoggingContextAwareScheduledExecutorService;
@@ -103,6 +104,7 @@
private final ReferenceManager<IndexSearcher> searcherManager;
private final ControlledRealTimeReopenThread<IndexSearcher> reopenThread;
private final Set<NrtFuture> notDoneNrtFutures;
+ private final AutoFlush autoFlush;
private ScheduledExecutorService autoCommitExecutor;
AbstractLuceneIndex(
@@ -112,12 +114,14 @@
String name,
String subIndex,
GerritIndexWriterConfig writerConfig,
- SearcherFactory searcherFactory)
+ SearcherFactory searcherFactory,
+ AutoFlush autoFlush)
throws IOException {
this.schema = schema;
this.sitePaths = sitePaths;
this.dir = dir;
this.name = name;
+ this.autoFlush = autoFlush;
String index = Joiner.on('_').skipNulls().join(name, subIndex);
long commitPeriod = writerConfig.getCommitWithinMs();
@@ -211,7 +215,9 @@
}
});
- reopenThread.start();
+ if (autoFlush.equals(AutoFlush.ENABLED)) {
+ reopenThread.start();
+ }
}
@Override
@@ -457,6 +463,9 @@
}
private boolean isGenAvailableNowForCurrentSearcher() {
+ if (autoFlush.equals(AutoFlush.DISABLED)) {
+ return true;
+ }
try {
return reopenThread.waitForGeneration(gen, 0);
} catch (InterruptedException e) {
diff --git a/java/com/google/gerrit/lucene/ChangeSubIndex.java b/java/com/google/gerrit/lucene/ChangeSubIndex.java
index 7d7cbef..5c78c34 100644
--- a/java/com/google/gerrit/lucene/ChangeSubIndex.java
+++ b/java/com/google/gerrit/lucene/ChangeSubIndex.java
@@ -29,6 +29,7 @@
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.index.AutoFlush;
import com.google.gerrit.server.index.change.ChangeField;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.query.change.ChangeData;
@@ -48,7 +49,8 @@
SitePaths sitePaths,
Path path,
GerritIndexWriterConfig writerConfig,
- SearcherFactory searcherFactory)
+ SearcherFactory searcherFactory,
+ AutoFlush autoFlush)
throws IOException {
this(
schema,
@@ -56,7 +58,8 @@
FSDirectory.open(path),
path.getFileName().toString(),
writerConfig,
- searcherFactory);
+ searcherFactory,
+ autoFlush);
}
ChangeSubIndex(
@@ -65,9 +68,10 @@
Directory dir,
String subIndex,
GerritIndexWriterConfig writerConfig,
- SearcherFactory searcherFactory)
+ SearcherFactory searcherFactory,
+ AutoFlush autoFlush)
throws IOException {
- super(schema, sitePaths, dir, NAME, subIndex, writerConfig, searcherFactory);
+ super(schema, sitePaths, dir, NAME, subIndex, writerConfig, searcherFactory, autoFlush);
}
@Override
diff --git a/java/com/google/gerrit/lucene/LuceneAccountIndex.java b/java/com/google/gerrit/lucene/LuceneAccountIndex.java
index 86a2111..84dd16d 100644
--- a/java/com/google/gerrit/lucene/LuceneAccountIndex.java
+++ b/java/com/google/gerrit/lucene/LuceneAccountIndex.java
@@ -31,6 +31,7 @@
import com.google.gerrit.server.account.AccountState;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.index.AutoFlush;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.account.AccountIndex;
import com.google.inject.Inject;
@@ -86,7 +87,8 @@
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Provider<AccountCache> accountCache,
- @Assisted Schema<AccountState> schema)
+ @Assisted Schema<AccountState> schema,
+ AutoFlush autoFlush)
throws IOException {
super(
schema,
@@ -95,7 +97,8 @@
ACCOUNTS,
null,
new GerritIndexWriterConfig(cfg, ACCOUNTS),
- new SearcherFactory());
+ new SearcherFactory(),
+ autoFlush);
this.accountCache = accountCache;
indexWriterConfig = new GerritIndexWriterConfig(cfg, ACCOUNTS);
diff --git a/java/com/google/gerrit/lucene/LuceneChangeIndex.java b/java/com/google/gerrit/lucene/LuceneChangeIndex.java
index 5194e5d..8c22dce 100644
--- a/java/com/google/gerrit/lucene/LuceneChangeIndex.java
+++ b/java/com/google/gerrit/lucene/LuceneChangeIndex.java
@@ -50,6 +50,7 @@
import com.google.gerrit.server.StarredChangesUtil;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.index.AutoFlush;
import com.google.gerrit.server.index.IndexExecutor;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.change.ChangeField;
@@ -155,7 +156,8 @@
@IndexExecutor(INTERACTIVE) ListeningExecutorService executor,
Provider<ReviewDb> db,
ChangeData.Factory changeDataFactory,
- @Assisted Schema<ChangeData> schema)
+ @Assisted Schema<ChangeData> schema,
+ AutoFlush autoFlush)
throws IOException {
this.executor = executor;
this.db = db;
@@ -171,18 +173,35 @@
if (LuceneIndexModule.isInMemoryTest(cfg)) {
openIndex =
new ChangeSubIndex(
- schema, sitePaths, new RAMDirectory(), "ramOpen", openConfig, searcherFactory);
+ schema,
+ sitePaths,
+ new RAMDirectory(),
+ "ramOpen",
+ openConfig,
+ searcherFactory,
+ autoFlush);
closedIndex =
new ChangeSubIndex(
- schema, sitePaths, new RAMDirectory(), "ramClosed", closedConfig, searcherFactory);
+ schema,
+ sitePaths,
+ new RAMDirectory(),
+ "ramClosed",
+ closedConfig,
+ searcherFactory,
+ autoFlush);
} else {
Path dir = LuceneVersionManager.getDir(sitePaths, CHANGES, schema);
openIndex =
new ChangeSubIndex(
- schema, sitePaths, dir.resolve(CHANGES_OPEN), openConfig, searcherFactory);
+ schema, sitePaths, dir.resolve(CHANGES_OPEN), openConfig, searcherFactory, autoFlush);
closedIndex =
new ChangeSubIndex(
- schema, sitePaths, dir.resolve(CHANGES_CLOSED), closedConfig, searcherFactory);
+ schema,
+ sitePaths,
+ dir.resolve(CHANGES_CLOSED),
+ closedConfig,
+ searcherFactory,
+ autoFlush);
}
}
diff --git a/java/com/google/gerrit/lucene/LuceneGroupIndex.java b/java/com/google/gerrit/lucene/LuceneGroupIndex.java
index 95e2ab9..40b455b 100644
--- a/java/com/google/gerrit/lucene/LuceneGroupIndex.java
+++ b/java/com/google/gerrit/lucene/LuceneGroupIndex.java
@@ -29,6 +29,7 @@
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.group.InternalGroup;
+import com.google.gerrit.server.index.AutoFlush;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.group.GroupIndex;
import com.google.inject.Inject;
@@ -82,7 +83,8 @@
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Provider<GroupCache> groupCache,
- @Assisted Schema<InternalGroup> schema)
+ @Assisted Schema<InternalGroup> schema,
+ AutoFlush autoFlush)
throws IOException {
super(
schema,
@@ -91,7 +93,8 @@
GROUPS,
null,
new GerritIndexWriterConfig(cfg, GROUPS),
- new SearcherFactory());
+ new SearcherFactory(),
+ autoFlush);
this.groupCache = groupCache;
indexWriterConfig = new GerritIndexWriterConfig(cfg, GROUPS);
diff --git a/java/com/google/gerrit/lucene/LuceneIndexModule.java b/java/com/google/gerrit/lucene/LuceneIndexModule.java
index 302a2da..10b58c0 100644
--- a/java/com/google/gerrit/lucene/LuceneIndexModule.java
+++ b/java/com/google/gerrit/lucene/LuceneIndexModule.java
@@ -19,6 +19,7 @@
import com.google.gerrit.index.project.ProjectIndex;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.index.AbstractIndexModule;
+import com.google.gerrit.server.index.AutoFlush;
import com.google.gerrit.server.index.VersionManager;
import com.google.gerrit.server.index.account.AccountIndex;
import com.google.gerrit.server.index.change.ChangeIndex;
@@ -28,25 +29,36 @@
import org.eclipse.jgit.lib.Config;
public class LuceneIndexModule extends AbstractIndexModule {
- public static LuceneIndexModule singleVersionAllLatest(int threads, boolean slave) {
- return new LuceneIndexModule(ImmutableMap.of(), threads, slave);
+ private final AutoFlush autoFlush;
+
+ public static LuceneIndexModule singleVersionAllLatest(
+ int threads, boolean slave, AutoFlush autoFlush) {
+ return new LuceneIndexModule(ImmutableMap.of(), threads, slave, autoFlush);
}
public static LuceneIndexModule singleVersionWithExplicitVersions(
- Map<String, Integer> versions, int threads, boolean slave) {
- return new LuceneIndexModule(versions, threads, slave);
+ Map<String, Integer> versions, int threads, boolean slave, AutoFlush autoFlush) {
+ return new LuceneIndexModule(versions, threads, slave, autoFlush);
}
- public static LuceneIndexModule latestVersion(boolean slave) {
- return new LuceneIndexModule(null, 0, slave);
+ public static LuceneIndexModule latestVersion(boolean slave, AutoFlush autoFlush) {
+ return new LuceneIndexModule(null, 0, slave, autoFlush);
}
static boolean isInMemoryTest(Config cfg) {
return cfg.getBoolean("index", "lucene", "testInmemory", false);
}
- private LuceneIndexModule(Map<String, Integer> singleVersions, int threads, boolean slave) {
+ private LuceneIndexModule(
+ Map<String, Integer> singleVersions, int threads, boolean slave, AutoFlush autoFlush) {
super(singleVersions, threads, slave);
+ this.autoFlush = autoFlush;
+ }
+
+ @Override
+ protected void configure() {
+ super.configure();
+ bind(AutoFlush.class).toInstance(autoFlush);
}
@Override
diff --git a/java/com/google/gerrit/lucene/LuceneProjectIndex.java b/java/com/google/gerrit/lucene/LuceneProjectIndex.java
index 02d8655..5b2a2b7 100644
--- a/java/com/google/gerrit/lucene/LuceneProjectIndex.java
+++ b/java/com/google/gerrit/lucene/LuceneProjectIndex.java
@@ -29,6 +29,7 @@
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.index.AutoFlush;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectState;
@@ -82,7 +83,8 @@
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Provider<ProjectCache> projectCache,
- @Assisted Schema<ProjectData> schema)
+ @Assisted Schema<ProjectData> schema,
+ AutoFlush autoFlush)
throws IOException {
super(
schema,
@@ -91,7 +93,8 @@
PROJECTS,
null,
new GerritIndexWriterConfig(cfg, PROJECTS),
- new SearcherFactory());
+ new SearcherFactory(),
+ autoFlush);
this.projectCache = projectCache;
indexWriterConfig = new GerritIndexWriterConfig(cfg, PROJECTS);
diff --git a/java/com/google/gerrit/pgm/Daemon.java b/java/com/google/gerrit/pgm/Daemon.java
index 5f6015b..31735e6 100644
--- a/java/com/google/gerrit/pgm/Daemon.java
+++ b/java/com/google/gerrit/pgm/Daemon.java
@@ -83,6 +83,7 @@
import com.google.gerrit.server.git.SearchingChangeCacheImpl;
import com.google.gerrit.server.git.WorkQueue;
import com.google.gerrit.server.group.PeriodicGroupIndexer;
+import com.google.gerrit.server.index.AutoFlush;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.IndexModule.IndexType;
import com.google.gerrit.server.index.OnlineUpgrader;
@@ -529,7 +530,7 @@
}
switch (indexType) {
case LUCENE:
- return LuceneIndexModule.latestVersion(slave);
+ return LuceneIndexModule.latestVersion(slave, AutoFlush.ENABLED);
case ELASTICSEARCH:
return ElasticIndexModule.latestVersion(slave);
default:
diff --git a/java/com/google/gerrit/pgm/Reindex.java b/java/com/google/gerrit/pgm/Reindex.java
index 85d1bd6..3789bc9 100644
--- a/java/com/google/gerrit/pgm/Reindex.java
+++ b/java/com/google/gerrit/pgm/Reindex.java
@@ -32,6 +32,7 @@
import com.google.gerrit.pgm.util.ThreadLimiter;
import com.google.gerrit.server.change.ChangeResource;
import com.google.gerrit.server.config.GerritServerConfig;
+import com.google.gerrit.server.index.AutoFlush;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.IndexModule.IndexType;
import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
@@ -155,7 +156,9 @@
Module indexModule;
switch (IndexModule.getIndexType(dbInjector)) {
case LUCENE:
- indexModule = LuceneIndexModule.singleVersionWithExplicitVersions(versions, threads, slave);
+ indexModule =
+ LuceneIndexModule.singleVersionWithExplicitVersions(
+ versions, threads, slave, AutoFlush.DISABLED);
break;
case ELASTICSEARCH:
indexModule =
diff --git a/java/com/google/gerrit/pgm/init/index/lucene/LuceneIndexModuleOnInit.java b/java/com/google/gerrit/pgm/init/index/lucene/LuceneIndexModuleOnInit.java
index 12a44dc..54618a1 100644
--- a/java/com/google/gerrit/pgm/init/index/lucene/LuceneIndexModuleOnInit.java
+++ b/java/com/google/gerrit/pgm/init/index/lucene/LuceneIndexModuleOnInit.java
@@ -17,6 +17,7 @@
import com.google.gerrit.lucene.LuceneAccountIndex;
import com.google.gerrit.lucene.LuceneGroupIndex;
import com.google.gerrit.pgm.init.index.IndexModuleOnInit;
+import com.google.gerrit.server.index.AutoFlush;
import com.google.gerrit.server.index.account.AccountIndex;
import com.google.gerrit.server.index.group.GroupIndex;
import com.google.inject.AbstractModule;
@@ -36,5 +37,7 @@
.build(GroupIndex.Factory.class));
install(new IndexModuleOnInit());
+
+ bind(AutoFlush.class).toInstance(AutoFlush.DISABLED);
}
}
diff --git a/java/com/google/gerrit/server/index/AutoFlush.java b/java/com/google/gerrit/server/index/AutoFlush.java
new file mode 100644
index 0000000..3d13110
--- /dev/null
+++ b/java/com/google/gerrit/server/index/AutoFlush.java
@@ -0,0 +1,20 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.index;
+
+public enum AutoFlush {
+ ENABLED,
+ DISABLED
+}
diff --git a/java/com/google/gerrit/testing/InMemoryModule.java b/java/com/google/gerrit/testing/InMemoryModule.java
index 18dfea0..3a7b8b4 100644
--- a/java/com/google/gerrit/testing/InMemoryModule.java
+++ b/java/com/google/gerrit/testing/InMemoryModule.java
@@ -64,6 +64,7 @@
import com.google.gerrit.server.git.PerThreadRequestScope;
import com.google.gerrit.server.git.SearchingChangeCacheImpl;
import com.google.gerrit.server.git.WorkQueue;
+import com.google.gerrit.server.index.AutoFlush;
import com.google.gerrit.server.index.IndexModule.IndexType;
import com.google.gerrit.server.index.account.AccountSchemaDefinitions;
import com.google.gerrit.server.index.account.AllAccountsIndexer;
@@ -320,8 +321,13 @@
boolean slave = cfg.getBoolean("container", "slave", false);
Class<?> clazz = Class.forName(moduleClassName);
Method m =
- clazz.getMethod("singleVersionWithExplicitVersions", Map.class, int.class, boolean.class);
- return (Module) m.invoke(null, getSingleSchemaVersions(), 0, slave);
+ clazz.getMethod(
+ "singleVersionWithExplicitVersions",
+ Map.class,
+ int.class,
+ boolean.class,
+ AutoFlush.class);
+ return (Module) m.invoke(null, getSingleSchemaVersions(), 0, slave, AutoFlush.ENABLED);
} catch (ClassNotFoundException
| SecurityException
| NoSuchMethodException
