Avoid auto-reindex of projects during init when unneeded
The forced reindex is not needed when the project index is present.
Preserve the automatic reindex of projects for new sites and defer
to reindex site program or online reindexing if project index already
exists.
The check is implemented in index backend agnostic manner, so that it
should work for both supported backends: Lucene and Elasticsearch. It
should also work independently whether or not a project index schema
migration is needed during upgrade.
Reindexing projects is a very expensive operation and can turn a simple
upgrade to a long and painful operation because of the increase of the
migration time and the amount of memory needed.
Bug: Issue 12680
Change-Id: I3a4f1d07405f7bb631467d1f005d48cb56ee867f
diff --git a/java/com/google/gerrit/pgm/Init.java b/java/com/google/gerrit/pgm/Init.java
index 799377c..3593d8a 100644
--- a/java/com/google/gerrit/pgm/Init.java
+++ b/java/com/google/gerrit/pgm/Init.java
@@ -30,6 +30,7 @@
import com.google.gerrit.pgm.util.ErrorLogFile;
import com.google.gerrit.server.config.GerritServerConfigModule;
import com.google.gerrit.server.config.SitePath;
+import com.google.gerrit.server.index.GerritIndexStatus;
import com.google.gerrit.server.ioutil.HostPlatform;
import com.google.gerrit.server.securestore.SecureStoreClassName;
import com.google.gerrit.server.util.ReplicaUtil;
@@ -60,9 +61,6 @@
@Option(name = "--no-auto-start", usage = "Don't automatically start daemon after init")
private boolean noAutoStart;
- @Option(name = "--no-reindex", usage = "Don't automatically reindex any entities")
- private boolean noReindex;
-
@Option(name = "--skip-plugins", usage = "Don't install plugins")
private boolean skipPlugins;
@@ -91,6 +89,8 @@
@Inject Browser browser;
+ private boolean projectsIndexExists;
+
public Init() {
super(new WarDistribution(), null);
}
@@ -103,6 +103,7 @@
@Override
protected boolean beforeInit(SiteInit init) throws Exception {
+ projectsIndexExists = new GerritIndexStatus(init.site).exists(ProjectSchemaDefinitions.NAME);
ErrorLogFile.errorOnlyConsole();
if (!skipPlugins) {
@@ -145,7 +146,7 @@
});
modules.add(new GerritServerConfigModule());
Guice.createInjector(modules).injectMembers(this);
- if (!ReplicaUtil.isReplica(run.flags.cfg)) {
+ if (!ReplicaUtil.isReplica(run.flags.cfg) && !projectsIndexExists) {
reindexProjects();
}
start(run);
@@ -260,9 +261,6 @@
}
private void reindexProjects() throws Exception {
- if (noReindex) {
- return;
- }
// Reindex all projects, so that we bootstrap the project index for new installations
List<String> reindexArgs =
ImmutableList.of(
diff --git a/java/com/google/gerrit/server/index/GerritIndexStatus.java b/java/com/google/gerrit/server/index/GerritIndexStatus.java
index 6d59100..9f0622e 100644
--- a/java/com/google/gerrit/server/index/GerritIndexStatus.java
+++ b/java/com/google/gerrit/server/index/GerritIndexStatus.java
@@ -48,6 +48,10 @@
return cfg.getBoolean(SECTION, indexDirName(indexName, version), KEY_READY, false);
}
+ public boolean exists(String indexName) {
+ return cfg.getSubsections(SECTION).stream().anyMatch(n -> n.startsWith(indexName));
+ }
+
public void save() throws IOException {
cfg.save();
}
diff --git a/javatests/com/google/gerrit/acceptance/pgm/InitIT.java b/javatests/com/google/gerrit/acceptance/pgm/InitIT.java
index e48088e..4caee64 100644
--- a/javatests/com/google/gerrit/acceptance/pgm/InitIT.java
+++ b/javatests/com/google/gerrit/acceptance/pgm/InitIT.java
@@ -26,7 +26,16 @@
import com.google.gerrit.index.project.ProjectIndexCollection;
import com.google.gerrit.server.config.AllProjectsName;
import com.google.gerrit.server.config.AllUsersName;
+import java.io.IOException;
+import java.nio.file.FileVisitOption;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.Comparator;
import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.eclipse.jgit.util.FS;
import org.junit.Test;
@NoHttpd
@@ -34,7 +43,7 @@
@Test
public void indexesAllProjectsAndAllUsers() throws Exception {
- runGerrit("init", "-d", sitePaths.site_path.toString(), "--show-stack-trace");
+ initSite();
try (ServerContext ctx = startServer()) {
ProjectIndexCollection projectIndex =
ctx.getInjector().getInstance(ProjectIndexCollection.class);
@@ -48,4 +57,52 @@
assertThat(allUsersData).isPresent();
}
}
+
+ @Test
+ public void initDoesNotReindexProjectsOnExistingSites() throws Exception {
+ initSite();
+
+ // Simulate a projects indexes files modified in the past by 3 seconds
+ Optional<Instant> projectsLastModified =
+ getProjectsIndexLastModified(sitePaths.index_dir).map(t -> t.minusSeconds(3));
+ assertThat(projectsLastModified).isPresent();
+ setProjectsIndexLastModifiedInThePast(sitePaths.index_dir, projectsLastModified.get());
+
+ initSite();
+ Optional<Instant> projectsLastModifiedAfterInit =
+ getProjectsIndexLastModified(sitePaths.index_dir);
+
+ // Verify that projects index files haven't been updated
+ assertThat(projectsLastModified).isEqualTo(projectsLastModifiedAfterInit);
+ }
+
+ private void initSite() throws Exception {
+ runGerrit("init", "-d", sitePaths.site_path.toString(), "--show-stack-trace");
+ }
+
+ private void setProjectsIndexLastModifiedInThePast(Path indexDir, Instant time)
+ throws IOException {
+ for (Path path : getAllProjectsIndexFiles(indexDir).collect(Collectors.toList())) {
+ FS.DETECTED.setLastModified(path, time);
+ }
+ }
+
+ private Optional<Instant> getProjectsIndexLastModified(Path indexDir) throws IOException {
+ return getAllProjectsIndexFiles(indexDir)
+ .map(FS.DETECTED::lastModifiedInstant)
+ .max(Comparator.comparingLong(Instant::toEpochMilli));
+ }
+
+ private Stream<Path> getAllProjectsIndexFiles(Path indexDir) throws IOException {
+ Optional<Path> projectsPath =
+ Files.walk(indexDir, 1)
+ .filter(Files::isDirectory)
+ .filter(p -> p.getFileName().toString().startsWith("projects_"))
+ .findFirst();
+ if (!projectsPath.isPresent()) {
+ return Stream.empty();
+ }
+
+ return Files.walk(projectsPath.get(), 1, FileVisitOption.FOLLOW_LINKS);
+ }
}
