Merge branch 'stable-3.0' into stable-3.1
* stable-3.0:
Simplify Init for Elasticsearch
Refactor reindexProjects in Init to be general
Avoid auto-reindex of projects during init when unneeded
Change-Id: Id945a48cebe8408e86e3c4025fc6fbb1cb09bdc6
diff --git a/java/com/google/gerrit/pgm/Init.java b/java/com/google/gerrit/pgm/Init.java
index 799377c..4e62a0f 100644
--- a/java/com/google/gerrit/pgm/Init.java
+++ b/java/com/google/gerrit/pgm/Init.java
@@ -22,6 +22,7 @@
import com.google.gerrit.common.IoUtil;
import com.google.gerrit.common.PageLinks;
import com.google.gerrit.common.PluginData;
+import com.google.gerrit.index.SchemaDefinitions;
import com.google.gerrit.index.project.ProjectSchemaDefinitions;
import com.google.gerrit.pgm.init.BaseInit;
import com.google.gerrit.pgm.init.Browser;
@@ -30,6 +31,10 @@
import com.google.gerrit.pgm.util.ErrorLogFile;
import com.google.gerrit.server.config.GerritServerConfigModule;
import com.google.gerrit.server.config.SitePath;
+import com.google.gerrit.server.index.GerritIndexStatus;
+import com.google.gerrit.server.index.account.AccountSchemaDefinitions;
+import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
+import com.google.gerrit.server.index.group.GroupSchemaDefinitions;
import com.google.gerrit.server.ioutil.HostPlatform;
import com.google.gerrit.server.securestore.SecureStoreClassName;
import com.google.gerrit.server.util.ReplicaUtil;
@@ -60,9 +65,6 @@
@Option(name = "--no-auto-start", usage = "Don't automatically start daemon after init")
private boolean noAutoStart;
- @Option(name = "--no-reindex", usage = "Don't automatically reindex any entities")
- private boolean noReindex;
-
@Option(name = "--skip-plugins", usage = "Don't install plugins")
private boolean skipPlugins;
@@ -91,6 +93,8 @@
@Inject Browser browser;
+ private GerritIndexStatus indexStatus;
+
public Init() {
super(new WarDistribution(), null);
}
@@ -103,6 +107,7 @@
@Override
protected boolean beforeInit(SiteInit init) throws Exception {
+ indexStatus = new GerritIndexStatus(init.site);
ErrorLogFile.errorOnlyConsole();
if (!skipPlugins) {
@@ -131,6 +136,12 @@
@Override
protected void afterInit(SiteRun run) throws Exception {
+ List<SchemaDefinitions<?>> schemaDefs =
+ ImmutableList.of(
+ AccountSchemaDefinitions.INSTANCE,
+ ChangeSchemaDefinitions.INSTANCE,
+ GroupSchemaDefinitions.INSTANCE,
+ ProjectSchemaDefinitions.INSTANCE);
List<Module> modules = new ArrayList<>();
modules.add(
new AbstractModule() {
@@ -146,7 +157,11 @@
modules.add(new GerritServerConfigModule());
Guice.createInjector(modules).injectMembers(this);
if (!ReplicaUtil.isReplica(run.flags.cfg)) {
- reindexProjects();
+ for (SchemaDefinitions<?> schemaDef : schemaDefs) {
+ if (!indexStatus.exists(schemaDef.getName())) {
+ reindex(schemaDef);
+ }
+ }
}
start(run);
}
@@ -259,11 +274,7 @@
}
}
- private void reindexProjects() throws Exception {
- if (noReindex) {
- return;
- }
- // Reindex all projects, so that we bootstrap the project index for new installations
+ private void reindex(SchemaDefinitions<?> schemaDef) throws Exception {
List<String> reindexArgs =
ImmutableList.of(
"--site-path",
@@ -271,8 +282,9 @@
"--threads",
Integer.toString(1),
"--index",
- ProjectSchemaDefinitions.NAME);
- getConsoleUI().message("Init complete, reindexing projects with:");
+ schemaDef.getName());
+ getConsoleUI()
+ .message(String.format("Init complete, reindexing %s with:", schemaDef.getName()));
getConsoleUI().message(" reindex " + reindexArgs.stream().collect(joining(" ")));
Reindex reindexPgm = new Reindex();
reindexPgm.main(reindexArgs.stream().toArray(String[]::new));
diff --git a/java/com/google/gerrit/server/index/GerritIndexStatus.java b/java/com/google/gerrit/server/index/GerritIndexStatus.java
index d835227..0941c6c 100644
--- a/java/com/google/gerrit/server/index/GerritIndexStatus.java
+++ b/java/com/google/gerrit/server/index/GerritIndexStatus.java
@@ -44,6 +44,10 @@
return cfg.getBoolean(SECTION, indexDirName(indexName, version), KEY_READY, false);
}
+ public boolean exists(String indexName) {
+ return cfg.getSubsections(SECTION).stream().anyMatch(n -> n.startsWith(indexName));
+ }
+
public void save() throws IOException {
cfg.save();
}
diff --git a/javatests/com/google/gerrit/acceptance/pgm/ElasticReindexIT.java b/javatests/com/google/gerrit/acceptance/pgm/ElasticReindexIT.java
index 38b7e0e..cf349ab 100644
--- a/javatests/com/google/gerrit/acceptance/pgm/ElasticReindexIT.java
+++ b/javatests/com/google/gerrit/acceptance/pgm/ElasticReindexIT.java
@@ -21,7 +21,6 @@
import com.google.gerrit.testing.ConfigSuite;
import com.google.inject.Injector;
import org.eclipse.jgit.lib.Config;
-import org.junit.Before;
public class ElasticReindexIT extends AbstractReindexTests {
@@ -39,10 +38,4 @@
public void configureIndex(Injector injector) {
createAllIndexes(injector);
}
-
- @Before
- public void reindexFirstSinceElastic() throws Exception {
- assertServerStartupFails();
- runGerrit("reindex", "-d", sitePaths.site_path.toString(), "--show-stack-trace");
- }
}
diff --git a/javatests/com/google/gerrit/acceptance/pgm/InitIT.java b/javatests/com/google/gerrit/acceptance/pgm/InitIT.java
index e48088e..4caee64 100644
--- a/javatests/com/google/gerrit/acceptance/pgm/InitIT.java
+++ b/javatests/com/google/gerrit/acceptance/pgm/InitIT.java
@@ -26,7 +26,16 @@
import com.google.gerrit.index.project.ProjectIndexCollection;
import com.google.gerrit.server.config.AllProjectsName;
import com.google.gerrit.server.config.AllUsersName;
+import java.io.IOException;
+import java.nio.file.FileVisitOption;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.Comparator;
import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.eclipse.jgit.util.FS;
import org.junit.Test;
@NoHttpd
@@ -34,7 +43,7 @@
@Test
public void indexesAllProjectsAndAllUsers() throws Exception {
- runGerrit("init", "-d", sitePaths.site_path.toString(), "--show-stack-trace");
+ initSite();
try (ServerContext ctx = startServer()) {
ProjectIndexCollection projectIndex =
ctx.getInjector().getInstance(ProjectIndexCollection.class);
@@ -48,4 +57,52 @@
assertThat(allUsersData).isPresent();
}
}
+
+ @Test
+ public void initDoesNotReindexProjectsOnExistingSites() throws Exception {
+ initSite();
+
+ // Simulate a projects indexes files modified in the past by 3 seconds
+ Optional<Instant> projectsLastModified =
+ getProjectsIndexLastModified(sitePaths.index_dir).map(t -> t.minusSeconds(3));
+ assertThat(projectsLastModified).isPresent();
+ setProjectsIndexLastModifiedInThePast(sitePaths.index_dir, projectsLastModified.get());
+
+ initSite();
+ Optional<Instant> projectsLastModifiedAfterInit =
+ getProjectsIndexLastModified(sitePaths.index_dir);
+
+ // Verify that projects index files haven't been updated
+ assertThat(projectsLastModified).isEqualTo(projectsLastModifiedAfterInit);
+ }
+
+ private void initSite() throws Exception {
+ runGerrit("init", "-d", sitePaths.site_path.toString(), "--show-stack-trace");
+ }
+
+ private void setProjectsIndexLastModifiedInThePast(Path indexDir, Instant time)
+ throws IOException {
+ for (Path path : getAllProjectsIndexFiles(indexDir).collect(Collectors.toList())) {
+ FS.DETECTED.setLastModified(path, time);
+ }
+ }
+
+ private Optional<Instant> getProjectsIndexLastModified(Path indexDir) throws IOException {
+ return getAllProjectsIndexFiles(indexDir)
+ .map(FS.DETECTED::lastModifiedInstant)
+ .max(Comparator.comparingLong(Instant::toEpochMilli));
+ }
+
+ private Stream<Path> getAllProjectsIndexFiles(Path indexDir) throws IOException {
+ Optional<Path> projectsPath =
+ Files.walk(indexDir, 1)
+ .filter(Files::isDirectory)
+ .filter(p -> p.getFileName().toString().startsWith("projects_"))
+ .findFirst();
+ if (!projectsPath.isPresent()) {
+ return Stream.empty();
+ }
+
+ return Files.walk(projectsPath.get(), 1, FileVisitOption.FOLLOW_LINKS);
+ }
}
