Merge changes I5a30dad3,I1fc60d2a,I3c5e7c15 * changes: Add REST endpoint for reindexing an index version Add REST API for getting info about indexes Optionally allow to reuse existing documents during reindexing
diff --git a/Documentation/config-gerrit.txt b/Documentation/config-gerrit.txt index cf2f2d9..9ce2eee 100644 --- a/Documentation/config-gerrit.txt +++ b/Documentation/config-gerrit.txt
@@ -3488,6 +3488,21 @@ Excluded projects can later be reindexed by for example using the link:cmd-index-changes-in-project.html[index changes in project command]. +[[index.reuseExistingDocuments]]index.reuseExistingDocuments:: ++ +Whether to reuse index documents that already exist during reindexing. ++ +Currently, only supported by the changes index. ++ +This feature is useful, if the Gerrit server has to be restarted +during an ongoing index online upgrade, since this would cause +a complete reindexing otherwise that might take an extensive time. ++ +Each existing document in the index will be checked for staleness +and reindexed if found to be stale. ++ +Defaults to false. + [[index.paginationType]]index.paginationType:: + The pagination type to use when index queries are repeated to
diff --git a/Documentation/rest-api-config.txt b/Documentation/rest-api-config.txt index aa88141..d8ff80a 100644 --- a/Documentation/rest-api-config.txt +++ b/Documentation/rest-api-config.txt
@@ -1444,8 +1444,8 @@ Content-Type: application/json; charset=UTF-8 )]}' - { - "accounts": { + [ + { "name": "accounts", "versions": { "13": { @@ -1454,7 +1454,7 @@ } } }, - "changes": { + { "name": "changes", "versions": { "83": { @@ -1467,7 +1467,7 @@ } } }, - "groups": { + { "name": "groups", "versions": { "10": { @@ -1476,7 +1476,7 @@ } } }, - "projects": { + { "name": "projects", "versions": { "8": { @@ -1485,33 +1485,125 @@ } } } + [ +---- + +=== Get Index +-- +'GET /config/server/indexes/changes' +-- + +Get an index used by Gerrit. It provides details about the index versions, which +index version is used to search and which versions are written to. + +.Request +---- + 'GET /config/server/indexes/changes' +---- + +.Response +---- + HTTP/1.1 200 OK + Content-Type: application/json; charset=UTF-8 + + )]}' + { + "name": "changes", + "versions": { + "83": { + "write": true, + "search": true + }, + "84": { + "write": true, + "search": false + } + } + } +---- + +=== List Index Versions +-- +'GET /config/server/indexes/changes/versions' +-- + +Lists versions of an index used by Gerrit. + +.Request +---- + 'GET /config/server/indexes/changes/versions' +---- + +.Response +---- + HTTP/1.1 200 OK + Content-Type: application/json; charset=UTF-8 + + )]}' + { + "83": { + "write": true, + "search": true + }, + "84": { + "write": true, + "search": false + } + } +---- + +=== Get Index Version +-- +'GET /config/server/indexes/changes/versions/85' +-- + +Get info about one version of an index used by Gerrit. + +.Request +---- + 'GET /config/server/indexes/changes/versions/84' +---- + +.Response +---- + HTTP/1.1 200 OK + Content-Type: application/json; charset=UTF-8 + + )]}' + { + "write": true, + "search": false } ---- [[snapshot-index]] === Create Index Snapshot -This endpoint allows Gerrit admins to create a snapshot of an index. -This snapshot can be used as a backup of the index. +These endpoints allow Gerrit admins to create index snapshots. +Created snapshots can be used as a backup of the index. -A snapshot of all versions of an index can be created by just using -the name of the index, e.g. `changes`. Only snapshots of indexes that -Gerrit currently writes to can be created. An index version can be -selected by using e.g. `changes~84`. Snapshots of all indexes can be -created by using `all` instead of an index name. +It is possible to create a snapshot of all indexes, snapshot of one index or +snapshot of one index version. + +The snapshots will be stored on the server at `$SITE/index/snapshots/$ID`. +The `$ID` can be optionally provided in link:#snapshot-index-input[SnapshotIndex.Input] +or will default to the current local time in ISO8601 format. + +Only snapshots of indexes that Gerrit currently writes to can be created. Note, that the creation of multiple snapshots, e.g. of different index versions, is not atomic. If a consistent state over multiple indexes is required, the server has to be put into read-only mode before creating the snapshot. -The snapshots will be stored on the server at `$SITE/index/snapshots/$ID`. -The `$ID` can be optionally provided in link:#snapshot-index-input[SnapshotIndex.Input] -or will default to the current local time in ISO8601 format. +==== Create Snapshot of All Indexes +-- +'POST /config/server/snapshot.indexes HTTP/1.0' +-- .Request ---- - PUT /config/server/indexes/all/snapshot HTTP/1.0 + POST /config/server/snapshot.indexes HTTP/1.0 Content-Type: application/json; charset=UTF-8 { @@ -1530,9 +1622,16 @@ } ---- +==== Create Snapshot of one Index +-- +'POST /config/server/indexes/link:#index-name[\{index-name\}]/snapshot' +-- + +This creates a snapshot of all write index versions of the specified index. + .Request ---- - PUT /config/server/indexes/accounts~13/snapshot HTTP/1.0 + POST /config/server/indexes/accounts/snapshot HTTP/1.0 Content-Type: application/json; charset=UTF-8 { @@ -1551,6 +1650,61 @@ } ---- +==== Create Snapshot of one Index Version +-- +'POST /config/server/indexes/link:#index-name[\{index-name\}]/versions/#index-version[\{index-version\}]/snapshot' +-- + +This creates a snapshot of one index version of the specified index. + +.Request +---- + POST /config/server/indexes/changes/versions/84/snapshot HTTP/1.0 + Content-Type: application/json; charset=UTF-8 + + { + "id": "snapshot-1" + } +---- + +.Response +---- + HTTP/1.1 200 OK + Content-Type: application/json; charset=UTF-8 + + )]}' + { + "id": "snapshot-1" + } +---- + +=== Reindex an Index Version +-- +'POST /config/server/indexes/link:#index-name[\{index-name\}]/versions/#index-version[\{index-version\}]/reindex' +-- + +This endpoint allows to trigger background reindexing of an index version. It is +also supported to specify whether to reuse existing up-to-date (non-stale) index +documents. + +.Request +---- + POST /config/server/indexes/changes/versions/84/reindex HTTP/1.0 + Content-Type: application/json; charset=UTF-8 + + { + "reuse": "true" + } +---- + +.Response +---- + HTTP/1.1 202 Accepted + Content-Type: application/json; charset=UTF-8 + + )]}' +---- + [[ids]] == IDs @@ -1568,6 +1722,13 @@ === \{task-id\} The ID of the task (hex string). +[[index-name]] +=== \{index-name\} +The name of the index. Can be any of: "accounts", "changes", "groups", "projects". + +[[index-version]] +=== \{index-version\} +The version of the index. This is an integer. [[json-entities]] == JSON Entities
diff --git a/java/com/google/gerrit/acceptance/ChangeIndexedCounter.java b/java/com/google/gerrit/acceptance/ChangeIndexedCounter.java index 5991646..98287c8 100644 --- a/java/com/google/gerrit/acceptance/ChangeIndexedCounter.java +++ b/java/com/google/gerrit/acceptance/ChangeIndexedCounter.java
@@ -41,6 +41,10 @@ return countsByChange.get(info._number); } + public long getTotalCount() { + return countsByChange.asMap().values().stream().reduce(0L, Long::sum); + } + public void assertReindexOf(ChangeInfo info) { assertReindexOf(info, 1); }
diff --git a/java/com/google/gerrit/index/IndexDefinition.java b/java/com/google/gerrit/index/IndexDefinition.java index f283bf1..cbcb34e 100644 --- a/java/com/google/gerrit/index/IndexDefinition.java +++ b/java/com/google/gerrit/index/IndexDefinition.java
@@ -67,7 +67,11 @@ } @Nullable - public final SiteIndexer<K, V, I> getSiteIndexer() { + public SiteIndexer<K, V, I> getSiteIndexer() { + return siteIndexer; + } + + public SiteIndexer<K, V, I> getSiteIndexer(boolean reuseExistingDocuments) { return siteIndexer; } }
diff --git a/java/com/google/gerrit/index/testing/FakeIndexVersionManager.java b/java/com/google/gerrit/index/testing/FakeIndexVersionManager.java index 5044e38..40d51fd 100644 --- a/java/com/google/gerrit/index/testing/FakeIndexVersionManager.java +++ b/java/com/google/gerrit/index/testing/FakeIndexVersionManager.java
@@ -40,7 +40,12 @@ SitePaths sitePaths, PluginSetContext<OnlineUpgradeListener> listeners, Collection<IndexDefinition<?, ?, ?>> defs) { - super(sitePaths, listeners, defs, VersionManager.getOnlineUpgrade(cfg)); + super( + sitePaths, + listeners, + defs, + VersionManager.getOnlineUpgrade(cfg), + cfg.getBoolean("index", "reuseExistingDocuments", false)); } @Override
diff --git a/java/com/google/gerrit/lucene/LuceneVersionManager.java b/java/com/google/gerrit/lucene/LuceneVersionManager.java index f3ba73d..265d3e0 100644 --- a/java/com/google/gerrit/lucene/LuceneVersionManager.java +++ b/java/com/google/gerrit/lucene/LuceneVersionManager.java
@@ -49,7 +49,12 @@ SitePaths sitePaths, PluginSetContext<OnlineUpgradeListener> listeners, Collection<IndexDefinition<?, ?, ?>> defs) { - super(sitePaths, listeners, defs, VersionManager.getOnlineUpgrade(cfg)); + super( + sitePaths, + listeners, + defs, + VersionManager.getOnlineUpgrade(cfg), + cfg.getBoolean("index", "reuseExistingDocuments", false)); } @Override
diff --git a/java/com/google/gerrit/pgm/Reindex.java b/java/com/google/gerrit/pgm/Reindex.java index 2ed2b76..e800d17 100644 --- a/java/com/google/gerrit/pgm/Reindex.java +++ b/java/com/google/gerrit/pgm/Reindex.java
@@ -98,6 +98,8 @@ @Option(name = "--build-bloom-filter", usage = "Build bloom filter for H2 disk caches.") private boolean buildBloomFilter; + private Boolean reuseExistingDocumentsOption; + private Injector dbInjector; private Injector sysInjector; private Injector cfgInjector; @@ -106,6 +108,11 @@ @Inject private Collection<IndexDefinition<?, ?, ?>> indexDefs; @Inject private DynamicMap<Cache<?, ?>> cacheMap; + @Option(name = "--reuse", usage = "Reindex only when existing index entry is stale") + public void setReuseExistingDocuments(boolean value) { + reuseExistingDocumentsOption = value; + } + @Override public int run() throws Exception { mustHaveValidSite(); @@ -258,9 +265,16 @@ requireNonNull( index, () -> String.format("no active search index configured for %s", def.getName())); index.markReady(false); - index.deleteAll(); + boolean reuseExistingDocuments = + reuseExistingDocumentsOption != null + ? reuseExistingDocumentsOption + : globalConfig.getBoolean("index", null, "reuseExistingDocuments", false); - SiteIndexer<K, V, I> siteIndexer = def.getSiteIndexer(); + if (!reuseExistingDocuments) { + index.deleteAll(); + } + + SiteIndexer<K, V, I> siteIndexer = def.getSiteIndexer(reuseExistingDocuments); siteIndexer.setProgressOut(System.err); siteIndexer.setVerboseOut(verbose ? System.out : NullOutputStream.INSTANCE); SiteIndexer.Result result = siteIndexer.indexAll(index);
diff --git a/java/com/google/gerrit/server/config/IndexResource.java b/java/com/google/gerrit/server/config/IndexResource.java index a65cc52..30d39c4 100644 --- a/java/com/google/gerrit/server/config/IndexResource.java +++ b/java/com/google/gerrit/server/config/IndexResource.java
@@ -14,44 +14,21 @@ package com.google.gerrit.server.config; -import com.google.common.collect.ImmutableList; -import com.google.gerrit.common.Nullable; -import com.google.gerrit.extensions.restapi.ResourceNotFoundException; import com.google.gerrit.extensions.restapi.RestView; import com.google.gerrit.index.Index; -import com.google.gerrit.index.IndexCollection; +import com.google.gerrit.index.IndexDefinition; import com.google.inject.TypeLiteral; -import java.util.Collection; -import java.util.List; public class IndexResource extends ConfigResource { public static final TypeLiteral<RestView<IndexResource>> INDEX_KIND = new TypeLiteral<>() {}; - private final ImmutableList<Index<?, ?>> indexes; + private IndexDefinition<?, ?, ?> def; - public IndexResource(IndexCollection<?, ?, ?> indexes, @Nullable Integer version) - throws ResourceNotFoundException { - if (version == null) { - this.indexes = ImmutableList.copyOf(indexes.getWriteIndexes()); - } else { - Index<?, ?> index = indexes.getWriteIndex(version); - if (index == null) { - throw new ResourceNotFoundException( - String.format("Unknown index version requested: %d", version)); - } - this.indexes = ImmutableList.of(index); - } + public IndexResource(IndexDefinition<?, ?, ?> def) { + this.def = def; } - public IndexResource(List<IndexCollection<?, ?, ?>> indexes) { - ImmutableList.Builder<Index<?, ?>> allIndexes = ImmutableList.builder(); - for (IndexCollection<?, ?, ?> index : indexes) { - allIndexes.addAll(index.getWriteIndexes()); - } - this.indexes = allIndexes.build(); - } - - public Collection<Index<?, ?>> getIndexes() { - return indexes; + public IndexDefinition<?, ?, ? extends Index<?, ?>> getIndexDefinition() { + return def; } }
diff --git a/java/com/google/gerrit/server/config/IndexVersionResource.java b/java/com/google/gerrit/server/config/IndexVersionResource.java new file mode 100644 index 0000000..8ccb0b0 --- /dev/null +++ b/java/com/google/gerrit/server/config/IndexVersionResource.java
@@ -0,0 +1,42 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.config; + +import com.google.gerrit.extensions.restapi.RestResource; +import com.google.gerrit.extensions.restapi.RestView; +import com.google.gerrit.index.Index; +import com.google.gerrit.index.IndexDefinition; +import com.google.inject.TypeLiteral; + +public class IndexVersionResource implements RestResource { + public static final TypeLiteral<RestView<IndexVersionResource>> INDEX_VERSION_KIND = + new TypeLiteral<>() {}; + + private final IndexDefinition<?, ?, ?> def; + private final Index<?, ?> index; + + public IndexVersionResource(IndexDefinition<?, ?, ?> def, Index<?, ?> index) { + this.def = def; + this.index = index; + } + + public IndexDefinition<?, ?, ?> getIndexDefinition() { + return def; + } + + public Index<?, ?> getIndex() { + return index; + } +}
diff --git a/java/com/google/gerrit/server/index/IndexModule.java b/java/com/google/gerrit/server/index/IndexModule.java index 96870ea..c048e3c 100644 --- a/java/com/google/gerrit/server/index/IndexModule.java +++ b/java/com/google/gerrit/server/index/IndexModule.java
@@ -25,6 +25,7 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.gerrit.extensions.events.LifecycleListener; import com.google.gerrit.extensions.registration.DynamicSet; +import com.google.gerrit.index.IndexConfig; import com.google.gerrit.index.IndexDefinition; import com.google.gerrit.index.IndexType; import com.google.gerrit.index.SchemaDefinitions; @@ -34,6 +35,7 @@ import com.google.gerrit.index.project.ProjectSchemaDefinitions; import com.google.gerrit.lifecycle.LifecycleModule; import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.git.GitRepositoryManager; import com.google.gerrit.server.git.MultiProgressMonitor; import com.google.gerrit.server.git.WorkQueue; import com.google.gerrit.server.index.account.AccountIndexCollection; @@ -42,11 +44,13 @@ import com.google.gerrit.server.index.account.AccountIndexer; import com.google.gerrit.server.index.account.AccountIndexerImpl; import com.google.gerrit.server.index.account.AccountSchemaDefinitions; +import com.google.gerrit.server.index.change.AllChangesIndexer; import com.google.gerrit.server.index.change.ChangeIndexCollection; import com.google.gerrit.server.index.change.ChangeIndexDefinition; import com.google.gerrit.server.index.change.ChangeIndexRewriter; import com.google.gerrit.server.index.change.ChangeIndexer; import com.google.gerrit.server.index.change.ChangeSchemaDefinitions; +import com.google.gerrit.server.index.change.StalenessChecker; import com.google.gerrit.server.index.group.GroupIndexCollection; import com.google.gerrit.server.index.group.GroupIndexDefinition; import com.google.gerrit.server.index.group.GroupIndexRewriter; @@ -129,6 +133,8 @@ bind(ChangeIndexCollection.class); listener().to(ChangeIndexCollection.class); factory(ChangeIndexer.Factory.class); + factory(StalenessChecker.Factory.class); + factory(AllChangesIndexer.Factory.class); bind(GroupIndexRewriter.class); // GroupIndexCollection is already bound very high up in SchemaModule. @@ -255,6 +261,13 @@ return MoreExecutors.listeningDecorator(workQueue.createQueue(threads, "Index-Batch", true)); } + @Provides + @Singleton + StalenessChecker getChangeStalenessChecker( + ChangeIndexCollection indexes, GitRepositoryManager repoManager, IndexConfig indexConfig) { + return new StalenessChecker(indexes, repoManager, indexConfig); + } + @Singleton private static class ShutdownIndexExecutors implements LifecycleListener { private final ListeningExecutorService interactiveExecutor;
diff --git a/java/com/google/gerrit/server/index/IndexVersionReindexer.java b/java/com/google/gerrit/server/index/IndexVersionReindexer.java new file mode 100644 index 0000000..5d136e8 --- /dev/null +++ b/java/com/google/gerrit/server/index/IndexVersionReindexer.java
@@ -0,0 +1,56 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index; + +import static com.google.gerrit.server.git.QueueProvider.QueueType.BATCH; + +import com.google.common.flogger.FluentLogger; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.gerrit.index.Index; +import com.google.gerrit.index.IndexDefinition; +import com.google.gerrit.index.SiteIndexer; +import com.google.inject.Inject; +import java.util.concurrent.Future; + +public class IndexVersionReindexer { + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + private ListeningExecutorService executor; + + @Inject + IndexVersionReindexer(@IndexExecutor(BATCH) ListeningExecutorService executor) { + this.executor = executor; + } + + public <K, V, I extends Index<K, V>> Future<SiteIndexer.Result> reindex( + IndexDefinition<K, V, I> def, int version, boolean reuse) { + I index = def.getIndexCollection().getWriteIndex(version); + SiteIndexer<K, V, I> siteIndexer = def.getSiteIndexer(reuse); + return executor.submit( + () -> { + String name = def.getName(); + logger.atInfo().log("Starting reindex of %s version %d", name, version); + SiteIndexer.Result result = siteIndexer.indexAll(index); + if (result.success()) { + logger.atInfo().log("Reindex %s version %s complete", name, version); + } else { + logger.atInfo().log( + "Reindex %s version %s failed. Successfully indexed %s, failed to index %s", + name, version, result.doneCount(), result.failedCount()); + } + return result; + }); + } +}
diff --git a/java/com/google/gerrit/server/index/OnlineReindexer.java b/java/com/google/gerrit/server/index/OnlineReindexer.java index eef394d..98abf46 100644 --- a/java/com/google/gerrit/server/index/OnlineReindexer.java +++ b/java/com/google/gerrit/server/index/OnlineReindexer.java
@@ -42,18 +42,21 @@ private final PluginSetContext<OnlineUpgradeListener> listeners; private I index; private final AtomicBoolean running = new AtomicBoolean(); + private final boolean reuseExistingDocuments; public OnlineReindexer( IndexDefinition<K, V, I> def, int oldVersion, int newVersion, - PluginSetContext<OnlineUpgradeListener> listeners) { + PluginSetContext<OnlineUpgradeListener> listeners, + boolean reuseExistingDocuments) { this.name = def.getName(); this.indexes = def.getIndexCollection(); this.batchIndexer = def.getSiteIndexer(); this.oldVersion = oldVersion; this.newVersion = newVersion; this.listeners = listeners; + this.reuseExistingDocuments = reuseExistingDocuments; } /** Starts the background process. */ @@ -106,7 +109,7 @@ "Starting online reindex of %s from schema version %s to %s", name, version(indexes.getSearchIndex()), version(index)); - if (oldVersion != newVersion) { + if (!reuseExistingDocuments && oldVersion != newVersion) { index.deleteAll(); } SiteIndexer.Result result = batchIndexer.indexAll(index);
diff --git a/java/com/google/gerrit/server/index/VersionManager.java b/java/com/google/gerrit/server/index/VersionManager.java index 39930a6..2c38caf 100644 --- a/java/com/google/gerrit/server/index/VersionManager.java +++ b/java/com/google/gerrit/server/index/VersionManager.java
@@ -59,6 +59,7 @@ } protected final boolean onlineUpgrade; + protected final boolean reuseExistingDocuments; protected final String runReindexMsg; protected final SitePaths sitePaths; @@ -72,7 +73,8 @@ SitePaths sitePaths, PluginSetContext<OnlineUpgradeListener> listeners, Collection<IndexDefinition<?, ?, ?>> defs, - boolean onlineUpgrade) { + boolean onlineUpgrade, + boolean reuseExistingDocuments) { this.sitePaths = sitePaths; this.listeners = listeners; this.defs = Maps.newHashMapWithExpectedSize(defs.size()); @@ -82,6 +84,7 @@ this.reindexers = Maps.newHashMapWithExpectedSize(defs.size()); this.onlineUpgrade = onlineUpgrade; + this.reuseExistingDocuments = reuseExistingDocuments; this.runReindexMsg = "No index versions for index '%s' ready; run java -jar " + sitePaths.gerrit_war.toAbsolutePath() @@ -190,7 +193,7 @@ if (!reindexers.containsKey(def.getName())) { int latest = write.get(0).version; OnlineReindexer<K, V, I> reindexer = - new OnlineReindexer<>(def, search.version, latest, listeners); + new OnlineReindexer<>(def, search.version, latest, listeners, reuseExistingDocuments); reindexers.put(def.getName(), reindexer); } }
diff --git a/java/com/google/gerrit/server/index/change/AllChangesIndexer.java b/java/com/google/gerrit/server/index/change/AllChangesIndexer.java index 3935108..925f68d 100644 --- a/java/com/google/gerrit/server/index/change/AllChangesIndexer.java +++ b/java/com/google/gerrit/server/index/change/AllChangesIndexer.java
@@ -43,7 +43,8 @@ import com.google.gerrit.server.notedb.ChangeNotes.Factory.ChangeNotesResult; import com.google.gerrit.server.project.ProjectCache; import com.google.gerrit.server.query.change.ChangeData; -import com.google.inject.Inject; +import com.google.inject.assistedinject.Assisted; +import com.google.inject.assistedinject.AssistedInject; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -52,6 +53,7 @@ import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; import java.util.stream.Collectors; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.ObjectId; @@ -65,6 +67,13 @@ */ public class AllChangesIndexer extends SiteIndexer<Change.Id, ChangeData, ChangeIndex> { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + public interface Factory { + AllChangesIndexer create(); + + AllChangesIndexer create(boolean reuseExistingDocuments); + } + private MultiProgressMonitor mpm; private VolatileTask doneTask; private Task failedTask; @@ -84,25 +93,54 @@ private final GitRepositoryManager repoManager; private final ListeningExecutorService executor; private final ChangeIndexer.Factory indexerFactory; + private final StalenessChecker.Factory stalenessCheckerFactory; private final ChangeNotes.Factory notesFactory; private final ProjectCache projectCache; private final Set<Project.NameKey> projectsToSkip; + private final boolean reuseExistingDocuments; - @Inject + @AssistedInject AllChangesIndexer( MultiProgressMonitor.Factory multiProgressMonitorFactory, ChangeData.Factory changeDataFactory, GitRepositoryManager repoManager, @IndexExecutor(BATCH) ListeningExecutorService executor, ChangeIndexer.Factory indexerFactory, + StalenessChecker.Factory stalenessCheckerFactory, ChangeNotes.Factory notesFactory, ProjectCache projectCache, @GerritServerConfig Config config) { + this( + multiProgressMonitorFactory, + changeDataFactory, + repoManager, + executor, + indexerFactory, + stalenessCheckerFactory, + notesFactory, + projectCache, + config, + config.getBoolean("index", null, "reuseExistingDocuments", false)); + } + + @AssistedInject + AllChangesIndexer( + MultiProgressMonitor.Factory multiProgressMonitorFactory, + ChangeData.Factory changeDataFactory, + GitRepositoryManager repoManager, + @IndexExecutor(BATCH) ListeningExecutorService executor, + ChangeIndexer.Factory indexerFactory, + StalenessChecker.Factory stalenessCheckerFactory, + ChangeNotes.Factory notesFactory, + ProjectCache projectCache, + @GerritServerConfig Config config, + @Assisted boolean reuseExistingDocuments) { this.multiProgressMonitorFactory = multiProgressMonitorFactory; this.changeDataFactory = changeDataFactory; this.repoManager = repoManager; this.executor = executor; this.indexerFactory = indexerFactory; + this.stalenessCheckerFactory = stalenessCheckerFactory; this.notesFactory = notesFactory; this.projectCache = projectCache; this.projectsToSkip = @@ -110,6 +148,7 @@ .stream() .map(p -> Project.NameKey.parse(p)) .collect(Collectors.toSet()); + this.reuseExistingDocuments = reuseExistingDocuments; } @AutoValue @@ -218,20 +257,27 @@ } private class ProjectSliceIndexer implements Callable<Void> { - private final ChangeIndexer indexer; private final ProjectSlice projectSlice; private final ProgressMonitor done; private final ProgressMonitor failed; + private final Consumer<ChangeData> indexAction; private ProjectSliceIndexer( ChangeIndexer indexer, ProjectSlice projectSlice, ProgressMonitor done, ProgressMonitor failed) { - this.indexer = indexer; this.projectSlice = projectSlice; this.done = done; this.failed = failed; + if (reuseExistingDocuments) { + indexAction = + cd -> { + var unused = indexer.reindexIfStale(cd); + }; + } else { + indexAction = cd -> indexer.index(cd); + } } @Override @@ -271,7 +317,7 @@ return; } try { - indexer.index(changeDataFactory.create(r.notes())); + indexAction.accept(changeDataFactory.create(r.notes())); done.update(1); verboseWriter.format( "Reindexed change %d (project: %s)\n", r.id().get(), r.notes().getProjectName().get()); @@ -385,13 +431,15 @@ for (int slice = 0; slice < slices; slice++) { ProjectSlice projectSlice = ProjectSlice.create(name, slice, slices, metaIdByChange); + ChangeIndexer indexer; + if (reuseExistingDocuments) { + indexer = + indexerFactory.create(executor, index, stalenessCheckerFactory.create(index)); + } else { + indexer = indexerFactory.create(executor, index); + } ListenableFuture<?> future = - executor.submit( - reindexProjectSlice( - indexerFactory.create(executor, index), - projectSlice, - doneTask, - failedTask)); + executor.submit(reindexProjectSlice(indexer, projectSlice, doneTask, failedTask)); String description = "project " + name + " (" + slice + "/" + slices + ")"; addErrorListener(future, description, projTask, ok); sliceIndexerFutures.add(future);
diff --git a/java/com/google/gerrit/server/index/change/ChangeIndexDefinition.java b/java/com/google/gerrit/server/index/change/ChangeIndexDefinition.java index dde9d86..342796b 100644 --- a/java/com/google/gerrit/server/index/change/ChangeIndexDefinition.java +++ b/java/com/google/gerrit/server/index/change/ChangeIndexDefinition.java
@@ -14,20 +14,34 @@ package com.google.gerrit.server.index.change; -import com.google.gerrit.common.Nullable; import com.google.gerrit.entities.Change; import com.google.gerrit.index.IndexDefinition; +import com.google.gerrit.index.SiteIndexer; import com.google.gerrit.server.query.change.ChangeData; import com.google.inject.Inject; /** Bundle of service classes that make up the change index. */ public class ChangeIndexDefinition extends IndexDefinition<Change.Id, ChangeData, ChangeIndex> { + private final AllChangesIndexer.Factory allChangesIndexerFactory; + @Inject ChangeIndexDefinition( ChangeIndexCollection indexCollection, ChangeIndex.Factory indexFactory, - @Nullable AllChangesIndexer allChangesIndexer) { - super(ChangeSchemaDefinitions.INSTANCE, indexCollection, indexFactory, allChangesIndexer); + AllChangesIndexer.Factory allChangesIndexerFactory) { + super(ChangeSchemaDefinitions.INSTANCE, indexCollection, indexFactory, null); + this.allChangesIndexerFactory = allChangesIndexerFactory; + } + + @Override + public SiteIndexer<Change.Id, ChangeData, ChangeIndex> getSiteIndexer() { + return allChangesIndexerFactory.create(); + } + + @Override + public SiteIndexer<Change.Id, ChangeData, ChangeIndex> getSiteIndexer( + boolean reuseExistingDocuments) { + return allChangesIndexerFactory.create(reuseExistingDocuments); } }
diff --git a/java/com/google/gerrit/server/index/change/ChangeIndexer.java b/java/com/google/gerrit/server/index/change/ChangeIndexer.java index 562f9c4..4ec390d 100644 --- a/java/com/google/gerrit/server/index/change/ChangeIndexer.java +++ b/java/com/google/gerrit/server/index/change/ChangeIndexer.java
@@ -70,6 +70,9 @@ public interface Factory { ChangeIndexer create(ListeningExecutorService executor, ChangeIndex index); + ChangeIndexer create( + ListeningExecutorService executor, ChangeIndex index, StalenessChecker stalenessChecker); + ChangeIndexer create(ListeningExecutorService executor, ChangeIndexCollection indexes); } @@ -121,6 +124,31 @@ ChangeNotes.Factory notesFactory, ThreadLocalRequestContext context, PluginSetContext<ChangeIndexedListener> indexedListeners, + @IndexExecutor(BATCH) ListeningExecutorService batchExecutor, + IsFirstInsertForEntry isFirstInsertForEntry, + @Assisted ListeningExecutorService executor, + @Assisted ChangeIndex index, + @Assisted StalenessChecker stalenessChecker) { + this.executor = executor; + this.changeDataFactory = changeDataFactory; + this.notesFactory = notesFactory; + this.context = context; + this.indexedListeners = indexedListeners; + this.batchExecutor = batchExecutor; + this.autoReindexIfStale = autoReindexIfStale(cfg); + this.isFirstInsertForEntry = isFirstInsertForEntry; + this.index = index; + this.indexes = null; + this.stalenessChecker = stalenessChecker; + } + + @AssistedInject + ChangeIndexer( + @GerritServerConfig Config cfg, + ChangeData.Factory changeDataFactory, + ChangeNotes.Factory notesFactory, + ThreadLocalRequestContext context, + PluginSetContext<ChangeIndexedListener> indexedListeners, StalenessChecker stalenessChecker, @IndexExecutor(BATCH) ListeningExecutorService batchExecutor, @Assisted ListeningExecutorService executor, @@ -350,7 +378,7 @@ * @param id ID of the change to index. * @return future for reindexing the change; returns true if the change was stale. */ - public ListenableFuture<Boolean> reindexIfStale(Project.NameKey project, Change.Id id) { + public ListenableFuture<Boolean> asyncReindexIfStale(Project.NameKey project, Change.Id id) { ReindexIfStaleTask task = new ReindexIfStaleTask(project, id); if (queuedReindexIfStaleTasks.add(task)) { return submit(task, batchExecutor); @@ -358,6 +386,41 @@ return Futures.immediateFuture(false); } + /** + * Synchronously check if a change is stale, and reindex if it is. + * + * @param cd the change data to be checked for staleness. + * @return true if the change was stale, false if it was up-to-date + */ + public boolean reindexIfStale(ChangeData cd) { + return reindexIfStale(cd.project(), cd.getId()); + } + + /** + * Synchronously check if a change is stale, and reindex if it is. + * + * @param project the project to which the change belongs. + * @param id ID of the change to index. + * @return true if the change was stale, false if it was up-to-date + */ + public boolean reindexIfStale(Project.NameKey project, Change.Id id) { + try { + StalenessCheckResult stalenessCheckResult = stalenessChecker.check(id); + if (stalenessCheckResult.isStale()) { + logger.atInfo().log("Reindexing stale document %s", stalenessCheckResult); + indexImpl(changeDataFactory.create(project, id)); + return true; + } + } catch (Exception e) { + if (!isCausedByRepositoryNotFoundException(e)) { + throw e; + } + logger.atFine().log( + "Change %s belongs to deleted project %s, aborting reindexing the change.", id, project); + } + return false; + } + private void autoReindexIfStale(ChangeData cd) { autoReindexIfStale(cd.project(), cd.getId()); } @@ -366,7 +429,7 @@ if (autoReindexIfStale) { // Don't retry indefinitely; if this fails the change will be stale. @SuppressWarnings("unused") - Future<?> possiblyIgnoredError = reindexIfStale(project, id); + Future<?> possiblyIgnoredError = asyncReindexIfStale(project, id); } } @@ -546,22 +609,7 @@ @Override public Boolean callImpl() throws Exception { remove(); - try { - StalenessCheckResult stalenessCheckResult = stalenessChecker.check(id); - if (stalenessCheckResult.isStale()) { - logger.atInfo().log("Reindexing stale document %s", stalenessCheckResult); - indexImpl(changeDataFactory.create(project, id)); - return true; - } - } catch (Exception e) { - if (!isCausedByRepositoryNotFoundException(e)) { - throw e; - } - logger.atFine().log( - "Change %s belongs to deleted project %s, aborting reindexing the change.", - id.get(), project.get()); - } - return false; + return reindexIfStale(project, id); } @Override
diff --git a/java/com/google/gerrit/server/index/change/IndexedChangeQuery.java b/java/com/google/gerrit/server/index/change/IndexedChangeQuery.java index 00642a9..f7ff13c 100644 --- a/java/com/google/gerrit/server/index/change/IndexedChangeQuery.java +++ b/java/com/google/gerrit/server/index/change/IndexedChangeQuery.java
@@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.gerrit.entities.Change; import com.google.gerrit.index.IndexConfig; @@ -51,6 +52,10 @@ */ public class IndexedChangeQuery extends IndexedQuery<Change.Id, ChangeData> implements ChangeDataSource, Matchable<ChangeData> { + public static QueryOptions oneResult() { + IndexConfig config = IndexConfig.createDefault(); + return createOptions(config, 0, 1, config.pageSizeMultiplier(), 1, ImmutableSet.of()); + } public static QueryOptions createOptions( IndexConfig config, int start, int limit, Set<String> fields) {
diff --git a/java/com/google/gerrit/server/index/change/StalenessChecker.java b/java/com/google/gerrit/server/index/change/StalenessChecker.java index eb4af01..83f6189 100644 --- a/java/com/google/gerrit/server/index/change/StalenessChecker.java +++ b/java/com/google/gerrit/server/index/change/StalenessChecker.java
@@ -27,6 +27,7 @@ import com.google.common.collect.SetMultimap; import com.google.common.collect.Sets; import com.google.common.flogger.FluentLogger; +import com.google.gerrit.common.Nullable; import com.google.gerrit.common.UsedAt; import com.google.gerrit.entities.Change; import com.google.gerrit.entities.Project; @@ -36,8 +37,8 @@ import com.google.gerrit.server.git.GitRepositoryManager; import com.google.gerrit.server.index.StalenessCheckResult; import com.google.gerrit.server.query.change.ChangeData; -import com.google.inject.Inject; -import com.google.inject.Singleton; +import com.google.inject.assistedinject.Assisted; +import com.google.inject.assistedinject.AssistedInject; import java.io.IOException; import java.util.List; import java.util.Optional; @@ -50,34 +51,58 @@ * Checker that compares values stored in the change index to metadata in NoteDb to detect index * documents that should have been updated (= stale). */ -@Singleton public class StalenessChecker { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + public interface Factory { + StalenessChecker create(ChangeIndex index); + } + public static final ImmutableSet<String> FIELDS = ImmutableSet.of( ChangeField.CHANGE_SPEC.getName(), ChangeField.REF_STATE_SPEC.getName(), ChangeField.REF_STATE_PATTERN_SPEC.getName()); - private final ChangeIndexCollection indexes; + @Nullable private final ChangeIndexCollection indexes; + @Nullable private final ChangeIndex index; private final GitRepositoryManager repoManager; private final IndexConfig indexConfig; - @Inject - StalenessChecker( + public StalenessChecker( ChangeIndexCollection indexes, GitRepositoryManager repoManager, IndexConfig indexConfig) { this.indexes = indexes; + this.index = null; this.repoManager = repoManager; this.indexConfig = indexConfig; } + @AssistedInject + StalenessChecker( + GitRepositoryManager repoManager, IndexConfig indexConfig, @Assisted ChangeIndex index) { + this.indexes = null; + this.repoManager = repoManager; + this.indexConfig = indexConfig; + this.index = index; + } + /** * Returns a {@link StalenessCheckResult} with structured information about staleness of the * provided {@link com.google.gerrit.entities.Change.Id}. */ public StalenessCheckResult check(Change.Id id) { - ChangeIndex i = indexes.getSearchIndex(); + if (index != null) { + return check(id, index); + } + return check(id, indexes.getSearchIndex()); + } + + /** + * Returns a {@link StalenessCheckResult} with structured information about staleness of the + * provided {@link com.google.gerrit.entities.Change.Id} in the provided {@link + * com.google.gerrit.server.index.change.ChangeIndex}. + */ + private StalenessCheckResult check(Change.Id id, ChangeIndex i) { if (i == null) { return StalenessCheckResult .notStale(); // No index; caller couldn't do anything if it is stale.
diff --git a/java/com/google/gerrit/server/restapi/config/ConfigRestApiModule.java b/java/com/google/gerrit/server/restapi/config/ConfigRestApiModule.java index 77caab4..fdb7d0e 100644 --- a/java/com/google/gerrit/server/restapi/config/ConfigRestApiModule.java +++ b/java/com/google/gerrit/server/restapi/config/ConfigRestApiModule.java
@@ -16,6 +16,7 @@ import static com.google.gerrit.server.config.ConfigResource.CONFIG_KIND; import static com.google.gerrit.server.config.IndexResource.INDEX_KIND; +import static com.google.gerrit.server.config.IndexVersionResource.INDEX_VERSION_KIND; import static com.google.gerrit.server.config.TaskResource.TASK_KIND; import com.google.gerrit.extensions.registration.DynamicMap; @@ -31,6 +32,7 @@ DynamicMap.mapOf(binder(), TASK_KIND); DynamicMap.mapOf(binder(), TopMenuResource.TOP_MENU_KIND); DynamicMap.mapOf(binder(), INDEX_KIND); + DynamicMap.mapOf(binder(), INDEX_VERSION_KIND); child(CONFIG_KIND, "capabilities").to(CapabilitiesCollection.class); post(CONFIG_KIND, "check.consistency").to(CheckConsistency.class); @@ -44,6 +46,7 @@ get(CONFIG_KIND, "preferences.edit").to(GetEditPreferences.class); put(CONFIG_KIND, "preferences.edit").to(SetEditPreferences.class); post(CONFIG_KIND, "reload").to(ReloadConfig.class); + post(CONFIG_KIND, "snapshot.indexes").to(SnapshotIndexes.class); child(CONFIG_KIND, "tasks").to(TasksCollection.class); delete(TASK_KIND).to(DeleteTask.class); @@ -53,7 +56,13 @@ get(CONFIG_KIND, "version").to(GetVersion.class); child(CONFIG_KIND, "indexes").to(IndexCollection.class); - put(INDEX_KIND, "snapshot").to(SnapshotIndex.class); + post(INDEX_KIND, "snapshot").to(SnapshotIndex.class); + get(INDEX_KIND).to(GetIndex.class); + + child(INDEX_KIND, "versions").to(IndexVersionsCollection.class); + get(INDEX_VERSION_KIND).to(GetIndexVersion.class); + post(INDEX_VERSION_KIND, "snapshot").to(SnapshotIndexVersion.class); + post(INDEX_VERSION_KIND, "reindex").to(ReindexIndexVersion.class); // The caches and summary REST endpoints are bound via RestCacheAdminModule. }
diff --git a/java/com/google/gerrit/server/restapi/config/GetIndex.java b/java/com/google/gerrit/server/restapi/config/GetIndex.java new file mode 100644 index 0000000..c96c66e --- /dev/null +++ b/java/com/google/gerrit/server/restapi/config/GetIndex.java
@@ -0,0 +1,31 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.restapi.config; + +import com.google.gerrit.extensions.restapi.AuthException; +import com.google.gerrit.extensions.restapi.BadRequestException; +import com.google.gerrit.extensions.restapi.ResourceConflictException; +import com.google.gerrit.extensions.restapi.Response; +import com.google.gerrit.extensions.restapi.RestReadView; +import com.google.gerrit.server.config.IndexResource; + +public class GetIndex implements RestReadView<IndexResource> { + + @Override + public Response<IndexInfo> apply(IndexResource rsrc) + throws AuthException, BadRequestException, ResourceConflictException, Exception { + return Response.ok(IndexInfo.fromIndexDefinition(rsrc.getIndexDefinition())); + } +}
diff --git a/java/com/google/gerrit/server/restapi/config/GetIndexVersion.java b/java/com/google/gerrit/server/restapi/config/GetIndexVersion.java new file mode 100644 index 0000000..1955511 --- /dev/null +++ b/java/com/google/gerrit/server/restapi/config/GetIndexVersion.java
@@ -0,0 +1,35 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.restapi.config; + +import com.google.gerrit.extensions.restapi.ResourceNotFoundException; +import com.google.gerrit.extensions.restapi.Response; +import com.google.gerrit.extensions.restapi.RestReadView; +import com.google.gerrit.index.IndexCollection; +import com.google.gerrit.server.config.IndexVersionResource; +import com.google.gerrit.server.restapi.config.IndexInfo.IndexVersionInfo; + +public class GetIndexVersion implements RestReadView<IndexVersionResource> { + + @Override + public Response<IndexVersionInfo> apply(IndexVersionResource rsrc) + throws ResourceNotFoundException { + IndexCollection<?, ?, ?> indexCollection = rsrc.getIndexDefinition().getIndexCollection(); + int version = rsrc.getIndex().getSchema().getVersion(); + boolean isSearch = indexCollection.getSearchIndex().getSchema().getVersion() == version; + boolean isWrite = indexCollection.getWriteIndex(version) != null; + return Response.ok(IndexInfo.IndexVersionInfo.create(isWrite, isSearch)); + } +}
diff --git a/java/com/google/gerrit/server/restapi/config/IndexCollection.java b/java/com/google/gerrit/server/restapi/config/IndexCollection.java index c5d295d..99a5718 100644 --- a/java/com/google/gerrit/server/restapi/config/IndexCollection.java +++ b/java/com/google/gerrit/server/restapi/config/IndexCollection.java
@@ -16,8 +16,6 @@ import static com.google.gerrit.common.data.GlobalCapability.MAINTAIN_SERVER; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableList; import com.google.gerrit.extensions.annotations.RequiresCapability; import com.google.gerrit.extensions.registration.DynamicMap; import com.google.gerrit.extensions.restapi.ChildCollection; @@ -32,8 +30,6 @@ import com.google.inject.Provider; import com.google.inject.Singleton; import java.util.Collection; -import java.util.List; -import java.util.Locale; @RequiresCapability(MAINTAIN_SERVER) @Singleton @@ -54,25 +50,10 @@ @Override public IndexResource parse(ConfigResource parent, IdString id) throws ResourceNotFoundException { - if (id.toString().toLowerCase(Locale.US).equals("all")) { - ImmutableList.Builder<com.google.gerrit.index.IndexCollection<?, ?, ?>> allIndexes = - ImmutableList.builder(); - for (IndexDefinition<?, ?, ?> def : defs) { - allIndexes.add(def.getIndexCollection()); - } - return new IndexResource(allIndexes.build()); - } - - List<String> segments = Splitter.on('~').splitToList(id.toString()); - if (segments.size() < 1 || 2 < segments.size()) { - throw new ResourceNotFoundException(id); - } - String indexName = segments.get(0); - Integer version = segments.size() == 2 ? Integer.valueOf(segments.get(1)) : null; - + String indexName = id.toString(); for (IndexDefinition<?, ?, ?> def : defs) { if (def.getName().equals(indexName)) { - return new IndexResource(def.getIndexCollection(), version); + return new IndexResource(def); } } throw new ResourceNotFoundException("Unknown index requested: " + indexName);
diff --git a/java/com/google/gerrit/server/restapi/config/IndexInfo.java b/java/com/google/gerrit/server/restapi/config/IndexInfo.java index 4a6bd3e53..5d52fe1 100644 --- a/java/com/google/gerrit/server/restapi/config/IndexInfo.java +++ b/java/com/google/gerrit/server/restapi/config/IndexInfo.java
@@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableSortedMap; import com.google.gerrit.index.Index; import com.google.gerrit.index.IndexCollection; +import com.google.gerrit.index.IndexDefinition; @AutoValue public abstract class IndexInfo { @@ -41,6 +42,10 @@ return new AutoValue_IndexInfo(name, versions.build()); } + public static IndexInfo fromIndexDefinition(IndexDefinition<?, ?, ?> def) { + return fromIndexCollection(def.getName(), def.getIndexCollection()); + } + public abstract String getName(); public abstract ImmutableMap<Integer, IndexVersionInfo> getVersions();
diff --git a/java/com/google/gerrit/server/restapi/config/IndexVersionsCollection.java b/java/com/google/gerrit/server/restapi/config/IndexVersionsCollection.java new file mode 100644 index 0000000..44c49f3 --- /dev/null +++ b/java/com/google/gerrit/server/restapi/config/IndexVersionsCollection.java
@@ -0,0 +1,83 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.restapi.config; + +import static com.google.gerrit.common.data.GlobalCapability.MAINTAIN_SERVER; + +import com.google.gerrit.extensions.annotations.RequiresCapability; +import com.google.gerrit.extensions.registration.DynamicMap; +import com.google.gerrit.extensions.restapi.ChildCollection; +import com.google.gerrit.extensions.restapi.IdString; +import com.google.gerrit.extensions.restapi.ResourceNotFoundException; +import com.google.gerrit.extensions.restapi.RestApiException; +import com.google.gerrit.extensions.restapi.RestView; +import com.google.gerrit.index.Index; +import com.google.gerrit.index.IndexCollection; +import com.google.gerrit.index.IndexDefinition; +import com.google.gerrit.server.config.IndexResource; +import com.google.gerrit.server.config.IndexVersionResource; +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.google.inject.Singleton; + +@RequiresCapability(MAINTAIN_SERVER) +@Singleton +public class IndexVersionsCollection + implements ChildCollection<IndexResource, IndexVersionResource> { + + private final DynamicMap<RestView<IndexVersionResource>> views; + private final Provider<ListIndexVersions> list; + + @Inject + IndexVersionsCollection( + DynamicMap<RestView<IndexVersionResource>> views, Provider<ListIndexVersions> list) { + this.views = views; + this.list = list; + } + + @Override + public RestView<IndexResource> list() throws RestApiException { + return list.get(); + } + + @Override + public IndexVersionResource parse(IndexResource parent, IdString id) + throws ResourceNotFoundException, Exception { + try { + int version = Integer.parseInt(id.get()); + IndexDefinition<?, ?, ?> def = parent.getIndexDefinition(); + IndexCollection<?, ?, ?> indexCollection = def.getIndexCollection(); + Index<?, ?> index = indexCollection.getWriteIndex(version); + if (index == null) { + Index<?, ?> searchIndex = indexCollection.getSearchIndex(); + if (searchIndex.getSchema().getVersion() == version) { + index = searchIndex; + } + } + if (index != null) { + return new IndexVersionResource(def, index); + } + } catch (NumberFormatException e) { + throw new ResourceNotFoundException("'" + id.get() + "' is not a number", e); + } + + throw new ResourceNotFoundException(); + } + + @Override + public DynamicMap<RestView<IndexVersionResource>> views() { + return views; + } +}
diff --git a/java/com/google/gerrit/server/restapi/config/ListIndexVersions.java b/java/com/google/gerrit/server/restapi/config/ListIndexVersions.java new file mode 100644 index 0000000..91bdae0 --- /dev/null +++ b/java/com/google/gerrit/server/restapi/config/ListIndexVersions.java
@@ -0,0 +1,37 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.restapi.config; + +import static com.google.gerrit.common.data.GlobalCapability.MAINTAIN_SERVER; + +import com.google.gerrit.extensions.annotations.RequiresCapability; +import com.google.gerrit.extensions.restapi.AuthException; +import com.google.gerrit.extensions.restapi.BadRequestException; +import com.google.gerrit.extensions.restapi.ResourceConflictException; +import com.google.gerrit.extensions.restapi.Response; +import com.google.gerrit.extensions.restapi.RestReadView; +import com.google.gerrit.server.config.IndexResource; +import java.util.Map; + +@RequiresCapability(MAINTAIN_SERVER) +public class ListIndexVersions implements RestReadView<IndexResource> { + + @Override + public Response<Map<Integer, IndexInfo.IndexVersionInfo>> apply(IndexResource rsrc) + throws AuthException, BadRequestException, ResourceConflictException, Exception { + IndexInfo info = IndexInfo.fromIndexDefinition(rsrc.getIndexDefinition()); + return Response.ok(info.getVersions()); + } +}
diff --git a/java/com/google/gerrit/server/restapi/config/ListIndexes.java b/java/com/google/gerrit/server/restapi/config/ListIndexes.java index 9710000..2e6664b 100644 --- a/java/com/google/gerrit/server/restapi/config/ListIndexes.java +++ b/java/com/google/gerrit/server/restapi/config/ListIndexes.java
@@ -16,6 +16,7 @@ import static com.google.gerrit.common.data.GlobalCapability.MAINTAIN_SERVER; +import com.google.common.collect.ImmutableList; import com.google.gerrit.extensions.annotations.RequiresCapability; import com.google.gerrit.extensions.restapi.Response; import com.google.gerrit.extensions.restapi.RestReadView; @@ -23,8 +24,6 @@ import com.google.gerrit.server.config.ConfigResource; import com.google.inject.Inject; import java.util.Collection; -import java.util.Map; -import java.util.TreeMap; @RequiresCapability(MAINTAIN_SERVER) public class ListIndexes implements RestReadView<ConfigResource> { @@ -35,13 +34,12 @@ this.defs = defs; } - private Map<String, IndexInfo> getIndexInfos() { - Map<String, IndexInfo> indexInfos = new TreeMap<>(); + private ImmutableList<IndexInfo> getIndexInfos() { + ImmutableList.Builder<IndexInfo> indexInfos = ImmutableList.builder(); for (IndexDefinition<?, ?, ?> def : defs) { - String name = def.getName(); - indexInfos.put(name, IndexInfo.fromIndexCollection(name, def.getIndexCollection())); + indexInfos.add(IndexInfo.fromIndexDefinition(def)); } - return indexInfos; + return indexInfos.build(); } @Override
diff --git a/java/com/google/gerrit/server/restapi/config/ReindexIndexVersion.java b/java/com/google/gerrit/server/restapi/config/ReindexIndexVersion.java new file mode 100644 index 0000000..d923155 --- /dev/null +++ b/java/com/google/gerrit/server/restapi/config/ReindexIndexVersion.java
@@ -0,0 +1,48 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.restapi.config; + +import com.google.gerrit.extensions.restapi.ResourceNotFoundException; +import com.google.gerrit.extensions.restapi.Response; +import com.google.gerrit.extensions.restapi.RestModifyView; +import com.google.gerrit.index.IndexDefinition; +import com.google.gerrit.server.config.IndexVersionResource; +import com.google.gerrit.server.index.IndexVersionReindexer; +import com.google.gerrit.server.restapi.config.ReindexIndexVersion.Input; +import com.google.inject.Inject; + +public class ReindexIndexVersion implements RestModifyView<IndexVersionResource, Input> { + public static class Input { + boolean reuse; + } + + private final IndexVersionReindexer indexVersionReindexer; + + @Inject + ReindexIndexVersion(IndexVersionReindexer indexVersionReindexer) { + this.indexVersionReindexer = indexVersionReindexer; + } + + @Override + public Response<?> apply(IndexVersionResource rsrc, Input input) + throws ResourceNotFoundException { + IndexDefinition<?, ?, ?> def = rsrc.getIndexDefinition(); + int version = rsrc.getIndex().getSchema().getVersion(); + @SuppressWarnings("unused") + var unused = indexVersionReindexer.reindex(def, version, input.reuse); + return Response.accepted( + String.format("Index %s version %d submitted for reindexing", def.getName(), version)); + } +}
diff --git a/java/com/google/gerrit/server/restapi/config/SnapshotIndex.java b/java/com/google/gerrit/server/restapi/config/SnapshotIndex.java index 72af97f..c50367f 100644 --- a/java/com/google/gerrit/server/restapi/config/SnapshotIndex.java +++ b/java/com/google/gerrit/server/restapi/config/SnapshotIndex.java
@@ -21,6 +21,7 @@ import com.google.gerrit.extensions.restapi.Response; import com.google.gerrit.extensions.restapi.RestModifyView; import com.google.gerrit.index.Index; +import com.google.gerrit.index.IndexDefinition; import com.google.gerrit.server.config.IndexResource; import com.google.gerrit.server.restapi.config.SnapshotIndex.Input; import com.google.inject.Singleton; @@ -29,7 +30,6 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import java.util.Collection; @RequiresCapability(MAINTAIN_SERVER) @Singleton @@ -38,12 +38,12 @@ @Override public Response<?> apply(IndexResource rsrc, Input input) throws IOException { - Collection<Index<?, ?>> indexes = rsrc.getIndexes(); String id = input.id; if (id == null) { id = LocalDateTime.now(ZoneId.systemDefault()).format(formatter); } - for (Index<?, ?> index : indexes) { + IndexDefinition<?, ?, ?> def = rsrc.getIndexDefinition(); + for (Index<?, ?> index : def.getIndexCollection().getWriteIndexes()) { try { @SuppressWarnings("unused") var unused = index.snapshot(id);
diff --git a/java/com/google/gerrit/server/restapi/config/SnapshotIndexVersion.java b/java/com/google/gerrit/server/restapi/config/SnapshotIndexVersion.java new file mode 100644 index 0000000..9f66ab3 --- /dev/null +++ b/java/com/google/gerrit/server/restapi/config/SnapshotIndexVersion.java
@@ -0,0 +1,54 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.restapi.config; + +import static com.google.gerrit.common.data.GlobalCapability.MAINTAIN_SERVER; + +import com.google.gerrit.extensions.annotations.RequiresCapability; +import com.google.gerrit.extensions.restapi.ResourceNotFoundException; +import com.google.gerrit.extensions.restapi.Response; +import com.google.gerrit.extensions.restapi.RestModifyView; +import com.google.gerrit.index.Index; +import com.google.gerrit.server.config.IndexVersionResource; +import com.google.gerrit.server.restapi.config.SnapshotIndexVersion.Input; +import com.google.inject.Singleton; +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; + +@RequiresCapability(MAINTAIN_SERVER) +@Singleton +public class SnapshotIndexVersion implements RestModifyView<IndexVersionResource, Input> { + private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss"); + + @Override + public Response<?> apply(IndexVersionResource rsrc, Input input) + throws IOException, ResourceNotFoundException { + String id = input.id; + if (id == null) { + id = LocalDateTime.now(ZoneId.systemDefault()).format(formatter); + } + Index<?, ?> index = rsrc.getIndex(); + var unused = index.snapshot(id); + SnapshotInfo info = new SnapshotInfo(); + info.id = id; + return Response.ok(info); + } + + public static class Input { + String id; + } +}
diff --git a/java/com/google/gerrit/server/restapi/config/SnapshotIndexes.java b/java/com/google/gerrit/server/restapi/config/SnapshotIndexes.java new file mode 100644 index 0000000..6a2c5f8 --- /dev/null +++ b/java/com/google/gerrit/server/restapi/config/SnapshotIndexes.java
@@ -0,0 +1,72 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.restapi.config; + +import static javax.servlet.http.HttpServletResponse.SC_CONFLICT; + +import com.google.gerrit.common.data.GlobalCapability; +import com.google.gerrit.extensions.annotations.RequiresCapability; +import com.google.gerrit.extensions.restapi.Response; +import com.google.gerrit.extensions.restapi.RestModifyView; +import com.google.gerrit.index.Index; +import com.google.gerrit.index.IndexDefinition; +import com.google.gerrit.server.config.ConfigResource; +import com.google.gerrit.server.restapi.config.SnapshotIndexes.Input; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import java.io.IOException; +import java.nio.file.FileAlreadyExistsException; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Collection; + +@RequiresCapability(GlobalCapability.MAINTAIN_SERVER) +@Singleton +public class SnapshotIndexes implements RestModifyView<ConfigResource, Input> { + private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss"); + + public static class Input { + String id; + } + + private final Collection<IndexDefinition<?, ?, ?>> defs; + + @Inject + SnapshotIndexes(Collection<IndexDefinition<?, ?, ?>> defs) { + this.defs = defs; + } + + @Override + public Response<?> apply(ConfigResource resource, Input input) throws IOException { + String id = input.id; + if (id == null) { + id = LocalDateTime.now(ZoneId.systemDefault()).format(formatter); + } + for (IndexDefinition<?, ?, ?> def : defs) { + for (Index<?, ?> index : def.getIndexCollection().getWriteIndexes()) { + try { + @SuppressWarnings("unused") + var unused = index.snapshot(id); + } catch (FileAlreadyExistsException e) { + return Response.withStatusCode(SC_CONFLICT, "Snapshot with same ID already exists."); + } + } + } + SnapshotInfo info = new SnapshotInfo(); + info.id = id; + return Response.ok(info); + } +}
diff --git a/java/com/google/gerrit/server/restapi/project/IndexChanges.java b/java/com/google/gerrit/server/restapi/project/IndexChanges.java index 6ad0005..532bd24 100644 --- a/java/com/google/gerrit/server/restapi/project/IndexChanges.java +++ b/java/com/google/gerrit/server/restapi/project/IndexChanges.java
@@ -32,7 +32,6 @@ import com.google.gerrit.server.index.change.ChangeIndexer; import com.google.gerrit.server.project.ProjectResource; import com.google.inject.Inject; -import com.google.inject.Provider; import com.google.inject.Singleton; import java.util.concurrent.Future; import org.eclipse.jgit.util.io.NullOutputStream; @@ -42,18 +41,18 @@ public class IndexChanges implements RestModifyView<ProjectResource, Input> { private final MultiProgressMonitor.Factory multiProgressMonitorFactory; - private final Provider<AllChangesIndexer> allChangesIndexerProvider; + private final AllChangesIndexer.Factory allChangesIndexerFactory; private final ChangeIndexer indexer; private final ListeningExecutorService executor; @Inject IndexChanges( MultiProgressMonitor.Factory multiProgressMonitorFactory, - Provider<AllChangesIndexer> allChangesIndexerProvider, + AllChangesIndexer.Factory allChangesIndexerFactory, ChangeIndexer indexer, @IndexExecutor(BATCH) ListeningExecutorService executor) { this.multiProgressMonitorFactory = multiProgressMonitorFactory; - this.allChangesIndexerProvider = allChangesIndexerProvider; + this.allChangesIndexerFactory = allChangesIndexerFactory; this.indexer = indexer; this.executor = executor; } @@ -65,7 +64,7 @@ multiProgressMonitorFactory .create(ByteStreams.nullOutputStream(), TaskKind.INDEXING, "Reindexing project") .beginSubTask("", MultiProgressMonitor.UNKNOWN); - AllChangesIndexer allChangesIndexer = allChangesIndexerProvider.get(); + AllChangesIndexer allChangesIndexer = allChangesIndexerFactory.create(); allChangesIndexer.setVerboseOut(NullOutputStream.INSTANCE); // The REST call is just a trigger for async reindexing, so it is safe to ignore the future's // return value.
diff --git a/javatests/com/google/gerrit/acceptance/pgm/AbstractReindexTests.java b/javatests/com/google/gerrit/acceptance/pgm/AbstractReindexTests.java index c5059c3..66c4078 100644 --- a/javatests/com/google/gerrit/acceptance/pgm/AbstractReindexTests.java +++ b/javatests/com/google/gerrit/acceptance/pgm/AbstractReindexTests.java
@@ -30,6 +30,7 @@ import com.google.gerrit.entities.Change; import com.google.gerrit.entities.Project; import com.google.gerrit.extensions.api.GerritApi; +import com.google.gerrit.extensions.api.changes.ReviewInput; import com.google.gerrit.extensions.common.ChangeInput; import com.google.gerrit.index.IndexDefinition; import com.google.gerrit.index.Schema; @@ -67,9 +68,45 @@ Files.createDirectory(sitePaths.index_dir); assertServerStartupFails(); - runGerrit("reindex", "-d", sitePaths.site_path.toString(), "--show-stack-trace"); + runGerrit("reindex", "-d", sitePaths.site_path.toString(), "--show-stack-trace", "--verbose"); + assertReady(ChangeSchemaDefinitions.INSTANCE.getLatest().getVersion()); + assertIndexQueries(); + } + + @Test + public void reindexWithSkipExistingDocumentsEnabled() throws Exception { + updateConfig(config -> config.setBoolean("index", null, "reuseExistingDocuments", true)); + setUpChange(); + + MoreFiles.deleteRecursively(sitePaths.index_dir, RecursiveDeleteOption.ALLOW_INSECURE); + Files.createDirectory(sitePaths.index_dir); + assertServerStartupFails(); + + runGerrit("reindex", "-d", sitePaths.site_path.toString(), "--show-stack-trace", "--verbose"); assertReady(ChangeSchemaDefinitions.INSTANCE.getLatest().getVersion()); + runGerrit("reindex", "-d", sitePaths.site_path.toString(), "--show-stack-trace", "--verbose"); + assertIndexQueries(); + + Files.copy(sitePaths.index_dir, sitePaths.resolve("index-backup")); + try (ServerContext ctx = startServer()) { + GerritApi gApi = ctx.getInjector().getInstance(GerritApi.class); + gApi.changes().id(changeId).revision(1).review(ReviewInput.approve()); + // Query change index + assertThat(gApi.changes().query("label:Code-Review+2").get().stream().map(c -> c.changeId)) + .containsExactly(changeId); + } + MoreFiles.deleteRecursively(sitePaths.index_dir, RecursiveDeleteOption.ALLOW_INSECURE); + Files.copy(sitePaths.resolve("index-backup"), sitePaths.index_dir); + runGerrit("reindex", "-d", sitePaths.site_path.toString(), "--show-stack-trace", "--verbose"); + try (ServerContext ctx = startServer()) { + GerritApi gApi = ctx.getInjector().getInstance(GerritApi.class); + assertThat(gApi.changes().query("label:Code-Review+2").get().stream().map(c -> c.changeId)) + .containsExactly(changeId); + } + } + + private void assertIndexQueries() throws Exception { try (ServerContext ctx = startServer()) { GerritApi gApi = ctx.getInjector().getInstance(GerritApi.class); // Query change index
diff --git a/javatests/com/google/gerrit/acceptance/rest/config/IndexSnapshotsIT.java b/javatests/com/google/gerrit/acceptance/rest/config/IndexSnapshotsIT.java index a88ee3e..904de9a 100644 --- a/javatests/com/google/gerrit/acceptance/rest/config/IndexSnapshotsIT.java +++ b/javatests/com/google/gerrit/acceptance/rest/config/IndexSnapshotsIT.java
@@ -16,21 +16,20 @@ import static com.google.common.truth.Truth.assertThat; -import com.google.common.collect.ImmutableList; import com.google.gerrit.acceptance.AbstractDaemonTest; import com.google.gerrit.acceptance.UseLocalDisk; import com.google.gerrit.extensions.restapi.Response; import com.google.gerrit.index.Index; -import com.google.gerrit.index.IndexCollection; -import com.google.gerrit.index.IndexDefinition; import com.google.gerrit.index.IndexType; -import com.google.gerrit.index.project.ProjectIndexCollection; +import com.google.gerrit.server.config.ConfigResource; import com.google.gerrit.server.config.IndexResource; import com.google.gerrit.server.config.SitePaths; -import com.google.gerrit.server.index.account.AccountIndexCollection; -import com.google.gerrit.server.index.change.ChangeIndexCollection; -import com.google.gerrit.server.index.group.GroupIndexCollection; +import com.google.gerrit.server.index.account.AccountIndexDefinition; +import com.google.gerrit.server.index.change.ChangeIndexDefinition; +import com.google.gerrit.server.index.group.GroupIndexDefinition; +import com.google.gerrit.server.index.project.ProjectIndexDefinition; import com.google.gerrit.server.restapi.config.SnapshotIndex; +import com.google.gerrit.server.restapi.config.SnapshotIndexes; import com.google.gerrit.server.restapi.config.SnapshotInfo; import com.google.gerrit.testing.SystemPropertiesTestRule; import com.google.inject.Inject; @@ -57,22 +56,25 @@ new SystemPropertiesTestRule(IndexType.SYS_PROP, "lucene"); @Inject private SnapshotIndex snapshotIndex; - @Inject private AccountIndexCollection accountIndexes; - @Inject private ChangeIndexCollection changeIndexes; - @Inject private GroupIndexCollection groupIndexes; - @Inject private ProjectIndexCollection projectIndexes; + @Inject private SnapshotIndexes snapshotIndexes; + @Inject private AccountIndexDefinition accountIndexDefinition; + @Inject private ChangeIndexDefinition changeIndexDefinition; + @Inject private GroupIndexDefinition groupIndexDefinition; + @Inject private ProjectIndexDefinition projectIndexDefinition; @Inject private SitePaths sitePaths; - @Inject private Collection<IndexDefinition<?, ?, ?>> defs; + + @Test + @UseLocalDisk + public void createAccountsIndexSnapshot() throws Exception { + Query query = new TermQuery(new Term("is", "active")); + createAndVerifySnapshot(new IndexResource(accountIndexDefinition), "accounts", query); + } @Test @UseLocalDisk public void createFullSnapshot() throws Exception { - ImmutableList.Builder<IndexCollection<?, ?, ?>> indexCollections = ImmutableList.builder(); - for (IndexDefinition<?, ?, ?> def : defs) { - indexCollections.add(def.getIndexCollection()); - } - File snapshot = createSnapshot(new IndexResource(indexCollections.build())); + File snapshot = createSnapshotOfAllIndexes(); File[] members = snapshot.listFiles(); for (File member : members) { assertThat(member.isDirectory()).isTrue(); @@ -82,30 +84,23 @@ @Test @UseLocalDisk - public void createAccountsIndexSnapshot() throws Exception { - Query query = new TermQuery(new Term("is", "active")); - createAndVerifySnapshot(new IndexResource(accountIndexes, null), "accounts", query); - } - - @Test - @UseLocalDisk public void createChangesIndexSnapshot() throws Exception { Query query = new TermQuery(new Term("status", "open")); - createAndVerifySnapshot(new IndexResource(changeIndexes, null), "changes", query); + createAndVerifySnapshot(new IndexResource(changeIndexDefinition), "changes", query); } @Test @UseLocalDisk public void createGroupsIndexSnapshot() throws Exception { Query query = new TermQuery(new Term("is", "active")); - createAndVerifySnapshot(new IndexResource(groupIndexes, null), "groups", query); + createAndVerifySnapshot(new IndexResource(groupIndexDefinition), "groups", query); } @Test @UseLocalDisk public void createProjectsIndexSnapshot() throws Exception { Query query = new TermQuery(new Term("name", "foo")); - createAndVerifySnapshot(new IndexResource(projectIndexes, null), "projects", query); + createAndVerifySnapshot(new IndexResource(projectIndexDefinition), "projects", query); } private File createAndVerifySnapshot(IndexResource rsrc, String prefix, Query query) @@ -113,8 +108,10 @@ File snapshot = createSnapshot(rsrc); File[] subdirs = snapshot.listFiles(); - assertThat(subdirs).hasLength(rsrc.getIndexes().size()); - for (Index<?, ?> i : rsrc.getIndexes()) { + Collection<? extends Index<?, ?>> indexes = + rsrc.getIndexDefinition().getIndexCollection().getWriteIndexes(); + assertThat(subdirs).hasLength(indexes.size()); + for (Index<?, ?> i : indexes) { String indexDirName = String.format("%s_%04d", prefix, i.getSchema().getVersion()); File[] result = snapshot.listFiles((d, n) -> n.equals(indexDirName)); assertThat(result).hasLength(1); @@ -126,6 +123,15 @@ private File createSnapshot(IndexResource rsrc) throws IOException { Response<?> rsp = snapshotIndex.apply(rsrc, new SnapshotIndex.Input()); + return verifySnapshot(rsp); + } + + private File createSnapshotOfAllIndexes() throws IOException { + Response<?> rsp = snapshotIndexes.apply(new ConfigResource(), new SnapshotIndexes.Input()); + return verifySnapshot(rsp); + } + + private File verifySnapshot(Response<?> rsp) { assertThat(rsp.value()).isInstanceOf(SnapshotInfo.class); SnapshotInfo snapshotInfo = (SnapshotInfo) rsp.value(); Path snapshotDir = sitePaths.index_dir.resolve("snapshots").resolve(snapshotInfo.id);
diff --git a/javatests/com/google/gerrit/acceptance/server/index/change/BUILD b/javatests/com/google/gerrit/acceptance/server/index/change/BUILD new file mode 100644 index 0000000..3be5249 --- /dev/null +++ b/javatests/com/google/gerrit/acceptance/server/index/change/BUILD
@@ -0,0 +1,7 @@ +load("//javatests/com/google/gerrit/acceptance:tests.bzl", "acceptance_tests") + +acceptance_tests( + srcs = glob(["*IT.java"]), + group = "server_index", + labels = ["server"], +)
diff --git a/javatests/com/google/gerrit/acceptance/server/index/change/LuceneChangeIndexerIT.java b/javatests/com/google/gerrit/acceptance/server/index/change/LuceneChangeIndexerIT.java new file mode 100644 index 0000000..b8af367 --- /dev/null +++ b/javatests/com/google/gerrit/acceptance/server/index/change/LuceneChangeIndexerIT.java
@@ -0,0 +1,137 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.acceptance.server.index.change; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.ListMultimap; +import com.google.gerrit.acceptance.AbstractDaemonTest; +import com.google.gerrit.acceptance.ChangeIndexedCounter; +import com.google.gerrit.acceptance.ExtensionRegistry; +import com.google.gerrit.acceptance.ExtensionRegistry.Registration; +import com.google.gerrit.acceptance.PushOneCommit; +import com.google.gerrit.acceptance.config.GerritConfig; +import com.google.gerrit.entities.Project.NameKey; +import com.google.gerrit.index.IndexDefinition; +import com.google.gerrit.index.RefState; +import com.google.gerrit.index.SiteIndexer.Result; +import com.google.gerrit.server.index.change.AllChangesIndexer; +import com.google.gerrit.server.index.change.ChangeIndex; +import com.google.gerrit.server.query.change.ChangeData; +import com.google.gerrit.testing.ConfigSuite; +import com.google.inject.Inject; +import java.util.Collection; +import java.util.Set; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.ObjectId; +import org.junit.Before; +import org.junit.Test; + +public class LuceneChangeIndexerIT extends AbstractDaemonTest { + @ConfigSuite.Default + public static Config defaultConfig() { + Config cfg = new Config(); + cfg.setBoolean("index", null, "autoReindexIfStale", false); + cfg.setString("index", null, "type", "lucene"); + return cfg; + } + + @Inject private ExtensionRegistry extensionRegistry; + + @Inject private Collection<IndexDefinition<?, ?, ?>> indexDefs; + private AllChangesIndexer allChangesIndexer; + private ChangeIndex index; + + @Before + public void setup() { + IndexDefinition<?, ?, ?> changeIndex = + indexDefs.stream().filter(i -> i.getName().equals("changes")).findFirst().get(); + allChangesIndexer = (AllChangesIndexer) changeIndex.getSiteIndexer(); + index = (ChangeIndex) changeIndex.getIndexCollection().getWriteIndexes().iterator().next(); + } + + @Test + @GerritConfig(name = "index.reuseExistingDocuments", value = "false") + public void testReindexWithoutReuse() throws Exception { + ChangeIndexedCounter changeIndexedCounter = new ChangeIndexedCounter(); + try (Registration registration = + extensionRegistry.newRegistration().add(changeIndexedCounter)) { + createChange(); + assertThat(changeIndexedCounter.getTotalCount()).isEqualTo(1); + changeIndexedCounter.clear(); + reindexChanges(); + assertThat(changeIndexedCounter.getTotalCount()).isEqualTo(1); + + createIndexWithMissingChangeAndReindex(changeIndexedCounter); + assertThat(changeIndexedCounter.getTotalCount()).isEqualTo(2); + + createIndexWithStaleChangeAndReindex(changeIndexedCounter); + assertThat(changeIndexedCounter.getTotalCount()).isEqualTo(3); + } + } + + @Test + @GerritConfig(name = "index.reuseExistingDocuments", value = "true") + public void testReindexWithReuse() throws Exception { + ChangeIndexedCounter changeIndexedCounter = new ChangeIndexedCounter(); + try (Registration registration = + extensionRegistry.newRegistration().add(changeIndexedCounter)) { + createChange(); + assertThat(changeIndexedCounter.getTotalCount()).isEqualTo(1); + changeIndexedCounter.clear(); + reindexChanges(); + assertThat(changeIndexedCounter.getTotalCount()).isEqualTo(0); + + createIndexWithMissingChangeAndReindex(changeIndexedCounter); + assertThat(changeIndexedCounter.getTotalCount()).isEqualTo(1); + + createIndexWithStaleChangeAndReindex(changeIndexedCounter); + assertThat(changeIndexedCounter.getTotalCount()).isEqualTo(1); + } + } + + private void createIndexWithMissingChangeAndReindex(ChangeIndexedCounter changeIndexedCounter) + throws Exception { + PushOneCommit.Result res = createChange(); + index.delete(res.getChange().getId()); + changeIndexedCounter.clear(); + reindexChanges(); + } + + private void createIndexWithStaleChangeAndReindex(ChangeIndexedCounter changeIndexedCounter) + throws Exception { + PushOneCommit.Result res = createChange(); + ChangeData wrongChangeData = res.getChange(); + ListMultimap<NameKey, RefState> refStates = + LinkedListMultimap.create(wrongChangeData.getRefStates()); + refStates.replaceValues( + project, + Set.of( + RefState.create( + "refs/changes/abcd", + ObjectId.fromString("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef")))); + wrongChangeData.setRefStates(ImmutableSetMultimap.copyOf(refStates)); + index.replace(wrongChangeData); + changeIndexedCounter.clear(); + reindexChanges(); + } + + private void reindexChanges() throws Exception { + Result res = allChangesIndexer.indexAll(index); + assertThat(res.success()).isTrue(); + } +}
diff --git a/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java b/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java index 6258b18..7a7cff5 100644 --- a/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java +++ b/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java
@@ -3576,7 +3576,7 @@ getChangeApi(change).addReviewer(anotherUser.toString()); assertQuery("reviewer:self", change); - assertThat(indexer.reindexIfStale(project, change.getId()).get()).isFalse(); + assertThat(indexer.reindexIfStale(project, change.getId())).isFalse(); // Remove reviewer behind index's back. ChangeUpdate update = newUpdate(change); @@ -3585,7 +3585,7 @@ // Index is stale. assertQuery("reviewer:self", change); - assertThat(indexer.reindexIfStale(project, change.getId()).get()).isTrue(); + assertThat(indexer.reindexIfStale(project, change.getId())).isTrue(); assertQuery("reviewer:self"); // Index is not stale when a draft comment exists @@ -3594,7 +3594,7 @@ in.message = "nit: trailing whitespace"; in.path = Patch.COMMIT_MSG; getChangeApi(change).current().createDraft(in); - assertThat(indexer.reindexIfStale(project, change.getId()).get()).isFalse(); + assertThat(indexer.reindexIfStale(project, change.getId())).isFalse(); } @Test