Add REST API to create snapshots of Lucene indexes
To safely backup indexes in Gerrit, Gerrit had to be shut down.
This is required, since an index consists of multiple files and
copying those while the index changes can lead to a broken
inconsistent state of the index. Lucene however provides a
mechanism to create a snapshot at runtime.
This change adds REST API endpoints to create snapshots of all
write indexes of a given index type. These snapshots will be
stored on the server in the Gerrit site under `$SITE/index/snapshots`
and can be used to restore the index state.
Release-Notes: REST API to create snapshot of Lucene indexes
Change-Id: I50f04e382d02d9dae9dee8b7c98467cd1c0c8cdb
diff --git a/Documentation/rest-api-config.txt b/Documentation/rest-api-config.txt
index 70c4b4d..918aba5 100644
--- a/Documentation/rest-api-config.txt
+++ b/Documentation/rest-api-config.txt
@@ -1488,6 +1488,69 @@
}
----
+[[snapshot-index]]
+=== Create Index Snapshot
+
+This endpoint allows Gerrit admins to create a snapshot of an index.
+This snapshot can be used as a backup of the index.
+
+A snapshot of all versions of an index can be created by just using
+the name of the index, e.g. `changes`. Only snapshots of indexes that
+Gerrit currently writes to can be created. An index version can be
+selected by using e.g. `changes~84`. Snapshots of all indexes can be
+created by using `all` instead of an index name.
+
+Note, that the creation of multiple snapshots, e.g. of different index
+versions, is not atomic. If a consistent state over multiple indexes is
+required, the server has to be put into read-only mode before creating
+the snapshot.
+
+The snapshots will be stored on the server at `$SITE/index/snapshots/$ID`.
+The `$ID` can be optionally provided in link:#snapshot-index-input[SnapshotIndex.Input]
+or will default to the current local time in ISO8601 format.
+
+.Request
+----
+ PUT /config/server/indexes/all/snapshot HTTP/1.0
+ Content-Type: application/json; charset=UTF-8
+
+ {
+ "id": "snapshot-1"
+ }
+----
+
+.Response
+----
+ HTTP/1.1 200 OK
+ Content-Type: application/json; charset=UTF-8
+
+ )]}'
+ {
+ "id": "snapshot-1"
+ }
+----
+
+.Request
+----
+ PUT /config/server/indexes/accounts~13/snapshot HTTP/1.0
+ Content-Type: application/json; charset=UTF-8
+
+ {
+ "id": "snapshot-1"
+ }
+----
+
+.Response
+----
+ HTTP/1.1 200 OK
+ Content-Type: application/json; charset=UTF-8
+
+ )]}'
+ {
+ "id": "snapshot-1"
+ }
+----
+
[[ids]]
== IDs
@@ -2134,6 +2197,19 @@
requirements that are applicable for changes appearing in the dashboard.
|=======================================
+[[snapshot-index-input]]
+=== SnapshotIndex.Input
+The `SnapshotIndex.Input` entity contains the parameters used to create an
+index snapshot.
+
+[options="header",cols="1,^1,5"]
+|=======================
+|Field Name ||Description
+|`id` | optional |
+A string ID that will be used as the folder name containing the
+snapshots. Defaults to current timestamp.
+|=======================
+
[[sshd-info]]
=== SshdInfo
The `SshdInfo` entity contains information about Gerrit
diff --git a/java/com/google/gerrit/index/Index.java b/java/com/google/gerrit/index/Index.java
index 3ed76ba..ec530c1 100644
--- a/java/com/google/gerrit/index/Index.java
+++ b/java/com/google/gerrit/index/Index.java
@@ -22,6 +22,7 @@
import com.google.gerrit.index.query.Matchable;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
+import java.io.IOException;
import java.util.Optional;
/**
@@ -169,4 +170,15 @@
default Optional<Matchable<V>> getIndexFilter() {
return Optional.empty();
}
+
+ /**
+ * Creates a snapshot of the index.
+ *
+ * @param id an ID used for the snapshot.
+ * @return {@code true} if the snapshot was successful.
+ * @throws IOException if writing the snapshot to disk fails.
+ */
+ default boolean snapshot(String id) throws IOException {
+ return false;
+ }
}
diff --git a/java/com/google/gerrit/lucene/AbstractLuceneIndex.java b/java/com/google/gerrit/lucene/AbstractLuceneIndex.java
index 938cd67..e00c394 100644
--- a/java/com/google/gerrit/lucene/AbstractLuceneIndex.java
+++ b/java/com/google/gerrit/lucene/AbstractLuceneIndex.java
@@ -25,6 +25,7 @@
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Sets;
import com.google.common.flogger.FluentLogger;
+import com.google.common.io.Files;
import com.google.common.primitives.Ints;
import com.google.common.util.concurrent.AbstractFuture;
import com.google.common.util.concurrent.Futures;
@@ -54,6 +55,8 @@
import com.google.gerrit.server.logging.LoggingContextAwareScheduledExecutorService;
import com.google.protobuf.MessageLite;
import java.io.IOException;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Path;
import java.sql.Timestamp;
import java.util.Set;
import java.util.concurrent.Callable;
@@ -74,8 +77,10 @@
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.SnapshotDeletionPolicy;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.ControlledRealTimeReopenThread;
import org.apache.lucene.search.IndexSearcher;
@@ -88,6 +93,7 @@
import org.apache.lucene.search.TopFieldDocs;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
/** Basic Lucene index implementation. */
public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> {
@@ -134,6 +140,9 @@
String index = Joiner.on('_').skipNulls().join(name, subIndex);
long commitPeriod = writerConfig.getCommitWithinMs();
+ writerConfig.setIndexDeletionPolicy(
+ new SnapshotDeletionPolicy(writerConfig.getIndexDeletionPolicy()));
+
if (commitPeriod < 0) {
writer = new AutoCommitWriter(dir, writerConfig.getLuceneConfig());
} else if (commitPeriod == 0) {
@@ -516,6 +525,34 @@
return schema;
}
+ @Override
+ public boolean snapshot(String id) throws IOException {
+ SnapshotDeletionPolicy snapshooter =
+ (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
+
+ IndexCommit commit = snapshooter.snapshot();
+ try {
+ Path sourceDir = canonical(((FSDirectory) commit.getDirectory()).getDirectory());
+ Path indexDir = canonical(sitePaths.index_dir);
+ Path targetDir =
+ indexDir.resolve("snapshots").resolve(id).resolve(indexDir.relativize(sourceDir));
+ if (targetDir.toFile().exists()) {
+ throw new FileAlreadyExistsException(targetDir.toString());
+ }
+ targetDir.toFile().mkdirs();
+ for (String file : commit.getFileNames()) {
+ Files.copy(sourceDir.resolve(file).toFile(), targetDir.resolve(file).toFile());
+ }
+ } finally {
+ snapshooter.release(commit);
+ }
+ return true;
+ }
+
+ private static Path canonical(Path p) throws IOException {
+ return p.toFile().getCanonicalFile().toPath();
+ }
+
protected class LuceneQuerySource implements DataSource<V> {
private final QueryOptions opts;
private final Query query;
diff --git a/java/com/google/gerrit/lucene/GerritIndexWriterConfig.java b/java/com/google/gerrit/lucene/GerritIndexWriterConfig.java
index f6b2f0e..bec63bd 100644
--- a/java/com/google/gerrit/lucene/GerritIndexWriterConfig.java
+++ b/java/com/google/gerrit/lucene/GerritIndexWriterConfig.java
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.ConcurrentMergeScheduler;
+import org.apache.lucene.index.IndexDeletionPolicy;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.eclipse.jgit.lib.Config;
@@ -77,6 +78,14 @@
}
}
+ void setIndexDeletionPolicy(IndexDeletionPolicy indexDeletionPolicy) {
+ luceneConfig.setIndexDeletionPolicy(indexDeletionPolicy);
+ }
+
+ IndexDeletionPolicy getIndexDeletionPolicy() {
+ return luceneConfig.getIndexDeletionPolicy();
+ }
+
CustomMappingAnalyzer getAnalyzer() {
return analyzer;
}
diff --git a/java/com/google/gerrit/lucene/LuceneChangeIndex.java b/java/com/google/gerrit/lucene/LuceneChangeIndex.java
index f97293e..45288d5 100644
--- a/java/com/google/gerrit/lucene/LuceneChangeIndex.java
+++ b/java/com/google/gerrit/lucene/LuceneChangeIndex.java
@@ -284,6 +284,11 @@
openIndex.markReady(ready);
}
+ @Override
+ public boolean snapshot(String id) throws IOException {
+ return openIndex.snapshot(id) && closedIndex.snapshot(id);
+ }
+
private Sort getSort() {
return new Sort(
new SortField(UPDATED_SORT_FIELD, SortField.Type.LONG, true),
diff --git a/java/com/google/gerrit/server/config/IndexResource.java b/java/com/google/gerrit/server/config/IndexResource.java
index ea1b57a..c1b70dc02 100644
--- a/java/com/google/gerrit/server/config/IndexResource.java
+++ b/java/com/google/gerrit/server/config/IndexResource.java
@@ -14,23 +14,44 @@
package com.google.gerrit.server.config;
+import com.google.common.collect.ImmutableList;
+import com.google.gerrit.common.Nullable;
+import com.google.gerrit.extensions.restapi.ResourceNotFoundException;
import com.google.gerrit.extensions.restapi.RestView;
import com.google.gerrit.index.Index;
import com.google.gerrit.index.IndexCollection;
import com.google.inject.TypeLiteral;
import java.util.Collection;
+import java.util.List;
public class IndexResource extends ConfigResource {
public static final TypeLiteral<RestView<IndexResource>> INDEX_KIND = new TypeLiteral<>() {};
- private final IndexCollection<?, ?, ?> indexes;
+ private final Collection<Index<?, ?>> indexes;
- public IndexResource(IndexCollection<?, ?, ?> indexes) {
- this.indexes = indexes;
+ public IndexResource(IndexCollection<?, ?, ?> indexes, @Nullable Integer version)
+ throws ResourceNotFoundException {
+ if (version == null) {
+ this.indexes = ImmutableList.copyOf(indexes.getWriteIndexes());
+ } else {
+ Index<?, ?> index = indexes.getWriteIndex(version);
+ if (index == null) {
+ throw new ResourceNotFoundException(
+ String.format("Unknown index version requested: %d", version));
+ }
+ this.indexes = ImmutableList.of(index);
+ }
}
- @SuppressWarnings("unchecked")
- public Collection<Index<?, ?>> getWriteIndexes() {
- return (Collection<Index<?, ?>>) indexes.getWriteIndexes();
+ public IndexResource(List<IndexCollection<?, ?, ?>> indexes) {
+ ImmutableList.Builder<Index<?, ?>> allIndexes = ImmutableList.builder();
+ for (IndexCollection<?, ?, ?> index : indexes) {
+ allIndexes.addAll(index.getWriteIndexes());
+ }
+ this.indexes = allIndexes.build();
+ }
+
+ public Collection<Index<?, ?>> getIndexes() {
+ return indexes;
}
}
diff --git a/java/com/google/gerrit/server/restapi/config/ConfigRestApiModule.java b/java/com/google/gerrit/server/restapi/config/ConfigRestApiModule.java
index f34a4a3..77caab4 100644
--- a/java/com/google/gerrit/server/restapi/config/ConfigRestApiModule.java
+++ b/java/com/google/gerrit/server/restapi/config/ConfigRestApiModule.java
@@ -53,6 +53,8 @@
get(CONFIG_KIND, "version").to(GetVersion.class);
child(CONFIG_KIND, "indexes").to(IndexCollection.class);
+ put(INDEX_KIND, "snapshot").to(SnapshotIndex.class);
+
// The caches and summary REST endpoints are bound via RestCacheAdminModule.
}
}
diff --git a/java/com/google/gerrit/server/restapi/config/IndexCollection.java b/java/com/google/gerrit/server/restapi/config/IndexCollection.java
index 3e00ebd..c5d295d 100644
--- a/java/com/google/gerrit/server/restapi/config/IndexCollection.java
+++ b/java/com/google/gerrit/server/restapi/config/IndexCollection.java
@@ -16,6 +16,8 @@
import static com.google.gerrit.common.data.GlobalCapability.MAINTAIN_SERVER;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
import com.google.gerrit.extensions.annotations.RequiresCapability;
import com.google.gerrit.extensions.registration.DynamicMap;
import com.google.gerrit.extensions.restapi.ChildCollection;
@@ -30,6 +32,8 @@
import com.google.inject.Provider;
import com.google.inject.Singleton;
import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
@RequiresCapability(MAINTAIN_SERVER)
@Singleton
@@ -50,11 +54,25 @@
@Override
public IndexResource parse(ConfigResource parent, IdString id) throws ResourceNotFoundException {
- String indexName = id.get();
+ if (id.toString().toLowerCase(Locale.US).equals("all")) {
+ ImmutableList.Builder<com.google.gerrit.index.IndexCollection<?, ?, ?>> allIndexes =
+ ImmutableList.builder();
+ for (IndexDefinition<?, ?, ?> def : defs) {
+ allIndexes.add(def.getIndexCollection());
+ }
+ return new IndexResource(allIndexes.build());
+ }
+
+ List<String> segments = Splitter.on('~').splitToList(id.toString());
+ if (segments.size() < 1 || 2 < segments.size()) {
+ throw new ResourceNotFoundException(id);
+ }
+ String indexName = segments.get(0);
+ Integer version = segments.size() == 2 ? Integer.valueOf(segments.get(1)) : null;
for (IndexDefinition<?, ?, ?> def : defs) {
if (def.getName().equals(indexName)) {
- return new IndexResource(def.getIndexCollection());
+ return new IndexResource(def.getIndexCollection(), version);
}
}
throw new ResourceNotFoundException("Unknown index requested: " + indexName);
diff --git a/java/com/google/gerrit/server/restapi/config/SnapshotIndex.java b/java/com/google/gerrit/server/restapi/config/SnapshotIndex.java
new file mode 100644
index 0000000..b5f1baf
--- /dev/null
+++ b/java/com/google/gerrit/server/restapi/config/SnapshotIndex.java
@@ -0,0 +1,61 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.restapi.config;
+
+import static com.google.gerrit.common.data.GlobalCapability.MAINTAIN_SERVER;
+import static javax.servlet.http.HttpServletResponse.SC_CONFLICT;
+
+import com.google.gerrit.extensions.annotations.RequiresCapability;
+import com.google.gerrit.extensions.restapi.Response;
+import com.google.gerrit.extensions.restapi.RestModifyView;
+import com.google.gerrit.index.Index;
+import com.google.gerrit.server.config.IndexResource;
+import com.google.gerrit.server.restapi.config.SnapshotIndex.Input;
+import com.google.inject.Singleton;
+import java.io.IOException;
+import java.nio.file.FileAlreadyExistsException;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.util.Collection;
+
+@RequiresCapability(MAINTAIN_SERVER)
+@Singleton
+public class SnapshotIndex implements RestModifyView<IndexResource, Input> {
+ private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss");
+
+ @Override
+ public Response<?> apply(IndexResource rsrc, Input input) throws IOException {
+ Collection<Index<?, ?>> indexes = rsrc.getIndexes();
+ String id = input.id;
+ if (id == null) {
+ id = LocalDateTime.now(ZoneId.systemDefault()).format(formatter);
+ }
+ for (Index<?, ?> index : indexes) {
+ try {
+ index.snapshot(id);
+ } catch (FileAlreadyExistsException e) {
+ return Response.withStatusCode(SC_CONFLICT, "Snapshot with same ID already exists.");
+ }
+ }
+ SnapshotInfo info = new SnapshotInfo();
+ info.id = id;
+ return Response.ok(info);
+ }
+
+ public static class Input {
+ String id;
+ }
+}
diff --git a/java/com/google/gerrit/server/restapi/config/SnapshotInfo.java b/java/com/google/gerrit/server/restapi/config/SnapshotInfo.java
new file mode 100644
index 0000000..addc559
--- /dev/null
+++ b/java/com/google/gerrit/server/restapi/config/SnapshotInfo.java
@@ -0,0 +1,19 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.restapi.config;
+
+public class SnapshotInfo {
+ public String id;
+}
diff --git a/javatests/com/google/gerrit/acceptance/rest/config/BUILD b/javatests/com/google/gerrit/acceptance/rest/config/BUILD
index 8550423..7a16841 100644
--- a/javatests/com/google/gerrit/acceptance/rest/config/BUILD
+++ b/javatests/com/google/gerrit/acceptance/rest/config/BUILD
@@ -6,5 +6,6 @@
labels = ["rest"],
deps = [
"//java/com/google/gerrit/server/restapi",
+ "//lib/lucene:lucene-core",
],
)
diff --git a/javatests/com/google/gerrit/acceptance/rest/config/IndexSnapshotsIT.java b/javatests/com/google/gerrit/acceptance/rest/config/IndexSnapshotsIT.java
new file mode 100644
index 0000000..a88ee3e
--- /dev/null
+++ b/javatests/com/google/gerrit/acceptance/rest/config/IndexSnapshotsIT.java
@@ -0,0 +1,222 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.acceptance.rest.config;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.collect.ImmutableList;
+import com.google.gerrit.acceptance.AbstractDaemonTest;
+import com.google.gerrit.acceptance.UseLocalDisk;
+import com.google.gerrit.extensions.restapi.Response;
+import com.google.gerrit.index.Index;
+import com.google.gerrit.index.IndexCollection;
+import com.google.gerrit.index.IndexDefinition;
+import com.google.gerrit.index.IndexType;
+import com.google.gerrit.index.project.ProjectIndexCollection;
+import com.google.gerrit.server.config.IndexResource;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.index.account.AccountIndexCollection;
+import com.google.gerrit.server.index.change.ChangeIndexCollection;
+import com.google.gerrit.server.index.group.GroupIndexCollection;
+import com.google.gerrit.server.restapi.config.SnapshotIndex;
+import com.google.gerrit.server.restapi.config.SnapshotInfo;
+import com.google.gerrit.testing.SystemPropertiesTestRule;
+import com.google.inject.Inject;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Collection;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+public class IndexSnapshotsIT extends AbstractDaemonTest {
+
+ @ClassRule
+ public static SystemPropertiesTestRule systemProperties =
+ new SystemPropertiesTestRule(IndexType.SYS_PROP, "lucene");
+
+ @Inject private SnapshotIndex snapshotIndex;
+ @Inject private AccountIndexCollection accountIndexes;
+ @Inject private ChangeIndexCollection changeIndexes;
+ @Inject private GroupIndexCollection groupIndexes;
+ @Inject private ProjectIndexCollection projectIndexes;
+
+ @Inject private SitePaths sitePaths;
+ @Inject private Collection<IndexDefinition<?, ?, ?>> defs;
+
+ @Test
+ @UseLocalDisk
+ public void createFullSnapshot() throws Exception {
+ ImmutableList.Builder<IndexCollection<?, ?, ?>> indexCollections = ImmutableList.builder();
+ for (IndexDefinition<?, ?, ?> def : defs) {
+ indexCollections.add(def.getIndexCollection());
+ }
+ File snapshot = createSnapshot(new IndexResource(indexCollections.build()));
+ File[] members = snapshot.listFiles();
+ for (File member : members) {
+ assertThat(member.isDirectory()).isTrue();
+ verifyIndexCanBeOpen(member);
+ }
+ }
+
+ @Test
+ @UseLocalDisk
+ public void createAccountsIndexSnapshot() throws Exception {
+ Query query = new TermQuery(new Term("is", "active"));
+ createAndVerifySnapshot(new IndexResource(accountIndexes, null), "accounts", query);
+ }
+
+ @Test
+ @UseLocalDisk
+ public void createChangesIndexSnapshot() throws Exception {
+ Query query = new TermQuery(new Term("status", "open"));
+ createAndVerifySnapshot(new IndexResource(changeIndexes, null), "changes", query);
+ }
+
+ @Test
+ @UseLocalDisk
+ public void createGroupsIndexSnapshot() throws Exception {
+ Query query = new TermQuery(new Term("is", "active"));
+ createAndVerifySnapshot(new IndexResource(groupIndexes, null), "groups", query);
+ }
+
+ @Test
+ @UseLocalDisk
+ public void createProjectsIndexSnapshot() throws Exception {
+ Query query = new TermQuery(new Term("name", "foo"));
+ createAndVerifySnapshot(new IndexResource(projectIndexes, null), "projects", query);
+ }
+
+ private File createAndVerifySnapshot(IndexResource rsrc, String prefix, Query query)
+ throws IOException {
+ File snapshot = createSnapshot(rsrc);
+
+ File[] subdirs = snapshot.listFiles();
+ assertThat(subdirs).hasLength(rsrc.getIndexes().size());
+ for (Index<?, ?> i : rsrc.getIndexes()) {
+ String indexDirName = String.format("%s_%04d", prefix, i.getSchema().getVersion());
+ File[] result = snapshot.listFiles((d, n) -> n.equals(indexDirName));
+ assertThat(result).hasLength(1);
+ File accountsIndexSnapshot = result[0];
+ openIndexAndQuery(accountsIndexSnapshot, query);
+ }
+ return snapshot;
+ }
+
+ private File createSnapshot(IndexResource rsrc) throws IOException {
+ Response<?> rsp = snapshotIndex.apply(rsrc, new SnapshotIndex.Input());
+ assertThat(rsp.value()).isInstanceOf(SnapshotInfo.class);
+ SnapshotInfo snapshotInfo = (SnapshotInfo) rsp.value();
+ Path snapshotDir = sitePaths.index_dir.resolve("snapshots").resolve(snapshotInfo.id);
+ File snapshot = snapshotDir.toFile();
+ assertThat(snapshot.exists()).isTrue();
+ assertThat(snapshot.isDirectory()).isTrue();
+ return snapshot;
+ }
+
+ private void verifyIndexCanBeOpen(File indexDir) throws IOException {
+ createIndex(indexDir).tryOpen();
+ }
+
+ private void openIndexAndQuery(File indexDir, Query query) throws IOException {
+ BaseIndex index = createIndex(indexDir);
+ index.openAndQuery(query);
+ }
+
+ private BaseIndex createIndex(File indexDir) {
+ BaseIndex index;
+ if (indexDir.getName().startsWith("changes")) {
+ index = new ChangeIndex(indexDir);
+ } else {
+ index = new SimpleIndex(indexDir);
+ }
+ return index;
+ }
+
+ private abstract static class BaseIndex {
+ protected File indexDir;
+
+ BaseIndex(File indexDir) {
+ this.indexDir = indexDir;
+ }
+
+ abstract void tryOpen() throws IOException;
+
+ abstract void openAndQuery(Query query) throws IOException;
+ }
+
+ private static class SimpleIndex extends BaseIndex {
+ SimpleIndex(File indexDir) {
+ super(indexDir);
+ }
+
+ @Override
+ void tryOpen() throws IOException {
+ Directory index = FSDirectory.open(indexDir.toPath());
+ try (IndexReader reader = DirectoryReader.open(index)) {}
+ }
+
+ @Override
+ void openAndQuery(Query query) throws IOException {
+ Directory index = FSDirectory.open(indexDir.toPath());
+ try (IndexReader reader = DirectoryReader.open(index)) {
+ IndexSearcher searcher = new IndexSearcher(reader);
+ TopDocs result = searcher.search(query, 10);
+ System.out.printf("query result length = %d\n", result.scoreDocs.length);
+ }
+ }
+ }
+
+ private static class ChangeIndex extends BaseIndex {
+ private SimpleIndex open;
+ private SimpleIndex closed;
+
+ ChangeIndex(File indexDir) {
+ super(indexDir);
+ File[] subDirs = indexDir.listFiles();
+ for (File subDir : subDirs) {
+ String name = subDir.getName();
+ if (name.equals("open")) {
+ this.open = new SimpleIndex(subDir);
+ } else if (name.equals("closed")) {
+ this.closed = new SimpleIndex(subDir);
+ } else {
+ throw new IllegalStateException("Unexpected subdir in changes index " + name);
+ }
+ }
+ }
+
+ @Override
+ void tryOpen() throws IOException {
+ open.tryOpen();
+ closed.tryOpen();
+ }
+
+ @Override
+ void openAndQuery(Query query) throws IOException {
+ open.openAndQuery(query);
+ closed.openAndQuery(query);
+ }
+ }
+}
