Add ref_states to ProjectFields
This commit adds a new field called ref_states to the ProjectIndex. This
field can be used to perform staleness detection based on NoteDb ref
states. We store the HEAD SHA1s in 40 digit hex notation for the project
in question as well as all of it's ancestors.
Storing the ancestors ref states as well is necessary because the index
stores the names of all ancestors, so we need to detect when the
project's hirarchy changes and we get out of sync.
This approach follows what was done for changes, accounts and groups.
A subsequent commit will build a staleness checker and wire it up.
Change-Id: I08efd4286b4fd5c577c0863802bcdf051972f5cf
diff --git a/java/com/google/gerrit/acceptance/ProjectResetter.java b/java/com/google/gerrit/acceptance/ProjectResetter.java
index f4a8da3..1fed8f8 100644
--- a/java/com/google/gerrit/acceptance/ProjectResetter.java
+++ b/java/com/google/gerrit/acceptance/ProjectResetter.java
@@ -23,6 +23,7 @@
import com.google.common.collect.MultimapBuilder;
import com.google.common.collect.Sets;
import com.google.gerrit.common.Nullable;
+import com.google.gerrit.index.RefState;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.AccountGroup;
import com.google.gerrit.reviewdb.client.Project;
@@ -32,7 +33,6 @@
import com.google.gerrit.server.account.GroupIncludeCache;
import com.google.gerrit.server.config.AllUsersName;
import com.google.gerrit.server.git.GitRepositoryManager;
-import com.google.gerrit.server.index.RefState;
import com.google.gerrit.server.index.account.AccountIndexer;
import com.google.gerrit.server.index.group.GroupIndexer;
import com.google.gerrit.server.project.ProjectCache;
diff --git a/java/com/google/gerrit/index/BUILD b/java/com/google/gerrit/index/BUILD
index 2442b593..6604ca1 100644
--- a/java/com/google/gerrit/index/BUILD
+++ b/java/com/google/gerrit/index/BUILD
@@ -37,6 +37,7 @@
"//java/com/google/gerrit/common:annotations",
"//java/com/google/gerrit/extensions:api",
"//java/com/google/gerrit/metrics",
+ "//java/com/google/gerrit/reviewdb:server",
"//lib:guava",
"//lib:gwtjsonrpc",
"//lib:gwtorm",
diff --git a/java/com/google/gerrit/server/index/RefState.java b/java/com/google/gerrit/index/RefState.java
similarity index 98%
rename from java/com/google/gerrit/server/index/RefState.java
rename to java/com/google/gerrit/index/RefState.java
index 6b893f0..f0e465d 100644
--- a/java/com/google/gerrit/server/index/RefState.java
+++ b/java/com/google/gerrit/index/RefState.java
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package com.google.gerrit.server.index;
+package com.google.gerrit.index;
import static com.google.common.base.MoreObjects.firstNonNull;
import static com.google.common.base.Preconditions.checkArgument;
diff --git a/java/com/google/gerrit/index/project/ProjectData.java b/java/com/google/gerrit/index/project/ProjectData.java
index 9d6b571..7365660 100644
--- a/java/com/google/gerrit/index/project/ProjectData.java
+++ b/java/com/google/gerrit/index/project/ProjectData.java
@@ -14,23 +14,43 @@
package com.google.gerrit.index.project;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+
import com.google.common.collect.ImmutableList;
import com.google.gerrit.reviewdb.client.Project;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
public class ProjectData {
private final Project project;
- private final ImmutableList<Project.NameKey> ancestors;
+ private final Optional<ProjectData> parent;
- public ProjectData(Project project, Iterable<Project.NameKey> ancestors) {
+ public ProjectData(Project project, Optional<ProjectData> parent) {
this.project = project;
- this.ancestors = ImmutableList.copyOf(ancestors);
+ this.parent = parent;
}
public Project getProject() {
return project;
}
- public ImmutableList<Project.NameKey> getAncestors() {
- return ancestors;
+ public Optional<ProjectData> getParent() {
+ return parent;
+ }
+
+ /** Returns all {@link ProjectData} in the hierarchy starting with the current one. */
+ public ImmutableList<ProjectData> tree() {
+ List<ProjectData> parents = new ArrayList<>();
+ Optional<ProjectData> curr = Optional.of(this);
+ while (curr.isPresent()) {
+ parents.add(curr.get());
+ curr = curr.get().parent;
+ }
+ return ImmutableList.copyOf(parents);
+ }
+
+ public ImmutableList<String> getParentNames() {
+ return tree().stream().skip(1).map(p -> p.getProject().getName()).collect(toImmutableList());
}
}
diff --git a/java/com/google/gerrit/index/project/ProjectField.java b/java/com/google/gerrit/index/project/ProjectField.java
index 1c2f629b..041813c 100644
--- a/java/com/google/gerrit/index/project/ProjectField.java
+++ b/java/com/google/gerrit/index/project/ProjectField.java
@@ -14,17 +14,24 @@
package com.google.gerrit.index.project;
+import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.gerrit.index.FieldDef.exact;
import static com.google.gerrit.index.FieldDef.fullText;
import static com.google.gerrit.index.FieldDef.prefix;
+import static com.google.gerrit.index.FieldDef.storedOnly;
-import com.google.common.collect.Iterables;
import com.google.gerrit.index.FieldDef;
+import com.google.gerrit.index.RefState;
import com.google.gerrit.index.SchemaUtil;
import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.reviewdb.client.RefNames;
/** Index schema for projects. */
public class ProjectField {
+ private static byte[] toRefState(Project project) {
+ return RefState.create(RefNames.REFS_CONFIG, project.getConfigRefState())
+ .toByteArray(project.getNameKey());
+ }
public static final FieldDef<ProjectData, String> NAME =
exact("name").stored().build(p -> p.getProject().getName());
@@ -39,6 +46,22 @@
prefix("name_part").buildRepeatable(p -> SchemaUtil.getNameParts(p.getProject().getName()));
public static final FieldDef<ProjectData, Iterable<String>> ANCESTOR_NAME =
- exact("ancestor_name")
- .buildRepeatable(p -> Iterables.transform(p.getAncestors(), Project.NameKey::get));
+ exact("ancestor_name").buildRepeatable(p -> p.getParentNames());
+
+ /**
+ * All values of all refs that were used in the course of indexing this document. This covers
+ * {@code refs/meta/config} of the current project and all of its parents.
+ *
+ * <p>Emitted as UTF-8 encoded strings of the form {@code project:ref/name:[hex sha]}.
+ */
+ public static final FieldDef<ProjectData, Iterable<byte[]>> REF_STATE =
+ storedOnly("ref_state")
+ .buildRepeatable(
+ projectData ->
+ projectData
+ .tree()
+ .stream()
+ .filter(p -> p.getProject().getConfigRefState() != null)
+ .map(p -> toRefState(p.getProject()))
+ .collect(toImmutableList()));
}
diff --git a/java/com/google/gerrit/index/project/ProjectSchemaDefinitions.java b/java/com/google/gerrit/index/project/ProjectSchemaDefinitions.java
index cbea4fe..4d212fb 100644
--- a/java/com/google/gerrit/index/project/ProjectSchemaDefinitions.java
+++ b/java/com/google/gerrit/index/project/ProjectSchemaDefinitions.java
@@ -21,6 +21,7 @@
public class ProjectSchemaDefinitions extends SchemaDefinitions<ProjectData> {
+ @Deprecated
static final Schema<ProjectData> V1 =
schema(
ProjectField.NAME,
@@ -29,6 +30,8 @@
ProjectField.NAME_PART,
ProjectField.ANCESTOR_NAME);
+ static final Schema<ProjectData> V2 = schema(V1, ProjectField.REF_STATE);
+
public static final ProjectSchemaDefinitions INSTANCE = new ProjectSchemaDefinitions();
private ProjectSchemaDefinitions() {
diff --git a/java/com/google/gerrit/reviewdb/client/Project.java b/java/com/google/gerrit/reviewdb/client/Project.java
index 921667e..996f1ec 100644
--- a/java/com/google/gerrit/reviewdb/client/Project.java
+++ b/java/com/google/gerrit/reviewdb/client/Project.java
@@ -99,6 +99,8 @@
protected String themeName;
+ protected String configRefState;
+
protected Project() {}
public Project(Project.NameKey nameKey) {
@@ -239,4 +241,14 @@
public void setParentName(NameKey n) {
parent = n;
}
+
+ /** Returns the {@code ObjectId} as 40 digit hex of {@code refs/meta/config}'s HEAD. */
+ public String getConfigRefState() {
+ return configRefState;
+ }
+
+ /** Sets the {@code ObjectId} as 40 digit hex of {@code refs/meta/config}'s HEAD. */
+ public void setConfigRefState(String state) {
+ configRefState = state;
+ }
}
diff --git a/java/com/google/gerrit/server/index/account/AccountField.java b/java/com/google/gerrit/server/index/account/AccountField.java
index 31e2ada..111991c 100644
--- a/java/com/google/gerrit/server/index/account/AccountField.java
+++ b/java/com/google/gerrit/server/index/account/AccountField.java
@@ -27,11 +27,11 @@
import com.google.common.collect.Iterables;
import com.google.gerrit.common.data.GlobalCapability;
import com.google.gerrit.index.FieldDef;
+import com.google.gerrit.index.RefState;
import com.google.gerrit.index.SchemaUtil;
import com.google.gerrit.reviewdb.client.RefNames;
import com.google.gerrit.server.account.AccountState;
import com.google.gerrit.server.account.externalids.ExternalId;
-import com.google.gerrit.server.index.RefState;
import java.sql.Timestamp;
import java.util.Arrays;
import java.util.Collections;
diff --git a/java/com/google/gerrit/server/index/account/StalenessChecker.java b/java/com/google/gerrit/server/index/account/StalenessChecker.java
index 6403d3d..0c3e329 100644
--- a/java/com/google/gerrit/server/index/account/StalenessChecker.java
+++ b/java/com/google/gerrit/server/index/account/StalenessChecker.java
@@ -24,6 +24,7 @@
import com.google.common.collect.MultimapBuilder;
import com.google.gerrit.index.IndexConfig;
import com.google.gerrit.index.QueryOptions;
+import com.google.gerrit.index.RefState;
import com.google.gerrit.index.query.FieldBundle;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.Project;
@@ -33,7 +34,6 @@
import com.google.gerrit.server.config.AllUsersName;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.index.IndexUtils;
-import com.google.gerrit.server.index.RefState;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.io.IOException;
diff --git a/java/com/google/gerrit/server/index/change/ChangeField.java b/java/com/google/gerrit/server/index/change/ChangeField.java
index 3375ce7..5d12e79 100644
--- a/java/com/google/gerrit/server/index/change/ChangeField.java
+++ b/java/com/google/gerrit/server/index/change/ChangeField.java
@@ -43,6 +43,7 @@
import com.google.gerrit.common.data.SubmitRecord;
import com.google.gerrit.common.data.SubmitRequirement;
import com.google.gerrit.index.FieldDef;
+import com.google.gerrit.index.RefState;
import com.google.gerrit.index.SchemaUtil;
import com.google.gerrit.mail.Address;
import com.google.gerrit.reviewdb.client.Account;
@@ -57,7 +58,6 @@
import com.google.gerrit.server.ReviewerSet;
import com.google.gerrit.server.StarredChangesUtil;
import com.google.gerrit.server.config.AllUsersName;
-import com.google.gerrit.server.index.RefState;
import com.google.gerrit.server.index.change.StalenessChecker.RefStatePattern;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.notedb.NoteDbChangeState.PrimaryStorage;
diff --git a/java/com/google/gerrit/server/index/change/StalenessChecker.java b/java/com/google/gerrit/server/index/change/StalenessChecker.java
index 642385c..0508b60 100644
--- a/java/com/google/gerrit/server/index/change/StalenessChecker.java
+++ b/java/com/google/gerrit/server/index/change/StalenessChecker.java
@@ -31,12 +31,12 @@
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.index.IndexConfig;
+import com.google.gerrit.index.RefState;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.UsedAt;
import com.google.gerrit.server.git.GitRepositoryManager;
-import com.google.gerrit.server.index.RefState;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.notedb.NoteDbChangeState.PrimaryStorage;
import com.google.gerrit.server.query.change.ChangeData;
diff --git a/java/com/google/gerrit/server/project/ProjectConfig.java b/java/com/google/gerrit/server/project/ProjectConfig.java
index 44aa83b..e3638b6 100644
--- a/java/com/google/gerrit/server/project/ProjectConfig.java
+++ b/java/com/google/gerrit/server/project/ProjectConfig.java
@@ -499,6 +499,9 @@
if (p.getDescription() == null) {
p.setDescription("");
}
+ if (revision != null) {
+ p.setConfigRefState(revision.toObjectId().name());
+ }
if (rc.getStringList(ACCESS, null, KEY_INHERIT_FROM).length > 1) {
// The config must not contain more than one parent to inherit from
diff --git a/java/com/google/gerrit/server/project/ProjectState.java b/java/com/google/gerrit/server/project/ProjectState.java
index 87dbcfc..e36ca4f 100644
--- a/java/com/google/gerrit/server/project/ProjectState.java
+++ b/java/com/google/gerrit/server/project/ProjectState.java
@@ -62,6 +62,7 @@
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Ref;
@@ -538,7 +539,11 @@
}
public ProjectData toProjectData() {
- return new ProjectData(getProject(), parents().transform(s -> s.getProject().getNameKey()));
+ ProjectData project = null;
+ for (ProjectState state : treeInOrder()) {
+ project = new ProjectData(state.getProject(), Optional.ofNullable(project));
+ }
+ return project;
}
private String readFile(Path p) throws IOException {
diff --git a/javatests/com/google/gerrit/acceptance/api/project/BUILD b/javatests/com/google/gerrit/acceptance/api/project/BUILD
index 768c20b..97c6f33 100644
--- a/javatests/com/google/gerrit/acceptance/api/project/BUILD
+++ b/javatests/com/google/gerrit/acceptance/api/project/BUILD
@@ -4,4 +4,5 @@
srcs = glob(["*IT.java"]),
group = "api_project",
labels = ["api"],
+ deps = ["//java/com/google/gerrit/index/project"],
)
diff --git a/javatests/com/google/gerrit/acceptance/api/project/ProjectIndexerIT.java b/javatests/com/google/gerrit/acceptance/api/project/ProjectIndexerIT.java
new file mode 100644
index 0000000..bb4502e
--- /dev/null
+++ b/javatests/com/google/gerrit/acceptance/api/project/ProjectIndexerIT.java
@@ -0,0 +1,75 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.acceptance.api.project;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.gerrit.acceptance.GitUtil.fetch;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.gerrit.acceptance.AbstractDaemonTest;
+import com.google.gerrit.index.IndexConfig;
+import com.google.gerrit.index.QueryOptions;
+import com.google.gerrit.index.RefState;
+import com.google.gerrit.index.project.ProjectField;
+import com.google.gerrit.index.project.ProjectIndex;
+import com.google.gerrit.index.project.ProjectIndexCollection;
+import com.google.gerrit.index.project.ProjectIndexer;
+import com.google.gerrit.index.query.FieldBundle;
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.inject.Inject;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Optional;
+import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
+import org.eclipse.jgit.junit.TestRepository;
+import org.eclipse.jgit.lib.Ref;
+import org.junit.Test;
+
+public class ProjectIndexerIT extends AbstractDaemonTest {
+ @Inject private ProjectIndexer projectIndexer;
+ @Inject private ProjectIndexCollection indexes;
+ @Inject private IndexConfig indexConfig;
+
+ private static final ImmutableSet<String> FIELDS =
+ ImmutableSet.of(ProjectField.NAME.getName(), ProjectField.REF_STATE.getName());
+
+ @Test
+ public void indexProject_indexesRefStateOfProjectAndParents() throws Exception {
+ projectIndexer.index(project);
+ ProjectIndex i = indexes.getSearchIndex();
+ assertThat(i.getSchema().hasField(ProjectField.REF_STATE)).isTrue();
+
+ Optional<FieldBundle> result =
+ i.getRaw(project, QueryOptions.create(indexConfig, 0, 1, FIELDS));
+
+ assertThat(result.isPresent()).isTrue();
+ Iterable<byte[]> refState = result.get().getValue(ProjectField.REF_STATE);
+ assertThat(refState).isNotEmpty();
+
+ Map<Project.NameKey, Collection<RefState>> states = RefState.parseStates(refState).asMap();
+
+ fetch(testRepo, "refs/meta/config:refs/meta/config");
+ Ref projectConfigRef = testRepo.getRepository().exactRef("refs/meta/config");
+ TestRepository<InMemoryRepository> allProjectsRepo = cloneProject(allProjects, admin);
+ fetch(allProjectsRepo, "refs/meta/config:refs/meta/config");
+ Ref allProjectConfigRef = allProjectsRepo.getRepository().exactRef("refs/meta/config");
+ assertThat(states)
+ .containsExactly(
+ project,
+ ImmutableSet.of(RefState.of(projectConfigRef)),
+ allProjects,
+ ImmutableSet.of(RefState.of(allProjectConfigRef)));
+ }
+}
diff --git a/javatests/com/google/gerrit/server/index/change/StalenessCheckerTest.java b/javatests/com/google/gerrit/server/index/change/StalenessCheckerTest.java
index acb33e9..51bda66 100644
--- a/javatests/com/google/gerrit/server/index/change/StalenessCheckerTest.java
+++ b/javatests/com/google/gerrit/server/index/change/StalenessCheckerTest.java
@@ -24,11 +24,11 @@
import com.google.common.collect.ImmutableListMultimap;
import com.google.common.collect.ImmutableSetMultimap;
import com.google.common.collect.ListMultimap;
+import com.google.gerrit.index.RefState;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.git.GitRepositoryManager;
-import com.google.gerrit.server.index.RefState;
import com.google.gerrit.server.index.change.StalenessChecker.RefStatePattern;
import com.google.gerrit.server.notedb.NoteDbChangeState;
import com.google.gerrit.testing.GerritBaseTests;
