Implement StalenessChecker for projects This commit implements a staleness checker for projects in the same fashion that we have it for groups and accounts. It uses the newly added ref_state field. Change-Id: Icbcf0bd1c700df789ef26501e4218956d718ea4c
diff --git a/java/com/google/gerrit/acceptance/AbstractDaemonTest.java b/java/com/google/gerrit/acceptance/AbstractDaemonTest.java index 36b0a65..53686a7 100644 --- a/java/com/google/gerrit/acceptance/AbstractDaemonTest.java +++ b/java/com/google/gerrit/acceptance/AbstractDaemonTest.java
@@ -68,6 +68,8 @@ import com.google.gerrit.extensions.restapi.BinaryResult; import com.google.gerrit.extensions.restapi.IdString; import com.google.gerrit.extensions.restapi.RestApiException; +import com.google.gerrit.index.project.ProjectIndex; +import com.google.gerrit.index.project.ProjectIndexCollection; import com.google.gerrit.mail.Address; import com.google.gerrit.mail.EmailHeader; import com.google.gerrit.reviewdb.client.Account; @@ -274,6 +276,7 @@ @Inject private ChangeIndexCollection changeIndexes; @Inject private AccountIndexCollection accountIndexes; + @Inject private ProjectIndexCollection projectIndexes; @Inject private EventRecorder.Factory eventRecorderFactory; @Inject private InProcessProtocol inProcessProtocol; @Inject private Provider<AnonymousUser> anonymousUser; @@ -900,6 +903,41 @@ }; } + protected AutoCloseable disableProjectIndex() { + disableProjectIndexWrites(); + ProjectIndex searchIndex = projectIndexes.getSearchIndex(); + if (!(searchIndex instanceof DisabledProjectIndex)) { + projectIndexes.setSearchIndex(new DisabledProjectIndex(searchIndex), false); + } + + return new AutoCloseable() { + @Override + public void close() { + enableProjectIndexWrites(); + ProjectIndex searchIndex = projectIndexes.getSearchIndex(); + if (searchIndex instanceof DisabledProjectIndex) { + projectIndexes.setSearchIndex(((DisabledProjectIndex) searchIndex).unwrap(), false); + } + } + }; + } + + protected void disableProjectIndexWrites() { + for (ProjectIndex i : projectIndexes.getWriteIndexes()) { + if (!(i instanceof DisabledProjectIndex)) { + projectIndexes.addWriteIndex(new DisabledProjectIndex(i)); + } + } + } + + protected void enableProjectIndexWrites() { + for (ProjectIndex i : projectIndexes.getWriteIndexes()) { + if (i instanceof DisabledProjectIndex) { + projectIndexes.addWriteIndex(((DisabledProjectIndex) i).unwrap()); + } + } + } + protected static Gson newGson() { return OutputFormat.JSON_COMPACT.newGson(); }
diff --git a/java/com/google/gerrit/acceptance/BUILD b/java/com/google/gerrit/acceptance/BUILD index efca382..0214cea 100644 --- a/java/com/google/gerrit/acceptance/BUILD +++ b/java/com/google/gerrit/acceptance/BUILD
@@ -99,6 +99,7 @@ "//java/com/google/gerrit/extensions:api", "//java/com/google/gerrit/httpd", "//java/com/google/gerrit/index", + "//java/com/google/gerrit/index/project", "//java/com/google/gerrit/lucene", "//java/com/google/gerrit/mail", "//java/com/google/gerrit/metrics",
diff --git a/java/com/google/gerrit/acceptance/DisabledProjectIndex.java b/java/com/google/gerrit/acceptance/DisabledProjectIndex.java new file mode 100644 index 0000000..2524a76 --- /dev/null +++ b/java/com/google/gerrit/acceptance/DisabledProjectIndex.java
@@ -0,0 +1,76 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.acceptance; + +import com.google.gerrit.index.QueryOptions; +import com.google.gerrit.index.Schema; +import com.google.gerrit.index.project.ProjectData; +import com.google.gerrit.index.project.ProjectIndex; +import com.google.gerrit.index.query.DataSource; +import com.google.gerrit.index.query.Predicate; +import com.google.gerrit.reviewdb.client.Project; + +/** + * This class wraps an index and assumes the search index can't handle any queries. However, it does + * return the current schema as the assumption is that we need a search index for starting Gerrit in + * the first place and only later lose the index connection (making it so that we can't send + * requests there anymore). + */ +public class DisabledProjectIndex implements ProjectIndex { + private final ProjectIndex index; + + public DisabledProjectIndex(ProjectIndex index) { + this.index = index; + } + + public ProjectIndex unwrap() { + return index; + } + + @Override + public Schema<ProjectData> getSchema() { + return index.getSchema(); + } + + @Override + public void close() { + index.close(); + } + + @Override + public void replace(ProjectData obj) { + throw new UnsupportedOperationException("ProjectIndex is disabled"); + } + + @Override + public void delete(Project.NameKey key) { + throw new UnsupportedOperationException("ProjectIndex is disabled"); + } + + @Override + public void deleteAll() { + throw new UnsupportedOperationException("ProjectIndex is disabled"); + } + + @Override + public DataSource<ProjectData> getSource(Predicate<ProjectData> p, QueryOptions opts) { + throw new UnsupportedOperationException("ProjectIndex is disabled"); + } + + @Override + public void markReady(boolean ready) { + throw new UnsupportedOperationException("ProjectIndex is disabled"); + } +}
diff --git a/java/com/google/gerrit/server/index/project/StalenessChecker.java b/java/com/google/gerrit/server/index/project/StalenessChecker.java new file mode 100644 index 0000000..5603f08 --- /dev/null +++ b/java/com/google/gerrit/server/index/project/StalenessChecker.java
@@ -0,0 +1,81 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index.project; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.MultimapBuilder; +import com.google.common.collect.SetMultimap; +import com.google.gerrit.index.IndexConfig; +import com.google.gerrit.index.QueryOptions; +import com.google.gerrit.index.RefState; +import com.google.gerrit.index.project.ProjectData; +import com.google.gerrit.index.project.ProjectField; +import com.google.gerrit.index.project.ProjectIndex; +import com.google.gerrit.index.project.ProjectIndexCollection; +import com.google.gerrit.index.query.FieldBundle; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.reviewdb.client.RefNames; +import com.google.gerrit.server.project.ProjectCache; +import java.io.IOException; +import java.util.Optional; +import javax.inject.Inject; + +public class StalenessChecker { + private static final ImmutableSet<String> FIELDS = + ImmutableSet.of(ProjectField.NAME.getName(), ProjectField.REF_STATE.getName()); + + private final ProjectCache projectCache; + private final ProjectIndexCollection indexes; + private final IndexConfig indexConfig; + + @Inject + StalenessChecker( + ProjectCache projectCache, ProjectIndexCollection indexes, IndexConfig indexConfig) { + this.projectCache = projectCache; + this.indexes = indexes; + this.indexConfig = indexConfig; + } + + public boolean isStale(Project.NameKey project) throws IOException { + ProjectData projectData = projectCache.get(project).toProjectData(); + ProjectIndex i = indexes.getSearchIndex(); + if (i == null) { + return false; // No index; caller couldn't do anything if it is stale. + } + + Optional<FieldBundle> result = + i.getRaw(project, QueryOptions.create(indexConfig, 0, 1, FIELDS)); + if (!result.isPresent()) { + return true; + } + + SetMultimap<Project.NameKey, RefState> indexedRefStates = + RefState.parseStates(result.get().getValue(ProjectField.REF_STATE)); + + SetMultimap<Project.NameKey, RefState> currentRefStates = + MultimapBuilder.hashKeys().hashSetValues().build(); + projectData + .tree() + .stream() + .filter(p -> p.getProject().getConfigRefState() != null) + .forEach( + p -> + currentRefStates.put( + p.getProject().getNameKey(), + RefState.create(RefNames.REFS_CONFIG, p.getProject().getConfigRefState()))); + + return !currentRefStates.equals(indexedRefStates); + } +}
diff --git a/javatests/com/google/gerrit/acceptance/api/project/ProjectIndexerIT.java b/javatests/com/google/gerrit/acceptance/api/project/ProjectIndexerIT.java index bb4502e..6fde012 100644 --- a/javatests/com/google/gerrit/acceptance/api/project/ProjectIndexerIT.java +++ b/javatests/com/google/gerrit/acceptance/api/project/ProjectIndexerIT.java
@@ -28,10 +28,13 @@ import com.google.gerrit.index.project.ProjectIndexer; import com.google.gerrit.index.query.FieldBundle; import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.index.project.StalenessChecker; +import com.google.gerrit.server.project.ProjectConfig; import com.google.inject.Inject; import java.util.Collection; import java.util.Map; import java.util.Optional; +import java.util.function.Consumer; import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository; import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.lib.Ref; @@ -41,6 +44,7 @@ @Inject private ProjectIndexer projectIndexer; @Inject private ProjectIndexCollection indexes; @Inject private IndexConfig indexConfig; + @Inject private StalenessChecker stalenessChecker; private static final ImmutableSet<String> FIELDS = ImmutableSet.of(ProjectField.NAME.getName(), ProjectField.REF_STATE.getName()); @@ -72,4 +76,54 @@ allProjects, ImmutableSet.of(RefState.of(allProjectConfigRef))); } + + @Test + public void stalenessChecker_currentProject_notStale() throws Exception { + assertThat(stalenessChecker.isStale(project)).isFalse(); + } + + @Test + public void stalenessChecker_currentProjectUpdates_isStale() throws Exception { + updateProjectConfigWithoutIndexUpdate(project); + assertThat(stalenessChecker.isStale(project)).isTrue(); + } + + @Test + public void stalenessChecker_parentProjectUpdates_isStale() throws Exception { + updateProjectConfigWithoutIndexUpdate(allProjects); + assertThat(stalenessChecker.isStale(project)).isTrue(); + } + + @Test + public void stalenessChecker_hierarchyChange_isStale() throws Exception { + Project.NameKey p1 = createProject("p1", allProjects); + Project.NameKey p2 = createProject("p2", allProjects); + try (ProjectConfigUpdate u = updateProject(project)) { + u.getConfig().getProject().setParentName(p1); + u.save(); + } + assertThat(stalenessChecker.isStale(project)).isFalse(); + + updateProjectConfigWithoutIndexUpdate(p1, c -> c.getProject().setParentName(p2)); + assertThat(stalenessChecker.isStale(project)).isTrue(); + } + + private void updateProjectConfigWithoutIndexUpdate(Project.NameKey project) throws Exception { + updateProjectConfigWithoutIndexUpdate( + project, c -> c.getProject().setDescription("making it stale")); + } + + private void updateProjectConfigWithoutIndexUpdate( + Project.NameKey project, Consumer<ProjectConfig> update) throws Exception { + try (AutoCloseable ignored = disableProjectIndex()) { + try (ProjectConfigUpdate u = updateProject(project)) { + update.accept(u.getConfig()); + u.save(); + } + } catch (UnsupportedOperationException e) { + // Drop, as we just wanted to drop the index update + return; + } + fail("should have a UnsupportedOperationException"); + } }