Merge "Merge branch 'stable-3.4' into stable-3.5" into stable-3.5
diff --git a/.bazelrc b/.bazelrc
index 6ccd56a..b4eafb1 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -13,10 +13,6 @@
build --announce_rc
-# Workaround Bazel worker crash (remove after upgrading to 4.1.0)
-# https://github.com/bazelbuild/bazel/issues/13333
-build --experimental_worker_multiplex=false
-
test --build_tests_only
test --test_output=errors
test --java_toolchain=//tools:error_prone_warnings_toolchain_java11
diff --git a/Documentation/dev-bazel.txt b/Documentation/dev-bazel.txt
index 1f28561..7c4ec18 100644
--- a/Documentation/dev-bazel.txt
+++ b/Documentation/dev-bazel.txt
@@ -268,7 +268,15 @@
bazel-bin/Documentation/searchfree.zip
----
-To build the executable WAR with the documentation included:
+To generate HTML files skipping the zip archiving:
+
+----
+ bazel build Documentation
+----
+
+And open `bazel-bin/Documentation/index.html`.
+
+To build the Gerrit executable WAR with the documentation included:
----
bazel build withdocs
diff --git a/java/com/google/gerrit/acceptance/AbstractDaemonTest.java b/java/com/google/gerrit/acceptance/AbstractDaemonTest.java
index a3b5e8d..c68a4fc 100644
--- a/java/com/google/gerrit/acceptance/AbstractDaemonTest.java
+++ b/java/com/google/gerrit/acceptance/AbstractDaemonTest.java
@@ -1158,7 +1158,7 @@
ProjectConfig config = projectConfigFactory.read(md);
config.updateProject(p -> p.setBooleanConfig(BooleanProjectConfig.USE_SIGNED_OFF_BY, value));
config.commit(md);
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
}
}
@@ -1167,7 +1167,7 @@
ProjectConfig config = projectConfigFactory.read(md);
config.updateProject(p -> p.setBooleanConfig(BooleanProjectConfig.REQUIRE_CHANGE_ID, value));
config.commit(md);
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
}
}
@@ -1681,7 +1681,7 @@
projectConfig.commit(metaDataUpdate);
metaDataUpdate.close();
metaDataUpdate = null;
- projectCache.evict(projectConfig.getProject());
+ projectCache.evictAndReindex(projectConfig.getProject());
}
@Override
diff --git a/java/com/google/gerrit/acceptance/DisabledAccountIndex.java b/java/com/google/gerrit/acceptance/DisabledAccountIndex.java
index 271d15c..7bd0c73 100644
--- a/java/com/google/gerrit/acceptance/DisabledAccountIndex.java
+++ b/java/com/google/gerrit/acceptance/DisabledAccountIndex.java
@@ -45,6 +45,11 @@
}
@Override
+ public void insert(AccountState obj) {
+ throw new UnsupportedOperationException("AccountIndex is disabled");
+ }
+
+ @Override
public void replace(AccountState obj) {
throw new UnsupportedOperationException("AccountIndex is disabled");
}
diff --git a/java/com/google/gerrit/acceptance/DisabledChangeIndex.java b/java/com/google/gerrit/acceptance/DisabledChangeIndex.java
index 34f72f5c..7671ad4 100644
--- a/java/com/google/gerrit/acceptance/DisabledChangeIndex.java
+++ b/java/com/google/gerrit/acceptance/DisabledChangeIndex.java
@@ -52,6 +52,11 @@
}
@Override
+ public void insert(ChangeData obj) {
+ throw new UnsupportedOperationException("ChangeIndex is disabled");
+ }
+
+ @Override
public void replace(ChangeData obj) {
throw new UnsupportedOperationException("ChangeIndex is disabled");
}
diff --git a/java/com/google/gerrit/acceptance/DisabledProjectIndex.java b/java/com/google/gerrit/acceptance/DisabledProjectIndex.java
index ed119ff..2e3dd90 100644
--- a/java/com/google/gerrit/acceptance/DisabledProjectIndex.java
+++ b/java/com/google/gerrit/acceptance/DisabledProjectIndex.java
@@ -50,6 +50,11 @@
}
@Override
+ public void insert(ProjectData obj) {
+ throw new UnsupportedOperationException("ProjectIndex is disabled");
+ }
+
+ @Override
public void replace(ProjectData obj) {
throw new UnsupportedOperationException("ProjectIndex is disabled");
}
diff --git a/java/com/google/gerrit/acceptance/ProjectResetter.java b/java/com/google/gerrit/acceptance/ProjectResetter.java
index d885303..46f7496 100644
--- a/java/com/google/gerrit/acceptance/ProjectResetter.java
+++ b/java/com/google/gerrit/acceptance/ProjectResetter.java
@@ -307,7 +307,7 @@
Sets.union(
projectsWithConfigChanges(restoredRefsByProject),
projectsWithConfigChanges(deletedRefsByProject))) {
- projectCache.evict(project);
+ projectCache.evictAndReindex(project);
}
}
diff --git a/java/com/google/gerrit/acceptance/ReadOnlyChangeIndex.java b/java/com/google/gerrit/acceptance/ReadOnlyChangeIndex.java
index e943519..f7a0669 100644
--- a/java/com/google/gerrit/acceptance/ReadOnlyChangeIndex.java
+++ b/java/com/google/gerrit/acceptance/ReadOnlyChangeIndex.java
@@ -45,6 +45,11 @@
}
@Override
+ public void insert(ChangeData obj) {
+ // do nothing
+ }
+
+ @Override
public void replace(ChangeData obj) {
// do nothing
}
diff --git a/java/com/google/gerrit/acceptance/testsuite/project/ProjectOperationsImpl.java b/java/com/google/gerrit/acceptance/testsuite/project/ProjectOperationsImpl.java
index 16dca66..18da4b3 100644
--- a/java/com/google/gerrit/acceptance/testsuite/project/ProjectOperationsImpl.java
+++ b/java/com/google/gerrit/acceptance/testsuite/project/ProjectOperationsImpl.java
@@ -147,7 +147,7 @@
setExclusiveGroupPermissions(projectConfig, projectUpdate.exclusiveGroupPermissions());
projectConfig.commit(metaDataUpdate);
}
- projectCache.evict(nameKey);
+ projectCache.evictAndReindex(nameKey);
}
private void removePermissions(
@@ -292,7 +292,7 @@
setConfig(projectConfig);
try {
- projectCache.evict(nameKey);
+ projectCache.evictAndReindex(nameKey);
} catch (Exception e) {
// Evicting the project from the cache, also triggers a reindex of the project.
// The reindex step fails if the project config is invalid. That's fine, since it was our
@@ -310,7 +310,7 @@
testProjectInvalidation.projectConfigUpdater().forEach(c -> c.accept(projectConfig));
setConfig(projectConfig);
try {
- projectCache.evict(nameKey);
+ projectCache.evictAndReindex(nameKey);
} catch (Exception e) {
// Evicting the project from the cache, also triggers a reindex of the project.
// The reindex step fails if the project config is invalid. That's fine, since it was our
diff --git a/java/com/google/gerrit/index/Index.java b/java/com/google/gerrit/index/Index.java
index ead302d..cc3117d 100644
--- a/java/com/google/gerrit/index/Index.java
+++ b/java/com/google/gerrit/index/Index.java
@@ -40,6 +40,16 @@
void close();
/**
+ * Insert a document into the index.
+ *
+ * <p>Results may not be immediately visible to searchers, but should be visible within a
+ * reasonable amount of time.
+ *
+ * @param obj document object
+ */
+ void insert(V obj);
+
+ /**
* Update a document in the index.
*
* <p>Semantically equivalent to deleting the document and reinserting it with new field values. A
diff --git a/java/com/google/gerrit/index/testing/AbstractFakeIndex.java b/java/com/google/gerrit/index/testing/AbstractFakeIndex.java
index b727e96..6ece45a 100644
--- a/java/com/google/gerrit/index/testing/AbstractFakeIndex.java
+++ b/java/com/google/gerrit/index/testing/AbstractFakeIndex.java
@@ -236,6 +236,9 @@
}
return cd;
}
+
+ @Override
+ public void insert(ChangeData obj) {}
}
/** Fake implementation of {@link AccountIndex} where all filtering happens in-memory. */
@@ -265,6 +268,9 @@
protected Comparator<AccountState> sortingComparator() {
return Comparator.comparing(a -> a.account().id().get());
}
+
+ @Override
+ public void insert(AccountState obj) {}
}
/** Fake implementation of {@link GroupIndex} where all filtering happens in-memory. */
@@ -295,6 +301,9 @@
protected Comparator<InternalGroup> sortingComparator() {
return Comparator.comparing(g -> g.getId().get());
}
+
+ @Override
+ public void insert(InternalGroup obj) {}
}
/** Fake implementation of {@link ProjectIndex} where all filtering happens in-memory. */
@@ -324,5 +333,8 @@
protected Comparator<ProjectData> sortingComparator() {
return Comparator.comparing(p -> p.getProject().getName());
}
+
+ @Override
+ public void insert(ProjectData obj) {}
}
}
diff --git a/java/com/google/gerrit/lucene/ChangeSubIndex.java b/java/com/google/gerrit/lucene/ChangeSubIndex.java
index 36f800f..475dac4 100644
--- a/java/com/google/gerrit/lucene/ChangeSubIndex.java
+++ b/java/com/google/gerrit/lucene/ChangeSubIndex.java
@@ -90,6 +90,11 @@
}
@Override
+ public void insert(ChangeData obj) {
+ throw new UnsupportedOperationException("don't use ChangeSubIndex directly");
+ }
+
+ @Override
public void replace(ChangeData obj) {
throw new UnsupportedOperationException("don't use ChangeSubIndex directly");
}
diff --git a/java/com/google/gerrit/lucene/LuceneAccountIndex.java b/java/com/google/gerrit/lucene/LuceneAccountIndex.java
index 87b7cce..c4a5240 100644
--- a/java/com/google/gerrit/lucene/LuceneAccountIndex.java
+++ b/java/com/google/gerrit/lucene/LuceneAccountIndex.java
@@ -144,6 +144,15 @@
}
@Override
+ public void insert(AccountState as) {
+ try {
+ insert(toDocument(as)).get();
+ } catch (ExecutionException | InterruptedException e) {
+ throw new StorageException(e);
+ }
+ }
+
+ @Override
public void delete(Account.Id key) {
try {
delete(idTerm(getSchema().useLegacyNumericFields(), key)).get();
diff --git a/java/com/google/gerrit/lucene/LuceneChangeIndex.java b/java/com/google/gerrit/lucene/LuceneChangeIndex.java
index cc58c65..e986d40 100644
--- a/java/com/google/gerrit/lucene/LuceneChangeIndex.java
+++ b/java/com/google/gerrit/lucene/LuceneChangeIndex.java
@@ -253,6 +253,24 @@
}
@Override
+ public void insert(ChangeData cd) {
+ @SuppressWarnings("unused") // TODO: Why do we need to find the lucene id?
+ Term id = LuceneChangeIndex.idTerm(idTerm, idField, cd);
+ // toDocument is essentially static and doesn't depend on the specific
+ // sub-index, so just pick one.
+ Document doc = openIndex.toDocument(cd);
+ try {
+ if (cd.change().isNew()) {
+ openIndex.insert(doc).get();
+ } else {
+ closedIndex.insert(doc).get();
+ }
+ } catch (ExecutionException | InterruptedException e) {
+ throw new StorageException(e);
+ }
+ }
+
+ @Override
public void delete(Change.Id changeId) {
Term id = LuceneChangeIndex.idTerm(idTerm, idField, changeId);
try {
diff --git a/java/com/google/gerrit/lucene/LuceneGroupIndex.java b/java/com/google/gerrit/lucene/LuceneGroupIndex.java
index 816739b..f7a2248 100644
--- a/java/com/google/gerrit/lucene/LuceneGroupIndex.java
+++ b/java/com/google/gerrit/lucene/LuceneGroupIndex.java
@@ -125,6 +125,15 @@
}
@Override
+ public void insert(InternalGroup group) {
+ try {
+ insert(toDocument(group)).get();
+ } catch (ExecutionException | InterruptedException e) {
+ throw new StorageException(e);
+ }
+ }
+
+ @Override
public void delete(AccountGroup.UUID key) {
try {
delete(idTerm(key)).get();
diff --git a/java/com/google/gerrit/lucene/LuceneProjectIndex.java b/java/com/google/gerrit/lucene/LuceneProjectIndex.java
index 7e7aecb..11707be 100644
--- a/java/com/google/gerrit/lucene/LuceneProjectIndex.java
+++ b/java/com/google/gerrit/lucene/LuceneProjectIndex.java
@@ -125,6 +125,15 @@
}
@Override
+ public void insert(ProjectData projectState) {
+ try {
+ insert(toDocument(projectState)).get();
+ } catch (ExecutionException | InterruptedException e) {
+ throw new StorageException(e);
+ }
+ }
+
+ @Override
public void delete(Project.NameKey nameKey) {
try {
delete(idTerm(nameKey)).get();
diff --git a/java/com/google/gerrit/pgm/Reindex.java b/java/com/google/gerrit/pgm/Reindex.java
index a12c08d..b4b5ffb 100644
--- a/java/com/google/gerrit/pgm/Reindex.java
+++ b/java/com/google/gerrit/pgm/Reindex.java
@@ -17,9 +17,11 @@
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toSet;
+import com.google.common.cache.Cache;
import com.google.common.collect.Sets;
import com.google.gerrit.common.Die;
import com.google.gerrit.extensions.config.FactoryModule;
+import com.google.gerrit.extensions.registration.DynamicMap;
import com.google.gerrit.index.Index;
import com.google.gerrit.index.IndexDefinition;
import com.google.gerrit.index.IndexType;
@@ -28,17 +30,24 @@
import com.google.gerrit.lucene.LuceneIndexModule;
import com.google.gerrit.pgm.util.BatchProgramModule;
import com.google.gerrit.pgm.util.SiteProgram;
+import com.google.gerrit.server.cache.CacheDisplay;
+import com.google.gerrit.server.cache.CacheInfo;
import com.google.gerrit.server.change.ChangeResource;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
import com.google.gerrit.server.index.options.AutoFlush;
+import com.google.gerrit.server.index.options.IsFirstInsertForEntry;
import com.google.gerrit.server.plugins.PluginGuiceEnvironment;
import com.google.gerrit.server.util.ReplicaUtil;
+import com.google.inject.AbstractModule;
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.Module;
+import com.google.inject.multibindings.OptionalBinder;
+import java.io.StringWriter;
+import java.io.Writer;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
@@ -49,6 +58,8 @@
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.util.io.NullOutputStream;
import org.kohsuke.args4j.Option;
@@ -79,6 +90,7 @@
private Config globalConfig;
@Inject private Collection<IndexDefinition<?, ?, ?>> indexDefs;
+ @Inject private DynamicMap<Cache<?, ?>> cacheMap;
@Override
public int run() throws Exception {
@@ -176,6 +188,16 @@
throw new IllegalStateException("unsupported index.type = " + indexType);
}
modules.add(indexModule);
+ modules.add(
+ new AbstractModule() {
+ @Override
+ protected void configure() {
+ super.configure();
+ OptionalBinder.newOptionalBinder(binder(), IsFirstInsertForEntry.class)
+ .setBinding()
+ .toInstance(IsFirstInsertForEntry.YES);
+ }
+ });
modules.add(new BatchProgramModule(dbInjector));
modules.add(
new FactoryModule() {
@@ -223,6 +245,20 @@
System.out.format(
"Index %s in version %d is %sready\n",
def.getName(), index.getSchema().getVersion(), result.success() ? "" : "NOT ");
+
+ try (Writer sw = new StringWriter()) {
+ sw.write(String.format("Cache Statistics at the end of reindexing %s\n", def.getName()));
+ new CacheDisplay(
+ sw,
+ StreamSupport.stream(cacheMap.spliterator(), false)
+ .map(e -> new CacheInfo(e.getExportName(), e.get()))
+ .collect(Collectors.toList()))
+ .displayCaches();
+ System.out.print(sw.toString());
+ } catch (Exception e) {
+ System.out.format("Error displaying the cache statistics\n" + e.getMessage());
+ }
+
return result.success();
}
}
diff --git a/java/com/google/gerrit/pgm/util/BatchProgramModule.java b/java/com/google/gerrit/pgm/util/BatchProgramModule.java
index b24fdad..64eef05 100644
--- a/java/com/google/gerrit/pgm/util/BatchProgramModule.java
+++ b/java/com/google/gerrit/pgm/util/BatchProgramModule.java
@@ -133,7 +133,7 @@
// We're just running through each change
// once, so don't worry about cache removal.
bind(new TypeLiteral<DynamicSet<CacheRemovalListener>>() {}).toInstance(DynamicSet.emptySet());
- bind(new TypeLiteral<DynamicMap<Cache<?, ?>>>() {}).toInstance(DynamicMap.emptyMap());
+ DynamicMap.mapOf(binder(), new TypeLiteral<Cache<?, ?>>() {});
bind(new TypeLiteral<List<CommentLinkInfo>>() {})
.toProvider(CommentLinkProvider.class)
.in(SINGLETON);
diff --git a/java/com/google/gerrit/server/CreateGroupPermissionSyncer.java b/java/com/google/gerrit/server/CreateGroupPermissionSyncer.java
index 6c76de7..60f3d4b 100644
--- a/java/com/google/gerrit/server/CreateGroupPermissionSyncer.java
+++ b/java/com/google/gerrit/server/CreateGroupPermissionSyncer.java
@@ -121,7 +121,7 @@
});
config.commit(md);
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
}
}
diff --git a/java/com/google/gerrit/server/cache/CacheDisplay.java b/java/com/google/gerrit/server/cache/CacheDisplay.java
new file mode 100644
index 0000000..60f5186
--- /dev/null
+++ b/java/com/google/gerrit/server/cache/CacheDisplay.java
@@ -0,0 +1,129 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.cache;
+
+import com.google.common.base.Strings;
+import java.io.IOException;
+import java.io.Writer;
+import java.util.Collection;
+
+public class CacheDisplay {
+
+ private final Writer stdout;
+ private final int nw;
+ private final Collection<CacheInfo> caches;
+
+ public CacheDisplay(Writer stdout, int nw, Collection<CacheInfo> caches) {
+ this.stdout = stdout;
+ this.nw = nw;
+ this.caches = caches;
+ }
+
+ public CacheDisplay(Writer stdout, Collection<CacheInfo> caches) {
+ this(stdout, 30, caches);
+ }
+
+ public void displayCaches() throws IOException {
+ stdout.write(
+ String.format( //
+ "%1s %-" + nw + "s|%-21s| %-5s |%-9s|\n" //
+ ,
+ "" //
+ ,
+ "Name" //
+ ,
+ "Entries" //
+ ,
+ "AvgGet" //
+ ,
+ "Hit Ratio" //
+ ));
+ stdout.write(
+ String.format( //
+ "%1s %-" + nw + "s|%6s %6s %7s| %-5s |%-4s %-4s|\n" //
+ ,
+ "" //
+ ,
+ "" //
+ ,
+ "Mem" //
+ ,
+ "Disk" //
+ ,
+ "Space" //
+ ,
+ "" //
+ ,
+ "Mem" //
+ ,
+ "Disk" //
+ ));
+ stdout.write("--");
+ for (int i = 0; i < nw; i++) {
+ stdout.write('-');
+ }
+ stdout.write("+---------------------+---------+---------+\n");
+ printMemoryCoreCaches(caches);
+ printMemoryPluginCaches(caches);
+ printDiskCaches(caches);
+ stdout.write('\n');
+ }
+
+ private void printMemoryCoreCaches(Collection<CacheInfo> caches) throws IOException {
+ for (CacheInfo cache : caches) {
+ if (!cache.name.contains("-") && CacheInfo.CacheType.MEM.equals(cache.type)) {
+ printCache(cache);
+ }
+ }
+ }
+
+ private void printMemoryPluginCaches(Collection<CacheInfo> caches) throws IOException {
+ for (CacheInfo cache : caches) {
+ if (cache.name.contains("-") && CacheInfo.CacheType.MEM.equals(cache.type)) {
+ printCache(cache);
+ }
+ }
+ }
+
+ private void printDiskCaches(Collection<CacheInfo> caches) throws IOException {
+ for (CacheInfo cache : caches) {
+ if (CacheInfo.CacheType.DISK.equals(cache.type)) {
+ printCache(cache);
+ }
+ }
+ }
+
+ private void printCache(CacheInfo cache) throws IOException {
+ stdout.write(
+ String.format(
+ "%1s %-" + nw + "s|%6s %6s %7s| %7s |%4s %4s|\n",
+ CacheInfo.CacheType.DISK.equals(cache.type) ? "D" : "",
+ cache.name,
+ nullToEmpty(cache.entries.mem),
+ nullToEmpty(cache.entries.disk),
+ Strings.nullToEmpty(cache.entries.space),
+ Strings.nullToEmpty(cache.averageGet),
+ formatAsPercent(cache.hitRatio.mem),
+ formatAsPercent(cache.hitRatio.disk)));
+ }
+
+ private static String nullToEmpty(Long l) {
+ return l != null ? String.valueOf(l) : "";
+ }
+
+ private static String formatAsPercent(Integer i) {
+ return i != null ? i + "%" : "";
+ }
+}
diff --git a/java/com/google/gerrit/server/cache/CacheInfo.java b/java/com/google/gerrit/server/cache/CacheInfo.java
new file mode 100644
index 0000000..d6eb065
--- /dev/null
+++ b/java/com/google/gerrit/server/cache/CacheInfo.java
@@ -0,0 +1,133 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.cache;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheStats;
+
+public class CacheInfo {
+
+ public String name;
+ public CacheType type;
+ public EntriesInfo entries;
+ public String averageGet;
+ public HitRatioInfo hitRatio;
+
+ public CacheInfo(Cache<?, ?> cache) {
+ this(null, cache);
+ }
+
+ public CacheInfo(String name, Cache<?, ?> cache) {
+ this.name = name;
+
+ CacheStats stat = cache.stats();
+
+ entries = new EntriesInfo();
+ entries.setMem(cache.size());
+
+ averageGet = duration(stat.averageLoadPenalty());
+
+ hitRatio = new HitRatioInfo();
+ hitRatio.setMem(stat.hitCount(), stat.requestCount());
+
+ if (cache instanceof PersistentCache) {
+ type = CacheType.DISK;
+ PersistentCache.DiskStats diskStats = ((PersistentCache) cache).diskStats();
+ entries.setDisk(diskStats.size());
+ entries.setSpace(diskStats.space());
+ hitRatio.setDisk(diskStats.hitCount(), diskStats.requestCount());
+ } else {
+ type = CacheType.MEM;
+ }
+ }
+
+ private static String duration(double ns) {
+ if (ns < 0.5) {
+ return null;
+ }
+ String suffix = "ns";
+ if (ns >= 1000.0) {
+ ns /= 1000.0;
+ suffix = "us";
+ }
+ if (ns >= 1000.0) {
+ ns /= 1000.0;
+ suffix = "ms";
+ }
+ if (ns >= 1000.0) {
+ ns /= 1000.0;
+ suffix = "s";
+ }
+ return String.format("%4.1f%s", ns, suffix).trim();
+ }
+
+ public static class EntriesInfo {
+ public Long mem;
+ public Long disk;
+ public String space;
+
+ public void setMem(long mem) {
+ this.mem = mem != 0 ? mem : null;
+ }
+
+ public void setDisk(long disk) {
+ this.disk = disk != 0 ? disk : null;
+ }
+
+ public void setSpace(double value) {
+ space = bytes(value);
+ }
+
+ private static String bytes(double value) {
+ value /= 1024;
+ String suffix = "k";
+
+ if (value > 1024) {
+ value /= 1024;
+ suffix = "m";
+ }
+ if (value > 1024) {
+ value /= 1024;
+ suffix = "g";
+ }
+ return String.format("%1$6.2f%2$s", value, suffix).trim();
+ }
+ }
+
+ public static class HitRatioInfo {
+ public Integer mem;
+ public Integer disk;
+
+ public void setMem(long value, long total) {
+ mem = percent(value, total);
+ }
+
+ public void setDisk(long value, long total) {
+ disk = percent(value, total);
+ }
+
+ private static Integer percent(long value, long total) {
+ if (total <= 0) {
+ return null;
+ }
+ return (int) ((100 * value) / total);
+ }
+ }
+
+ public enum CacheType {
+ MEM,
+ DISK
+ }
+}
diff --git a/java/com/google/gerrit/server/git/receive/ReceiveCommits.java b/java/com/google/gerrit/server/git/receive/ReceiveCommits.java
index f6c462c..55d7af0 100644
--- a/java/com/google/gerrit/server/git/receive/ReceiveCommits.java
+++ b/java/com/google/gerrit/server/git/receive/ReceiveCommits.java
@@ -3279,7 +3279,7 @@
}
if (isConfig(cmd)) {
logger.atFine().log("Reloading project in cache");
- projectCache.evict(project);
+ projectCache.evictAndReindex(project);
ProjectState ps =
projectCache.get(project.getNameKey()).orElseThrow(illegalState(project.getNameKey()));
try {
diff --git a/java/com/google/gerrit/server/group/db/RenameGroupOp.java b/java/com/google/gerrit/server/group/db/RenameGroupOp.java
index 843b346..257bc16 100644
--- a/java/com/google/gerrit/server/group/db/RenameGroupOp.java
+++ b/java/com/google/gerrit/server/group/db/RenameGroupOp.java
@@ -123,7 +123,7 @@
//
GroupReference ref = config.getGroup(uuid);
if (ref == null || newName.equals(ref.getName())) {
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
return;
}
@@ -132,7 +132,7 @@
md.setMessage("Rename group " + oldName + " to " + newName + "\n");
try {
config.commit(md);
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
success = true;
} catch (IOException e) {
logger.atSevere().withCause(e).log(
diff --git a/java/com/google/gerrit/server/index/IndexModule.java b/java/com/google/gerrit/server/index/IndexModule.java
index 4050f43d..a729863 100644
--- a/java/com/google/gerrit/server/index/IndexModule.java
+++ b/java/com/google/gerrit/server/index/IndexModule.java
@@ -52,6 +52,7 @@
import com.google.gerrit.server.index.group.GroupIndexer;
import com.google.gerrit.server.index.group.GroupIndexerImpl;
import com.google.gerrit.server.index.group.GroupSchemaDefinitions;
+import com.google.gerrit.server.index.options.IsFirstInsertForEntry;
import com.google.gerrit.server.index.project.ProjectIndexDefinition;
import com.google.gerrit.server.index.project.ProjectIndexerImpl;
import com.google.inject.Inject;
@@ -60,6 +61,7 @@
import com.google.inject.Provides;
import com.google.inject.ProvisionException;
import com.google.inject.Singleton;
+import com.google.inject.multibindings.OptionalBinder;
import java.util.Collection;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -145,6 +147,9 @@
}
DynamicSet.setOf(binder(), OnlineUpgradeListener.class);
+ OptionalBinder.newOptionalBinder(binder(), IsFirstInsertForEntry.class)
+ .setDefault()
+ .toInstance(IsFirstInsertForEntry.NO);
}
@Provides
diff --git a/java/com/google/gerrit/server/index/account/AllAccountsIndexer.java b/java/com/google/gerrit/server/index/account/AllAccountsIndexer.java
index 63889b7..1f48e35 100644
--- a/java/com/google/gerrit/server/index/account/AllAccountsIndexer.java
+++ b/java/com/google/gerrit/server/index/account/AllAccountsIndexer.java
@@ -27,6 +27,7 @@
import com.google.gerrit.server.account.AccountState;
import com.google.gerrit.server.account.Accounts;
import com.google.gerrit.server.index.IndexExecutor;
+import com.google.gerrit.server.index.options.IsFirstInsertForEntry;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.io.IOException;
@@ -50,15 +51,18 @@
private final ListeningExecutorService executor;
private final Accounts accounts;
private final AccountCache accountCache;
+ private final IsFirstInsertForEntry isFirstInsertForEntry;
@Inject
AllAccountsIndexer(
@IndexExecutor(BATCH) ListeningExecutorService executor,
Accounts accounts,
- AccountCache accountCache) {
+ AccountCache accountCache,
+ IsFirstInsertForEntry isFirstInsertForEntry) {
this.executor = executor;
this.accounts = accounts;
this.accountCache = accountCache;
+ this.isFirstInsertForEntry = isFirstInsertForEntry;
}
@Override
@@ -92,7 +96,11 @@
try {
Optional<AccountState> a = accountCache.get(id);
if (a.isPresent()) {
- index.replace(a.get());
+ if (isFirstInsertForEntry.equals(IsFirstInsertForEntry.YES)) {
+ index.insert(a.get());
+ } else {
+ index.replace(a.get());
+ }
} else {
index.delete(id);
}
diff --git a/java/com/google/gerrit/server/index/change/AllChangesIndexer.java b/java/com/google/gerrit/server/index/change/AllChangesIndexer.java
index 83b4e61..bde614f 100644
--- a/java/com/google/gerrit/server/index/change/AllChangesIndexer.java
+++ b/java/com/google/gerrit/server/index/change/AllChangesIndexer.java
@@ -20,15 +20,16 @@
import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
import static com.google.gerrit.server.git.QueueProvider.QueueType.BATCH;
+import com.google.auto.value.AutoValue;
import com.google.common.base.Stopwatch;
+import com.google.common.collect.Sets;
import com.google.common.flogger.FluentLogger;
-import com.google.common.primitives.Ints;
+import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.UncheckedExecutionException;
import com.google.gerrit.entities.Change;
import com.google.gerrit.entities.Project;
-import com.google.gerrit.entities.RefNames;
import com.google.gerrit.index.SiteIndexer;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.MultiProgressMonitor;
@@ -38,6 +39,7 @@
import com.google.gerrit.server.index.OnlineReindexMode;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.notedb.ChangeNotes.Factory.ChangeNotesResult;
+import com.google.gerrit.server.notedb.ChangeNotes.Factory.ScanResult;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.inject.Inject;
@@ -45,11 +47,13 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Objects;
+import java.util.Set;
import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.eclipse.jgit.errors.RepositoryNotFoundException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
import org.eclipse.jgit.lib.ProgressMonitor;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.TextProgressMonitor;
@@ -64,6 +68,15 @@
private static final int PROJECT_SLICE_MAX_REFS = 1000;
private final MultiProgressMonitor.Factory multiProgressMonitorFactory;
+
+ private static class ProjectsCollectionFailure extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ public ProjectsCollectionFailure(String message) {
+ super(message);
+ }
+ }
+
private final ChangeData.Factory changeDataFactory;
private final GitRepositoryManager repoManager;
private final ListeningExecutorService executor;
@@ -89,103 +102,58 @@
this.projectCache = projectCache;
}
- private static class ProjectSlice {
- private final Project.NameKey name;
- private final int slice;
- private final int slices;
+ @AutoValue
+ public abstract static class ProjectSlice {
+ public abstract Project.NameKey name();
- ProjectSlice(Project.NameKey name, int slice, int slices) {
- this.name = name;
- this.slice = slice;
- this.slices = slices;
- }
+ public abstract int slice();
- public Project.NameKey getName() {
- return name;
- }
+ public abstract int slices();
- public int getSlice() {
- return slice;
- }
+ public abstract ScanResult scanResult();
- public int getSlices() {
- return slices;
+ private static ProjectSlice create(Project.NameKey name, int slice, int slices, ScanResult sr) {
+ return new AutoValue_AllChangesIndexer_ProjectSlice(name, slice, slices, sr);
}
}
@Override
public Result indexAll(ChangeIndex index) {
- ProgressMonitor pm = new TextProgressMonitor();
- pm.beginTask("Collecting projects", ProgressMonitor.UNKNOWN);
- List<ProjectSlice> projectSlices = new ArrayList<>();
- int changeCount = 0;
- Stopwatch sw = Stopwatch.createStarted();
- int projectsFailed = 0;
- for (Project.NameKey name : projectCache.all()) {
- try (Repository repo = repoManager.openRepository(name)) {
- // The simplest approach to distribute indexing would be to let each thread grab a project
- // and index it fully. But if a site has one big project and 100s of small projects, then
- // in the beginning all CPUs would be busy reindexing projects. But soon enough all small
- // projects have been reindexed, and only the thread that reindexes the big project is
- // still working. The other threads would idle. Reindexing the big project on a single
- // thread becomes the critical path. Bringing in more CPUs would not speed up things.
- //
- // To avoid such situations, we split big repos into smaller parts and let
- // the thread pool index these smaller parts. This splitting introduces an overhead in the
- // workload setup and there might be additional slow-downs from multiple threads
- // concurrently working on different parts of the same project. But for Wikimedia's Gerrit,
- // which had 2 big projects, many middle sized ones, and lots of smaller ones, the
- // splitting of repos into smaller parts reduced indexing time from 1.5 hours to 55 minutes
- // in 2020.
- int size = estimateSize(repo);
- if (size == 0) {
- pm.update(1);
- continue;
- }
- changeCount += size;
- int slices = 1 + size / PROJECT_SLICE_MAX_REFS;
- if (slices > 1) {
- verboseWriter.println("Submitting " + name + " for indexing in " + slices + " slices");
- }
- for (int slice = 0; slice < slices; slice++) {
- projectSlices.add(new ProjectSlice(name, slice, slices));
- }
- } catch (IOException e) {
- logger.atSevere().withCause(e).log("Error collecting project %s", name);
- projectsFailed++;
- if (projectsFailed > projectCache.all().size() / 2) {
- logger.atSevere().log("Over 50%% of the projects could not be collected: aborted");
- return Result.create(sw, false, 0, 0);
- }
- }
- pm.update(1);
- }
- pm.endTask();
- setTotalWork(changeCount);
+ // The simplest approach to distribute indexing would be to let each thread grab a project
+ // and index it fully. But if a site has one big project and 100s of small projects, then
+ // in the beginning all CPUs would be busy reindexing projects. But soon enough all small
+ // projects have been reindexed, and only the thread that reindexes the big project is
+ // still working. The other threads would idle. Reindexing the big project on a single
+ // thread becomes the critical path. Bringing in more CPUs would not speed up things.
+ //
+ // To avoid such situations, we split big repos into smaller parts and let
+ // the thread pool index these smaller parts. This splitting introduces an overhead in the
+ // workload setup and there might be additional slow-downs from multiple threads
+ // concurrently working on different parts of the same project. But for Wikimedia's Gerrit,
+ // which had 2 big projects, many middle sized ones, and lots of smaller ones, the
+ // splitting of repos into smaller parts reduced indexing time from 1.5 hours to 55 minutes
+ // in 2020.
- // projectSlices are currently grouped by projects. First all slices for project1, followed
- // by all slices for project2, and so on. As workers pick tasks sequentially, multiple threads
- // would typically work concurrently on different slices of the same project. While this is not
- // a big issue, shuffling the list beforehand helps with ungrouping the project slices, so
- // different slices are less likely to be worked on concurrently.
+ Stopwatch sw = Stopwatch.createStarted();
+ List<ProjectSlice> projectSlices;
+ try {
+ projectSlices = new SliceCreator().create();
+ } catch (ProjectsCollectionFailure | InterruptedException | ExecutionException e) {
+ logger.atSevere().log(e.getMessage());
+ return Result.create(sw, false, 0, 0);
+ }
+
+ // Since project slices are created in parallel, they are somewhat shuffled already. However,
+ // the number of threads used to create the project slices doesn't guarantee good randomization.
+ // If the slices are not shuffled well, then multiple threads would typically work concurrently
+ // on different slices of the same project. While this is not a big issue, shuffling the list
+ // beforehand helps with ungrouping the project slices, so different slices are less likely to
+ // be worked on concurrently.
// This shuffling gave a 6% runtime reduction for Wikimedia's Gerrit in 2020.
Collections.shuffle(projectSlices);
return indexAll(index, projectSlices);
}
- private int estimateSize(Repository repo) throws IOException {
- // Estimate size based on IDs that show up in ref names. This is not perfect, since patch set
- // refs may exist for changes whose metadata was never successfully stored. But that's ok, as
- // the estimate is just used as a heuristic for sorting projects.
- long size =
- repo.getRefDatabase().getRefsByPrefix(RefNames.REFS_CHANGES).stream()
- .map(r -> Change.Id.fromRef(r.getName()))
- .filter(Objects::nonNull)
- .distinct()
- .count();
- return Ints.saturatedCast(size);
- }
-
private SiteIndexer.Result indexAll(ChangeIndex index, List<ProjectSlice> projectSlices) {
Stopwatch sw = Stopwatch.createStarted();
MultiProgressMonitor mpm =
@@ -199,9 +167,9 @@
AtomicBoolean ok = new AtomicBoolean(true);
for (ProjectSlice projectSlice : projectSlices) {
- Project.NameKey name = projectSlice.getName();
- int slice = projectSlice.getSlice();
- int slices = projectSlice.getSlices();
+ Project.NameKey name = projectSlice.name();
+ int slice = projectSlice.slice();
+ int slices = projectSlice.slices();
ListenableFuture<?> future =
executor.submit(
reindexProject(
@@ -209,6 +177,7 @@
name,
slice,
slices,
+ projectSlice.scanResult(),
doneTask,
failedTask));
String description = "project " + name + " (" + slice + "/" + slices + ")";
@@ -249,7 +218,13 @@
public Callable<Void> reindexProject(
ChangeIndexer indexer, Project.NameKey project, Task done, Task failed) {
- return reindexProject(indexer, project, 0, 1, done, failed);
+ try (Repository repo = repoManager.openRepository(project)) {
+ return reindexProject(
+ indexer, project, 0, 1, ChangeNotes.Factory.scanChangeIds(repo), done, failed);
+ } catch (IOException e) {
+ logger.atSevere().log(e.getMessage());
+ return null;
+ }
}
public Callable<Void> reindexProject(
@@ -257,9 +232,10 @@
Project.NameKey project,
int slice,
int slices,
+ ScanResult scanResult,
Task done,
Task failed) {
- return new ProjectIndexer(indexer, project, slice, slices, done, failed);
+ return new ProjectIndexer(indexer, project, slice, slices, scanResult, done, failed);
}
private class ProjectIndexer implements Callable<Void> {
@@ -267,6 +243,7 @@
private final Project.NameKey project;
private final int slice;
private final int slices;
+ private final ScanResult scanResult;
private final ProgressMonitor done;
private final ProgressMonitor failed;
@@ -275,32 +252,30 @@
Project.NameKey project,
int slice,
int slices,
+ ScanResult scanResult,
ProgressMonitor done,
ProgressMonitor failed) {
this.indexer = indexer;
this.project = project;
this.slice = slice;
this.slices = slices;
+ this.scanResult = scanResult;
this.done = done;
this.failed = failed;
}
@Override
public Void call() throws Exception {
- try (Repository repo = repoManager.openRepository(project)) {
- OnlineReindexMode.begin();
-
- // Order of scanning changes is undefined. This is ok if we assume that packfile locality is
- // not important for indexing, since sites should have a fully populated DiffSummary cache.
- // It does mean that reindexing after invalidating the DiffSummary cache will be expensive,
- // but the goal is to invalidate that cache as infrequently as we possibly can. And besides,
- // we don't have concrete proof that improving packfile locality would help.
- notesFactory.scan(repo, project, id -> (id.get() % slices) == slice).forEach(r -> index(r));
- } catch (RepositoryNotFoundException rnfe) {
- logger.atSevere().log(rnfe.getMessage());
- } finally {
- OnlineReindexMode.end();
- }
+ OnlineReindexMode.begin();
+ // Order of scanning changes is undefined. This is ok if we assume that packfile locality is
+ // not important for indexing, since sites should have a fully populated DiffSummary cache.
+ // It does mean that reindexing after invalidating the DiffSummary cache will be expensive,
+ // but the goal is to invalidate that cache as infrequently as we possibly can. And besides,
+ // we don't have concrete proof that improving packfile locality would help.
+ notesFactory
+ .scan(scanResult, project, id -> (id.get() % slices) == slice)
+ .forEach(r -> index(r));
+ OnlineReindexMode.end();
return null;
}
@@ -340,4 +315,63 @@
return "Index all changes of project " + project.get();
}
}
+
+ private class SliceCreator {
+ final Set<ProjectSlice> projectSlices = Sets.newConcurrentHashSet();
+ final AtomicInteger changeCount = new AtomicInteger(0);
+ final AtomicInteger projectsFailed = new AtomicInteger(0);
+ final ProgressMonitor pm = new TextProgressMonitor();
+
+ private List<ProjectSlice> create()
+ throws ProjectsCollectionFailure, InterruptedException, ExecutionException {
+ List<ListenableFuture<?>> futures = new ArrayList<>();
+ pm.beginTask("Collecting projects", ProgressMonitor.UNKNOWN);
+ for (Project.NameKey name : projectCache.all()) {
+ futures.add(executor.submit(new ProjectSliceCreator(name)));
+ }
+
+ Futures.allAsList(futures).get();
+
+ if (projectsFailed.get() > projectCache.all().size() / 2) {
+ throw new ProjectsCollectionFailure(
+ "Over 50%% of the projects could not be collected: aborted");
+ }
+
+ pm.endTask();
+ setTotalWork(changeCount.get());
+ return projectSlices.stream().collect(Collectors.toList());
+ }
+
+ private class ProjectSliceCreator implements Callable<Void> {
+ final Project.NameKey name;
+
+ public ProjectSliceCreator(Project.NameKey name) {
+ this.name = name;
+ }
+
+ @Override
+ public Void call() throws IOException {
+ try (Repository repo = repoManager.openRepository(name)) {
+ ScanResult sr = ChangeNotes.Factory.scanChangeIds(repo);
+ int size = sr.all().size();
+ if (size > 0) {
+ changeCount.addAndGet(size);
+ int slices = 1 + size / PROJECT_SLICE_MAX_REFS;
+ if (slices > 1) {
+ verboseWriter.println(
+ "Submitting " + name + " for indexing in " + slices + " slices");
+ }
+ for (int slice = 0; slice < slices; slice++) {
+ projectSlices.add(ProjectSlice.create(name, slice, slices, sr));
+ }
+ }
+ } catch (IOException e) {
+ logger.atSevere().withCause(e).log("Error collecting project %s", name);
+ projectsFailed.incrementAndGet();
+ }
+ pm.update(1);
+ return null;
+ }
+ }
+ }
}
diff --git a/java/com/google/gerrit/server/index/change/ChangeIndexer.java b/java/com/google/gerrit/server/index/change/ChangeIndexer.java
index f7f0f33..8f68904 100644
--- a/java/com/google/gerrit/server/index/change/ChangeIndexer.java
+++ b/java/com/google/gerrit/server/index/change/ChangeIndexer.java
@@ -30,6 +30,7 @@
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.index.IndexExecutor;
import com.google.gerrit.server.index.StalenessCheckResult;
+import com.google.gerrit.server.index.options.IsFirstInsertForEntry;
import com.google.gerrit.server.logging.Metadata;
import com.google.gerrit.server.logging.TraceContext;
import com.google.gerrit.server.logging.TraceContext.TraceTimer;
@@ -79,6 +80,7 @@
private final PluginSetContext<ChangeIndexedListener> indexedListeners;
private final StalenessChecker stalenessChecker;
private final boolean autoReindexIfStale;
+ private final IsFirstInsertForEntry isFirstInsertForEntry;
private final Map<Change.Id, IndexTask> queuedIndexTasks = new ConcurrentHashMap<>();
private final Set<ReindexIfStaleTask> queuedReindexIfStaleTasks =
@@ -94,7 +96,8 @@
StalenessChecker stalenessChecker,
@IndexExecutor(BATCH) ListeningExecutorService batchExecutor,
@Assisted ListeningExecutorService executor,
- @Assisted ChangeIndex index) {
+ @Assisted ChangeIndex index,
+ IsFirstInsertForEntry isFirstInsertForEntry) {
this.executor = executor;
this.changeDataFactory = changeDataFactory;
this.notesFactory = notesFactory;
@@ -105,6 +108,7 @@
this.autoReindexIfStale = autoReindexIfStale(cfg);
this.index = index;
this.indexes = null;
+ this.isFirstInsertForEntry = isFirstInsertForEntry;
}
@AssistedInject
@@ -117,7 +121,8 @@
StalenessChecker stalenessChecker,
@IndexExecutor(BATCH) ListeningExecutorService batchExecutor,
@Assisted ListeningExecutorService executor,
- @Assisted ChangeIndexCollection indexes) {
+ @Assisted ChangeIndexCollection indexes,
+ IsFirstInsertForEntry isFirstInsertForEntry) {
this.executor = executor;
this.changeDataFactory = changeDataFactory;
this.notesFactory = notesFactory;
@@ -128,6 +133,7 @@
this.autoReindexIfStale = autoReindexIfStale(cfg);
this.index = null;
this.indexes = indexes;
+ this.isFirstInsertForEntry = isFirstInsertForEntry;
}
private static boolean autoReindexIfStale(Config cfg) {
@@ -227,21 +233,25 @@
}
private void indexImpl(ChangeData cd) {
- logger.atFine().log("Replace change %d in index.", cd.getId().get());
+ logger.atFine().log("Reindex change %d in index.", cd.getId().get());
for (Index<?, ChangeData> i : getWriteIndexes()) {
try (TraceTimer traceTimer =
TraceContext.newTimer(
- "Replacing change in index",
+ "Reindexing change in index",
Metadata.builder()
.changeId(cd.getId().get())
.patchSetId(cd.currentPatchSet().number())
.indexVersion(i.getSchema().getVersion())
.build())) {
- i.replace(cd);
+ if (isFirstInsertForEntry.equals(IsFirstInsertForEntry.YES)) {
+ i.insert(cd);
+ } else {
+ i.replace(cd);
+ }
} catch (RuntimeException e) {
throw new StorageException(
String.format(
- "Failed to replace change %d in index version %d (current patch set = %d)",
+ "Failed to reindex change %d in index version %d (current patch set = %d)",
cd.getId().get(), i.getSchema().getVersion(), cd.currentPatchSet().number()),
e);
}
diff --git a/java/com/google/gerrit/server/index/group/AllGroupsIndexer.java b/java/com/google/gerrit/server/index/group/AllGroupsIndexer.java
index b3ef679..3773d435 100644
--- a/java/com/google/gerrit/server/index/group/AllGroupsIndexer.java
+++ b/java/com/google/gerrit/server/index/group/AllGroupsIndexer.java
@@ -30,6 +30,7 @@
import com.google.gerrit.server.group.db.Groups;
import com.google.gerrit.server.group.db.GroupsNoteDbConsistencyChecker;
import com.google.gerrit.server.index.IndexExecutor;
+import com.google.gerrit.server.index.options.IsFirstInsertForEntry;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.io.IOException;
@@ -54,15 +55,18 @@
private final ListeningExecutorService executor;
private final GroupCache groupCache;
private final Groups groups;
+ private final IsFirstInsertForEntry isFirstInsertForEntry;
@Inject
AllGroupsIndexer(
@IndexExecutor(BATCH) ListeningExecutorService executor,
GroupCache groupCache,
- Groups groups) {
+ Groups groups,
+ IsFirstInsertForEntry isFirstInsertForEntry) {
this.executor = executor;
this.groupCache = groupCache;
this.groups = groups;
+ this.isFirstInsertForEntry = isFirstInsertForEntry;
}
@Override
@@ -96,9 +100,14 @@
executor.submit(
() -> {
try {
+ groupCache.evict(uuid);
InternalGroup internalGroup = reindexedGroups.get(uuid);
if (internalGroup != null) {
- index.replace(internalGroup);
+ if (isFirstInsertForEntry.equals(IsFirstInsertForEntry.YES)) {
+ index.insert(internalGroup);
+ } else {
+ index.replace(internalGroup);
+ }
} else {
index.delete(uuid);
diff --git a/java/com/google/gerrit/server/index/options/IsFirstInsertForEntry.java b/java/com/google/gerrit/server/index/options/IsFirstInsertForEntry.java
new file mode 100644
index 0000000..f943309
--- /dev/null
+++ b/java/com/google/gerrit/server/index/options/IsFirstInsertForEntry.java
@@ -0,0 +1,26 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.index.options;
+
+/**
+ * This enum should be injected and checked to decide on which operation ({@link
+ * com.google.gerrit.index.Index#replace(Object) replace()} or {@link
+ * com.google.gerrit.index.Index#insert(Object) insert()}) should be performed on a specific {@link
+ * com.google.gerrit.index.Index index}.
+ */
+public enum IsFirstInsertForEntry {
+ YES,
+ NO
+}
diff --git a/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java b/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java
index 0e4b688..1c977d1 100644
--- a/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java
+++ b/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java
@@ -27,6 +27,7 @@
import com.google.gerrit.index.project.ProjectData;
import com.google.gerrit.index.project.ProjectIndex;
import com.google.gerrit.server.index.IndexExecutor;
+import com.google.gerrit.server.index.options.IsFirstInsertForEntry;
import com.google.gerrit.server.project.ProjectCache;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -48,12 +49,16 @@
private final ListeningExecutorService executor;
private final ProjectCache projectCache;
+ private final IsFirstInsertForEntry isFirstInsertForEntry;
@Inject
AllProjectsIndexer(
- @IndexExecutor(BATCH) ListeningExecutorService executor, ProjectCache projectCache) {
+ @IndexExecutor(BATCH) ListeningExecutorService executor,
+ ProjectCache projectCache,
+ IsFirstInsertForEntry isFirstInsertForEntry) {
this.executor = executor;
this.projectCache = projectCache;
+ this.isFirstInsertForEntry = isFirstInsertForEntry;
}
@Override
@@ -79,8 +84,13 @@
() -> {
try {
projectCache.evict(name);
- index.replace(
- projectCache.get(name).orElseThrow(illegalState(name)).toProjectData());
+ ProjectData projectData =
+ projectCache.get(name).orElseThrow(illegalState(name)).toProjectData();
+ if (isFirstInsertForEntry.equals(IsFirstInsertForEntry.YES)) {
+ index.insert(projectData);
+ } else {
+ index.replace(projectData);
+ }
verboseWriter.println("Reindexed " + desc);
done.incrementAndGet();
} catch (Exception e) {
diff --git a/java/com/google/gerrit/server/notedb/ChangeNotes.java b/java/com/google/gerrit/server/notedb/ChangeNotes.java
index 5db3764..a1d6b29 100644
--- a/java/com/google/gerrit/server/notedb/ChangeNotes.java
+++ b/java/com/google/gerrit/server/notedb/ChangeNotes.java
@@ -115,6 +115,29 @@
this.projectCache = projectCache;
}
+ @AutoValue
+ public abstract static class ScanResult {
+ abstract ImmutableSet<Change.Id> fromPatchSetRefs();
+
+ abstract ImmutableSet<Change.Id> fromMetaRefs();
+
+ public SetView<Change.Id> all() {
+ return Sets.union(fromPatchSetRefs(), fromMetaRefs());
+ }
+ }
+
+ public static ScanResult scanChangeIds(Repository repo) throws IOException {
+ ImmutableSet.Builder<Change.Id> fromPs = ImmutableSet.builder();
+ ImmutableSet.Builder<Change.Id> fromMeta = ImmutableSet.builder();
+ for (Ref r : repo.getRefDatabase().getRefsByPrefix(RefNames.REFS_CHANGES)) {
+ Change.Id id = Change.Id.fromRef(r.getName());
+ if (id != null) {
+ (r.getName().endsWith(RefNames.META_SUFFIX) ? fromMeta : fromPs).add(id);
+ }
+ }
+ return new AutoValue_ChangeNotes_Factory_ScanResult(fromPs.build(), fromMeta.build());
+ }
+
public ChangeNotes createChecked(Change c) {
return createChecked(c.getProject(), c.getId());
}
@@ -274,8 +297,11 @@
public Stream<ChangeNotesResult> scan(
Repository repo, Project.NameKey project, Predicate<Change.Id> changeIdPredicate)
throws IOException {
- ScanResult sr = scanChangeIds(repo);
+ return scan(scanChangeIds(repo), project, changeIdPredicate);
+ }
+ public Stream<ChangeNotesResult> scan(
+ ScanResult sr, Project.NameKey project, Predicate<Change.Id> changeIdPredicate) {
Stream<Change.Id> idStream = sr.all().stream();
if (changeIdPredicate != null) {
idStream = idStream.filter(changeIdPredicate);
@@ -347,29 +373,6 @@
@Nullable
abstract ChangeNotes maybeNotes();
}
-
- @AutoValue
- abstract static class ScanResult {
- abstract ImmutableSet<Change.Id> fromPatchSetRefs();
-
- abstract ImmutableSet<Change.Id> fromMetaRefs();
-
- SetView<Change.Id> all() {
- return Sets.union(fromPatchSetRefs(), fromMetaRefs());
- }
- }
-
- private static ScanResult scanChangeIds(Repository repo) throws IOException {
- ImmutableSet.Builder<Change.Id> fromPs = ImmutableSet.builder();
- ImmutableSet.Builder<Change.Id> fromMeta = ImmutableSet.builder();
- for (Ref r : repo.getRefDatabase().getRefsByPrefix(RefNames.REFS_CHANGES)) {
- Change.Id id = Change.Id.fromRef(r.getName());
- if (id != null) {
- (r.getName().endsWith(RefNames.META_SUFFIX) ? fromMeta : fromPs).add(id);
- }
- }
- return new AutoValue_ChangeNotes_Factory_ScanResult(fromPs.build(), fromMeta.build());
- }
}
private final boolean shouldExist;
diff --git a/java/com/google/gerrit/server/project/NullProjectCache.java b/java/com/google/gerrit/server/project/NullProjectCache.java
index 1d5f5b7..d19a726 100644
--- a/java/com/google/gerrit/server/project/NullProjectCache.java
+++ b/java/com/google/gerrit/server/project/NullProjectCache.java
@@ -42,11 +42,6 @@
}
@Override
- public void evict(Project p) {
- throw new UnsupportedOperationException();
- }
-
- @Override
public void evict(NameKey p) {
throw new UnsupportedOperationException();
}
@@ -80,4 +75,14 @@
public void onCreateProject(NameKey newProjectName) throws IOException {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public void evictAndReindex(Project p) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void evictAndReindex(NameKey p) {
+ throw new UnsupportedOperationException();
+ }
}
diff --git a/java/com/google/gerrit/server/project/ProjectCache.java b/java/com/google/gerrit/server/project/ProjectCache.java
index fee7105..fb0a4ec 100644
--- a/java/com/google/gerrit/server/project/ProjectCache.java
+++ b/java/com/google/gerrit/server/project/ProjectCache.java
@@ -59,18 +59,25 @@
Optional<ProjectState> get(@Nullable Project.NameKey projectName) throws StorageException;
/**
+ * Invalidate the cached information about the given project.
+ *
+ * @param p the NameKey of the project that is being evicted
+ */
+ void evict(Project.NameKey p);
+
+ /**
* Invalidate the cached information about the given project, and triggers reindexing for it
*
* @param p project that is being evicted
*/
- void evict(Project p);
+ void evictAndReindex(Project p);
/**
* Invalidate the cached information about the given project, and triggers reindexing for it
*
* @param p the NameKey of the project that is being evicted
*/
- void evict(Project.NameKey p);
+ void evictAndReindex(Project.NameKey p);
/**
* Remove information about the given project from the cache. It will no longer be returned from
diff --git a/java/com/google/gerrit/server/project/ProjectCacheImpl.java b/java/com/google/gerrit/server/project/ProjectCacheImpl.java
index de27afa..7e7c7be 100644
--- a/java/com/google/gerrit/server/project/ProjectCacheImpl.java
+++ b/java/com/google/gerrit/server/project/ProjectCacheImpl.java
@@ -214,16 +214,21 @@
}
@Override
- public void evict(Project p) {
- evict(p.getNameKey());
- }
-
- @Override
public void evict(Project.NameKey p) {
if (p != null) {
logger.atFine().log("Evict project '%s'", p.get());
inMemoryProjectCache.invalidate(p);
}
+ }
+
+ @Override
+ public void evictAndReindex(Project p) {
+ evictAndReindex(p.getNameKey());
+ }
+
+ @Override
+ public void evictAndReindex(Project.NameKey p) {
+ evict(p);
indexer.get().index(p);
}
@@ -244,7 +249,7 @@
} finally {
listLock.unlock();
}
- evict(name);
+ evictAndReindex(name);
}
@Override
diff --git a/java/com/google/gerrit/server/restapi/config/GetCache.java b/java/com/google/gerrit/server/restapi/config/GetCache.java
index 93600ea..5dd3d3d 100644
--- a/java/com/google/gerrit/server/restapi/config/GetCache.java
+++ b/java/com/google/gerrit/server/restapi/config/GetCache.java
@@ -16,6 +16,7 @@
import com.google.gerrit.extensions.restapi.Response;
import com.google.gerrit.extensions.restapi.RestReadView;
+import com.google.gerrit.server.cache.CacheInfo;
import com.google.gerrit.server.config.CacheResource;
import com.google.inject.Singleton;
@@ -23,7 +24,7 @@
public class GetCache implements RestReadView<CacheResource> {
@Override
- public Response<ListCaches.CacheInfo> apply(CacheResource rsrc) {
- return Response.ok(new ListCaches.CacheInfo(rsrc.getName(), rsrc.getCache()));
+ public Response<CacheInfo> apply(CacheResource rsrc) {
+ return Response.ok(new CacheInfo(rsrc.getName(), rsrc.getCache()));
}
}
diff --git a/java/com/google/gerrit/server/restapi/config/ListCaches.java b/java/com/google/gerrit/server/restapi/config/ListCaches.java
index ccafbe8..ffc65c9 100644
--- a/java/com/google/gerrit/server/restapi/config/ListCaches.java
+++ b/java/com/google/gerrit/server/restapi/config/ListCaches.java
@@ -22,7 +22,6 @@
import static java.util.stream.Collectors.joining;
import com.google.common.cache.Cache;
-import com.google.common.cache.CacheStats;
import com.google.common.collect.Streams;
import com.google.gerrit.extensions.annotations.RequiresAnyCapability;
import com.google.gerrit.extensions.registration.DynamicMap;
@@ -30,7 +29,7 @@
import com.google.gerrit.extensions.restapi.BinaryResult;
import com.google.gerrit.extensions.restapi.Response;
import com.google.gerrit.extensions.restapi.RestReadView;
-import com.google.gerrit.server.cache.PersistentCache;
+import com.google.gerrit.server.cache.CacheInfo;
import com.google.gerrit.server.config.ConfigResource;
import com.google.inject.Inject;
import java.util.Map;
@@ -87,118 +86,4 @@
}
return Response.ok(cacheNames.collect(toImmutableList()));
}
-
- public enum CacheType {
- MEM,
- DISK
- }
-
- public static class CacheInfo {
- public String name;
- public CacheType type;
- public EntriesInfo entries;
- public String averageGet;
- public HitRatioInfo hitRatio;
-
- public CacheInfo(Cache<?, ?> cache) {
- this(null, cache);
- }
-
- public CacheInfo(String name, Cache<?, ?> cache) {
- this.name = name;
-
- CacheStats stat = cache.stats();
-
- entries = new EntriesInfo();
- entries.setMem(cache.size());
-
- averageGet = duration(stat.averageLoadPenalty());
-
- hitRatio = new HitRatioInfo();
- hitRatio.setMem(stat.hitCount(), stat.requestCount());
-
- if (cache instanceof PersistentCache) {
- type = CacheType.DISK;
- PersistentCache.DiskStats diskStats = ((PersistentCache) cache).diskStats();
- entries.setDisk(diskStats.size());
- entries.setSpace(diskStats.space());
- hitRatio.setDisk(diskStats.hitCount(), diskStats.requestCount());
- } else {
- type = CacheType.MEM;
- }
- }
-
- private static String duration(double ns) {
- if (ns < 0.5) {
- return null;
- }
- String suffix = "ns";
- if (ns >= 1000.0) {
- ns /= 1000.0;
- suffix = "us";
- }
- if (ns >= 1000.0) {
- ns /= 1000.0;
- suffix = "ms";
- }
- if (ns >= 1000.0) {
- ns /= 1000.0;
- suffix = "s";
- }
- return String.format("%4.1f%s", ns, suffix).trim();
- }
- }
-
- public static class EntriesInfo {
- public Long mem;
- public Long disk;
- public String space;
-
- public void setMem(long mem) {
- this.mem = mem != 0 ? mem : null;
- }
-
- public void setDisk(long disk) {
- this.disk = disk != 0 ? disk : null;
- }
-
- public void setSpace(double value) {
- space = bytes(value);
- }
-
- private static String bytes(double value) {
- value /= 1024;
- String suffix = "k";
-
- if (value > 1024) {
- value /= 1024;
- suffix = "m";
- }
- if (value > 1024) {
- value /= 1024;
- suffix = "g";
- }
- return String.format("%1$6.2f%2$s", value, suffix).trim();
- }
- }
-
- public static class HitRatioInfo {
- public Integer mem;
- public Integer disk;
-
- public void setMem(long value, long total) {
- mem = percent(value, total);
- }
-
- public void setDisk(long value, long total) {
- disk = percent(value, total);
- }
-
- private static Integer percent(long value, long total) {
- if (total <= 0) {
- return null;
- }
- return (int) ((100 * value) / total);
- }
- }
}
diff --git a/java/com/google/gerrit/server/restapi/project/CreateLabel.java b/java/com/google/gerrit/server/restapi/project/CreateLabel.java
index d2f4161..e3f293a 100644
--- a/java/com/google/gerrit/server/restapi/project/CreateLabel.java
+++ b/java/com/google/gerrit/server/restapi/project/CreateLabel.java
@@ -106,7 +106,7 @@
config.commit(md);
- projectCache.evict(rsrc.getProjectState().getProject());
+ projectCache.evictAndReindex(rsrc.getProjectState().getProject());
return Response.created(LabelDefinitionJson.format(rsrc.getNameKey(), labelType));
}
diff --git a/java/com/google/gerrit/server/restapi/project/DeleteLabel.java b/java/com/google/gerrit/server/restapi/project/DeleteLabel.java
index 531640c..8a1927a 100644
--- a/java/com/google/gerrit/server/restapi/project/DeleteLabel.java
+++ b/java/com/google/gerrit/server/restapi/project/DeleteLabel.java
@@ -90,7 +90,7 @@
config.commit(md);
}
- projectCache.evict(rsrc.getProject().getProjectState().getProject());
+ projectCache.evictAndReindex(rsrc.getProject().getProjectState().getProject());
return Response.none();
}
diff --git a/java/com/google/gerrit/server/restapi/project/GetAccess.java b/java/com/google/gerrit/server/restapi/project/GetAccess.java
index a79439c..651e7f0 100644
--- a/java/com/google/gerrit/server/restapi/project/GetAccess.java
+++ b/java/com/google/gerrit/server/restapi/project/GetAccess.java
@@ -153,12 +153,12 @@
if (config.updateGroupNames(groupBackend)) {
md.setMessage("Update group names\n");
config.commit(md);
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
projectState = projectCache.get(projectName).orElseThrow(illegalState(projectName));
perm = permissionBackend.currentUser().project(projectName);
} else if (config.getRevision() != null
&& !config.getRevision().equals(projectState.getConfig().getRevision().orElse(null))) {
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
projectState = projectCache.get(projectName).orElseThrow(illegalState(projectName));
perm = permissionBackend.currentUser().project(projectName);
}
diff --git a/java/com/google/gerrit/server/restapi/project/PostLabels.java b/java/com/google/gerrit/server/restapi/project/PostLabels.java
index a56cfe6..6cb912e 100644
--- a/java/com/google/gerrit/server/restapi/project/PostLabels.java
+++ b/java/com/google/gerrit/server/restapi/project/PostLabels.java
@@ -139,7 +139,7 @@
if (dirty) {
config.commit(md);
- projectCache.evict(rsrc.getProjectState().getProject());
+ projectCache.evictAndReindex(rsrc.getProjectState().getProject());
}
}
diff --git a/java/com/google/gerrit/server/restapi/project/PutConfig.java b/java/com/google/gerrit/server/restapi/project/PutConfig.java
index fa8638e..30d667c 100644
--- a/java/com/google/gerrit/server/restapi/project/PutConfig.java
+++ b/java/com/google/gerrit/server/restapi/project/PutConfig.java
@@ -163,7 +163,7 @@
md.setMessage("Modified project settings\n");
try {
projectConfig.commit(md);
- projectCache.evict(projectConfig.getProject());
+ projectCache.evictAndReindex(projectConfig.getProject());
md.getRepository().setGitwebDescription(projectConfig.getProject().getDescription());
} catch (IOException e) {
if (e.getCause() instanceof ConfigInvalidException) {
diff --git a/java/com/google/gerrit/server/restapi/project/PutDescription.java b/java/com/google/gerrit/server/restapi/project/PutDescription.java
index a65c626..ec42035 100644
--- a/java/com/google/gerrit/server/restapi/project/PutDescription.java
+++ b/java/com/google/gerrit/server/restapi/project/PutDescription.java
@@ -84,7 +84,7 @@
md.setAuthor(user);
md.setMessage(msg);
config.commit(md);
- cache.evict(resource.getProjectState().getProject());
+ cache.evictAndReindex(resource.getProjectState().getProject());
md.getRepository().setGitwebDescription(config.getProject().getDescription());
return Strings.isNullOrEmpty(config.getProject().getDescription())
diff --git a/java/com/google/gerrit/server/restapi/project/SetAccess.java b/java/com/google/gerrit/server/restapi/project/SetAccess.java
index 794cae8..07dbeca 100644
--- a/java/com/google/gerrit/server/restapi/project/SetAccess.java
+++ b/java/com/google/gerrit/server/restapi/project/SetAccess.java
@@ -125,7 +125,7 @@
}
config.commit(md);
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
createGroupPermissionSyncer.syncIfNeeded();
} catch (InvalidNameException e) {
throw new BadRequestException(e.toString());
diff --git a/java/com/google/gerrit/server/restapi/project/SetDefaultDashboard.java b/java/com/google/gerrit/server/restapi/project/SetDefaultDashboard.java
index 5aef76a..853d7df 100644
--- a/java/com/google/gerrit/server/restapi/project/SetDefaultDashboard.java
+++ b/java/com/google/gerrit/server/restapi/project/SetDefaultDashboard.java
@@ -115,7 +115,7 @@
md.setAuthor(rsrc.getUser().asIdentifiedUser());
md.setMessage(msg);
config.commit(md);
- cache.evict(rsrc.getProjectState().getProject());
+ cache.evictAndReindex(rsrc.getProjectState().getProject());
if (target != null) {
Response<DashboardInfo> response = get.get().apply(target);
diff --git a/java/com/google/gerrit/server/restapi/project/SetLabel.java b/java/com/google/gerrit/server/restapi/project/SetLabel.java
index d69abef..801fac7 100644
--- a/java/com/google/gerrit/server/restapi/project/SetLabel.java
+++ b/java/com/google/gerrit/server/restapi/project/SetLabel.java
@@ -98,7 +98,7 @@
config.getLabelSections().get(newName.isEmpty() ? labelType.getName() : newName);
config.commit(md);
- projectCache.evict(rsrc.getProject().getProjectState().getProject());
+ projectCache.evictAndReindex(rsrc.getProject().getProjectState().getProject());
}
}
return Response.ok(LabelDefinitionJson.format(rsrc.getProject().getNameKey(), labelType));
diff --git a/java/com/google/gerrit/server/restapi/project/SetParent.java b/java/com/google/gerrit/server/restapi/project/SetParent.java
index 91c29f5..ef31dc5 100644
--- a/java/com/google/gerrit/server/restapi/project/SetParent.java
+++ b/java/com/google/gerrit/server/restapi/project/SetParent.java
@@ -114,7 +114,7 @@
md.setAuthor(user);
md.setMessage(msg);
config.commit(md);
- cache.evict(rsrc.getProjectState().getProject());
+ cache.evictAndReindex(rsrc.getProjectState().getProject());
Project.NameKey parent = config.getProject().getParent(allProjects);
requireNonNull(parent);
diff --git a/java/com/google/gerrit/server/submit/SubmitStrategyOp.java b/java/com/google/gerrit/server/submit/SubmitStrategyOp.java
index 7d428eb..f26ec17 100644
--- a/java/com/google/gerrit/server/submit/SubmitStrategyOp.java
+++ b/java/com/google/gerrit/server/submit/SubmitStrategyOp.java
@@ -482,7 +482,7 @@
// TODO(dborowitz): Move to BatchUpdate? Would also allow us to run once
// per project even if multiple changes to refs/meta/config are submitted.
if (RefNames.REFS_CONFIG.equals(getDest().branch())) {
- args.projectCache.evict(getProject());
+ args.projectCache.evictAndReindex(getProject());
ProjectState p =
args.projectCache.get(getProject()).orElseThrow(illegalState(getProject()));
try (Repository git = args.repoManager.openRepository(getProject())) {
diff --git a/java/com/google/gerrit/sshd/commands/ShowCaches.java b/java/com/google/gerrit/sshd/commands/ShowCaches.java
index 979be1b..02956f7 100644
--- a/java/com/google/gerrit/sshd/commands/ShowCaches.java
+++ b/java/com/google/gerrit/sshd/commands/ShowCaches.java
@@ -24,6 +24,8 @@
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.extensions.restapi.AuthException;
import com.google.gerrit.server.CurrentUser;
+import com.google.gerrit.server.cache.CacheDisplay;
+import com.google.gerrit.server.cache.CacheInfo;
import com.google.gerrit.server.config.ConfigResource;
import com.google.gerrit.server.permissions.GlobalPermission;
import com.google.gerrit.server.permissions.PermissionBackend;
@@ -35,8 +37,6 @@
import com.google.gerrit.server.restapi.config.GetSummary.TaskSummaryInfo;
import com.google.gerrit.server.restapi.config.GetSummary.ThreadSummaryInfo;
import com.google.gerrit.server.restapi.config.ListCaches;
-import com.google.gerrit.server.restapi.config.ListCaches.CacheInfo;
-import com.google.gerrit.server.restapi.config.ListCaches.CacheType;
import com.google.gerrit.server.util.time.TimeUtil;
import com.google.gerrit.sshd.CommandMetaData;
import com.google.gerrit.sshd.SshCommand;
@@ -123,52 +123,8 @@
stdout.format("%-25s %-20s uptime %16s\n", "", "", uptime(now.getTime() - serverStarted));
stdout.print('\n');
- stdout.print(
- String.format( //
- "%1s %-" + nw + "s|%-21s| %-5s |%-9s|\n" //
- ,
- "" //
- ,
- "Name" //
- ,
- "Entries" //
- ,
- "AvgGet" //
- ,
- "Hit Ratio" //
- ));
- stdout.print(
- String.format( //
- "%1s %-" + nw + "s|%6s %6s %7s| %-5s |%-4s %-4s|\n" //
- ,
- "" //
- ,
- "" //
- ,
- "Mem" //
- ,
- "Disk" //
- ,
- "Space" //
- ,
- "" //
- ,
- "Mem" //
- ,
- "Disk" //
- ));
- stdout.print("--");
- for (int i = 0; i < nw; i++) {
- stdout.print('-');
- }
- stdout.print("+---------------------+---------+---------+\n");
-
try {
- Collection<CacheInfo> caches = getCaches();
- printMemoryCoreCaches(caches);
- printMemoryPluginCaches(caches);
- printDiskCaches(caches);
- stdout.print('\n');
+ new CacheDisplay(stdout, nw, getCaches()).displayCaches();
boolean showJvm;
try {
@@ -209,52 +165,6 @@
return caches.values();
}
- private void printMemoryCoreCaches(Collection<CacheInfo> caches) {
- for (CacheInfo cache : caches) {
- if (!cache.name.contains("-") && CacheType.MEM.equals(cache.type)) {
- printCache(cache);
- }
- }
- }
-
- private void printMemoryPluginCaches(Collection<CacheInfo> caches) {
- for (CacheInfo cache : caches) {
- if (cache.name.contains("-") && CacheType.MEM.equals(cache.type)) {
- printCache(cache);
- }
- }
- }
-
- private void printDiskCaches(Collection<CacheInfo> caches) {
- for (CacheInfo cache : caches) {
- if (CacheType.DISK.equals(cache.type)) {
- printCache(cache);
- }
- }
- }
-
- private void printCache(CacheInfo cache) {
- stdout.print(
- String.format(
- "%1s %-" + nw + "s|%6s %6s %7s| %7s |%4s %4s|\n",
- CacheType.DISK.equals(cache.type) ? "D" : "",
- cache.name,
- nullToEmpty(cache.entries.mem),
- nullToEmpty(cache.entries.disk),
- Strings.nullToEmpty(cache.entries.space),
- Strings.nullToEmpty(cache.averageGet),
- formatAsPercent(cache.hitRatio.mem),
- formatAsPercent(cache.hitRatio.disk)));
- }
-
- private static String nullToEmpty(Long l) {
- return l != null ? String.valueOf(l) : "";
- }
-
- private static String formatAsPercent(Integer i) {
- return i != null ? String.valueOf(i) + "%" : "";
- }
-
private void memSummary(MemSummaryInfo memSummary) {
stdout.format(
"Mem: %s total = %s used + %s free + %s buffers\n",
diff --git a/javatests/com/google/gerrit/acceptance/ProjectResetterTest.java b/javatests/com/google/gerrit/acceptance/ProjectResetterTest.java
index ff9bac9..7d04558 100644
--- a/javatests/com/google/gerrit/acceptance/ProjectResetterTest.java
+++ b/javatests/com/google/gerrit/acceptance/ProjectResetterTest.java
@@ -231,7 +231,7 @@
updateRef(repo2, metaConfig);
}
- verify(projectCache, only()).evict(project2);
+ verify(projectCache, only()).evictAndReindex(project2);
}
@Test
@@ -248,7 +248,7 @@
createRef(repo2, RefNames.REFS_CONFIG);
}
- verify(projectCache, only()).evict(project2);
+ verify(projectCache, only()).evictAndReindex(project2);
}
@Test
diff --git a/javatests/com/google/gerrit/acceptance/api/accounts/AgreementsIT.java b/javatests/com/google/gerrit/acceptance/api/accounts/AgreementsIT.java
index 5279ba1..9b77b01 100644
--- a/javatests/com/google/gerrit/acceptance/api/accounts/AgreementsIT.java
+++ b/javatests/com/google/gerrit/acceptance/api/accounts/AgreementsIT.java
@@ -71,7 +71,7 @@
config.updateProject(
p -> p.setBooleanConfig(BooleanProjectConfig.USE_CONTRIBUTOR_AGREEMENTS, value));
config.commit(md);
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
}
}
diff --git a/javatests/com/google/gerrit/acceptance/api/project/ProjectIT.java b/javatests/com/google/gerrit/acceptance/api/project/ProjectIT.java
index 6b2f4f1..9c74c48 100644
--- a/javatests/com/google/gerrit/acceptance/api/project/ProjectIT.java
+++ b/javatests/com/google/gerrit/acceptance/api/project/ProjectIT.java
@@ -1000,7 +1000,7 @@
ProjectConfig config = projectConfigFactory.read(md);
config.renameGroup(AccountGroup.uuid(group.id), newName);
config.commit(md);
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
}
Optional<String> afterRename =
diff --git a/javatests/com/google/gerrit/acceptance/git/AbstractSubmoduleSubscription.java b/javatests/com/google/gerrit/acceptance/git/AbstractSubmoduleSubscription.java
index 415aa79..c3bcbd3 100644
--- a/javatests/com/google/gerrit/acceptance/git/AbstractSubmoduleSubscription.java
+++ b/javatests/com/google/gerrit/acceptance/git/AbstractSubmoduleSubscription.java
@@ -226,7 +226,7 @@
ObjectId oldId = pc.getRevision();
ObjectId newId = pc.commit(md);
assertThat(newId).isNotEqualTo(oldId);
- projectCache.evict(pc.getProject());
+ projectCache.evictAndReindex(pc.getProject());
}
}
diff --git a/javatests/com/google/gerrit/acceptance/rest/config/CacheOperationsIT.java b/javatests/com/google/gerrit/acceptance/rest/config/CacheOperationsIT.java
index ef5e7dc..ab8e4d8 100644
--- a/javatests/com/google/gerrit/acceptance/rest/config/CacheOperationsIT.java
+++ b/javatests/com/google/gerrit/acceptance/rest/config/CacheOperationsIT.java
@@ -25,7 +25,7 @@
import com.google.gerrit.acceptance.RestResponse;
import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
import com.google.gerrit.common.data.GlobalCapability;
-import com.google.gerrit.server.restapi.config.ListCaches.CacheInfo;
+import com.google.gerrit.server.cache.CacheInfo;
import com.google.gerrit.server.restapi.config.PostCaches;
import com.google.inject.Inject;
import java.util.Arrays;
diff --git a/javatests/com/google/gerrit/acceptance/rest/config/FlushCacheIT.java b/javatests/com/google/gerrit/acceptance/rest/config/FlushCacheIT.java
index a161ec4..164f683 100644
--- a/javatests/com/google/gerrit/acceptance/rest/config/FlushCacheIT.java
+++ b/javatests/com/google/gerrit/acceptance/rest/config/FlushCacheIT.java
@@ -23,7 +23,7 @@
import com.google.gerrit.acceptance.RestResponse;
import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
import com.google.gerrit.common.data.GlobalCapability;
-import com.google.gerrit.server.restapi.config.ListCaches.CacheInfo;
+import com.google.gerrit.server.cache.CacheInfo;
import com.google.inject.Inject;
import org.junit.Test;
diff --git a/javatests/com/google/gerrit/acceptance/rest/config/GetCacheIT.java b/javatests/com/google/gerrit/acceptance/rest/config/GetCacheIT.java
index 247d63b..8765360 100644
--- a/javatests/com/google/gerrit/acceptance/rest/config/GetCacheIT.java
+++ b/javatests/com/google/gerrit/acceptance/rest/config/GetCacheIT.java
@@ -18,8 +18,7 @@
import com.google.gerrit.acceptance.AbstractDaemonTest;
import com.google.gerrit.acceptance.RestResponse;
-import com.google.gerrit.server.restapi.config.ListCaches.CacheInfo;
-import com.google.gerrit.server.restapi.config.ListCaches.CacheType;
+import com.google.gerrit.server.cache.CacheInfo;
import org.junit.Test;
public class GetCacheIT extends AbstractDaemonTest {
@@ -31,7 +30,7 @@
CacheInfo result = newGson().fromJson(r.getReader(), CacheInfo.class);
assertThat(result.name).isEqualTo("accounts");
- assertThat(result.type).isEqualTo(CacheType.MEM);
+ assertThat(result.type).isEqualTo(CacheInfo.CacheType.MEM);
assertThat(result.entries.mem).isAtLeast(1L);
assertThat(result.averageGet).isNotNull();
assertThat(result.averageGet).endsWith("s");
diff --git a/javatests/com/google/gerrit/acceptance/rest/config/ListCachesIT.java b/javatests/com/google/gerrit/acceptance/rest/config/ListCachesIT.java
index 8baeffc..be21436 100644
--- a/javatests/com/google/gerrit/acceptance/rest/config/ListCachesIT.java
+++ b/javatests/com/google/gerrit/acceptance/rest/config/ListCachesIT.java
@@ -21,8 +21,7 @@
import com.google.common.io.BaseEncoding;
import com.google.gerrit.acceptance.AbstractDaemonTest;
import com.google.gerrit.acceptance.RestResponse;
-import com.google.gerrit.server.restapi.config.ListCaches.CacheInfo;
-import com.google.gerrit.server.restapi.config.ListCaches.CacheType;
+import com.google.gerrit.server.cache.CacheInfo;
import com.google.gson.reflect.TypeToken;
import java.util.Arrays;
import java.util.List;
@@ -40,7 +39,7 @@
assertThat(result).containsKey("accounts");
CacheInfo accountsCacheInfo = result.get("accounts");
- assertThat(accountsCacheInfo.type).isEqualTo(CacheType.MEM);
+ assertThat(accountsCacheInfo.type).isEqualTo(CacheInfo.CacheType.MEM);
assertThat(accountsCacheInfo.entries.mem).isAtLeast(1L);
assertThat(accountsCacheInfo.averageGet).isNotNull();
assertThat(accountsCacheInfo.averageGet).endsWith("s");
diff --git a/javatests/com/google/gerrit/server/change/LabelNormalizerTest.java b/javatests/com/google/gerrit/server/change/LabelNormalizerTest.java
index 5e3be9a..8f341aa 100644
--- a/javatests/com/google/gerrit/server/change/LabelNormalizerTest.java
+++ b/javatests/com/google/gerrit/server/change/LabelNormalizerTest.java
@@ -205,7 +205,7 @@
private void save(ProjectConfig pc) throws Exception {
try (MetaDataUpdate md = metaDataUpdateFactory.create(pc.getProject().getNameKey(), user)) {
pc.commit(md);
- projectCache.evict(pc.getProject().getNameKey());
+ projectCache.evictAndReindex(pc.getProject().getNameKey());
}
}
diff --git a/javatests/com/google/gerrit/server/index/change/FakeChangeIndex.java b/javatests/com/google/gerrit/server/index/change/FakeChangeIndex.java
index a23ccab..fc56a3c 100644
--- a/javatests/com/google/gerrit/server/index/change/FakeChangeIndex.java
+++ b/javatests/com/google/gerrit/server/index/change/FakeChangeIndex.java
@@ -74,6 +74,11 @@
}
@Override
+ public void insert(ChangeData obj) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public void replace(ChangeData cd) {
throw new UnsupportedOperationException();
}
diff --git a/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java b/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java
index 332548a..9acce8f 100644
--- a/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java
+++ b/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java
@@ -1135,7 +1135,7 @@
cfg.upsertLabelType(verified);
cfg.commit(md);
}
- projectCache.evict(project);
+ projectCache.evictAndReindex(project);
String heads = RefNames.REFS_HEADS + "*";
projectOperations
@@ -2265,7 +2265,7 @@
});
config.commit(md);
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
}
}
diff --git a/plugins/delete-project b/plugins/delete-project
index 5dcb1a6..612f143 160000
--- a/plugins/delete-project
+++ b/plugins/delete-project
@@ -1 +1 @@
-Subproject commit 5dcb1a64e779def0309354dd0e9886189845b020
+Subproject commit 612f143792652d571ecfcb19915ad5754a3ba1a7
diff --git a/plugins/webhooks b/plugins/webhooks
index 9e4cd70..8df6303 160000
--- a/plugins/webhooks
+++ b/plugins/webhooks
@@ -1 +1 @@
-Subproject commit 9e4cd708b96e4fab79b4101eaf3f3e6a8b872dca
+Subproject commit 8df6303e652857c3d7cee3348d2cb7d40a7db1af
diff --git a/polygerrit-ui/app/elements/shared/gr-rest-api-interface/gr-rest-api-interface.ts b/polygerrit-ui/app/elements/shared/gr-rest-api-interface/gr-rest-api-interface.ts
index 475a69c..dd2c8d3 100644
--- a/polygerrit-ui/app/elements/shared/gr-rest-api-interface/gr-rest-api-interface.ts
+++ b/polygerrit-ui/app/elements/shared/gr-rest-api-interface/gr-rest-api-interface.ts
@@ -1608,7 +1608,7 @@
params.n = n;
}
if (project) {
- params.p = encodeURIComponent(project);
+ params.p = project;
}
return this._restApiHelper.fetchJSON({
url: '/groups/',
