Merge "Improve comment describing ExternalId#SCHEME_GERRIT" into stable-3.2
diff --git a/.bazelrc b/.bazelrc
index aece2e0..e560f2397 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -11,10 +11,6 @@
# this flag here once flipped in Bazel again.
build --incompatible_strict_action_env
-# Workaround Bazel worker crash (remove after upgrading to 4.1.0)
-# https://github.com/bazelbuild/bazel/issues/13333
-build --experimental_worker_multiplex=false
-
test --build_tests_only
test --test_output=errors
diff --git a/Documentation/config-gerrit.txt b/Documentation/config-gerrit.txt
index 0c0626e..b964040 100644
--- a/Documentation/config-gerrit.txt
+++ b/Documentation/config-gerrit.txt
@@ -2979,7 +2979,7 @@
[[index.batchThreads]]index.batchThreads::
+
Number of threads to use for indexing in background operations, such as
-online schema upgrades, and also for offline reindexing.
+online schema upgrades, and also the default for offline reindexing.
+
If not set or set to a zero, defaults to the number of logical CPUs as returned
by the JVM. If set to a negative value, defaults to a direct executor.
@@ -3182,6 +3182,11 @@
+
Defaults to true (throttling enabled).
+During offline reindexing, setting ramBufferSize greater than the size
+of index (size of specific index folder under <site_dir>/index) and
+maxBufferedDocs as -1 avoids unnecessary flushes and triggers only a
+single flush at the end of the process.
+
Sample Lucene index configuration:
----
[index]
diff --git a/Documentation/dev-plugins.txt b/Documentation/dev-plugins.txt
index b3d7a45..9a5ce03 100644
--- a/Documentation/dev-plugins.txt
+++ b/Documentation/dev-plugins.txt
@@ -808,6 +808,63 @@
}
----
+Plugins can receive a bean object for each of the gerrit ssh and the REST API
+commands by implementing BeanParseListener interface and registering it to a
+command class name in the plugin module's `configure()` method. The below
+example shows a plugin that always limits the number of projects returned
+by the ls-projects SSH command.
+
+[source, java]
+----
+protected static class PluginModule extends AbstractModule {
+ @Override
+ public void configure() {
+ bind(DynamicOptions.DynamicBean.class)
+ .annotatedWith(Exports.named(ListProjectsCommand.class))
+ .to(ListProjectsCommandBeanListener.class);
+ }
+
+ protected static class ListProjectsCommandBeanListener
+ implements DynamicOptions.BeanParseListener {
+ @Override
+ public void onBeanParseStart(String plugin, Object bean) {
+ ListProjectsCommand command = (ListProjectsCommand) bean;
+ command.impl.setLimit(1);
+ }
+
+ @Override
+ public void onBeanParseEnd(String plugin, Object bean) {}
+ }
+}
+----
+
+The below example shows a plugin that always limits the number of projects
+returned by the /projects/ REST API.
+
+[source, java]
+----
+protected static class PluginModule extends AbstractModule {
+ @Override
+ public void configure() {
+ bind(DynamicOptions.DynamicBean.class)
+ .annotatedWith(Exports.named(ListProjects.class))
+ .to(ListProjectsBeanListener.class);
+ }
+
+ protected static class ListProjectsBeanListener
+ implements DynamicOptions.BeanParseListener {
+ @Override
+ public void onBeanParseStart(String plugin, Object bean) {
+ ListProjects listProjects = (ListProjects) bean;
+ listProjects.setLimit(1);
+ }
+
+ @Override
+ public void onBeanParseEnd(String plugin, Object bean) {}
+ }
+}
+----
+
=== Calling Command Options ===
Within an OptionHandler, during the processing of an option, plugins can
diff --git a/Documentation/logs.txt b/Documentation/logs.txt
index 43a8e62..a95956c 100644
--- a/Documentation/logs.txt
+++ b/Documentation/logs.txt
@@ -48,6 +48,11 @@
* `referer`: the `Referer` HTTP request header. This gives the site that
the client reports having been referred from.
* `client agent`: the client agent which sent the request.
+* `command status`: the overall result of the git command over HTTP. Currently
+ populated only for the transfer phase of `git-upload-pack` commands.
+ Possible values:
+** `-1`: The `git-upload-pack` transfer was ultimately not successful
+** `0`: The `git-upload-pack` transfer was ultimately successful
Example:
```
diff --git a/Documentation/pgm-reindex.txt b/Documentation/pgm-reindex.txt
index 5167277..0653d8d 100644
--- a/Documentation/pgm-reindex.txt
+++ b/Documentation/pgm-reindex.txt
@@ -20,7 +20,8 @@
== OPTIONS
--threads::
- Number of threads to use for indexing.
+ Number of threads to use for indexing. Default is
+ link:config-gerrit.html#index.batchThreads[index.batchThreads]
--changes-schema-version::
Schema version to reindex; default is most recent version.
@@ -35,6 +36,10 @@
Reindex only index with given name. This option can be supplied
more than once to reindex multiple indices.
+--disable-cache-stats::
+ Disables printing cache statistics at the end of program to reduce
+ noise. Defaulted when reindex is run from init on a new site.
+
== CONTEXT
The secondary index must be enabled. See
link:config-gerrit.html#index.type[index.type].
diff --git a/Documentation/rest-api-projects.txt b/Documentation/rest-api-projects.txt
index de2c550..7bb8084 100644
--- a/Documentation/rest-api-projects.txt
+++ b/Documentation/rest-api-projects.txt
@@ -421,11 +421,11 @@
----
The `/projects/` URL also accepts a start integer in the `start`
-parameter. The results will skip `start` groups from project list.
+parameter. The results will skip `start` projects from project list.
Query 25 projects starting from index 50.
----
- GET /groups/?query=<query>&limit=25&start=50 HTTP/1.0
+ GET /projects/?query=<query>&limit=25&start=50 HTTP/1.0
----
[[project-query-options]]
diff --git a/java/com/google/gerrit/acceptance/AbstractDaemonTest.java b/java/com/google/gerrit/acceptance/AbstractDaemonTest.java
index f29cdb2..bc6a71c 100644
--- a/java/com/google/gerrit/acceptance/AbstractDaemonTest.java
+++ b/java/com/google/gerrit/acceptance/AbstractDaemonTest.java
@@ -1002,7 +1002,7 @@
ProjectConfig config = projectConfigFactory.read(md);
config.getProject().setBooleanConfig(BooleanProjectConfig.USE_SIGNED_OFF_BY, value);
config.commit(md);
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
}
}
@@ -1011,7 +1011,7 @@
ProjectConfig config = projectConfigFactory.read(md);
config.getProject().setBooleanConfig(BooleanProjectConfig.REQUIRE_CHANGE_ID, value);
config.commit(md);
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
}
}
@@ -1500,7 +1500,7 @@
projectConfig.commit(metaDataUpdate);
metaDataUpdate.close();
metaDataUpdate = null;
- projectCache.evict(projectConfig.getProject());
+ projectCache.evictAndReindex(projectConfig.getProject());
}
@Override
diff --git a/java/com/google/gerrit/acceptance/DisabledAccountIndex.java b/java/com/google/gerrit/acceptance/DisabledAccountIndex.java
index 271d15c..7bd0c73 100644
--- a/java/com/google/gerrit/acceptance/DisabledAccountIndex.java
+++ b/java/com/google/gerrit/acceptance/DisabledAccountIndex.java
@@ -45,6 +45,11 @@
}
@Override
+ public void insert(AccountState obj) {
+ throw new UnsupportedOperationException("AccountIndex is disabled");
+ }
+
+ @Override
public void replace(AccountState obj) {
throw new UnsupportedOperationException("AccountIndex is disabled");
}
diff --git a/java/com/google/gerrit/acceptance/DisabledChangeIndex.java b/java/com/google/gerrit/acceptance/DisabledChangeIndex.java
index 34f72f5c..7671ad4 100644
--- a/java/com/google/gerrit/acceptance/DisabledChangeIndex.java
+++ b/java/com/google/gerrit/acceptance/DisabledChangeIndex.java
@@ -52,6 +52,11 @@
}
@Override
+ public void insert(ChangeData obj) {
+ throw new UnsupportedOperationException("ChangeIndex is disabled");
+ }
+
+ @Override
public void replace(ChangeData obj) {
throw new UnsupportedOperationException("ChangeIndex is disabled");
}
diff --git a/java/com/google/gerrit/acceptance/DisabledProjectIndex.java b/java/com/google/gerrit/acceptance/DisabledProjectIndex.java
index ed119ff..2e3dd90 100644
--- a/java/com/google/gerrit/acceptance/DisabledProjectIndex.java
+++ b/java/com/google/gerrit/acceptance/DisabledProjectIndex.java
@@ -50,6 +50,11 @@
}
@Override
+ public void insert(ProjectData obj) {
+ throw new UnsupportedOperationException("ProjectIndex is disabled");
+ }
+
+ @Override
public void replace(ProjectData obj) {
throw new UnsupportedOperationException("ProjectIndex is disabled");
}
diff --git a/java/com/google/gerrit/acceptance/GerritServer.java b/java/com/google/gerrit/acceptance/GerritServer.java
index 986f549..f1598e7 100644
--- a/java/com/google/gerrit/acceptance/GerritServer.java
+++ b/java/com/google/gerrit/acceptance/GerritServer.java
@@ -45,6 +45,7 @@
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePath;
import com.google.gerrit.server.git.receive.AsyncReceiveCommits;
+import com.google.gerrit.server.index.options.AutoFlush;
import com.google.gerrit.server.schema.JdbcAccountPatchReviewStore;
import com.google.gerrit.server.ssh.NoSshModule;
import com.google.gerrit.server.util.ReplicaUtil;
@@ -419,7 +420,8 @@
"accountPatchReviewDb", null, "url", JdbcAccountPatchReviewStore.TEST_IN_MEMORY_URL);
daemon.setEnableHttpd(desc.httpd());
daemon.setLuceneModule(
- LuceneIndexModule.singleVersionAllLatest(0, ReplicaUtil.isReplica(baseConfig)));
+ LuceneIndexModule.singleVersionAllLatest(
+ 0, ReplicaUtil.isReplica(baseConfig), AutoFlush.ENABLED));
daemon.setDatabaseForTesting(
ImmutableList.of(
new InMemoryTestingDatabaseModule(cfg, site, inMemoryRepoManager),
diff --git a/java/com/google/gerrit/acceptance/ProjectResetter.java b/java/com/google/gerrit/acceptance/ProjectResetter.java
index d885303..46f7496 100644
--- a/java/com/google/gerrit/acceptance/ProjectResetter.java
+++ b/java/com/google/gerrit/acceptance/ProjectResetter.java
@@ -307,7 +307,7 @@
Sets.union(
projectsWithConfigChanges(restoredRefsByProject),
projectsWithConfigChanges(deletedRefsByProject))) {
- projectCache.evict(project);
+ projectCache.evictAndReindex(project);
}
}
diff --git a/java/com/google/gerrit/acceptance/ReadOnlyChangeIndex.java b/java/com/google/gerrit/acceptance/ReadOnlyChangeIndex.java
index e943519..f7a0669 100644
--- a/java/com/google/gerrit/acceptance/ReadOnlyChangeIndex.java
+++ b/java/com/google/gerrit/acceptance/ReadOnlyChangeIndex.java
@@ -45,6 +45,11 @@
}
@Override
+ public void insert(ChangeData obj) {
+ // do nothing
+ }
+
+ @Override
public void replace(ChangeData obj) {
// do nothing
}
diff --git a/java/com/google/gerrit/acceptance/testsuite/project/ProjectOperationsImpl.java b/java/com/google/gerrit/acceptance/testsuite/project/ProjectOperationsImpl.java
index b0f46e9..59a2ed3 100644
--- a/java/com/google/gerrit/acceptance/testsuite/project/ProjectOperationsImpl.java
+++ b/java/com/google/gerrit/acceptance/testsuite/project/ProjectOperationsImpl.java
@@ -148,7 +148,7 @@
setExclusiveGroupPermissions(projectConfig, projectUpdate.exclusiveGroupPermissions());
projectConfig.commit(metaDataUpdate);
}
- projectCache.evict(nameKey);
+ projectCache.evictAndReindex(nameKey);
}
private void removePermissions(
@@ -295,7 +295,7 @@
setConfig(projectConfig);
try {
- projectCache.evict(nameKey);
+ projectCache.evictAndReindex(nameKey);
} catch (Exception e) {
// Evicting the project from the cache, also triggers a reindex of the project.
// The reindex step fails if the project config is invalid. That's fine, since it was our
@@ -313,7 +313,7 @@
testProjectInvalidation.projectConfigUpdater().forEach(c -> c.accept(projectConfig));
setConfig(projectConfig);
try {
- projectCache.evict(nameKey);
+ projectCache.evictAndReindex(nameKey);
} catch (Exception e) {
// Evicting the project from the cache, also triggers a reindex of the project.
// The reindex step fails if the project config is invalid. That's fine, since it was our
diff --git a/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java b/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
index e56f470..d53490b 100644
--- a/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
+++ b/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
@@ -175,6 +175,12 @@
}
@Override
+ public void insert(V obj) {
+ // TODO: Implement real insert() if it helps performance
+ replace(obj);
+ }
+
+ @Override
public void deleteAll() {
// Delete the index, if it exists.
String endpoint = indexName + client.adapter().indicesExistParams();
diff --git a/java/com/google/gerrit/httpd/GitOverHttpServlet.java b/java/com/google/gerrit/httpd/GitOverHttpServlet.java
index 6e22704..3e2d3ef 100644
--- a/java/com/google/gerrit/httpd/GitOverHttpServlet.java
+++ b/java/com/google/gerrit/httpd/GitOverHttpServlet.java
@@ -14,11 +14,15 @@
package com.google.gerrit.httpd;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static org.eclipse.jgit.http.server.GitSmartHttpTools.sendError;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.Cache;
import com.google.common.collect.ImmutableListMultimap;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
+import com.google.common.flogger.FluentLogger;
import com.google.gerrit.common.data.Capable;
import com.google.gerrit.entities.Project;
import com.google.gerrit.extensions.registration.DynamicSet;
@@ -54,6 +58,7 @@
import com.google.inject.TypeLiteral;
import com.google.inject.name.Named;
import java.io.IOException;
+import java.text.MessageFormat;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collections;
@@ -70,10 +75,13 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpServletResponseWrapper;
+import org.eclipse.jgit.errors.PackProtocolException;
import org.eclipse.jgit.errors.RepositoryNotFoundException;
import org.eclipse.jgit.http.server.GitServlet;
import org.eclipse.jgit.http.server.GitSmartHttpTools;
+import org.eclipse.jgit.http.server.HttpServerText;
import org.eclipse.jgit.http.server.ServletUtils;
+import org.eclipse.jgit.http.server.UploadPackErrorHandler;
import org.eclipse.jgit.http.server.resolver.AsIsFileService;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
@@ -100,6 +108,23 @@
private static final String ID_CACHE = "adv_bases";
public static final String URL_REGEX;
+ public static final String GIT_COMMAND_STATUS_HEADER = "X-git-command-status";
+
+ private enum GIT_COMMAND_STATUS {
+ OK(0),
+ FAIL(-1);
+
+ private final int exitStatus;
+
+ GIT_COMMAND_STATUS(int exitStatus) {
+ this.exitStatus = exitStatus;
+ }
+
+ @Override
+ public String toString() {
+ return Integer.toString(exitStatus);
+ }
+ }
static {
StringBuilder url = new StringBuilder();
@@ -212,12 +237,14 @@
Resolver resolver,
UploadFactory upload,
UploadFilter uploadFilter,
+ GerritUploadPackErrorHandler uploadPackErrorHandler,
ReceivePackFactory<HttpServletRequest> receive,
ReceiveFilter receiveFilter) {
setRepositoryResolver(resolver);
setAsIsFileService(AsIsFileService.DISABLED);
setUploadPackFactory(upload);
+ setUploadPackErrorHandler(uploadPackErrorHandler);
addUploadPackFilter(uploadFilter);
setReceivePackFactory(receive);
@@ -456,6 +483,35 @@
public void destroy() {}
}
+ static class GerritUploadPackErrorHandler implements UploadPackErrorHandler {
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+ @Override
+ public void upload(HttpServletRequest req, HttpServletResponse rsp, UploadPackRunnable r)
+ throws IOException {
+ rsp.setHeader(GIT_COMMAND_STATUS_HEADER, GIT_COMMAND_STATUS.FAIL.toString());
+ try {
+ r.upload();
+ rsp.setHeader(GIT_COMMAND_STATUS_HEADER, GIT_COMMAND_STATUS.OK.toString());
+ } catch (ServiceMayNotContinueException e) {
+ if (!e.isOutput() && !rsp.isCommitted()) {
+ rsp.reset();
+ sendError(req, rsp, e.getStatusCode(), e.getMessage());
+ }
+ } catch (Throwable e) {
+ logger.atSevere().withCause(e).log(
+ MessageFormat.format(
+ HttpServerText.get().internalErrorDuringUploadPack,
+ ServletUtils.getRepository(req)));
+ if (!rsp.isCommitted()) {
+ rsp.reset();
+ String msg = e instanceof PackProtocolException ? e.getMessage() : null;
+ sendError(req, rsp, SC_INTERNAL_SERVER_ERROR, msg);
+ }
+ }
+ }
+ }
+
static class ReceiveFactory implements ReceivePackFactory<HttpServletRequest> {
private final AsyncReceiveCommits.Factory factory;
private final Provider<CurrentUser> userProvider;
diff --git a/java/com/google/gerrit/httpd/init/WebAppInitializer.java b/java/com/google/gerrit/httpd/init/WebAppInitializer.java
index 05992d4..35c33be 100644
--- a/java/com/google/gerrit/httpd/init/WebAppInitializer.java
+++ b/java/com/google/gerrit/httpd/init/WebAppInitializer.java
@@ -79,6 +79,7 @@
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.OnlineUpgrader;
import com.google.gerrit.server.index.VersionManager;
+import com.google.gerrit.server.index.options.AutoFlush;
import com.google.gerrit.server.mail.SignedTokenEmailTokenVerifier;
import com.google.gerrit.server.mail.receive.MailReceiver;
import com.google.gerrit.server.mail.send.SmtpEmailSender;
@@ -353,7 +354,7 @@
private Module createIndexModule() {
if (indexType.isLucene()) {
- return LuceneIndexModule.latestVersion(false);
+ return LuceneIndexModule.latestVersion(false, AutoFlush.ENABLED);
} else if (indexType.isElasticsearch()) {
return ElasticIndexModule.latestVersion(false);
} else {
diff --git a/java/com/google/gerrit/index/Index.java b/java/com/google/gerrit/index/Index.java
index 44f8b42..e662bc8 100644
--- a/java/com/google/gerrit/index/Index.java
+++ b/java/com/google/gerrit/index/Index.java
@@ -40,6 +40,16 @@
void close();
/**
+ * Insert a document into the index.
+ *
+ * <p>Results may not be immediately visible to searchers, but should be visible within a
+ * reasonable amount of time.
+ *
+ * @param obj document object
+ */
+ void insert(V obj);
+
+ /**
* Update a document in the index.
*
* <p>Semantically equivalent to deleting the document and reinserting it with new field values. A
diff --git a/java/com/google/gerrit/lucene/AbstractLuceneIndex.java b/java/com/google/gerrit/lucene/AbstractLuceneIndex.java
index 5392ab4..1e6ccac 100644
--- a/java/com/google/gerrit/lucene/AbstractLuceneIndex.java
+++ b/java/com/google/gerrit/lucene/AbstractLuceneIndex.java
@@ -45,6 +45,7 @@
import com.google.gerrit.index.query.ResultSet;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
+import com.google.gerrit.server.index.options.AutoFlush;
import com.google.gerrit.server.logging.LoggingContextAwareExecutorService;
import com.google.gerrit.server.logging.LoggingContextAwareScheduledExecutorService;
import java.io.IOException;
@@ -105,6 +106,7 @@
private final ReferenceManager<IndexSearcher> searcherManager;
private final ControlledRealTimeReopenThread<IndexSearcher> reopenThread;
private final Set<NrtFuture> notDoneNrtFutures;
+ private final AutoFlush autoFlush;
private ScheduledExecutorService autoCommitExecutor;
AbstractLuceneIndex(
@@ -115,13 +117,15 @@
ImmutableSet<String> skipFields,
String subIndex,
GerritIndexWriterConfig writerConfig,
- SearcherFactory searcherFactory)
+ SearcherFactory searcherFactory,
+ AutoFlush autoFlush)
throws IOException {
this.schema = schema;
this.sitePaths = sitePaths;
this.dir = dir;
this.name = name;
this.skipFields = skipFields;
+ this.autoFlush = autoFlush;
String index = Joiner.on('_').skipNulls().join(name, subIndex);
long commitPeriod = writerConfig.getCommitWithinMs();
@@ -215,7 +219,9 @@
}
});
- reopenThread.start();
+ if (autoFlush.equals(AutoFlush.ENABLED)) {
+ reopenThread.start();
+ }
}
@Override
@@ -484,6 +490,9 @@
}
private boolean isGenAvailableNowForCurrentSearcher() {
+ if (autoFlush.equals(AutoFlush.DISABLED)) {
+ return true;
+ }
try {
return reopenThread.waitForGeneration(gen, 0);
} catch (InterruptedException e) {
diff --git a/java/com/google/gerrit/lucene/ChangeSubIndex.java b/java/com/google/gerrit/lucene/ChangeSubIndex.java
index e51a91a7..360331a 100644
--- a/java/com/google/gerrit/lucene/ChangeSubIndex.java
+++ b/java/com/google/gerrit/lucene/ChangeSubIndex.java
@@ -33,6 +33,7 @@
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.change.ChangeField;
import com.google.gerrit.server.index.change.ChangeIndex;
+import com.google.gerrit.server.index.options.AutoFlush;
import com.google.gerrit.server.query.change.ChangeData;
import java.io.IOException;
import java.nio.file.Path;
@@ -51,7 +52,8 @@
Path path,
ImmutableSet<String> skipFields,
GerritIndexWriterConfig writerConfig,
- SearcherFactory searcherFactory)
+ SearcherFactory searcherFactory,
+ AutoFlush autoFlush)
throws IOException {
this(
schema,
@@ -60,7 +62,8 @@
path.getFileName().toString(),
skipFields,
writerConfig,
- searcherFactory);
+ searcherFactory,
+ autoFlush);
}
ChangeSubIndex(
@@ -70,9 +73,24 @@
String subIndex,
ImmutableSet<String> skipFields,
GerritIndexWriterConfig writerConfig,
- SearcherFactory searcherFactory)
+ SearcherFactory searcherFactory,
+ AutoFlush autoFlush)
throws IOException {
- super(schema, sitePaths, dir, NAME, skipFields, subIndex, writerConfig, searcherFactory);
+ super(
+ schema,
+ sitePaths,
+ dir,
+ NAME,
+ skipFields,
+ subIndex,
+ writerConfig,
+ searcherFactory,
+ autoFlush);
+ }
+
+ @Override
+ public void insert(ChangeData obj) {
+ throw new UnsupportedOperationException("don't use ChangeSubIndex directly");
}
@Override
diff --git a/java/com/google/gerrit/lucene/LuceneAccountIndex.java b/java/com/google/gerrit/lucene/LuceneAccountIndex.java
index 242cffd..c4a5240 100644
--- a/java/com/google/gerrit/lucene/LuceneAccountIndex.java
+++ b/java/com/google/gerrit/lucene/LuceneAccountIndex.java
@@ -36,6 +36,7 @@
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.account.AccountIndex;
+import com.google.gerrit.server.index.options.AutoFlush;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
@@ -94,7 +95,8 @@
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Provider<AccountCache> accountCache,
- @Assisted Schema<AccountState> schema)
+ @Assisted Schema<AccountState> schema,
+ AutoFlush autoFlush)
throws IOException {
super(
schema,
@@ -104,7 +106,8 @@
ImmutableSet.of(),
null,
new GerritIndexWriterConfig(cfg, ACCOUNTS),
- new SearcherFactory());
+ new SearcherFactory(),
+ autoFlush);
this.accountCache = accountCache;
indexWriterConfig = new GerritIndexWriterConfig(cfg, ACCOUNTS);
@@ -141,6 +144,15 @@
}
@Override
+ public void insert(AccountState as) {
+ try {
+ insert(toDocument(as)).get();
+ } catch (ExecutionException | InterruptedException e) {
+ throw new StorageException(e);
+ }
+ }
+
+ @Override
public void delete(Account.Id key) {
try {
delete(idTerm(getSchema().useLegacyNumericFields(), key)).get();
diff --git a/java/com/google/gerrit/lucene/LuceneChangeIndex.java b/java/com/google/gerrit/lucene/LuceneChangeIndex.java
index bf1a166..c162c77 100644
--- a/java/com/google/gerrit/lucene/LuceneChangeIndex.java
+++ b/java/com/google/gerrit/lucene/LuceneChangeIndex.java
@@ -64,6 +64,7 @@
import com.google.gerrit.server.index.change.ChangeField;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.change.ChangeIndexRewriter;
+import com.google.gerrit.server.index.options.AutoFlush;
import com.google.gerrit.server.project.SubmitRuleOptions;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.ChangeDataSource;
@@ -179,7 +180,8 @@
SitePaths sitePaths,
@IndexExecutor(INTERACTIVE) ListeningExecutorService executor,
ChangeData.Factory changeDataFactory,
- @Assisted Schema<ChangeData> schema)
+ @Assisted Schema<ChangeData> schema,
+ AutoFlush autoFlush)
throws IOException {
this.executor = executor;
this.changeDataFactory = changeDataFactory;
@@ -204,7 +206,8 @@
"ramOpen",
skipFields,
openConfig,
- searcherFactory);
+ searcherFactory,
+ autoFlush);
closedIndex =
new ChangeSubIndex(
schema,
@@ -213,7 +216,8 @@
"ramClosed",
skipFields,
closedConfig,
- searcherFactory);
+ searcherFactory,
+ autoFlush);
} else {
Path dir = LuceneVersionManager.getDir(sitePaths, CHANGES, schema);
openIndex =
@@ -223,7 +227,8 @@
dir.resolve(CHANGES_OPEN),
skipFields,
openConfig,
- searcherFactory);
+ searcherFactory,
+ autoFlush);
closedIndex =
new ChangeSubIndex(
schema,
@@ -231,7 +236,8 @@
dir.resolve(CHANGES_CLOSED),
skipFields,
closedConfig,
- searcherFactory);
+ searcherFactory,
+ autoFlush);
}
idField = this.schema.useLegacyNumericFields() ? LEGACY_ID : LEGACY_ID_STR;
@@ -281,6 +287,22 @@
}
@Override
+ public void insert(ChangeData cd) {
+ // toDocument is essentially static and doesn't depend on the specific
+ // sub-index, so just pick one.
+ Document doc = openIndex.toDocument(cd);
+ try {
+ if (cd.change().isNew()) {
+ openIndex.insert(doc).get();
+ } else {
+ closedIndex.insert(doc).get();
+ }
+ } catch (ExecutionException | InterruptedException e) {
+ throw new StorageException(e);
+ }
+ }
+
+ @Override
public void delete(Change.Id changeId) {
Term id = LuceneChangeIndex.idTerm(idTerm, idField, changeId);
try {
diff --git a/java/com/google/gerrit/lucene/LuceneGroupIndex.java b/java/com/google/gerrit/lucene/LuceneGroupIndex.java
index 3d1d471..9f8b62e 100644
--- a/java/com/google/gerrit/lucene/LuceneGroupIndex.java
+++ b/java/com/google/gerrit/lucene/LuceneGroupIndex.java
@@ -33,6 +33,7 @@
import com.google.gerrit.server.group.InternalGroup;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.group.GroupIndex;
+import com.google.gerrit.server.index.options.AutoFlush;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
@@ -84,7 +85,8 @@
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Provider<GroupCache> groupCache,
- @Assisted Schema<InternalGroup> schema)
+ @Assisted Schema<InternalGroup> schema,
+ AutoFlush autoFlush)
throws IOException {
super(
schema,
@@ -94,7 +96,8 @@
ImmutableSet.of(),
null,
new GerritIndexWriterConfig(cfg, GROUPS),
- new SearcherFactory());
+ new SearcherFactory(),
+ autoFlush);
this.groupCache = groupCache;
indexWriterConfig = new GerritIndexWriterConfig(cfg, GROUPS);
@@ -122,6 +125,15 @@
}
@Override
+ public void insert(InternalGroup group) {
+ try {
+ insert(toDocument(group)).get();
+ } catch (ExecutionException | InterruptedException e) {
+ throw new StorageException(e);
+ }
+ }
+
+ @Override
public void delete(AccountGroup.UUID key) {
try {
delete(idTerm(key)).get();
diff --git a/java/com/google/gerrit/lucene/LuceneIndexModule.java b/java/com/google/gerrit/lucene/LuceneIndexModule.java
index 302a2da..73256b1 100644
--- a/java/com/google/gerrit/lucene/LuceneIndexModule.java
+++ b/java/com/google/gerrit/lucene/LuceneIndexModule.java
@@ -14,6 +14,7 @@
package com.google.gerrit.lucene;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.gerrit.index.IndexConfig;
import com.google.gerrit.index.project.ProjectIndex;
@@ -23,30 +24,48 @@
import com.google.gerrit.server.index.account.AccountIndex;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.group.GroupIndex;
+import com.google.gerrit.server.index.options.AutoFlush;
import java.util.Map;
import org.apache.lucene.search.BooleanQuery;
import org.eclipse.jgit.lib.Config;
public class LuceneIndexModule extends AbstractIndexModule {
- public static LuceneIndexModule singleVersionAllLatest(int threads, boolean slave) {
- return new LuceneIndexModule(ImmutableMap.of(), threads, slave);
+ private final AutoFlush autoFlush;
+
+ public static LuceneIndexModule singleVersionAllLatest(
+ int threads, boolean slave, AutoFlush autoFlush) {
+ return new LuceneIndexModule(ImmutableMap.of(), threads, slave, autoFlush);
+ }
+
+ @VisibleForTesting
+ public static LuceneIndexModule singleVersionWithExplicitVersions(
+ Map<String, Integer> versions, int threads, boolean slave) {
+ return new LuceneIndexModule(versions, threads, slave, AutoFlush.ENABLED);
}
public static LuceneIndexModule singleVersionWithExplicitVersions(
- Map<String, Integer> versions, int threads, boolean slave) {
- return new LuceneIndexModule(versions, threads, slave);
+ Map<String, Integer> versions, int threads, boolean slave, AutoFlush autoFlush) {
+ return new LuceneIndexModule(versions, threads, slave, autoFlush);
}
- public static LuceneIndexModule latestVersion(boolean slave) {
- return new LuceneIndexModule(null, 0, slave);
+ public static LuceneIndexModule latestVersion(boolean slave, AutoFlush autoFlush) {
+ return new LuceneIndexModule(null, 0, slave, autoFlush);
}
static boolean isInMemoryTest(Config cfg) {
return cfg.getBoolean("index", "lucene", "testInmemory", false);
}
- private LuceneIndexModule(Map<String, Integer> singleVersions, int threads, boolean slave) {
+ private LuceneIndexModule(
+ Map<String, Integer> singleVersions, int threads, boolean slave, AutoFlush autoFlush) {
super(singleVersions, threads, slave);
+ this.autoFlush = autoFlush;
+ }
+
+ @Override
+ protected void configure() {
+ super.configure();
+ bind(AutoFlush.class).toInstance(autoFlush);
}
@Override
diff --git a/java/com/google/gerrit/lucene/LuceneProjectIndex.java b/java/com/google/gerrit/lucene/LuceneProjectIndex.java
index 2a418ca..11707be 100644
--- a/java/com/google/gerrit/lucene/LuceneProjectIndex.java
+++ b/java/com/google/gerrit/lucene/LuceneProjectIndex.java
@@ -32,6 +32,7 @@
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
+import com.google.gerrit.server.index.options.AutoFlush;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectState;
import com.google.inject.Inject;
@@ -84,7 +85,8 @@
@GerritServerConfig Config cfg,
SitePaths sitePaths,
Provider<ProjectCache> projectCache,
- @Assisted Schema<ProjectData> schema)
+ @Assisted Schema<ProjectData> schema,
+ AutoFlush autoFlush)
throws IOException {
super(
schema,
@@ -94,7 +96,8 @@
ImmutableSet.of(),
null,
new GerritIndexWriterConfig(cfg, PROJECTS),
- new SearcherFactory());
+ new SearcherFactory(),
+ autoFlush);
this.projectCache = projectCache;
indexWriterConfig = new GerritIndexWriterConfig(cfg, PROJECTS);
@@ -122,6 +125,15 @@
}
@Override
+ public void insert(ProjectData projectState) {
+ try {
+ insert(toDocument(projectState)).get();
+ } catch (ExecutionException | InterruptedException e) {
+ throw new StorageException(e);
+ }
+ }
+
+ @Override
public void delete(Project.NameKey nameKey) {
try {
delete(idTerm(nameKey)).get();
diff --git a/java/com/google/gerrit/pgm/Daemon.java b/java/com/google/gerrit/pgm/Daemon.java
index 0eaf75d..449a868 100644
--- a/java/com/google/gerrit/pgm/Daemon.java
+++ b/java/com/google/gerrit/pgm/Daemon.java
@@ -87,6 +87,7 @@
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.OnlineUpgrader;
import com.google.gerrit.server.index.VersionManager;
+import com.google.gerrit.server.index.options.AutoFlush;
import com.google.gerrit.server.mail.SignedTokenEmailTokenVerifier;
import com.google.gerrit.server.mail.receive.MailReceiver;
import com.google.gerrit.server.mail.send.SmtpEmailSender;
@@ -513,7 +514,7 @@
return luceneModule;
}
if (indexType.isLucene()) {
- return LuceneIndexModule.latestVersion(replica);
+ return LuceneIndexModule.latestVersion(replica, AutoFlush.ENABLED);
}
if (indexType.isElasticsearch()) {
return ElasticIndexModule.latestVersion(replica);
diff --git a/java/com/google/gerrit/pgm/Init.java b/java/com/google/gerrit/pgm/Init.java
index 4e62a0f..19d19d4 100644
--- a/java/com/google/gerrit/pgm/Init.java
+++ b/java/com/google/gerrit/pgm/Init.java
@@ -18,6 +18,7 @@
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.gerrit.common.IoUtil;
import com.google.gerrit.common.PageLinks;
@@ -157,11 +158,13 @@
modules.add(new GerritServerConfigModule());
Guice.createInjector(modules).injectMembers(this);
if (!ReplicaUtil.isReplica(run.flags.cfg)) {
+ List<String> indicesToReindex = new ArrayList<>();
for (SchemaDefinitions<?> schemaDef : schemaDefs) {
if (!indexStatus.exists(schemaDef.getName())) {
- reindex(schemaDef);
+ indicesToReindex.add(schemaDef.getName());
}
}
+ reindex(indicesToReindex, run.flags.isNew);
}
start(run);
}
@@ -274,17 +277,23 @@
}
}
- private void reindex(SchemaDefinitions<?> schemaDef) throws Exception {
+ private void reindex(List<String> indices, boolean isNewSite) throws Exception {
+ if (indices.isEmpty()) {
+ return;
+ }
List<String> reindexArgs =
- ImmutableList.of(
- "--site-path",
- getSitePath().toString(),
- "--threads",
- Integer.toString(1),
- "--index",
- schemaDef.getName());
+ Lists.newArrayList(
+ "--site-path", getSitePath().toString(), "--threads", Integer.toString(1));
+ for (String index : indices) {
+ reindexArgs.add("--index");
+ reindexArgs.add(index);
+ }
+ if (isNewSite) {
+ reindexArgs.add("--disable-cache-stats");
+ }
+
getConsoleUI()
- .message(String.format("Init complete, reindexing %s with:", schemaDef.getName()));
+ .message(String.format("Init complete, reindexing %s with:", String.join(",", indices)));
getConsoleUI().message(" reindex " + reindexArgs.stream().collect(joining(" ")));
Reindex reindexPgm = new Reindex();
reindexPgm.main(reindexArgs.stream().toArray(String[]::new));
diff --git a/java/com/google/gerrit/pgm/Reindex.java b/java/com/google/gerrit/pgm/Reindex.java
index 966801f..0eb1b67 100644
--- a/java/com/google/gerrit/pgm/Reindex.java
+++ b/java/com/google/gerrit/pgm/Reindex.java
@@ -17,10 +17,12 @@
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toSet;
+import com.google.common.cache.Cache;
import com.google.common.collect.Sets;
import com.google.gerrit.common.Die;
import com.google.gerrit.elasticsearch.ElasticIndexModule;
import com.google.gerrit.extensions.config.FactoryModule;
+import com.google.gerrit.extensions.registration.DynamicMap;
import com.google.gerrit.index.Index;
import com.google.gerrit.index.IndexDefinition;
import com.google.gerrit.index.IndexType;
@@ -29,16 +31,24 @@
import com.google.gerrit.lucene.LuceneIndexModule;
import com.google.gerrit.pgm.util.BatchProgramModule;
import com.google.gerrit.pgm.util.SiteProgram;
+import com.google.gerrit.server.cache.CacheDisplay;
+import com.google.gerrit.server.cache.CacheInfo;
import com.google.gerrit.server.change.ChangeResource;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
+import com.google.gerrit.server.index.options.AutoFlush;
+import com.google.gerrit.server.index.options.IsFirstInsertForEntry;
import com.google.gerrit.server.plugins.PluginGuiceEnvironment;
import com.google.gerrit.server.util.ReplicaUtil;
+import com.google.inject.AbstractModule;
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.Module;
+import com.google.inject.multibindings.OptionalBinder;
+import java.io.StringWriter;
+import java.io.Writer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -47,13 +57,17 @@
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.util.io.NullOutputStream;
import org.kohsuke.args4j.Option;
public class Reindex extends SiteProgram {
- @Option(name = "--threads", usage = "Number of threads to use for indexing")
- private int threads = Runtime.getRuntime().availableProcessors();
+ @Option(
+ name = "--threads",
+ usage = "Number of threads to use for indexing. Default is index.batchThreads from config.")
+ private int threads = 0;
@Option(
name = "--changes-schema-version",
@@ -69,12 +83,20 @@
@Option(name = "--index", usage = "Only reindex specified indices")
private List<String> indices = new ArrayList<>();
+ @Option(
+ name = "--disable-cache-stats",
+ usage =
+ "Disables printing the cache statistics."
+ + "Defaults to true when reindex is run from init on a new site, false otherwise")
+ private boolean disableCacheStats;
+
private Injector dbInjector;
private Injector sysInjector;
private Injector cfgInjector;
private Config globalConfig;
@Inject private Collection<IndexDefinition<?, ?, ?>> indexDefs;
+ @Inject private DynamicMap<Cache<?, ?>> cacheMap;
@Override
public int run() throws Exception {
@@ -98,6 +120,9 @@
try {
boolean ok = list ? list() : reindex();
+ if (!disableCacheStats) {
+ printCacheStats();
+ }
return ok ? 0 : 1;
} catch (Exception e) {
throw die(e.getMessage(), e);
@@ -150,7 +175,9 @@
Module indexModule;
IndexType indexType = IndexModule.getIndexType(dbInjector);
if (indexType.isLucene()) {
- indexModule = LuceneIndexModule.singleVersionWithExplicitVersions(versions, threads, replica);
+ indexModule =
+ LuceneIndexModule.singleVersionWithExplicitVersions(
+ versions, threads, replica, AutoFlush.DISABLED);
} else if (indexType.isElasticsearch()) {
indexModule =
ElasticIndexModule.singleVersionWithExplicitVersions(versions, threads, replica);
@@ -158,6 +185,16 @@
throw new IllegalStateException("unsupported index.type = " + indexType);
}
modules.add(indexModule);
+ modules.add(
+ new AbstractModule() {
+ @Override
+ protected void configure() {
+ super.configure();
+ OptionalBinder.newOptionalBinder(binder(), IsFirstInsertForEntry.class)
+ .setBinding()
+ .toInstance(IsFirstInsertForEntry.YES);
+ }
+ });
modules.add(new BatchProgramModule());
modules.add(
new FactoryModule() {
@@ -205,6 +242,22 @@
System.out.format(
"Index %s in version %d is %sready\n",
def.getName(), index.getSchema().getVersion(), result.success() ? "" : "NOT ");
+
return result.success();
}
+
+ private void printCacheStats() {
+ try (Writer sw = new StringWriter()) {
+ sw.write("Cache Statistics at the end of reindexing\n");
+ new CacheDisplay(
+ sw,
+ StreamSupport.stream(cacheMap.spliterator(), false)
+ .map(e -> new CacheInfo(e.getExportName(), e.get()))
+ .collect(Collectors.toList()))
+ .displayCaches();
+ System.out.print(sw.toString());
+ } catch (Exception e) {
+ System.out.format("Error displaying the cache statistics\n" + e.getMessage());
+ }
+ }
}
diff --git a/java/com/google/gerrit/pgm/http/jetty/HttpLog.java b/java/com/google/gerrit/pgm/http/jetty/HttpLog.java
index 4e4c93b..27a53c2 100644
--- a/java/com/google/gerrit/pgm/http/jetty/HttpLog.java
+++ b/java/com/google/gerrit/pgm/http/jetty/HttpLog.java
@@ -14,6 +14,8 @@
package com.google.gerrit.pgm.http.jetty;
+import static com.google.gerrit.httpd.GitOverHttpServlet.GIT_COMMAND_STATUS_HEADER;
+
import com.google.common.base.Strings;
import com.google.gerrit.httpd.GetUserFilter;
import com.google.gerrit.httpd.restapi.LogRedactUtil;
@@ -51,6 +53,7 @@
protected static final String P_LATENCY = "Latency";
protected static final String P_REFERER = "Referer";
protected static final String P_USER_AGENT = "User-Agent";
+ protected static final String P_COMMAND_STATUS = "Command-Status";
private final AsyncAppender async;
@@ -113,6 +116,7 @@
set(event, P_LATENCY, System.currentTimeMillis() - req.getTimeStamp());
set(event, P_REFERER, req.getHeader("Referer"));
set(event, P_USER_AGENT, req.getHeader("User-Agent"));
+ set(event, P_COMMAND_STATUS, rsp.getHeader(GIT_COMMAND_STATUS_HEADER));
async.append(event);
}
diff --git a/java/com/google/gerrit/pgm/http/jetty/HttpLogJsonLayout.java b/java/com/google/gerrit/pgm/http/jetty/HttpLogJsonLayout.java
index 95a5b07..807b311 100644
--- a/java/com/google/gerrit/pgm/http/jetty/HttpLogJsonLayout.java
+++ b/java/com/google/gerrit/pgm/http/jetty/HttpLogJsonLayout.java
@@ -14,6 +14,7 @@
package com.google.gerrit.pgm.http.jetty;
+import static com.google.gerrit.pgm.http.jetty.HttpLog.P_COMMAND_STATUS;
import static com.google.gerrit.pgm.http.jetty.HttpLog.P_CONTENT_LENGTH;
import static com.google.gerrit.pgm.http.jetty.HttpLog.P_HOST;
import static com.google.gerrit.pgm.http.jetty.HttpLog.P_LATENCY;
@@ -50,6 +51,7 @@
public String latency;
public String referer;
public String userAgent;
+ public String commandStatus;
public HttpJsonLogEntry(LoggingEvent event) {
this.host = getMdcString(event, P_HOST);
@@ -64,6 +66,7 @@
this.latency = getMdcString(event, P_LATENCY);
this.referer = getMdcString(event, P_REFERER);
this.userAgent = getMdcString(event, P_USER_AGENT);
+ this.commandStatus = getMdcString(event, P_COMMAND_STATUS);
}
}
}
diff --git a/java/com/google/gerrit/pgm/http/jetty/HttpLogLayout.java b/java/com/google/gerrit/pgm/http/jetty/HttpLogLayout.java
index 268f59f..b83fcc5 100644
--- a/java/com/google/gerrit/pgm/http/jetty/HttpLogLayout.java
+++ b/java/com/google/gerrit/pgm/http/jetty/HttpLogLayout.java
@@ -71,6 +71,9 @@
buf.append(' ');
dq_opt(buf, event, HttpLog.P_USER_AGENT);
+ buf.append(' ');
+ dq_opt(buf, event, HttpLog.P_COMMAND_STATUS);
+
buf.append('\n');
return buf.toString();
}
diff --git a/java/com/google/gerrit/pgm/init/index/lucene/LuceneIndexModuleOnInit.java b/java/com/google/gerrit/pgm/init/index/lucene/LuceneIndexModuleOnInit.java
index 12a44dc..078e648 100644
--- a/java/com/google/gerrit/pgm/init/index/lucene/LuceneIndexModuleOnInit.java
+++ b/java/com/google/gerrit/pgm/init/index/lucene/LuceneIndexModuleOnInit.java
@@ -19,6 +19,7 @@
import com.google.gerrit.pgm.init.index.IndexModuleOnInit;
import com.google.gerrit.server.index.account.AccountIndex;
import com.google.gerrit.server.index.group.GroupIndex;
+import com.google.gerrit.server.index.options.AutoFlush;
import com.google.inject.AbstractModule;
import com.google.inject.assistedinject.FactoryModuleBuilder;
@@ -36,5 +37,7 @@
.build(GroupIndex.Factory.class));
install(new IndexModuleOnInit());
+
+ bind(AutoFlush.class).toInstance(AutoFlush.DISABLED);
}
}
diff --git a/java/com/google/gerrit/pgm/util/BatchProgramModule.java b/java/com/google/gerrit/pgm/util/BatchProgramModule.java
index a831b8e..dd76859 100644
--- a/java/com/google/gerrit/pgm/util/BatchProgramModule.java
+++ b/java/com/google/gerrit/pgm/util/BatchProgramModule.java
@@ -112,7 +112,7 @@
// We're just running through each change
// once, so don't worry about cache removal.
bind(new TypeLiteral<DynamicSet<CacheRemovalListener>>() {}).toInstance(DynamicSet.emptySet());
- bind(new TypeLiteral<DynamicMap<Cache<?, ?>>>() {}).toInstance(DynamicMap.emptyMap());
+ DynamicMap.mapOf(binder(), new TypeLiteral<Cache<?, ?>>() {});
bind(new TypeLiteral<List<CommentLinkInfo>>() {})
.toProvider(CommentLinkProvider.class)
.in(SINGLETON);
diff --git a/java/com/google/gerrit/server/CreateGroupPermissionSyncer.java b/java/com/google/gerrit/server/CreateGroupPermissionSyncer.java
index 17313e4..9b2ba84 100644
--- a/java/com/google/gerrit/server/CreateGroupPermissionSyncer.java
+++ b/java/com/google/gerrit/server/CreateGroupPermissionSyncer.java
@@ -120,7 +120,7 @@
}
config.commit(md);
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
}
}
diff --git a/java/com/google/gerrit/server/DynamicOptions.java b/java/com/google/gerrit/server/DynamicOptions.java
index 41dc082..ebf4ec6 100644
--- a/java/com/google/gerrit/server/DynamicOptions.java
+++ b/java/com/google/gerrit/server/DynamicOptions.java
@@ -213,7 +213,7 @@
Class<?> beanClass =
(bean instanceof BeanReceiver)
? ((BeanReceiver) bean).getExportedBeanReceiver()
- : getClass();
+ : bean.getClass();
for (String plugin : dynamicBeans.plugins()) {
Provider<DynamicBean> provider =
dynamicBeans.byPlugin(plugin).get(beanClass.getCanonicalName());
diff --git a/java/com/google/gerrit/server/cache/CacheDisplay.java b/java/com/google/gerrit/server/cache/CacheDisplay.java
new file mode 100644
index 0000000..60f5186
--- /dev/null
+++ b/java/com/google/gerrit/server/cache/CacheDisplay.java
@@ -0,0 +1,129 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.cache;
+
+import com.google.common.base.Strings;
+import java.io.IOException;
+import java.io.Writer;
+import java.util.Collection;
+
+public class CacheDisplay {
+
+ private final Writer stdout;
+ private final int nw;
+ private final Collection<CacheInfo> caches;
+
+ public CacheDisplay(Writer stdout, int nw, Collection<CacheInfo> caches) {
+ this.stdout = stdout;
+ this.nw = nw;
+ this.caches = caches;
+ }
+
+ public CacheDisplay(Writer stdout, Collection<CacheInfo> caches) {
+ this(stdout, 30, caches);
+ }
+
+ public void displayCaches() throws IOException {
+ stdout.write(
+ String.format( //
+ "%1s %-" + nw + "s|%-21s| %-5s |%-9s|\n" //
+ ,
+ "" //
+ ,
+ "Name" //
+ ,
+ "Entries" //
+ ,
+ "AvgGet" //
+ ,
+ "Hit Ratio" //
+ ));
+ stdout.write(
+ String.format( //
+ "%1s %-" + nw + "s|%6s %6s %7s| %-5s |%-4s %-4s|\n" //
+ ,
+ "" //
+ ,
+ "" //
+ ,
+ "Mem" //
+ ,
+ "Disk" //
+ ,
+ "Space" //
+ ,
+ "" //
+ ,
+ "Mem" //
+ ,
+ "Disk" //
+ ));
+ stdout.write("--");
+ for (int i = 0; i < nw; i++) {
+ stdout.write('-');
+ }
+ stdout.write("+---------------------+---------+---------+\n");
+ printMemoryCoreCaches(caches);
+ printMemoryPluginCaches(caches);
+ printDiskCaches(caches);
+ stdout.write('\n');
+ }
+
+ private void printMemoryCoreCaches(Collection<CacheInfo> caches) throws IOException {
+ for (CacheInfo cache : caches) {
+ if (!cache.name.contains("-") && CacheInfo.CacheType.MEM.equals(cache.type)) {
+ printCache(cache);
+ }
+ }
+ }
+
+ private void printMemoryPluginCaches(Collection<CacheInfo> caches) throws IOException {
+ for (CacheInfo cache : caches) {
+ if (cache.name.contains("-") && CacheInfo.CacheType.MEM.equals(cache.type)) {
+ printCache(cache);
+ }
+ }
+ }
+
+ private void printDiskCaches(Collection<CacheInfo> caches) throws IOException {
+ for (CacheInfo cache : caches) {
+ if (CacheInfo.CacheType.DISK.equals(cache.type)) {
+ printCache(cache);
+ }
+ }
+ }
+
+ private void printCache(CacheInfo cache) throws IOException {
+ stdout.write(
+ String.format(
+ "%1s %-" + nw + "s|%6s %6s %7s| %7s |%4s %4s|\n",
+ CacheInfo.CacheType.DISK.equals(cache.type) ? "D" : "",
+ cache.name,
+ nullToEmpty(cache.entries.mem),
+ nullToEmpty(cache.entries.disk),
+ Strings.nullToEmpty(cache.entries.space),
+ Strings.nullToEmpty(cache.averageGet),
+ formatAsPercent(cache.hitRatio.mem),
+ formatAsPercent(cache.hitRatio.disk)));
+ }
+
+ private static String nullToEmpty(Long l) {
+ return l != null ? String.valueOf(l) : "";
+ }
+
+ private static String formatAsPercent(Integer i) {
+ return i != null ? i + "%" : "";
+ }
+}
diff --git a/java/com/google/gerrit/server/cache/CacheInfo.java b/java/com/google/gerrit/server/cache/CacheInfo.java
new file mode 100644
index 0000000..d6eb065
--- /dev/null
+++ b/java/com/google/gerrit/server/cache/CacheInfo.java
@@ -0,0 +1,133 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.cache;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheStats;
+
+public class CacheInfo {
+
+ public String name;
+ public CacheType type;
+ public EntriesInfo entries;
+ public String averageGet;
+ public HitRatioInfo hitRatio;
+
+ public CacheInfo(Cache<?, ?> cache) {
+ this(null, cache);
+ }
+
+ public CacheInfo(String name, Cache<?, ?> cache) {
+ this.name = name;
+
+ CacheStats stat = cache.stats();
+
+ entries = new EntriesInfo();
+ entries.setMem(cache.size());
+
+ averageGet = duration(stat.averageLoadPenalty());
+
+ hitRatio = new HitRatioInfo();
+ hitRatio.setMem(stat.hitCount(), stat.requestCount());
+
+ if (cache instanceof PersistentCache) {
+ type = CacheType.DISK;
+ PersistentCache.DiskStats diskStats = ((PersistentCache) cache).diskStats();
+ entries.setDisk(diskStats.size());
+ entries.setSpace(diskStats.space());
+ hitRatio.setDisk(diskStats.hitCount(), diskStats.requestCount());
+ } else {
+ type = CacheType.MEM;
+ }
+ }
+
+ private static String duration(double ns) {
+ if (ns < 0.5) {
+ return null;
+ }
+ String suffix = "ns";
+ if (ns >= 1000.0) {
+ ns /= 1000.0;
+ suffix = "us";
+ }
+ if (ns >= 1000.0) {
+ ns /= 1000.0;
+ suffix = "ms";
+ }
+ if (ns >= 1000.0) {
+ ns /= 1000.0;
+ suffix = "s";
+ }
+ return String.format("%4.1f%s", ns, suffix).trim();
+ }
+
+ public static class EntriesInfo {
+ public Long mem;
+ public Long disk;
+ public String space;
+
+ public void setMem(long mem) {
+ this.mem = mem != 0 ? mem : null;
+ }
+
+ public void setDisk(long disk) {
+ this.disk = disk != 0 ? disk : null;
+ }
+
+ public void setSpace(double value) {
+ space = bytes(value);
+ }
+
+ private static String bytes(double value) {
+ value /= 1024;
+ String suffix = "k";
+
+ if (value > 1024) {
+ value /= 1024;
+ suffix = "m";
+ }
+ if (value > 1024) {
+ value /= 1024;
+ suffix = "g";
+ }
+ return String.format("%1$6.2f%2$s", value, suffix).trim();
+ }
+ }
+
+ public static class HitRatioInfo {
+ public Integer mem;
+ public Integer disk;
+
+ public void setMem(long value, long total) {
+ mem = percent(value, total);
+ }
+
+ public void setDisk(long value, long total) {
+ disk = percent(value, total);
+ }
+
+ private static Integer percent(long value, long total) {
+ if (total <= 0) {
+ return null;
+ }
+ return (int) ((100 * value) / total);
+ }
+ }
+
+ public enum CacheType {
+ MEM,
+ DISK
+ }
+}
diff --git a/java/com/google/gerrit/server/events/EventGsonProvider.java b/java/com/google/gerrit/server/events/EventGsonProvider.java
index 688507b..ab51518 100644
--- a/java/com/google/gerrit/server/events/EventGsonProvider.java
+++ b/java/com/google/gerrit/server/events/EventGsonProvider.java
@@ -30,7 +30,7 @@
.registerTypeAdapter(Supplier.class, new SupplierSerializer())
.registerTypeAdapter(Supplier.class, new SupplierDeserializer())
.registerTypeAdapter(Change.Key.class, new ChangeKeyAdapter())
- .registerTypeAdapter(Project.NameKey.class, new ProjectNameKeyAdapter())
+ .registerTypeHierarchyAdapter(Project.NameKey.class, new ProjectNameKeyAdapter())
.create();
}
}
diff --git a/java/com/google/gerrit/server/git/GitRepositoryManagerModule.java b/java/com/google/gerrit/server/git/GitRepositoryManagerModule.java
index 354b69f..e4137b0 100644
--- a/java/com/google/gerrit/server/git/GitRepositoryManagerModule.java
+++ b/java/com/google/gerrit/server/git/GitRepositoryManagerModule.java
@@ -15,6 +15,7 @@
package com.google.gerrit.server.git;
import com.google.gerrit.lifecycle.LifecycleModule;
+import com.google.gerrit.server.ModuleImpl;
import com.google.gerrit.server.config.RepositoryConfig;
import com.google.inject.Inject;
@@ -22,7 +23,9 @@
* Module to install {@link MultiBaseLocalDiskRepositoryManager} rather than {@link
* LocalDiskRepositoryManager} if needed.
*/
+@ModuleImpl(name = GitRepositoryManagerModule.MANAGER_MODULE)
public class GitRepositoryManagerModule extends LifecycleModule {
+ public static final String MANAGER_MODULE = "git-manager";
private final RepositoryConfig repoConfig;
diff --git a/java/com/google/gerrit/server/git/PermissionAwareReadOnlyRefDatabase.java b/java/com/google/gerrit/server/git/PermissionAwareReadOnlyRefDatabase.java
index b7dc2b3..7950dc6 100644
--- a/java/com/google/gerrit/server/git/PermissionAwareReadOnlyRefDatabase.java
+++ b/java/com/google/gerrit/server/git/PermissionAwareReadOnlyRefDatabase.java
@@ -14,7 +14,6 @@
package com.google.gerrit.server.git;
-import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toMap;
import com.google.common.base.Preconditions;
@@ -95,20 +94,11 @@
return Iterables.getOnlyElement(result);
}
+ // WARNING: This method is deprecated in JGit's RefDatabase and it will be removed on master.
+ // Do not add any logic here but rather enrich the getRefsByPrefix method below.
@Override
public Map<String, Ref> getRefs(String prefix) throws IOException {
- List<Ref> refs = getDelegate().getRefDatabase().getRefsByPrefix(prefix);
- if (refs.isEmpty()) {
- return Collections.emptyMap();
- }
-
- Collection<Ref> result;
- try {
- result = forProject.filter(refs, getDelegate(), RefFilterOptions.defaults());
- } catch (PermissionBackendException e) {
- throw new IOException("", e);
- }
- return buildPrefixRefMap(prefix, result);
+ return buildPrefixRefMap(prefix, getRefsByPrefix(prefix));
}
private Map<String, Ref> buildPrefixRefMap(String prefix, Collection<Ref> refs) {
@@ -125,26 +115,18 @@
@Override
public List<Ref> getRefsByPrefix(String prefix) throws IOException {
- Map<String, Ref> coarseRefs;
- int lastSlash = prefix.lastIndexOf('/');
- if (lastSlash == -1) {
- coarseRefs = getRefs(ALL);
- } else {
- coarseRefs = getRefs(prefix.substring(0, lastSlash + 1));
+ List<Ref> refs = getDelegate().getRefDatabase().getRefsByPrefix(prefix);
+ if (refs.isEmpty()) {
+ return Collections.emptyList();
}
- List<Ref> result;
- if (lastSlash + 1 == prefix.length()) {
- result = coarseRefs.values().stream().collect(toList());
- } else {
- String p = prefix.substring(lastSlash + 1);
- result =
- coarseRefs.entrySet().stream()
- .filter(e -> e.getKey().startsWith(p))
- .map(e -> e.getValue())
- .collect(toList());
+ Collection<Ref> result;
+ try {
+ result = forProject.filter(refs, getDelegate(), RefFilterOptions.defaults());
+ } catch (PermissionBackendException e) {
+ throw new IOException("", e);
}
- return Collections.unmodifiableList(result);
+ return result.stream().collect(Collectors.toList());
}
@Override
diff --git a/java/com/google/gerrit/server/git/meta/VersionedMetaData.java b/java/com/google/gerrit/server/git/meta/VersionedMetaData.java
index feb038a..ea594dd 100644
--- a/java/com/google/gerrit/server/git/meta/VersionedMetaData.java
+++ b/java/com/google/gerrit/server/git/meta/VersionedMetaData.java
@@ -217,6 +217,27 @@
}
/**
+ * Update this metadata branch, recording a new commit on its reference. This method mutates its
+ * receiver.
+ *
+ * @param update helper information to define the update that will occur.
+ * @param objInserter Shared object inserter.
+ * @param objReader Shared object reader.
+ * @param revWalk Shared rev walk.
+ * @return the commit that was created
+ * @throws IOException if there is a storage problem and the update cannot be executed as
+ * requested or if it failed because of a concurrent update to the same reference
+ */
+ public RevCommit commit(
+ MetaDataUpdate update, ObjectInserter objInserter, ObjectReader objReader, RevWalk revWalk)
+ throws IOException {
+ try (BatchMetaDataUpdate batch = openUpdate(update, objInserter, objReader, revWalk)) {
+ batch.write(update.getCommitBuilder());
+ return batch.commit();
+ }
+ }
+
+ /**
* Creates a new commit and a new ref based on this commit. This method mutates its receiver.
*
* @param update helper information to define the update that will occur.
@@ -263,11 +284,39 @@
* @throws IOException if the update failed.
*/
public BatchMetaDataUpdate openUpdate(MetaDataUpdate update) throws IOException {
+ return openUpdate(update, null, null, null);
+ }
+
+ /**
+ * Open a batch of updates to the same metadata ref.
+ *
+ * <p>This allows making multiple commits to a single metadata ref, at the end of which is a
+ * single ref update. For batching together updates to multiple refs (each consisting of one or
+ * more commits against their respective refs), create the {@link MetaDataUpdate} with a {@link
+ * BatchRefUpdate}.
+ *
+ * <p>A ref update produced by this {@link BatchMetaDataUpdate} is only committed if there is no
+ * associated {@link BatchRefUpdate}. As a result, the configured ref updated event is not fired
+ * if there is an associated batch.
+ *
+ * <p>If object inserter, reader and revwalk are provided, then the updates are not flushed,
+ * allowing callers the flexibility to flush only once after several updates.
+ *
+ * @param update helper info about the update.
+ * @param objInserter Shared object inserter.
+ * @param objReader Shared object reader.
+ * @param revWalk Shared rev walk.
+ * @throws IOException if the update failed.
+ */
+ public BatchMetaDataUpdate openUpdate(
+ MetaDataUpdate update, ObjectInserter objInserter, ObjectReader objReader, RevWalk revWalk)
+ throws IOException {
final Repository db = update.getRepository();
- inserter = db.newObjectInserter();
- reader = inserter.newReader();
- final RevWalk rw = new RevWalk(reader);
+ inserter = objInserter == null ? db.newObjectInserter() : objInserter;
+ reader = objReader == null ? inserter.newReader() : objReader;
+ final RevWalk rw = revWalk == null ? new RevWalk(reader) : revWalk;
+
final RevTree tree = revision != null ? rw.parseTree(revision) : null;
newTree = readTree(tree);
return new BatchMetaDataUpdate() {
@@ -379,13 +428,16 @@
public void close() {
newTree = null;
- rw.close();
- if (inserter != null) {
+ if (revWalk == null) {
+ rw.close();
+ }
+
+ if (objInserter == null && inserter != null) {
inserter.close();
inserter = null;
}
- if (reader != null) {
+ if (objReader == null && reader != null) {
reader.close();
reader = null;
}
@@ -396,7 +448,9 @@
BatchRefUpdate bru = update.getBatch();
if (bru != null) {
bru.addCommand(new ReceiveCommand(oldId.toObjectId(), newId.toObjectId(), refName));
- inserter.flush();
+ if (objInserter == null) {
+ inserter.flush();
+ }
revision = rw.parseCommit(newId);
return revision;
}
diff --git a/java/com/google/gerrit/server/git/receive/ReceiveCommits.java b/java/com/google/gerrit/server/git/receive/ReceiveCommits.java
index 100bd5b..f0a9652 100644
--- a/java/com/google/gerrit/server/git/receive/ReceiveCommits.java
+++ b/java/com/google/gerrit/server/git/receive/ReceiveCommits.java
@@ -3080,7 +3080,7 @@
}
if (isConfig(cmd)) {
logger.atFine().log("Reloading project in cache");
- projectCache.evict(project);
+ projectCache.evictAndReindex(project);
ProjectState ps =
projectCache.get(project.getNameKey()).orElseThrow(illegalState(project.getNameKey()));
try {
diff --git a/java/com/google/gerrit/server/group/db/RenameGroupOp.java b/java/com/google/gerrit/server/group/db/RenameGroupOp.java
index 420dd33e..2f24233 100644
--- a/java/com/google/gerrit/server/group/db/RenameGroupOp.java
+++ b/java/com/google/gerrit/server/group/db/RenameGroupOp.java
@@ -121,7 +121,7 @@
//
GroupReference ref = config.getGroup(uuid);
if (ref == null || newName.equals(ref.getName())) {
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
return;
}
@@ -130,7 +130,7 @@
md.setMessage("Rename group " + oldName + " to " + newName + "\n");
try {
config.commit(md);
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
success = true;
} catch (IOException e) {
logger.atSevere().withCause(e).log(
diff --git a/java/com/google/gerrit/server/index/IndexModule.java b/java/com/google/gerrit/server/index/IndexModule.java
index 6db00f5..8b04c8d 100644
--- a/java/com/google/gerrit/server/index/IndexModule.java
+++ b/java/com/google/gerrit/server/index/IndexModule.java
@@ -51,6 +51,7 @@
import com.google.gerrit.server.index.group.GroupIndexer;
import com.google.gerrit.server.index.group.GroupIndexerImpl;
import com.google.gerrit.server.index.group.GroupSchemaDefinitions;
+import com.google.gerrit.server.index.options.IsFirstInsertForEntry;
import com.google.gerrit.server.index.project.ProjectIndexDefinition;
import com.google.gerrit.server.index.project.ProjectIndexerImpl;
import com.google.inject.Inject;
@@ -59,6 +60,7 @@
import com.google.inject.Provides;
import com.google.inject.ProvisionException;
import com.google.inject.Singleton;
+import com.google.inject.multibindings.OptionalBinder;
import java.util.Collection;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -143,6 +145,9 @@
}
DynamicSet.setOf(binder(), OnlineUpgradeListener.class);
+ OptionalBinder.newOptionalBinder(binder(), IsFirstInsertForEntry.class)
+ .setDefault()
+ .toInstance(IsFirstInsertForEntry.NO);
}
@Provides
@@ -208,13 +213,14 @@
return interactiveExecutor;
}
int threads = this.threads;
- if (threads < 0) {
- return MoreExecutors.newDirectExecutorService();
- } else if (threads == 0) {
+ if (threads == 0) {
threads =
config.getInt(
"index", null, "threads", Runtime.getRuntime().availableProcessors() / 2 + 1);
}
+ if (threads < 0) {
+ return MoreExecutors.newDirectExecutorService();
+ }
return MoreExecutors.listeningDecorator(
workQueue.createQueue(threads, "Index-Interactive", true));
}
@@ -227,11 +233,13 @@
if (batchExecutor != null) {
return batchExecutor;
}
- int threads = config.getInt("index", null, "batchThreads", 0);
+ int threads = this.threads;
+ if (threads == 0) {
+ threads =
+ config.getInt("index", null, "batchThreads", Runtime.getRuntime().availableProcessors());
+ }
if (threads < 0) {
return MoreExecutors.newDirectExecutorService();
- } else if (threads == 0) {
- threads = Runtime.getRuntime().availableProcessors();
}
return MoreExecutors.listeningDecorator(workQueue.createQueue(threads, "Index-Batch", true));
}
diff --git a/java/com/google/gerrit/server/index/account/AllAccountsIndexer.java b/java/com/google/gerrit/server/index/account/AllAccountsIndexer.java
index 63889b7..ec27db0 100644
--- a/java/com/google/gerrit/server/index/account/AllAccountsIndexer.java
+++ b/java/com/google/gerrit/server/index/account/AllAccountsIndexer.java
@@ -27,6 +27,7 @@
import com.google.gerrit.server.account.AccountState;
import com.google.gerrit.server.account.Accounts;
import com.google.gerrit.server.index.IndexExecutor;
+import com.google.gerrit.server.index.options.IsFirstInsertForEntry;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.io.IOException;
@@ -50,15 +51,18 @@
private final ListeningExecutorService executor;
private final Accounts accounts;
private final AccountCache accountCache;
+ private final IsFirstInsertForEntry isFirstInsertForEntry;
@Inject
AllAccountsIndexer(
@IndexExecutor(BATCH) ListeningExecutorService executor,
Accounts accounts,
- AccountCache accountCache) {
+ AccountCache accountCache,
+ IsFirstInsertForEntry isFirstInsertForEntry) {
this.executor = executor;
this.accounts = accounts;
this.accountCache = accountCache;
+ this.isFirstInsertForEntry = isFirstInsertForEntry;
}
@Override
@@ -92,7 +96,11 @@
try {
Optional<AccountState> a = accountCache.get(id);
if (a.isPresent()) {
- index.replace(a.get());
+ if (isFirstInsertForEntry.equals(isFirstInsertForEntry.YES)) {
+ index.insert(a.get());
+ } else {
+ index.replace(a.get());
+ }
} else {
index.delete(id);
}
diff --git a/java/com/google/gerrit/server/index/change/AllChangesIndexer.java b/java/com/google/gerrit/server/index/change/AllChangesIndexer.java
index e9349c4..494aa84 100644
--- a/java/com/google/gerrit/server/index/change/AllChangesIndexer.java
+++ b/java/com/google/gerrit/server/index/change/AllChangesIndexer.java
@@ -20,15 +20,16 @@
import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
import static com.google.gerrit.server.git.QueueProvider.QueueType.BATCH;
+import com.google.auto.value.AutoValue;
import com.google.common.base.Stopwatch;
+import com.google.common.collect.Sets;
import com.google.common.flogger.FluentLogger;
-import com.google.common.primitives.Ints;
+import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.UncheckedExecutionException;
import com.google.gerrit.entities.Change;
import com.google.gerrit.entities.Project;
-import com.google.gerrit.entities.RefNames;
import com.google.gerrit.index.SiteIndexer;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.MultiProgressMonitor;
@@ -37,6 +38,7 @@
import com.google.gerrit.server.index.OnlineReindexMode;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.notedb.ChangeNotes.Factory.ChangeNotesResult;
+import com.google.gerrit.server.notedb.ChangeNotes.Factory.ScanResult;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.inject.Inject;
@@ -44,11 +46,13 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Objects;
+import java.util.Set;
import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.eclipse.jgit.errors.RepositoryNotFoundException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
import org.eclipse.jgit.lib.ProgressMonitor;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.TextProgressMonitor;
@@ -62,6 +66,14 @@
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static final int PROJECT_SLICE_MAX_REFS = 1000;
+ private static class ProjectsCollectionFailure extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ public ProjectsCollectionFailure(String message) {
+ super(message);
+ }
+ }
+
private final ChangeData.Factory changeDataFactory;
private final GitRepositoryManager repoManager;
private final ListeningExecutorService executor;
@@ -85,99 +97,58 @@
this.projectCache = projectCache;
}
- private static class ProjectSlice {
- private final Project.NameKey name;
- private final int slice;
- private final int slices;
+ @AutoValue
+ public abstract static class ProjectSlice {
+ public abstract Project.NameKey name();
- ProjectSlice(Project.NameKey name, int slice, int slices) {
- this.name = name;
- this.slice = slice;
- this.slices = slices;
- }
+ public abstract int slice();
- public Project.NameKey getName() {
- return name;
- }
+ public abstract int slices();
- public int getSlice() {
- return slice;
- }
+ public abstract ScanResult scanResult();
- public int getSlices() {
- return slices;
+ private static ProjectSlice create(Project.NameKey name, int slice, int slices, ScanResult sr) {
+ return new AutoValue_AllChangesIndexer_ProjectSlice(name, slice, slices, sr);
}
}
@Override
public Result indexAll(ChangeIndex index) {
- ProgressMonitor pm = new TextProgressMonitor();
- pm.beginTask("Collecting projects", ProgressMonitor.UNKNOWN);
- List<ProjectSlice> projectSlices = new ArrayList<>();
- int changeCount = 0;
- Stopwatch sw = Stopwatch.createStarted();
- int projectsFailed = 0;
- for (Project.NameKey name : projectCache.all()) {
- try (Repository repo = repoManager.openRepository(name)) {
- // The simplest approach to distribute indexing would be to let each thread grab a project
- // and index it fully. But if a site has one big project and 100s of small projects, then
- // in the beginning all CPUs would be busy reindexing projects. But soon enough all small
- // projects have been reindexed, and only the thread that reindexes the big project is
- // still working. The other threads would idle. Reindexing the big project on a single
- // thread becomes the critical path. Bringing in more CPUs would not speed up things.
- //
- // To avoid such situations, we split big repos into smaller parts and let
- // the thread pool index these smaller parts. This splitting introduces an overhead in the
- // workload setup and there might be additional slow-downs from multiple threads
- // concurrently working on different parts of the same project. But for Wikimedia's Gerrit,
- // which had 2 big projects, many middle sized ones, and lots of smaller ones, the
- // splitting of repos into smaller parts reduced indexing time from 1.5 hours to 55 minutes
- // in 2020.
- int size = estimateSize(repo);
- changeCount += size;
- int slices = 1 + size / PROJECT_SLICE_MAX_REFS;
- if (slices > 1) {
- verboseWriter.println("Submitting " + name + " for indexing in " + slices + " slices");
- }
- for (int slice = 0; slice < slices; slice++) {
- projectSlices.add(new ProjectSlice(name, slice, slices));
- }
- } catch (IOException e) {
- logger.atSevere().withCause(e).log("Error collecting project %s", name);
- projectsFailed++;
- if (projectsFailed > projectCache.all().size() / 2) {
- logger.atSevere().log("Over 50%% of the projects could not be collected: aborted");
- return Result.create(sw, false, 0, 0);
- }
- }
- pm.update(1);
- }
- pm.endTask();
- setTotalWork(changeCount);
+ // The simplest approach to distribute indexing would be to let each thread grab a project
+ // and index it fully. But if a site has one big project and 100s of small projects, then
+ // in the beginning all CPUs would be busy reindexing projects. But soon enough all small
+ // projects have been reindexed, and only the thread that reindexes the big project is
+ // still working. The other threads would idle. Reindexing the big project on a single
+ // thread becomes the critical path. Bringing in more CPUs would not speed up things.
+ //
+ // To avoid such situations, we split big repos into smaller parts and let
+ // the thread pool index these smaller parts. This splitting introduces an overhead in the
+ // workload setup and there might be additional slow-downs from multiple threads
+ // concurrently working on different parts of the same project. But for Wikimedia's Gerrit,
+ // which had 2 big projects, many middle sized ones, and lots of smaller ones, the
+ // splitting of repos into smaller parts reduced indexing time from 1.5 hours to 55 minutes
+ // in 2020.
- // projectSlices are currently grouped by projects. First all slices for project1, followed
- // by all slices for project2, and so on. As workers pick tasks sequentially, multiple threads
- // would typically work concurrently on different slices of the same project. While this is not
- // a big issue, shuffling the list beforehand helps with ungrouping the project slices, so
- // different slices are less likely to be worked on concurrently.
+ Stopwatch sw = Stopwatch.createStarted();
+ List<ProjectSlice> projectSlices;
+ try {
+ projectSlices = new SliceCreator().create();
+ } catch (ProjectsCollectionFailure | InterruptedException | ExecutionException e) {
+ logger.atSevere().log(e.getMessage());
+ return Result.create(sw, false, 0, 0);
+ }
+
+ // Since project slices are created in parallel, they are somewhat shuffled already. However,
+ // the number of threads used to create the project slices doesn't guarantee good randomization.
+ // If the slices are not shuffled well, then multiple threads would typically work concurrently
+ // on different slices of the same project. While this is not a big issue, shuffling the list
+ // beforehand helps with ungrouping the project slices, so different slices are less likely to
+ // be worked on concurrently.
// This shuffling gave a 6% runtime reduction for Wikimedia's Gerrit in 2020.
Collections.shuffle(projectSlices);
return indexAll(index, projectSlices);
}
- private int estimateSize(Repository repo) throws IOException {
- // Estimate size based on IDs that show up in ref names. This is not perfect, since patch set
- // refs may exist for changes whose metadata was never successfully stored. But that's ok, as
- // the estimate is just used as a heuristic for sorting projects.
- long size =
- repo.getRefDatabase().getRefsByPrefix(RefNames.REFS_CHANGES).stream()
- .map(r -> Change.Id.fromRef(r.getName()))
- .filter(Objects::nonNull)
- .distinct()
- .count();
- return Ints.saturatedCast(size);
- }
-
private SiteIndexer.Result indexAll(ChangeIndex index, List<ProjectSlice> projectSlices) {
Stopwatch sw = Stopwatch.createStarted();
MultiProgressMonitor mpm = new MultiProgressMonitor(progressOut, "Reindexing changes");
@@ -190,9 +161,9 @@
AtomicBoolean ok = new AtomicBoolean(true);
for (ProjectSlice projectSlice : projectSlices) {
- Project.NameKey name = projectSlice.getName();
- int slice = projectSlice.getSlice();
- int slices = projectSlice.getSlices();
+ Project.NameKey name = projectSlice.name();
+ int slice = projectSlice.slice();
+ int slices = projectSlice.slices();
ListenableFuture<?> future =
executor.submit(
reindexProject(
@@ -200,6 +171,7 @@
name,
slice,
slices,
+ projectSlice.scanResult(),
doneTask,
failedTask));
String description = "project " + name + " (" + slice + "/" + slices + ")";
@@ -238,7 +210,13 @@
public Callable<Void> reindexProject(
ChangeIndexer indexer, Project.NameKey project, Task done, Task failed) {
- return reindexProject(indexer, project, 0, 1, done, failed);
+ try (Repository repo = repoManager.openRepository(project)) {
+ return reindexProject(
+ indexer, project, 0, 1, ChangeNotes.Factory.scanChangeIds(repo), done, failed);
+ } catch (IOException e) {
+ logger.atSevere().log(e.getMessage());
+ return null;
+ }
}
public Callable<Void> reindexProject(
@@ -246,9 +224,10 @@
Project.NameKey project,
int slice,
int slices,
+ ScanResult scanResult,
Task done,
Task failed) {
- return new ProjectIndexer(indexer, project, slice, slices, done, failed);
+ return new ProjectIndexer(indexer, project, slice, slices, scanResult, done, failed);
}
private class ProjectIndexer implements Callable<Void> {
@@ -256,6 +235,7 @@
private final Project.NameKey project;
private final int slice;
private final int slices;
+ private final ScanResult scanResult;
private final ProgressMonitor done;
private final ProgressMonitor failed;
@@ -264,32 +244,30 @@
Project.NameKey project,
int slice,
int slices,
+ ScanResult scanResult,
ProgressMonitor done,
ProgressMonitor failed) {
this.indexer = indexer;
this.project = project;
this.slice = slice;
this.slices = slices;
+ this.scanResult = scanResult;
this.done = done;
this.failed = failed;
}
@Override
public Void call() throws Exception {
- try (Repository repo = repoManager.openRepository(project)) {
- OnlineReindexMode.begin();
-
- // Order of scanning changes is undefined. This is ok if we assume that packfile locality is
- // not important for indexing, since sites should have a fully populated DiffSummary cache.
- // It does mean that reindexing after invalidating the DiffSummary cache will be expensive,
- // but the goal is to invalidate that cache as infrequently as we possibly can. And besides,
- // we don't have concrete proof that improving packfile locality would help.
- notesFactory.scan(repo, project, id -> (id.get() % slices) == slice).forEach(r -> index(r));
- } catch (RepositoryNotFoundException rnfe) {
- logger.atSevere().log(rnfe.getMessage());
- } finally {
- OnlineReindexMode.end();
- }
+ OnlineReindexMode.begin();
+ // Order of scanning changes is undefined. This is ok if we assume that packfile locality is
+ // not important for indexing, since sites should have a fully populated DiffSummary cache.
+ // It does mean that reindexing after invalidating the DiffSummary cache will be expensive,
+ // but the goal is to invalidate that cache as infrequently as we possibly can. And besides,
+ // we don't have concrete proof that improving packfile locality would help.
+ notesFactory
+ .scan(scanResult, project, id -> (id.get() % slices) == slice)
+ .forEach(r -> index(r));
+ OnlineReindexMode.end();
return null;
}
@@ -329,4 +307,63 @@
return "Index all changes of project " + project.get();
}
}
+
+ private class SliceCreator {
+ final Set<ProjectSlice> projectSlices = Sets.newConcurrentHashSet();
+ final AtomicInteger changeCount = new AtomicInteger(0);
+ final AtomicInteger projectsFailed = new AtomicInteger(0);
+ final ProgressMonitor pm = new TextProgressMonitor();
+
+ private List<ProjectSlice> create()
+ throws ProjectsCollectionFailure, InterruptedException, ExecutionException {
+ List<ListenableFuture<?>> futures = new ArrayList<>();
+ pm.beginTask("Collecting projects", ProgressMonitor.UNKNOWN);
+ for (Project.NameKey name : projectCache.all()) {
+ futures.add(executor.submit(new ProjectSliceCreator(name)));
+ }
+
+ Futures.allAsList(futures).get();
+
+ if (projectsFailed.get() > projectCache.all().size() / 2) {
+ throw new ProjectsCollectionFailure(
+ "Over 50%% of the projects could not be collected: aborted");
+ }
+
+ pm.endTask();
+ setTotalWork(changeCount.get());
+ return projectSlices.stream().collect(Collectors.toList());
+ }
+
+ private class ProjectSliceCreator implements Callable<Void> {
+ final Project.NameKey name;
+
+ public ProjectSliceCreator(Project.NameKey name) {
+ this.name = name;
+ }
+
+ @Override
+ public Void call() throws IOException {
+ try (Repository repo = repoManager.openRepository(name)) {
+ ScanResult sr = ChangeNotes.Factory.scanChangeIds(repo);
+ int size = sr.all().size();
+ if (size > 0) {
+ changeCount.addAndGet(size);
+ int slices = 1 + size / PROJECT_SLICE_MAX_REFS;
+ if (slices > 1) {
+ verboseWriter.println(
+ "Submitting " + name + " for indexing in " + slices + " slices");
+ }
+ for (int slice = 0; slice < slices; slice++) {
+ projectSlices.add(ProjectSlice.create(name, slice, slices, sr));
+ }
+ }
+ } catch (IOException e) {
+ logger.atSevere().withCause(e).log("Error collecting project %s", name);
+ projectsFailed.incrementAndGet();
+ }
+ pm.update(1);
+ return null;
+ }
+ }
+ }
}
diff --git a/java/com/google/gerrit/server/index/change/ChangeIndexer.java b/java/com/google/gerrit/server/index/change/ChangeIndexer.java
index e0f6bec..a088af0 100644
--- a/java/com/google/gerrit/server/index/change/ChangeIndexer.java
+++ b/java/com/google/gerrit/server/index/change/ChangeIndexer.java
@@ -30,6 +30,7 @@
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.index.IndexExecutor;
import com.google.gerrit.server.index.StalenessCheckResult;
+import com.google.gerrit.server.index.options.IsFirstInsertForEntry;
import com.google.gerrit.server.logging.Metadata;
import com.google.gerrit.server.logging.TraceContext;
import com.google.gerrit.server.logging.TraceContext.TraceTimer;
@@ -78,6 +79,7 @@
private final PluginSetContext<ChangeIndexedListener> indexedListeners;
private final StalenessChecker stalenessChecker;
private final boolean autoReindexIfStale;
+ private final IsFirstInsertForEntry isFirstInsertForEntry;
private final Set<IndexTask> queuedIndexTasks =
Collections.newSetFromMap(new ConcurrentHashMap<>());
@@ -94,7 +96,8 @@
StalenessChecker stalenessChecker,
@IndexExecutor(BATCH) ListeningExecutorService batchExecutor,
@Assisted ListeningExecutorService executor,
- @Assisted ChangeIndex index) {
+ @Assisted ChangeIndex index,
+ IsFirstInsertForEntry isFirstInsertForEntry) {
this.executor = executor;
this.changeDataFactory = changeDataFactory;
this.notesFactory = notesFactory;
@@ -105,6 +108,7 @@
this.autoReindexIfStale = autoReindexIfStale(cfg);
this.index = index;
this.indexes = null;
+ this.isFirstInsertForEntry = isFirstInsertForEntry;
}
@AssistedInject
@@ -117,7 +121,8 @@
StalenessChecker stalenessChecker,
@IndexExecutor(BATCH) ListeningExecutorService batchExecutor,
@Assisted ListeningExecutorService executor,
- @Assisted ChangeIndexCollection indexes) {
+ @Assisted ChangeIndexCollection indexes,
+ IsFirstInsertForEntry isFirstInsertForEntry) {
this.executor = executor;
this.changeDataFactory = changeDataFactory;
this.notesFactory = notesFactory;
@@ -128,6 +133,7 @@
this.autoReindexIfStale = autoReindexIfStale(cfg);
this.index = null;
this.indexes = indexes;
+ this.isFirstInsertForEntry = isFirstInsertForEntry;
}
private static boolean autoReindexIfStale(Config cfg) {
@@ -198,21 +204,25 @@
}
private void indexImpl(ChangeData cd) {
- logger.atFine().log("Replace change %d in index.", cd.getId().get());
+ logger.atFine().log("Reindex change %d in index.", cd.getId().get());
for (Index<?, ChangeData> i : getWriteIndexes()) {
try (TraceTimer traceTimer =
TraceContext.newTimer(
- "Replacing change in index",
+ "Reindexing change in index",
Metadata.builder()
.changeId(cd.getId().get())
.patchSetId(cd.currentPatchSet().number())
.indexVersion(i.getSchema().getVersion())
.build())) {
- i.replace(cd);
+ if (isFirstInsertForEntry.equals(isFirstInsertForEntry.YES)) {
+ i.insert(cd);
+ } else {
+ i.replace(cd);
+ }
} catch (RuntimeException e) {
throw new StorageException(
String.format(
- "Failed to replace change %d in index version %d (current patch set = %d)",
+ "Failed to reindex change %d in index version %d (current patch set = %d)",
cd.getId().get(), i.getSchema().getVersion(), cd.currentPatchSet().number()),
e);
}
diff --git a/java/com/google/gerrit/server/index/group/AllGroupsIndexer.java b/java/com/google/gerrit/server/index/group/AllGroupsIndexer.java
index 51c7730..84d6a9d 100644
--- a/java/com/google/gerrit/server/index/group/AllGroupsIndexer.java
+++ b/java/com/google/gerrit/server/index/group/AllGroupsIndexer.java
@@ -30,6 +30,7 @@
import com.google.gerrit.server.group.db.Groups;
import com.google.gerrit.server.group.db.GroupsNoteDbConsistencyChecker;
import com.google.gerrit.server.index.IndexExecutor;
+import com.google.gerrit.server.index.options.IsFirstInsertForEntry;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.io.IOException;
@@ -54,15 +55,18 @@
private final ListeningExecutorService executor;
private final GroupCache groupCache;
private final Groups groups;
+ private final IsFirstInsertForEntry isFirstInsertForEntry;
@Inject
AllGroupsIndexer(
@IndexExecutor(BATCH) ListeningExecutorService executor,
GroupCache groupCache,
- Groups groups) {
+ Groups groups,
+ IsFirstInsertForEntry isFirstInsertForEntry) {
this.executor = executor;
this.groupCache = groupCache;
this.groups = groups;
+ this.isFirstInsertForEntry = isFirstInsertForEntry;
}
@Override
@@ -97,7 +101,11 @@
groupCache.evict(uuid);
Optional<InternalGroup> internalGroup = groupCache.get(uuid);
if (internalGroup.isPresent()) {
- index.replace(internalGroup.get());
+ if (isFirstInsertForEntry.equals(isFirstInsertForEntry.YES)) {
+ index.insert(internalGroup.get());
+ } else {
+ index.replace(internalGroup.get());
+ }
} else {
index.delete(uuid);
diff --git a/java/com/google/gerrit/server/index/options/AutoFlush.java b/java/com/google/gerrit/server/index/options/AutoFlush.java
new file mode 100644
index 0000000..7b82edb
--- /dev/null
+++ b/java/com/google/gerrit/server/index/options/AutoFlush.java
@@ -0,0 +1,20 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.index.options;
+
+public enum AutoFlush {
+ ENABLED,
+ DISABLED
+}
diff --git a/java/com/google/gerrit/server/index/options/IsFirstInsertForEntry.java b/java/com/google/gerrit/server/index/options/IsFirstInsertForEntry.java
new file mode 100644
index 0000000..f943309
--- /dev/null
+++ b/java/com/google/gerrit/server/index/options/IsFirstInsertForEntry.java
@@ -0,0 +1,26 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.index.options;
+
+/**
+ * This enum should be injected and checked to decide on which operation ({@link
+ * com.google.gerrit.index.Index#replace(Object) replace()} or {@link
+ * com.google.gerrit.index.Index#insert(Object) insert()}) should be performed on a specific {@link
+ * com.google.gerrit.index.Index index}.
+ */
+public enum IsFirstInsertForEntry {
+ YES,
+ NO
+}
diff --git a/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java b/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java
index 0e4b688..86c7e94 100644
--- a/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java
+++ b/java/com/google/gerrit/server/index/project/AllProjectsIndexer.java
@@ -27,6 +27,7 @@
import com.google.gerrit.index.project.ProjectData;
import com.google.gerrit.index.project.ProjectIndex;
import com.google.gerrit.server.index.IndexExecutor;
+import com.google.gerrit.server.index.options.IsFirstInsertForEntry;
import com.google.gerrit.server.project.ProjectCache;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -48,12 +49,16 @@
private final ListeningExecutorService executor;
private final ProjectCache projectCache;
+ private final IsFirstInsertForEntry isFirstInsertForEntry;
@Inject
AllProjectsIndexer(
- @IndexExecutor(BATCH) ListeningExecutorService executor, ProjectCache projectCache) {
+ @IndexExecutor(BATCH) ListeningExecutorService executor,
+ ProjectCache projectCache,
+ IsFirstInsertForEntry isFirstInsertForEntry) {
this.executor = executor;
this.projectCache = projectCache;
+ this.isFirstInsertForEntry = isFirstInsertForEntry;
}
@Override
@@ -79,8 +84,13 @@
() -> {
try {
projectCache.evict(name);
- index.replace(
- projectCache.get(name).orElseThrow(illegalState(name)).toProjectData());
+ ProjectData projectData =
+ projectCache.get(name).orElseThrow(illegalState(name)).toProjectData();
+ if (isFirstInsertForEntry.equals(isFirstInsertForEntry.YES)) {
+ index.insert(projectData);
+ } else {
+ index.replace(projectData);
+ }
verboseWriter.println("Reindexed " + desc);
done.incrementAndGet();
} catch (Exception e) {
diff --git a/java/com/google/gerrit/server/mail/send/OutgoingEmail.java b/java/com/google/gerrit/server/mail/send/OutgoingEmail.java
index b35bbec..8c56469 100644
--- a/java/com/google/gerrit/server/mail/send/OutgoingEmail.java
+++ b/java/com/google/gerrit/server/mail/send/OutgoingEmail.java
@@ -404,9 +404,13 @@
* username. If no username is set, this function returns null.
*
* @param accountId user to fetch.
- * @return name/email of account, username, or null if unset.
+ * @return name/email of account, username, or null if unset or the accountId is null.
*/
- protected String getUserNameEmailFor(Account.Id accountId) {
+ protected String getUserNameEmailFor(@Nullable Account.Id accountId) {
+ if (accountId == null) {
+ return null;
+ }
+
Optional<AccountState> accountState = args.accountCache.get(accountId);
if (!accountState.isPresent()) {
return null;
diff --git a/java/com/google/gerrit/server/notedb/ChangeNotes.java b/java/com/google/gerrit/server/notedb/ChangeNotes.java
index 36a61cc0..51acf16e 100644
--- a/java/com/google/gerrit/server/notedb/ChangeNotes.java
+++ b/java/com/google/gerrit/server/notedb/ChangeNotes.java
@@ -106,6 +106,29 @@
this.projectCache = projectCache;
}
+ @AutoValue
+ public abstract static class ScanResult {
+ abstract ImmutableSet<Change.Id> fromPatchSetRefs();
+
+ abstract ImmutableSet<Change.Id> fromMetaRefs();
+
+ public SetView<Change.Id> all() {
+ return Sets.union(fromPatchSetRefs(), fromMetaRefs());
+ }
+ }
+
+ public static ScanResult scanChangeIds(Repository repo) throws IOException {
+ ImmutableSet.Builder<Change.Id> fromPs = ImmutableSet.builder();
+ ImmutableSet.Builder<Change.Id> fromMeta = ImmutableSet.builder();
+ for (Ref r : repo.getRefDatabase().getRefsByPrefix(RefNames.REFS_CHANGES)) {
+ Change.Id id = Change.Id.fromRef(r.getName());
+ if (id != null) {
+ (r.getName().endsWith(RefNames.META_SUFFIX) ? fromMeta : fromPs).add(id);
+ }
+ }
+ return new AutoValue_ChangeNotes_Factory_ScanResult(fromPs.build(), fromMeta.build());
+ }
+
public ChangeNotes createChecked(Change c) {
return createChecked(c.getProject(), c.getId());
}
@@ -213,8 +236,11 @@
public Stream<ChangeNotesResult> scan(
Repository repo, Project.NameKey project, Predicate<Change.Id> changeIdPredicate)
throws IOException {
- ScanResult sr = scanChangeIds(repo);
+ return scan(scanChangeIds(repo), project, changeIdPredicate);
+ }
+ public Stream<ChangeNotesResult> scan(
+ ScanResult sr, Project.NameKey project, Predicate<Change.Id> changeIdPredicate) {
Stream<Change.Id> idStream = sr.all().stream();
if (changeIdPredicate != null) {
idStream = idStream.filter(changeIdPredicate);
@@ -286,29 +312,6 @@
@Nullable
abstract ChangeNotes maybeNotes();
}
-
- @AutoValue
- abstract static class ScanResult {
- abstract ImmutableSet<Change.Id> fromPatchSetRefs();
-
- abstract ImmutableSet<Change.Id> fromMetaRefs();
-
- SetView<Change.Id> all() {
- return Sets.union(fromPatchSetRefs(), fromMetaRefs());
- }
- }
-
- private static ScanResult scanChangeIds(Repository repo) throws IOException {
- ImmutableSet.Builder<Change.Id> fromPs = ImmutableSet.builder();
- ImmutableSet.Builder<Change.Id> fromMeta = ImmutableSet.builder();
- for (Ref r : repo.getRefDatabase().getRefsByPrefix(RefNames.REFS_CHANGES)) {
- Change.Id id = Change.Id.fromRef(r.getName());
- if (id != null) {
- (r.getName().endsWith(RefNames.META_SUFFIX) ? fromMeta : fromPs).add(id);
- }
- }
- return new AutoValue_ChangeNotes_Factory_ScanResult(fromPs.build(), fromMeta.build());
- }
}
private final boolean shouldExist;
diff --git a/java/com/google/gerrit/server/project/ProjectCache.java b/java/com/google/gerrit/server/project/ProjectCache.java
index 3fba7d3..6a2ae50 100644
--- a/java/com/google/gerrit/server/project/ProjectCache.java
+++ b/java/com/google/gerrit/server/project/ProjectCache.java
@@ -59,18 +59,25 @@
Optional<ProjectState> get(@Nullable Project.NameKey projectName) throws StorageException;
/**
+ * Invalidate the cached information about the given project.
+ *
+ * @param p the NameKey of the project that is being evicted
+ */
+ void evict(Project.NameKey p);
+
+ /**
* Invalidate the cached information about the given project, and triggers reindexing for it
*
* @param p project that is being evicted
*/
- void evict(Project p);
+ void evictAndReindex(Project p);
/**
* Invalidate the cached information about the given project, and triggers reindexing for it
*
* @param p the NameKey of the project that is being evicted
*/
- void evict(Project.NameKey p);
+ void evictAndReindex(Project.NameKey p);
/**
* Remove information about the given project from the cache. It will no longer be returned from
diff --git a/java/com/google/gerrit/server/project/ProjectCacheImpl.java b/java/com/google/gerrit/server/project/ProjectCacheImpl.java
index 9d09eeb..1523b1b 100644
--- a/java/com/google/gerrit/server/project/ProjectCacheImpl.java
+++ b/java/com/google/gerrit/server/project/ProjectCacheImpl.java
@@ -159,16 +159,21 @@
}
@Override
- public void evict(Project p) {
- evict(p.getNameKey());
- }
-
- @Override
public void evict(Project.NameKey p) {
if (p != null) {
logger.atFine().log("Evict project '%s'", p.get());
byName.invalidate(p.get());
}
+ }
+
+ @Override
+ public void evictAndReindex(Project p) {
+ evictAndReindex(p.getNameKey());
+ }
+
+ @Override
+ public void evictAndReindex(Project.NameKey p) {
+ evict(p);
indexer.get().index(p);
}
@@ -189,7 +194,7 @@
} finally {
listLock.unlock();
}
- evict(name);
+ evictAndReindex(name);
}
@Override
diff --git a/java/com/google/gerrit/server/restapi/config/GetCache.java b/java/com/google/gerrit/server/restapi/config/GetCache.java
index 93600ea..5dd3d3d 100644
--- a/java/com/google/gerrit/server/restapi/config/GetCache.java
+++ b/java/com/google/gerrit/server/restapi/config/GetCache.java
@@ -16,6 +16,7 @@
import com.google.gerrit.extensions.restapi.Response;
import com.google.gerrit.extensions.restapi.RestReadView;
+import com.google.gerrit.server.cache.CacheInfo;
import com.google.gerrit.server.config.CacheResource;
import com.google.inject.Singleton;
@@ -23,7 +24,7 @@
public class GetCache implements RestReadView<CacheResource> {
@Override
- public Response<ListCaches.CacheInfo> apply(CacheResource rsrc) {
- return Response.ok(new ListCaches.CacheInfo(rsrc.getName(), rsrc.getCache()));
+ public Response<CacheInfo> apply(CacheResource rsrc) {
+ return Response.ok(new CacheInfo(rsrc.getName(), rsrc.getCache()));
}
}
diff --git a/java/com/google/gerrit/server/restapi/config/ListCaches.java b/java/com/google/gerrit/server/restapi/config/ListCaches.java
index ccafbe8..ffc65c9 100644
--- a/java/com/google/gerrit/server/restapi/config/ListCaches.java
+++ b/java/com/google/gerrit/server/restapi/config/ListCaches.java
@@ -22,7 +22,6 @@
import static java.util.stream.Collectors.joining;
import com.google.common.cache.Cache;
-import com.google.common.cache.CacheStats;
import com.google.common.collect.Streams;
import com.google.gerrit.extensions.annotations.RequiresAnyCapability;
import com.google.gerrit.extensions.registration.DynamicMap;
@@ -30,7 +29,7 @@
import com.google.gerrit.extensions.restapi.BinaryResult;
import com.google.gerrit.extensions.restapi.Response;
import com.google.gerrit.extensions.restapi.RestReadView;
-import com.google.gerrit.server.cache.PersistentCache;
+import com.google.gerrit.server.cache.CacheInfo;
import com.google.gerrit.server.config.ConfigResource;
import com.google.inject.Inject;
import java.util.Map;
@@ -87,118 +86,4 @@
}
return Response.ok(cacheNames.collect(toImmutableList()));
}
-
- public enum CacheType {
- MEM,
- DISK
- }
-
- public static class CacheInfo {
- public String name;
- public CacheType type;
- public EntriesInfo entries;
- public String averageGet;
- public HitRatioInfo hitRatio;
-
- public CacheInfo(Cache<?, ?> cache) {
- this(null, cache);
- }
-
- public CacheInfo(String name, Cache<?, ?> cache) {
- this.name = name;
-
- CacheStats stat = cache.stats();
-
- entries = new EntriesInfo();
- entries.setMem(cache.size());
-
- averageGet = duration(stat.averageLoadPenalty());
-
- hitRatio = new HitRatioInfo();
- hitRatio.setMem(stat.hitCount(), stat.requestCount());
-
- if (cache instanceof PersistentCache) {
- type = CacheType.DISK;
- PersistentCache.DiskStats diskStats = ((PersistentCache) cache).diskStats();
- entries.setDisk(diskStats.size());
- entries.setSpace(diskStats.space());
- hitRatio.setDisk(diskStats.hitCount(), diskStats.requestCount());
- } else {
- type = CacheType.MEM;
- }
- }
-
- private static String duration(double ns) {
- if (ns < 0.5) {
- return null;
- }
- String suffix = "ns";
- if (ns >= 1000.0) {
- ns /= 1000.0;
- suffix = "us";
- }
- if (ns >= 1000.0) {
- ns /= 1000.0;
- suffix = "ms";
- }
- if (ns >= 1000.0) {
- ns /= 1000.0;
- suffix = "s";
- }
- return String.format("%4.1f%s", ns, suffix).trim();
- }
- }
-
- public static class EntriesInfo {
- public Long mem;
- public Long disk;
- public String space;
-
- public void setMem(long mem) {
- this.mem = mem != 0 ? mem : null;
- }
-
- public void setDisk(long disk) {
- this.disk = disk != 0 ? disk : null;
- }
-
- public void setSpace(double value) {
- space = bytes(value);
- }
-
- private static String bytes(double value) {
- value /= 1024;
- String suffix = "k";
-
- if (value > 1024) {
- value /= 1024;
- suffix = "m";
- }
- if (value > 1024) {
- value /= 1024;
- suffix = "g";
- }
- return String.format("%1$6.2f%2$s", value, suffix).trim();
- }
- }
-
- public static class HitRatioInfo {
- public Integer mem;
- public Integer disk;
-
- public void setMem(long value, long total) {
- mem = percent(value, total);
- }
-
- public void setDisk(long value, long total) {
- disk = percent(value, total);
- }
-
- private static Integer percent(long value, long total) {
- if (total <= 0) {
- return null;
- }
- return (int) ((100 * value) / total);
- }
- }
}
diff --git a/java/com/google/gerrit/server/restapi/project/CreateLabel.java b/java/com/google/gerrit/server/restapi/project/CreateLabel.java
index a85ad39..51cbcea 100644
--- a/java/com/google/gerrit/server/restapi/project/CreateLabel.java
+++ b/java/com/google/gerrit/server/restapi/project/CreateLabel.java
@@ -101,7 +101,7 @@
config.commit(md);
- projectCache.evict(rsrc.getProjectState().getProject());
+ projectCache.evictAndReindex(rsrc.getProjectState().getProject());
return Response.created(LabelDefinitionJson.format(rsrc.getNameKey(), labelType));
}
diff --git a/java/com/google/gerrit/server/restapi/project/DeleteLabel.java b/java/com/google/gerrit/server/restapi/project/DeleteLabel.java
index 531640c..8a1927a 100644
--- a/java/com/google/gerrit/server/restapi/project/DeleteLabel.java
+++ b/java/com/google/gerrit/server/restapi/project/DeleteLabel.java
@@ -90,7 +90,7 @@
config.commit(md);
}
- projectCache.evict(rsrc.getProject().getProjectState().getProject());
+ projectCache.evictAndReindex(rsrc.getProject().getProjectState().getProject());
return Response.none();
}
diff --git a/java/com/google/gerrit/server/restapi/project/GetAccess.java b/java/com/google/gerrit/server/restapi/project/GetAccess.java
index 5a3dbcd..0432150 100644
--- a/java/com/google/gerrit/server/restapi/project/GetAccess.java
+++ b/java/com/google/gerrit/server/restapi/project/GetAccess.java
@@ -153,12 +153,12 @@
if (config.updateGroupNames(groupBackend)) {
md.setMessage("Update group names\n");
config.commit(md);
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
projectState = projectCache.get(projectName).orElseThrow(illegalState(projectName));
perm = permissionBackend.currentUser().project(projectName);
} else if (config.getRevision() != null
&& !config.getRevision().equals(projectState.getConfig().getRevision())) {
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
projectState = projectCache.get(projectName).orElseThrow(illegalState(projectName));
perm = permissionBackend.currentUser().project(projectName);
}
@@ -331,7 +331,7 @@
}
AccountGroup.UUID group = r.getGroup().getUUID();
if (group != null) {
- pInfo.rules.put(group.get(), info);
+ pInfo.rules.putIfAbsent(group.get(), info); // First entry for the group wins
loadGroup(groups, group);
}
}
diff --git a/java/com/google/gerrit/server/restapi/project/PostLabels.java b/java/com/google/gerrit/server/restapi/project/PostLabels.java
index 8835359..69e31b4 100644
--- a/java/com/google/gerrit/server/restapi/project/PostLabels.java
+++ b/java/com/google/gerrit/server/restapi/project/PostLabels.java
@@ -139,7 +139,7 @@
if (dirty) {
config.commit(md);
- projectCache.evict(rsrc.getProjectState().getProject());
+ projectCache.evictAndReindex(rsrc.getProjectState().getProject());
}
}
diff --git a/java/com/google/gerrit/server/restapi/project/PutConfig.java b/java/com/google/gerrit/server/restapi/project/PutConfig.java
index 9f9433b..1d5c151 100644
--- a/java/com/google/gerrit/server/restapi/project/PutConfig.java
+++ b/java/com/google/gerrit/server/restapi/project/PutConfig.java
@@ -168,7 +168,7 @@
md.setMessage("Modified project settings\n");
try {
projectConfig.commit(md);
- projectCache.evict(projectConfig.getProject());
+ projectCache.evictAndReindex(projectConfig.getProject());
md.getRepository().setGitwebDescription(p.getDescription());
} catch (IOException e) {
if (e.getCause() instanceof ConfigInvalidException) {
diff --git a/java/com/google/gerrit/server/restapi/project/PutDescription.java b/java/com/google/gerrit/server/restapi/project/PutDescription.java
index a0b9feb..3acbb61 100644
--- a/java/com/google/gerrit/server/restapi/project/PutDescription.java
+++ b/java/com/google/gerrit/server/restapi/project/PutDescription.java
@@ -85,7 +85,7 @@
md.setAuthor(user);
md.setMessage(msg);
config.commit(md);
- cache.evict(resource.getProjectState().getProject());
+ cache.evictAndReindex(resource.getProjectState().getProject());
md.getRepository().setGitwebDescription(project.getDescription());
return Strings.isNullOrEmpty(project.getDescription())
diff --git a/java/com/google/gerrit/server/restapi/project/SetAccess.java b/java/com/google/gerrit/server/restapi/project/SetAccess.java
index 02c1b54..c3a332b 100644
--- a/java/com/google/gerrit/server/restapi/project/SetAccess.java
+++ b/java/com/google/gerrit/server/restapi/project/SetAccess.java
@@ -125,7 +125,7 @@
}
config.commit(md);
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
createGroupPermissionSyncer.syncIfNeeded();
} catch (InvalidNameException e) {
throw new BadRequestException(e.toString());
diff --git a/java/com/google/gerrit/server/restapi/project/SetDefaultDashboard.java b/java/com/google/gerrit/server/restapi/project/SetDefaultDashboard.java
index 9920be0..b105442 100644
--- a/java/com/google/gerrit/server/restapi/project/SetDefaultDashboard.java
+++ b/java/com/google/gerrit/server/restapi/project/SetDefaultDashboard.java
@@ -116,7 +116,7 @@
md.setAuthor(rsrc.getUser().asIdentifiedUser());
md.setMessage(msg);
config.commit(md);
- cache.evict(rsrc.getProjectState().getProject());
+ cache.evictAndReindex(rsrc.getProjectState().getProject());
if (target != null) {
Response<DashboardInfo> response = get.get().apply(target);
diff --git a/java/com/google/gerrit/server/restapi/project/SetLabel.java b/java/com/google/gerrit/server/restapi/project/SetLabel.java
index 0a35865..13a873c 100644
--- a/java/com/google/gerrit/server/restapi/project/SetLabel.java
+++ b/java/com/google/gerrit/server/restapi/project/SetLabel.java
@@ -90,7 +90,7 @@
}
config.commit(md);
- projectCache.evict(rsrc.getProject().getProjectState().getProject());
+ projectCache.evictAndReindex(rsrc.getProject().getProjectState().getProject());
}
}
return Response.ok(LabelDefinitionJson.format(rsrc.getProject().getNameKey(), labelType));
diff --git a/java/com/google/gerrit/server/restapi/project/SetParent.java b/java/com/google/gerrit/server/restapi/project/SetParent.java
index 42790aa..13564c1 100644
--- a/java/com/google/gerrit/server/restapi/project/SetParent.java
+++ b/java/com/google/gerrit/server/restapi/project/SetParent.java
@@ -115,7 +115,7 @@
md.setAuthor(user);
md.setMessage(msg);
config.commit(md);
- cache.evict(rsrc.getProjectState().getProject());
+ cache.evictAndReindex(rsrc.getProjectState().getProject());
Project.NameKey parent = project.getParent(allProjects);
requireNonNull(parent);
diff --git a/java/com/google/gerrit/server/submit/SubmitStrategyOp.java b/java/com/google/gerrit/server/submit/SubmitStrategyOp.java
index a4141be..f35bf89 100644
--- a/java/com/google/gerrit/server/submit/SubmitStrategyOp.java
+++ b/java/com/google/gerrit/server/submit/SubmitStrategyOp.java
@@ -485,7 +485,7 @@
// TODO(dborowitz): Move to BatchUpdate? Would also allow us to run once
// per project even if multiple changes to refs/meta/config are submitted.
if (RefNames.REFS_CONFIG.equals(getDest().branch())) {
- args.projectCache.evict(getProject());
+ args.projectCache.evictAndReindex(getProject());
ProjectState p =
args.projectCache.get(getProject()).orElseThrow(illegalState(getProject()));
try (Repository git = args.repoManager.openRepository(getProject())) {
diff --git a/java/com/google/gerrit/sshd/commands/ShowCaches.java b/java/com/google/gerrit/sshd/commands/ShowCaches.java
index ba84179..219ceaa 100644
--- a/java/com/google/gerrit/sshd/commands/ShowCaches.java
+++ b/java/com/google/gerrit/sshd/commands/ShowCaches.java
@@ -24,6 +24,8 @@
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.extensions.restapi.AuthException;
import com.google.gerrit.server.CurrentUser;
+import com.google.gerrit.server.cache.CacheDisplay;
+import com.google.gerrit.server.cache.CacheInfo;
import com.google.gerrit.server.config.ConfigResource;
import com.google.gerrit.server.permissions.GlobalPermission;
import com.google.gerrit.server.permissions.PermissionBackend;
@@ -35,8 +37,6 @@
import com.google.gerrit.server.restapi.config.GetSummary.TaskSummaryInfo;
import com.google.gerrit.server.restapi.config.GetSummary.ThreadSummaryInfo;
import com.google.gerrit.server.restapi.config.ListCaches;
-import com.google.gerrit.server.restapi.config.ListCaches.CacheInfo;
-import com.google.gerrit.server.restapi.config.ListCaches.CacheType;
import com.google.gerrit.server.util.time.TimeUtil;
import com.google.gerrit.sshd.CommandMetaData;
import com.google.gerrit.sshd.SshCommand;
@@ -123,52 +123,8 @@
stdout.format("%-25s %-20s uptime %16s\n", "", "", uptime(now.getTime() - serverStarted));
stdout.print('\n');
- stdout.print(
- String.format( //
- "%1s %-" + nw + "s|%-21s| %-5s |%-9s|\n" //
- ,
- "" //
- ,
- "Name" //
- ,
- "Entries" //
- ,
- "AvgGet" //
- ,
- "Hit Ratio" //
- ));
- stdout.print(
- String.format( //
- "%1s %-" + nw + "s|%6s %6s %7s| %-5s |%-4s %-4s|\n" //
- ,
- "" //
- ,
- "" //
- ,
- "Mem" //
- ,
- "Disk" //
- ,
- "Space" //
- ,
- "" //
- ,
- "Mem" //
- ,
- "Disk" //
- ));
- stdout.print("--");
- for (int i = 0; i < nw; i++) {
- stdout.print('-');
- }
- stdout.print("+---------------------+---------+---------+\n");
-
try {
- Collection<CacheInfo> caches = getCaches();
- printMemoryCoreCaches(caches);
- printMemoryPluginCaches(caches);
- printDiskCaches(caches);
- stdout.print('\n');
+ new CacheDisplay(stdout, nw, getCaches()).displayCaches();
boolean showJvm;
try {
@@ -209,52 +165,6 @@
return caches.values();
}
- private void printMemoryCoreCaches(Collection<CacheInfo> caches) {
- for (CacheInfo cache : caches) {
- if (!cache.name.contains("-") && CacheType.MEM.equals(cache.type)) {
- printCache(cache);
- }
- }
- }
-
- private void printMemoryPluginCaches(Collection<CacheInfo> caches) {
- for (CacheInfo cache : caches) {
- if (cache.name.contains("-") && CacheType.MEM.equals(cache.type)) {
- printCache(cache);
- }
- }
- }
-
- private void printDiskCaches(Collection<CacheInfo> caches) {
- for (CacheInfo cache : caches) {
- if (CacheType.DISK.equals(cache.type)) {
- printCache(cache);
- }
- }
- }
-
- private void printCache(CacheInfo cache) {
- stdout.print(
- String.format(
- "%1s %-" + nw + "s|%6s %6s %7s| %7s |%4s %4s|\n",
- CacheType.DISK.equals(cache.type) ? "D" : "",
- cache.name,
- nullToEmpty(cache.entries.mem),
- nullToEmpty(cache.entries.disk),
- Strings.nullToEmpty(cache.entries.space),
- Strings.nullToEmpty(cache.averageGet),
- formatAsPercent(cache.hitRatio.mem),
- formatAsPercent(cache.hitRatio.disk)));
- }
-
- private static String nullToEmpty(Long l) {
- return l != null ? String.valueOf(l) : "";
- }
-
- private static String formatAsPercent(Integer i) {
- return i != null ? String.valueOf(i) + "%" : "";
- }
-
private void memSummary(MemSummaryInfo memSummary) {
stdout.format(
"Mem: %s total = %s used + %s free + %s buffers\n",
diff --git a/javatests/com/google/gerrit/acceptance/ProjectResetterTest.java b/javatests/com/google/gerrit/acceptance/ProjectResetterTest.java
index ff9bac9..7d04558 100644
--- a/javatests/com/google/gerrit/acceptance/ProjectResetterTest.java
+++ b/javatests/com/google/gerrit/acceptance/ProjectResetterTest.java
@@ -231,7 +231,7 @@
updateRef(repo2, metaConfig);
}
- verify(projectCache, only()).evict(project2);
+ verify(projectCache, only()).evictAndReindex(project2);
}
@Test
@@ -248,7 +248,7 @@
createRef(repo2, RefNames.REFS_CONFIG);
}
- verify(projectCache, only()).evict(project2);
+ verify(projectCache, only()).evictAndReindex(project2);
}
@Test
diff --git a/javatests/com/google/gerrit/acceptance/api/accounts/AgreementsIT.java b/javatests/com/google/gerrit/acceptance/api/accounts/AgreementsIT.java
index 11ca391..2b0bdf9 100644
--- a/javatests/com/google/gerrit/acceptance/api/accounts/AgreementsIT.java
+++ b/javatests/com/google/gerrit/acceptance/api/accounts/AgreementsIT.java
@@ -69,7 +69,7 @@
ProjectConfig config = projectConfigFactory.read(md);
config.getProject().setBooleanConfig(BooleanProjectConfig.USE_CONTRIBUTOR_AGREEMENTS, value);
config.commit(md);
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
}
}
diff --git a/javatests/com/google/gerrit/acceptance/git/AbstractSubmoduleSubscription.java b/javatests/com/google/gerrit/acceptance/git/AbstractSubmoduleSubscription.java
index a0725c3..c5c54c0 100644
--- a/javatests/com/google/gerrit/acceptance/git/AbstractSubmoduleSubscription.java
+++ b/javatests/com/google/gerrit/acceptance/git/AbstractSubmoduleSubscription.java
@@ -226,7 +226,7 @@
ObjectId oldId = pc.getRevision();
ObjectId newId = pc.commit(md);
assertThat(newId).isNotEqualTo(oldId);
- projectCache.evict(pc.getProject());
+ projectCache.evictAndReindex(pc.getProject());
}
}
diff --git a/javatests/com/google/gerrit/acceptance/rest/DynamicOptionsBeanParseListenerIT.java b/javatests/com/google/gerrit/acceptance/rest/DynamicOptionsBeanParseListenerIT.java
new file mode 100644
index 0000000..39f1e8d
--- /dev/null
+++ b/javatests/com/google/gerrit/acceptance/rest/DynamicOptionsBeanParseListenerIT.java
@@ -0,0 +1,67 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.acceptance.rest;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.gerrit.acceptance.AbstractDaemonTest;
+import com.google.gerrit.acceptance.RestResponse;
+import com.google.gerrit.extensions.annotations.Exports;
+import com.google.gerrit.json.OutputFormat;
+import com.google.gerrit.server.DynamicOptions;
+import com.google.gerrit.server.restapi.project.ListProjects;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import com.google.inject.AbstractModule;
+import java.util.Map;
+import org.junit.Test;
+
+public class DynamicOptionsBeanParseListenerIT extends AbstractDaemonTest {
+ private static final Gson GSON = OutputFormat.JSON.newGson();
+
+ @Test
+ public void testBeanParseListener() throws Exception {
+ createProjectOverAPI("project1", project, true, null);
+ createProjectOverAPI("project2", project, true, null);
+ try (AutoCloseable ignored = installPlugin("my-plugin", PluginModule.class)) {
+ assertThat(getProjects(adminRestSession.get("/projects/"))).hasSize(1);
+ }
+ }
+
+ protected Map<String, Object> getProjects(RestResponse res) throws Exception {
+ res.assertOK();
+ return GSON.fromJson(res.getReader(), new TypeToken<Map<String, Object>>() {}.getType());
+ }
+
+ protected static class ListProjectsBeanListener implements DynamicOptions.BeanParseListener {
+ @Override
+ public void onBeanParseStart(String plugin, Object bean) {
+ ListProjects listProjects = (ListProjects) bean;
+ listProjects.setLimit(1);
+ }
+
+ @Override
+ public void onBeanParseEnd(String plugin, Object bean) {}
+ }
+
+ protected static class PluginModule extends AbstractModule {
+ @Override
+ public void configure() {
+ bind(DynamicOptions.DynamicBean.class)
+ .annotatedWith(Exports.named(ListProjects.class))
+ .to(ListProjectsBeanListener.class);
+ }
+ }
+}
diff --git a/javatests/com/google/gerrit/acceptance/rest/config/CacheOperationsIT.java b/javatests/com/google/gerrit/acceptance/rest/config/CacheOperationsIT.java
index daeb032..e4b4e4a 100644
--- a/javatests/com/google/gerrit/acceptance/rest/config/CacheOperationsIT.java
+++ b/javatests/com/google/gerrit/acceptance/rest/config/CacheOperationsIT.java
@@ -25,7 +25,7 @@
import com.google.gerrit.acceptance.RestResponse;
import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
import com.google.gerrit.common.data.GlobalCapability;
-import com.google.gerrit.server.restapi.config.ListCaches.CacheInfo;
+import com.google.gerrit.server.cache.CacheInfo;
import com.google.gerrit.server.restapi.config.PostCaches;
import com.google.inject.Inject;
import java.util.Arrays;
diff --git a/javatests/com/google/gerrit/acceptance/rest/config/FlushCacheIT.java b/javatests/com/google/gerrit/acceptance/rest/config/FlushCacheIT.java
index a161ec4..164f683 100644
--- a/javatests/com/google/gerrit/acceptance/rest/config/FlushCacheIT.java
+++ b/javatests/com/google/gerrit/acceptance/rest/config/FlushCacheIT.java
@@ -23,7 +23,7 @@
import com.google.gerrit.acceptance.RestResponse;
import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
import com.google.gerrit.common.data.GlobalCapability;
-import com.google.gerrit.server.restapi.config.ListCaches.CacheInfo;
+import com.google.gerrit.server.cache.CacheInfo;
import com.google.inject.Inject;
import org.junit.Test;
diff --git a/javatests/com/google/gerrit/acceptance/rest/config/GetCacheIT.java b/javatests/com/google/gerrit/acceptance/rest/config/GetCacheIT.java
index 247d63b..8765360 100644
--- a/javatests/com/google/gerrit/acceptance/rest/config/GetCacheIT.java
+++ b/javatests/com/google/gerrit/acceptance/rest/config/GetCacheIT.java
@@ -18,8 +18,7 @@
import com.google.gerrit.acceptance.AbstractDaemonTest;
import com.google.gerrit.acceptance.RestResponse;
-import com.google.gerrit.server.restapi.config.ListCaches.CacheInfo;
-import com.google.gerrit.server.restapi.config.ListCaches.CacheType;
+import com.google.gerrit.server.cache.CacheInfo;
import org.junit.Test;
public class GetCacheIT extends AbstractDaemonTest {
@@ -31,7 +30,7 @@
CacheInfo result = newGson().fromJson(r.getReader(), CacheInfo.class);
assertThat(result.name).isEqualTo("accounts");
- assertThat(result.type).isEqualTo(CacheType.MEM);
+ assertThat(result.type).isEqualTo(CacheInfo.CacheType.MEM);
assertThat(result.entries.mem).isAtLeast(1L);
assertThat(result.averageGet).isNotNull();
assertThat(result.averageGet).endsWith("s");
diff --git a/javatests/com/google/gerrit/acceptance/rest/config/ListCachesIT.java b/javatests/com/google/gerrit/acceptance/rest/config/ListCachesIT.java
index 8baeffc..be21436 100644
--- a/javatests/com/google/gerrit/acceptance/rest/config/ListCachesIT.java
+++ b/javatests/com/google/gerrit/acceptance/rest/config/ListCachesIT.java
@@ -21,8 +21,7 @@
import com.google.common.io.BaseEncoding;
import com.google.gerrit.acceptance.AbstractDaemonTest;
import com.google.gerrit.acceptance.RestResponse;
-import com.google.gerrit.server.restapi.config.ListCaches.CacheInfo;
-import com.google.gerrit.server.restapi.config.ListCaches.CacheType;
+import com.google.gerrit.server.cache.CacheInfo;
import com.google.gson.reflect.TypeToken;
import java.util.Arrays;
import java.util.List;
@@ -40,7 +39,7 @@
assertThat(result).containsKey("accounts");
CacheInfo accountsCacheInfo = result.get("accounts");
- assertThat(accountsCacheInfo.type).isEqualTo(CacheType.MEM);
+ assertThat(accountsCacheInfo.type).isEqualTo(CacheInfo.CacheType.MEM);
assertThat(accountsCacheInfo.entries.mem).isAtLeast(1L);
assertThat(accountsCacheInfo.averageGet).isNotNull();
assertThat(accountsCacheInfo.averageGet).endsWith("s");
diff --git a/javatests/com/google/gerrit/acceptance/rest/project/AccessIT.java b/javatests/com/google/gerrit/acceptance/rest/project/AccessIT.java
index 0514e03..0e7e07f 100644
--- a/javatests/com/google/gerrit/acceptance/rest/project/AccessIT.java
+++ b/javatests/com/google/gerrit/acceptance/rest/project/AccessIT.java
@@ -23,6 +23,8 @@
import static com.google.gerrit.testing.GerritJUnit.assertThrows;
import static com.google.gerrit.truth.ConfigSubject.assertThat;
import static com.google.gerrit.truth.MapSubject.assertThatMap;
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertEquals;
import com.google.gerrit.acceptance.AbstractDaemonTest;
import com.google.gerrit.acceptance.ExtensionRegistry;
@@ -62,6 +64,7 @@
import com.google.gerrit.server.schema.GrantRevertPermission;
import com.google.inject.Inject;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
import org.eclipse.jgit.junit.TestRepository;
@@ -167,7 +170,7 @@
ProjectAccessInfo expected = pApi().access();
grantRevertPermission.execute(newProjectName);
- projectCache.evict(newProjectName);
+ projectCache.evictAndReindex(newProjectName);
ProjectAccessInfo actual = pApi().access();
// Permissions don't change
assertThat(expected.local).isEqualTo(actual.local);
@@ -934,6 +937,90 @@
return gApi.projects().name(newProjectName.get());
}
+ @Test
+ public void grantAllowAndDenyForSameGroup() throws Exception {
+ GroupReference registeredUsers = systemGroupBackend.getGroup(REGISTERED_USERS);
+ String access = "access";
+ List<String> allowThenDeny =
+ asList(registeredUsers.toConfigValue(), "deny " + registeredUsers.toConfigValue());
+ // Clone repository to forcefully add permission
+ TestRepository<InMemoryRepository> allProjectsRepo = cloneProject(allProjects, admin);
+
+ // Fetch permission ref
+ GitUtil.fetch(allProjectsRepo, "refs/meta/config:cfg");
+ allProjectsRepo.reset("cfg");
+
+ // Load current permissions
+ String config =
+ gApi.projects()
+ .name(allProjects.get())
+ .branch(RefNames.REFS_CONFIG)
+ .file(ProjectConfig.PROJECT_CONFIG)
+ .asString();
+
+ // Append and push allowThenDeny permissions
+ Config cfg = new Config();
+ cfg.fromText(config);
+ cfg.setStringList(access, AccessSection.HEADS, Permission.READ, allowThenDeny);
+ config = cfg.toText();
+ PushOneCommit push =
+ pushFactory.create(
+ admin.newIdent(), allProjectsRepo, "Subject", ProjectConfig.PROJECT_CONFIG, config);
+ push.to(RefNames.REFS_CONFIG).assertOkStatus();
+
+ ProjectAccessInfo pai = gApi.projects().name(allProjects.get()).access();
+ Map<String, AccessSectionInfo> local = pai.local;
+ AccessSectionInfo heads = local.get(AccessSection.HEADS);
+ Map<String, PermissionInfo> permissions = heads.permissions;
+ PermissionInfo read = permissions.get(Permission.READ);
+ Map<String, PermissionRuleInfo> rules = read.rules;
+ assertEquals(
+ rules.get(registeredUsers.getUUID().get()),
+ new PermissionRuleInfo(PermissionRuleInfo.Action.ALLOW, false));
+ }
+
+ @Test
+ public void grantDenyAndAllowForSameGroup() throws Exception {
+ GroupReference registeredUsers = systemGroupBackend.getGroup(REGISTERED_USERS);
+ String access = "access";
+ List<String> denyThenAllow =
+ asList("deny " + registeredUsers.toConfigValue(), registeredUsers.toConfigValue());
+ // Clone repository to forcefully add permission
+ TestRepository<InMemoryRepository> allProjectsRepo = cloneProject(allProjects, admin);
+
+ // Fetch permission ref
+ GitUtil.fetch(allProjectsRepo, "refs/meta/config:cfg");
+ allProjectsRepo.reset("cfg");
+
+ // Load current permissions
+ String config =
+ gApi.projects()
+ .name(allProjects.get())
+ .branch(RefNames.REFS_CONFIG)
+ .file(ProjectConfig.PROJECT_CONFIG)
+ .asString();
+
+ // Append and push denyThenAllow permissions
+ Config cfg = new Config();
+ cfg.fromText(config);
+ cfg.setStringList(access, AccessSection.HEADS, Permission.READ, denyThenAllow);
+ config = cfg.toText();
+ PushOneCommit push =
+ pushFactory.create(
+ admin.newIdent(), allProjectsRepo, "Subject", ProjectConfig.PROJECT_CONFIG, config);
+ push.to(RefNames.REFS_CONFIG).assertOkStatus();
+
+ ProjectAccessInfo pai = gApi.projects().name(allProjects.get()).access();
+ Map<String, AccessSectionInfo> local = pai.local;
+ AccessSectionInfo heads = local.get(AccessSection.HEADS);
+ Map<String, PermissionInfo> permissions = heads.permissions;
+ PermissionInfo read = permissions.get(Permission.READ);
+ Map<String, PermissionRuleInfo> rules = read.rules;
+ assertEquals(
+ rules.get(registeredUsers.getUUID().get()),
+ new PermissionRuleInfo(PermissionRuleInfo.Action.DENY, false));
+ }
+
private ProjectAccessInput newProjectAccessInput() {
ProjectAccessInput p = new ProjectAccessInput();
p.add = new HashMap<>();
diff --git a/javatests/com/google/gerrit/acceptance/ssh/DynamicOptionsBeanParseListenerIT.java b/javatests/com/google/gerrit/acceptance/ssh/DynamicOptionsBeanParseListenerIT.java
new file mode 100644
index 0000000..0afdbc6
--- /dev/null
+++ b/javatests/com/google/gerrit/acceptance/ssh/DynamicOptionsBeanParseListenerIT.java
@@ -0,0 +1,69 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.acceptance.ssh;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.gerrit.acceptance.AbstractDaemonTest;
+import com.google.gerrit.acceptance.NoHttpd;
+import com.google.gerrit.acceptance.UseSsh;
+import com.google.gerrit.extensions.annotations.Exports;
+import com.google.gerrit.server.DynamicOptions;
+import com.google.gerrit.sshd.commands.ListProjectsCommand;
+import com.google.inject.AbstractModule;
+import java.util.Arrays;
+import java.util.List;
+import org.junit.Test;
+
+@NoHttpd
+@UseSsh
+public class DynamicOptionsBeanParseListenerIT extends AbstractDaemonTest {
+
+ @Test
+ public void testBeanParseListener() throws Exception {
+ createProjectOverAPI("project1", project, true, null);
+ createProjectOverAPI("project2", project, true, null);
+ try (AutoCloseable ignored = installPlugin("my-plugin", PluginModule.class)) {
+ String output = adminSshSession.exec("gerrit ls-projects");
+ adminSshSession.assertSuccess();
+ assertThat(getProjects(output)).hasSize(1);
+ }
+ }
+
+ protected List<String> getProjects(String sshOutput) {
+ return Arrays.asList(sshOutput.split("\n"));
+ }
+
+ protected static class ListProjectsCommandBeanListener
+ implements DynamicOptions.BeanParseListener {
+ @Override
+ public void onBeanParseStart(String plugin, Object bean) {
+ ListProjectsCommand command = (ListProjectsCommand) bean;
+ command.impl.setLimit(1);
+ }
+
+ @Override
+ public void onBeanParseEnd(String plugin, Object bean) {}
+ }
+
+ protected static class PluginModule extends AbstractModule {
+ @Override
+ public void configure() {
+ bind(DynamicOptions.DynamicBean.class)
+ .annotatedWith(Exports.named(ListProjectsCommand.class))
+ .to(ListProjectsCommandBeanListener.class);
+ }
+ }
+}
diff --git a/javatests/com/google/gerrit/integration/git/GitProtocolV2IT.java b/javatests/com/google/gerrit/integration/git/GitProtocolV2IT.java
index 8577c16..6c4c53d 100644
--- a/javatests/com/google/gerrit/integration/git/GitProtocolV2IT.java
+++ b/javatests/com/google/gerrit/integration/git/GitProtocolV2IT.java
@@ -357,7 +357,7 @@
}
private static void assertGitProtocolV2Refs(String commit, String out) {
- assertThat(out).contains("git< version 2");
+ assertThat(out).containsMatch("(git|ls-remote)< version 2");
assertThat(out).contains("refs/changes/01/1/1");
assertThat(out).contains("refs/changes/01/1/meta");
assertThat(out).contains(commit);
diff --git a/javatests/com/google/gerrit/server/change/LabelNormalizerTest.java b/javatests/com/google/gerrit/server/change/LabelNormalizerTest.java
index c259e60..39914b9 100644
--- a/javatests/com/google/gerrit/server/change/LabelNormalizerTest.java
+++ b/javatests/com/google/gerrit/server/change/LabelNormalizerTest.java
@@ -186,7 +186,7 @@
private void save(ProjectConfig pc) throws Exception {
try (MetaDataUpdate md = metaDataUpdateFactory.create(pc.getProject().getNameKey(), user)) {
pc.commit(md);
- projectCache.evict(pc.getProject().getNameKey());
+ projectCache.evictAndReindex(pc.getProject().getNameKey());
}
}
diff --git a/javatests/com/google/gerrit/server/events/EventDeserializerTest.java b/javatests/com/google/gerrit/server/events/EventDeserializerTest.java
index e0223e4..97f6e4e 100644
--- a/javatests/com/google/gerrit/server/events/EventDeserializerTest.java
+++ b/javatests/com/google/gerrit/server/events/EventDeserializerTest.java
@@ -22,6 +22,8 @@
import com.google.gerrit.entities.BranchNameKey;
import com.google.gerrit.entities.Change;
import com.google.gerrit.entities.Project;
+import com.google.gerrit.server.config.AllProjectsName;
+import com.google.gerrit.server.config.AllUsersName;
import com.google.gerrit.server.data.AccountAttribute;
import com.google.gerrit.server.data.ChangeAttribute;
import com.google.gerrit.server.data.RefUpdateAttribute;
@@ -240,6 +242,31 @@
assertSameChangeEvent(e, orig);
}
+ @Test
+ public void shouldSerializeAllProjectsToString() {
+ String allProjectsString = "foobar";
+ AllProjectsName allProjectsNameKey = new AllProjectsName(allProjectsString);
+
+ assertThat(gson.toJson(allProjectsNameKey))
+ .isEqualTo(String.format("\"%s\"", allProjectsString));
+ }
+
+ @Test
+ public void shouldSerializeAllUsersToString() {
+ String allUsersString = "foobar";
+ AllUsersName allUsersNameKey = new AllUsersName(allUsersString);
+
+ assertThat(gson.toJson(allUsersNameKey)).isEqualTo(String.format("\"%s\"", allUsersString));
+ }
+
+ @Test
+ public void shouldSerializeProjectNameKeyToString() {
+ String projectString = "foobar";
+ Project.NameKey projectNameKey = Project.nameKey(projectString);
+
+ assertThat(gson.toJson(projectNameKey)).isEqualTo(String.format("\"%s\"", projectString));
+ }
+
private <T> Supplier<T> createSupplier(T value) {
return Suppliers.memoize(() -> value);
}
diff --git a/javatests/com/google/gerrit/server/index/change/FakeChangeIndex.java b/javatests/com/google/gerrit/server/index/change/FakeChangeIndex.java
index a23ccab..fc56a3c 100644
--- a/javatests/com/google/gerrit/server/index/change/FakeChangeIndex.java
+++ b/javatests/com/google/gerrit/server/index/change/FakeChangeIndex.java
@@ -74,6 +74,11 @@
}
@Override
+ public void insert(ChangeData obj) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public void replace(ChangeData cd) {
throw new UnsupportedOperationException();
}
diff --git a/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java b/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java
index 1aa0f35..8113f64 100644
--- a/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java
+++ b/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java
@@ -1029,7 +1029,7 @@
cfg.getLabelSections().put(verified.getName(), verified);
cfg.commit(md);
}
- projectCache.evict(project);
+ projectCache.evictAndReindex(project);
String heads = RefNames.REFS_HEADS + "*";
projectOperations
@@ -1865,7 +1865,7 @@
rule.setForce(force);
p.add(rule);
config.commit(md);
- projectCache.evict(config.getProject());
+ projectCache.evictAndReindex(config.getProject());
}
}
diff --git a/plugins/delete-project b/plugins/delete-project
index 76f5b25..407c15e 160000
--- a/plugins/delete-project
+++ b/plugins/delete-project
@@ -1 +1 @@
-Subproject commit 76f5b2573343d1b565477678a58641697003248a
+Subproject commit 407c15e7949ad5a086ba5c8998be68964ff17de5
diff --git a/plugins/gitiles b/plugins/gitiles
index 32012fb..d5fba00 160000
--- a/plugins/gitiles
+++ b/plugins/gitiles
@@ -1 +1 @@
-Subproject commit 32012fb81fd1e047d30a7f9092c12ce1c40a7aa6
+Subproject commit d5fba00a6d1e233434eaef38927f8a8b58bc830a
diff --git a/plugins/replication b/plugins/replication
index cc72dc5..d9f7c93 160000
--- a/plugins/replication
+++ b/plugins/replication
@@ -1 +1 @@
-Subproject commit cc72dc5cb3e4198ebd4f487a3fcc04846e1ceb43
+Subproject commit d9f7c9387119e67bef20fe795975763f3bf50be9
diff --git a/plugins/webhooks b/plugins/webhooks
index 9fc9c2d..76b829e 160000
--- a/plugins/webhooks
+++ b/plugins/webhooks
@@ -1 +1 @@
-Subproject commit 9fc9c2d4e69f7e2701cbcd873977d3312a231a81
+Subproject commit 76b829e09dcab780462652cb33c882a6d0074dac
diff --git a/polygerrit-ui/app/behaviors/keyboard-shortcut-behavior/keyboard-shortcut-behavior.js b/polygerrit-ui/app/behaviors/keyboard-shortcut-behavior/keyboard-shortcut-behavior.js
index cb21a9f..5d9c74f 100644
--- a/polygerrit-ui/app/behaviors/keyboard-shortcut-behavior/keyboard-shortcut-behavior.js
+++ b/polygerrit-ui/app/behaviors/keyboard-shortcut-behavior/keyboard-shortcut-behavior.js
@@ -523,8 +523,9 @@
// Suppress shortcuts if the key is 'enter' and target is an anchor.
return true;
}
- for (let i = 0; e.path && i < e.path.length; i++) {
- if (e.path[i].tagName === 'GR-OVERLAY') { return true; }
+ const path = e.composedPath();
+ for (let i = 0; path && i < path.length; i++) {
+ if (path[i].tagName === 'GR-OVERLAY') { return true; }
}
this.dispatchEvent(new CustomEvent('shortcut-triggered', {
diff --git a/polygerrit-ui/app/elements/admin/gr-access-section/gr-access-section.js b/polygerrit-ui/app/elements/admin/gr-access-section/gr-access-section.js
index e07a64e..2865f0d 100644
--- a/polygerrit-ui/app/elements/admin/gr-access-section/gr-access-section.js
+++ b/polygerrit-ui/app/elements/admin/gr-access-section/gr-access-section.js
@@ -64,6 +64,7 @@
static get properties() {
return {
+ repo: String,
capabilities: Object,
/** @type {?} */
section: {
diff --git a/polygerrit-ui/app/elements/admin/gr-access-section/gr-access-section_html.js b/polygerrit-ui/app/elements/admin/gr-access-section/gr-access-section_html.js
index c46cf30..5be78df 100644
--- a/polygerrit-ui/app/elements/admin/gr-access-section/gr-access-section_html.js
+++ b/polygerrit-ui/app/elements/admin/gr-access-section/gr-access-section_html.js
@@ -120,6 +120,7 @@
section="[[section.id]]"
editing="[[editing]]"
groups="[[groups]]"
+ repo="[[repo]]"
on-added-permission-removed="_handleAddedPermissionRemoved"
>
</gr-permission>
diff --git a/polygerrit-ui/app/elements/admin/gr-permission/gr-permission.js b/polygerrit-ui/app/elements/admin/gr-permission/gr-permission.js
index ea4e05c..3f577ae 100644
--- a/polygerrit-ui/app/elements/admin/gr-permission/gr-permission.js
+++ b/polygerrit-ui/app/elements/admin/gr-permission/gr-permission.js
@@ -61,6 +61,7 @@
static get properties() {
return {
+ repo: String,
labels: Object,
name: String,
/** @type {?} */
@@ -254,6 +255,7 @@
_getGroupSuggestions() {
return this.$.restAPI.getSuggestedGroups(
this._groupFilter,
+ this.repo,
MAX_AUTOCOMPLETE_RESULTS)
.then(response => {
const groups = [];
diff --git a/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor.js b/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor.js
index 318c2c3..0e148a3 100644
--- a/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor.js
+++ b/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor.js
@@ -42,12 +42,13 @@
static get properties() {
return {
- /** @type {?} */
+ /** @type {?} */
pluginOption: Object,
/** @type {boolean} */
disabled: {
type: Boolean,
- computed: '_computeDisabled(pluginOption.*)',
+ value: false,
+ reflectToAttribute: true,
},
/** @type {?} */
_newValue: {
@@ -57,11 +58,6 @@
};
}
- _computeDisabled(record) {
- return !(record && record.base && record.base.info &&
- record.base.info.editable);
- }
-
_handleAddTap(e) {
e.preventDefault();
this._handleAdd();
diff --git a/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor_html.js b/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor_html.js
index be35035..7a8da94 100644
--- a/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor_html.js
+++ b/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor_html.js
@@ -85,6 +85,7 @@
id="input"
on-keydown="_handleInputKeydown"
bind-value="{{_newValue}}"
+ disabled$="[[disabled]]"
/>
</iron-input>
<gr-button
diff --git a/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor_test.html b/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor_test.html
index 5eff42d..42f1d2a 100644
--- a/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor_test.html
+++ b/polygerrit-ui/app/elements/admin/gr-plugin-config-array-editor/gr-plugin-config-array-editor_test.html
@@ -60,16 +60,6 @@
assert.equal(element._computeShowInputRow(false), '');
});
- test('_computeDisabled', () => {
- assert.isTrue(element._computeDisabled({}));
- assert.isTrue(element._computeDisabled({base: {}}));
- assert.isTrue(element._computeDisabled({base: {info: {}}}));
- assert.isTrue(
- element._computeDisabled({base: {info: {editable: false}}}));
- assert.isFalse(
- element._computeDisabled({base: {info: {editable: true}}}));
- });
-
suite('adding', () => {
setup(() => {
dispatchStub = sandbox.stub(element, '_dispatchChanged');
@@ -78,11 +68,13 @@
test('with enter', () => {
element._newValue = '';
MockInteractions.pressAndReleaseKeyOn(element.$.input, 13); // Enter
+ assert.isFalse(element.$.input.hasAttribute('disabled'));
flushAsynchronousOperations();
assert.isFalse(dispatchStub.called);
element._newValue = 'test';
MockInteractions.pressAndReleaseKeyOn(element.$.input, 13); // Enter
+ assert.isFalse(element.$.input.hasAttribute('disabled'));
flushAsynchronousOperations();
assert.isTrue(dispatchStub.called);
@@ -109,6 +101,7 @@
test('deleting', () => {
dispatchStub = sandbox.stub(element, '_dispatchChanged');
element.pluginOption = {info: {values: ['test', 'test2']}};
+ element.disabled = true;
flushAsynchronousOperations();
const rows = getAll('.existingItems .row');
@@ -119,7 +112,7 @@
flushAsynchronousOperations();
assert.isFalse(dispatchStub.called);
- element.pluginOption.info.editable = true;
+ element.disabled = false;
element.notifyPath('pluginOption.info.editable');
flushAsynchronousOperations();
diff --git a/polygerrit-ui/app/elements/admin/gr-repo-access/gr-repo-access_html.js b/polygerrit-ui/app/elements/admin/gr-repo-access/gr-repo-access_html.js
index 5f0739a..cf77233 100644
--- a/polygerrit-ui/app/elements/admin/gr-repo-access/gr-repo-access_html.js
+++ b/polygerrit-ui/app/elements/admin/gr-repo-access/gr-repo-access_html.js
@@ -125,6 +125,7 @@
editing="[[_editing]]"
owner-of="[[_ownerOf]]"
groups="[[_groups]]"
+ repo="[[repo]]"
on-added-section-removed="_handleAddedSectionRemoved"
></gr-access-section>
</template>
diff --git a/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config.js b/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config.js
index dcabdfc..c419a53 100644
--- a/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config.js
+++ b/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config.js
@@ -59,6 +59,11 @@
type: Array,
computed: '_computePluginConfigOptions(pluginData.*)',
},
+ disabled: {
+ type: Boolean,
+ value: false,
+ reflectToAttribute: true,
+ },
};
}
@@ -90,8 +95,8 @@
type === this.ENTRY_TYPES.LONG;
}
- _computeDisabled(editable) {
- return !editable;
+ _computeDisabled(disabled, editable) {
+ return disabled || !editable;
}
/**
diff --git a/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config_html.js b/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config_html.js
index ee633463..937e67c 100644
--- a/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config_html.js
+++ b/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config_html.js
@@ -55,6 +55,7 @@
<gr-plugin-config-array-editor
on-plugin-config-option-changed="_handleArrayChange"
plugin-option="[[option]]"
+ disabled$="[[_computeDisabled(disabled, option.info.editable)]]"
></gr-plugin-config-array-editor>
</template>
<template is="dom-if" if="[[_isBoolean(option.info.type)]]">
@@ -62,7 +63,7 @@
checked="[[_computeChecked(option.info.value)]]"
on-change="_handleBooleanChange"
data-option-key$="[[option._key]]"
- disabled$="[[_computeDisabled(option.info.editable)]]"
+ disabled$="[[_computeDisabled(disabled, option.info.editable)]]"
on-tap="_onTapPluginBoolean"
></paper-toggle-button>
</template>
@@ -73,7 +74,7 @@
>
<select
data-option-key$="[[option._key]]"
- disabled$="[[_computeDisabled(option.info.editable)]]"
+ disabled$="[[_computeDisabled(disabled, option.info.editable)]]"
>
<template
is="dom-repeat"
@@ -90,14 +91,14 @@
bind-value="[[option.info.value]]"
on-input="_handleStringChange"
data-option-key$="[[option._key]]"
- disabled$="[[_computeDisabled(option.info.editable)]]"
+ disabled$="[[_computeDisabled(disabled, option.info.editable)]]"
>
<input
is="iron-input"
value="[[option.info.value]]"
on-input="_handleStringChange"
data-option-key$="[[option._key]]"
- disabled$="[[_computeDisabled(option.info.editable)]]"
+ disabled$="[[_computeDisabled(disabled, option.info.editable)]]"
/>
</iron-input>
</template>
diff --git a/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config_test.html b/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config_test.html
index b124d9b..121d029 100644
--- a/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config_test.html
+++ b/polygerrit-ui/app/elements/admin/gr-repo-plugin-config/gr-repo-plugin-config_test.html
@@ -57,10 +57,11 @@
});
test('_computeDisabled', () => {
- assert.isFalse(element._computeDisabled(true));
- assert.isTrue(element._computeDisabled(undefined));
- assert.isTrue(element._computeDisabled(null));
- assert.isTrue(element._computeDisabled(false));
+ assert.isFalse(element._computeDisabled(false, true));
+ assert.isTrue(element._computeDisabled(false, undefined));
+ assert.isTrue(element._computeDisabled(false, null));
+ assert.isTrue(element._computeDisabled(false, false));
+ assert.isTrue(element._computeDisabled(true, true));
});
test('_handleChange', () => {
diff --git a/polygerrit-ui/app/elements/admin/gr-repo/gr-repo_html.js b/polygerrit-ui/app/elements/admin/gr-repo/gr-repo_html.js
index de36e73..4c0833f 100644
--- a/polygerrit-ui/app/elements/admin/gr-repo/gr-repo_html.js
+++ b/polygerrit-ui/app/elements/admin/gr-repo/gr-repo_html.js
@@ -411,6 +411,7 @@
<template is="dom-repeat" items="[[_pluginData]]" as="data">
<gr-repo-plugin-config
plugin-data="[[data]]"
+ disabled$="[[_readOnly]]"
></gr-repo-plugin-config>
</template>
</div>
diff --git a/polygerrit-ui/app/elements/core/gr-smart-search/gr-smart-search.js b/polygerrit-ui/app/elements/core/gr-smart-search/gr-smart-search.js
index dcece30..c1b2e42 100644
--- a/polygerrit-ui/app/elements/core/gr-smart-search/gr-smart-search.js
+++ b/polygerrit-ui/app/elements/core/gr-smart-search/gr-smart-search.js
@@ -117,6 +117,7 @@
if (expression.length === 0) { return Promise.resolve([]); }
return this.$.restAPI.getSuggestedGroups(
expression,
+ undefined,
MAX_AUTOCOMPLETE_RESULTS)
.then(groups => {
if (!groups) { return []; }
diff --git a/polygerrit-ui/app/elements/shared/gr-rest-api-interface/gr-rest-api-interface.js b/polygerrit-ui/app/elements/shared/gr-rest-api-interface/gr-rest-api-interface.js
index 72c8058..c10e594 100644
--- a/polygerrit-ui/app/elements/shared/gr-rest-api-interface/gr-rest-api-interface.js
+++ b/polygerrit-ui/app/elements/shared/gr-rest-api-interface/gr-rest-api-interface.js
@@ -1495,12 +1495,14 @@
/**
* @param {string} inputVal
+ * @param{string} opt_p
* @param {number} opt_n
* @param {function(?Response, string=)=} opt_errFn
*/
- getSuggestedGroups(inputVal, opt_n, opt_errFn) {
+ getSuggestedGroups(inputVal, opt_p, opt_n, opt_errFn) {
const params = {s: inputVal};
if (opt_n) { params.n = opt_n; }
+ if (opt_p) { params.p = opt_p; }
return this._restApiHelper.fetchJSON({
url: '/groups/',
errFn: opt_errFn,
diff --git a/resources/com/google/gerrit/httpd/auth/container/LoginRedirect.html b/resources/com/google/gerrit/httpd/auth/container/LoginRedirect.html
index 0567468..54c3661 100644
--- a/resources/com/google/gerrit/httpd/auth/container/LoginRedirect.html
+++ b/resources/com/google/gerrit/httpd/auth/container/LoginRedirect.html
@@ -4,6 +4,12 @@
<title>Gerrit Code Review</title>
<script type="text/javascript">
var href = window.location.href;
+ var query = "";
+ var q = href.indexOf('?');
+ if (q >= 0) {
+ query = href.substring(q);
+ href = href.substring(0,q);
+ }
var p = href.indexOf('#');
var token;
if (p >= 0) {
@@ -12,7 +18,7 @@
} else {
token = '';
}
- window.location.replace(href + 'login/' + token);
+ window.location.replace(href + 'login/' + token + query);
</script>
</head>
<body>
diff --git a/tools/bzl/license-map.py b/tools/bzl/license-map.py
index daa8582..022f209 100644
--- a/tools/bzl/license-map.py
+++ b/tools/bzl/license-map.py
@@ -24,7 +24,7 @@
def read_file(filename):
"Reads file and returns its content"
- with open(filename) as fd:
+ with open(filename, encoding='utf-8') as fd:
return fd.read()
# List of files in package to which license is applied.
@@ -56,6 +56,8 @@
LicenseMapItem = namedtuple("LicenseMapItem",
["name", "safename", "packages", "license_text"])
+def print_utf8(str=""):
+ stdout.buffer.write((str + "\n").encode('utf-8'))
def load_xmls(xml_filenames):
"""Load xml files produced by bazel query
@@ -134,7 +136,7 @@
if args.asciidoctor:
# We don't want any blank line before "= Gerrit Code Review - Licenses"
- print("""= Gerrit Code Review - Licenses
+ print_utf8("""= Gerrit Code Review - Licenses
// DO NOT EDIT - GENERATED AUTOMATICALLY.
@@ -178,10 +180,10 @@
for data in xml_data + json_map_data:
name = data.name
safename = data.safename
- print()
- print("[[%s]]" % safename)
- print(name)
- print()
+ print_utf8()
+ print_utf8("[[%s]]" % safename)
+ print_utf8(name)
+ print_utf8()
for p in data.packages:
package_notice = ""
if p.licensed_files.kind == "OnlySpecificFiles":
@@ -189,20 +191,20 @@
elif p.licensed_files.kind == "AllFilesExceptSpecific":
package_notice = " - except the following file(s):"
- print("* " + get_package_display_name(p) + package_notice)
+ print_utf8("* " + get_package_display_name(p) + package_notice)
for file in p.licensed_files.files:
- print("** " + file)
- print()
- print("[[%s_license]]" % safename)
- print("----")
+ print_utf8("** " + file)
+ print_utf8()
+ print_utf8("[[%s_license]]" % safename)
+ print_utf8("----")
license_text = data.license_text
- print(data.license_text.rstrip("\r\n"))
- print()
- print("----")
- print()
+ print_utf8(data.license_text.rstrip("\r\n"))
+ print_utf8()
+ print_utf8("----")
+ print_utf8()
if args.asciidoctor:
- print("""
+ print_utf8("""
GERRIT
------
Part of link:index.html[Gerrit Code Review]
@@ -219,7 +221,7 @@
"""
result = []
for json_map in json_filenames:
- with open(json_map, 'r') as f:
+ with open(json_map, 'r', encoding='utf-8') as f:
licenses_list = json.load(f)
for license_id, license in licenses_list.items():
name = license["licenseName"]
diff --git a/tools/maven/gerrit-acceptance-framework_pom.xml b/tools/maven/gerrit-acceptance-framework_pom.xml
index 8efba2e..25bb226 100644
--- a/tools/maven/gerrit-acceptance-framework_pom.xml
+++ b/tools/maven/gerrit-acceptance-framework_pom.xml
@@ -2,7 +2,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.google.gerrit</groupId>
<artifactId>gerrit-acceptance-framework</artifactId>
- <version>3.2.13-SNAPSHOT</version>
+ <version>3.2.15-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Gerrit Code Review - Acceptance Test Framework</name>
<description>Framework for Gerrit's acceptance tests</description>
diff --git a/tools/maven/gerrit-extension-api_pom.xml b/tools/maven/gerrit-extension-api_pom.xml
index 63df76e..b930f41 100644
--- a/tools/maven/gerrit-extension-api_pom.xml
+++ b/tools/maven/gerrit-extension-api_pom.xml
@@ -2,7 +2,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.google.gerrit</groupId>
<artifactId>gerrit-extension-api</artifactId>
- <version>3.2.13-SNAPSHOT</version>
+ <version>3.2.15-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Gerrit Code Review - Extension API</name>
<description>API for Gerrit Extensions</description>
diff --git a/tools/maven/gerrit-plugin-api_pom.xml b/tools/maven/gerrit-plugin-api_pom.xml
index e845d9f..d3fe4aa9 100644
--- a/tools/maven/gerrit-plugin-api_pom.xml
+++ b/tools/maven/gerrit-plugin-api_pom.xml
@@ -2,7 +2,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.google.gerrit</groupId>
<artifactId>gerrit-plugin-api</artifactId>
- <version>3.2.13-SNAPSHOT</version>
+ <version>3.2.15-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Gerrit Code Review - Plugin API</name>
<description>API for Gerrit Plugins</description>
diff --git a/tools/maven/gerrit-war_pom.xml b/tools/maven/gerrit-war_pom.xml
index b66903c..1afd6de 100644
--- a/tools/maven/gerrit-war_pom.xml
+++ b/tools/maven/gerrit-war_pom.xml
@@ -2,7 +2,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.google.gerrit</groupId>
<artifactId>gerrit-war</artifactId>
- <version>3.2.13-SNAPSHOT</version>
+ <version>3.2.15-SNAPSHOT</version>
<packaging>war</packaging>
<name>Gerrit Code Review - WAR</name>
<description>Gerrit WAR</description>
diff --git a/version.bzl b/version.bzl
index 6fa7b9e..c003386 100644
--- a/version.bzl
+++ b/version.bzl
@@ -2,4 +2,4 @@
# Used by :api_install and :api_deploy targets
# when talking to the destination repository.
#
-GERRIT_VERSION = "3.2.13-SNAPSHOT"
+GERRIT_VERSION = "3.2.15-SNAPSHOT"
