Merge branch 'stable-3.1' into stable-3.2 * stable-3.1: Bazel: Consume bazel-toolchains release from bazel mirror Align the recommended buildifier version with CI's crafting-changes: Add some info on Optional/Nullable Bump bazel-toolchains to 5.0.0 Set version to 2.16.28-SNAPSHOT Set version to 2.16.27 ForRef#check should permit internal users to read all refs Update git submodules Update highlightjs to 10.6.0 Use NoteDb sequence seed for groups sequence initialization GroupsOnInit: Remove unused dependency on ReviewDb Use NoteDb sequence seed for accounts sequence initialization Do not use ReviewDb anymore when is disabled Elasticsearch: Discontinue EOL versions 7.2 and 7.3 support e2e-tests: Add SubmitChangeInBranch scenario Update developers based on gerritcodereview's list ForRef#check should permit internal users to read all refs Change-Id: I72f7184e0de6ed8225c135ec21c8a83c9d211009
diff --git a/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java b/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java index ed32ce5..e56f470 100644 --- a/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java +++ b/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
@@ -26,6 +26,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Streams; import com.google.common.flogger.FluentLogger; +import com.google.common.io.BaseEncoding; import com.google.common.io.CharStreams; import com.google.gerrit.common.Nullable; import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties; @@ -68,7 +69,6 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; -import org.apache.commons.codec.binary.Base64; import org.apache.http.HttpEntity; import org.apache.http.HttpStatus; import org.apache.http.StatusLine; @@ -88,7 +88,7 @@ protected static final String SETTINGS = "settings"; protected static byte[] decodeBase64(String base64String) { - return Base64.decodeBase64(base64String); + return BaseEncoding.base64().decode(base64String); } protected static <T> List<T> decodeProtos( @@ -248,7 +248,7 @@ } else if (type == FieldType.TIMESTAMP) { rawFields.put(element.getKey(), new Timestamp(inner.getAsLong())); } else if (type == FieldType.STORED_ONLY) { - rawFields.put(element.getKey(), Base64.decodeBase64(inner.getAsString())); + rawFields.put(element.getKey(), decodeBase64(inner.getAsString())); } else { throw FieldType.badFieldType(type); }
diff --git a/java/com/google/gerrit/elasticsearch/BUILD b/java/com/google/gerrit/elasticsearch/BUILD index edbd82c..8bab80b 100644 --- a/java/com/google/gerrit/elasticsearch/BUILD +++ b/java/com/google/gerrit/elasticsearch/BUILD
@@ -19,7 +19,6 @@ "//lib:guava", "//lib:jgit", "//lib:protobuf", - "//lib/commons:codec", "//lib/commons:lang", "//lib/elasticsearch-rest-client", "//lib/flogger:api",
diff --git a/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java b/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java index bde3ad5..8967789 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java +++ b/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
@@ -14,6 +14,7 @@ package com.google.gerrit.elasticsearch; +import com.google.common.collect.ImmutableSet; import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties; import com.google.gerrit.elasticsearch.bulk.BulkRequest; import com.google.gerrit.elasticsearch.bulk.IndexRequest; @@ -72,7 +73,9 @@ @Override public void replace(AccountState as) { - BulkRequest bulk = new IndexRequest(getId(as), indexName).add(new UpdateRequest<>(schema, as)); + BulkRequest bulk = + new IndexRequest(getId(as), indexName) + .add(new UpdateRequest<>(schema, as, ImmutableSet.of())); String uri = getURI(BULK); Response response = postRequest(uri, bulk, getRefreshParam());
diff --git a/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java b/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java index 2be1585..625a598 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java +++ b/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
@@ -19,6 +19,7 @@ import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.ListMultimap; import com.google.common.collect.MultimapBuilder; @@ -43,6 +44,8 @@ import com.google.gerrit.server.ReviewerByEmailSet; import com.google.gerrit.server.ReviewerSet; import com.google.gerrit.server.StarredChangesUtil; +import com.google.gerrit.server.change.MergeabilityComputationBehavior; +import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.index.IndexUtils; import com.google.gerrit.server.index.change.ChangeField; @@ -58,6 +61,7 @@ import java.util.Optional; import java.util.Set; import org.apache.http.HttpStatus; +import org.eclipse.jgit.lib.Config; import org.elasticsearch.client.Response; /** Secondary index implementation using Elasticsearch. */ @@ -82,6 +86,7 @@ private final ChangeData.Factory changeDataFactory; private final Schema<ChangeData> schema; private final FieldDef<ChangeData, ?> idField; + private final ImmutableSet<String> skipFields; @Inject ElasticChangeIndex( @@ -89,6 +94,7 @@ ChangeData.Factory changeDataFactory, SitePaths sitePaths, ElasticRestClientProvider clientBuilder, + @GerritServerConfig Config gerritConfig, @Assisted Schema<ChangeData> schema) { super(cfg, sitePaths, schema, clientBuilder, CHANGES); this.changeDataFactory = changeDataFactory; @@ -96,11 +102,16 @@ this.mapping = new ChangeMapping(schema, client.adapter()); this.idField = this.schema.useLegacyNumericFields() ? ChangeField.LEGACY_ID : ChangeField.LEGACY_ID_STR; + this.skipFields = + MergeabilityComputationBehavior.fromConfig(gerritConfig).includeInIndex() + ? ImmutableSet.of() + : ImmutableSet.of(ChangeField.MERGEABLE.getName()); } @Override public void replace(ChangeData cd) { - BulkRequest bulk = new IndexRequest(getId(cd), indexName).add(new UpdateRequest<>(schema, cd)); + BulkRequest bulk = + new IndexRequest(getId(cd), indexName).add(new UpdateRequest<>(schema, cd, skipFields)); String uri = getURI(BULK); Response response = postRequest(uri, bulk, getRefreshParam()); @@ -209,7 +220,7 @@ // Mergeable. JsonElement mergeableElement = source.get(ChangeField.MERGEABLE.getName()); - if (mergeableElement != null) { + if (mergeableElement != null && !skipFields.contains(ChangeField.MERGEABLE.getName())) { String mergeable = mergeableElement.getAsString(); if ("1".equals(mergeable)) { cd.setMergeable(true); @@ -341,6 +352,15 @@ // Unresolved-comment-count. decodeUnresolvedCommentCount(source, ChangeField.UNRESOLVED_COMMENT_COUNT.getName(), cd); + // Attention set. + if (fields.contains(ChangeField.ATTENTION_SET_FULL.getName())) { + ChangeField.parseAttentionSet( + FluentIterable.from(source.getAsJsonArray(ChangeField.ATTENTION_SET_FULL.getName())) + .transform(ElasticChangeIndex::decodeBase64JsonElement) + .toSet(), + cd); + } + return cd; } @@ -359,12 +379,16 @@ } ChangeField.parseSubmitRecords( FluentIterable.from(records) - .transform(i -> new String(decodeBase64(i.getAsString()), UTF_8)) + .transform(ElasticChangeIndex::decodeBase64JsonElement) .toList(), opts, out); } + private static String decodeBase64JsonElement(JsonElement input) { + return new String(decodeBase64(input.getAsString()), UTF_8); + } + private void decodeUnresolvedCommentCount(JsonObject doc, String fieldName, ChangeData out) { JsonElement count = doc.get(fieldName); if (count == null) {
diff --git a/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java b/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java index 241e7fd..f8c2ec5 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java +++ b/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
@@ -14,6 +14,7 @@ package com.google.gerrit.elasticsearch; +import com.google.common.collect.ImmutableSet; import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties; import com.google.gerrit.elasticsearch.bulk.BulkRequest; import com.google.gerrit.elasticsearch.bulk.IndexRequest; @@ -73,7 +74,8 @@ @Override public void replace(InternalGroup group) { BulkRequest bulk = - new IndexRequest(getId(group), indexName).add(new UpdateRequest<>(schema, group)); + new IndexRequest(getId(group), indexName) + .add(new UpdateRequest<>(schema, group, ImmutableSet.of())); String uri = getURI(BULK); Response response = postRequest(uri, bulk, getRefreshParam());
diff --git a/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java b/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java index 36eeca1..b8bfc38 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java +++ b/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java
@@ -14,6 +14,7 @@ package com.google.gerrit.elasticsearch; +import com.google.common.collect.ImmutableSet; import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties; import com.google.gerrit.elasticsearch.bulk.BulkRequest; import com.google.gerrit.elasticsearch.bulk.IndexRequest; @@ -31,12 +32,14 @@ import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.index.IndexUtils; import com.google.gerrit.server.project.ProjectCache; +import com.google.gerrit.server.project.ProjectState; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.assistedinject.Assisted; +import java.util.Optional; import java.util.Set; import org.apache.http.HttpStatus; import org.elasticsearch.client.Response; @@ -74,7 +77,7 @@ public void replace(ProjectData projectState) { BulkRequest bulk = new IndexRequest(projectState.getProject().getName(), indexName) - .add(new UpdateRequest<>(schema, projectState)); + .add(new UpdateRequest<>(schema, projectState, ImmutableSet.of())); String uri = getURI(BULK); Response response = postRequest(uri, bulk, getRefreshParam()); @@ -118,6 +121,10 @@ Project.NameKey nameKey = Project.nameKey(source.getAsJsonObject().get(ProjectField.NAME.getName()).getAsString()); - return projectCache.get().get(nameKey).toProjectData(); + Optional<ProjectState> state = projectCache.get().get(nameKey); + if (!state.isPresent()) { + return null; + } + return state.get().toProjectData(); } }
diff --git a/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java b/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java index 2f0bd01..196b8d6 100644 --- a/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java +++ b/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java
@@ -16,6 +16,7 @@ import static java.util.stream.Collectors.toList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Streams; import com.google.gerrit.elasticsearch.builders.XContentBuilder; @@ -27,17 +28,19 @@ private final Schema<V> schema; private final V v; + private final ImmutableSet<String> skipFields; - public UpdateRequest(Schema<V> schema, V v) { + public UpdateRequest(Schema<V> schema, V v, ImmutableSet<String> skipFields) { this.schema = schema; this.v = v; + this.skipFields = skipFields; } @Override protected String getRequest() { try (XContentBuilder closeable = new XContentBuilder()) { XContentBuilder builder = closeable.startObject(); - for (Values<V> values : schema.buildFields(v)) { + for (Values<V> values : schema.buildFields(v, skipFields)) { String name = values.getField().getName(); if (values.getField().isRepeatable()) { builder.field(name, Streams.stream(values.getValues()).collect(toList()));