Merge branch 'stable-3.4'
* stable-3.4:
Elasticsearch: Discontinue EOL version 7.4 support
Change-Id: Iff6820b9280950e3f756ed3dab209a260fd58e0e
diff --git a/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java b/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
index 44a377a..562464d 100644
--- a/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
+++ b/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
@@ -90,7 +90,7 @@
protected static final String SEARCH = "_search";
protected static final String SETTINGS = "settings";
- protected static byte[] decodeBase64(String base64String) {
+ static byte[] decodeBase64(String base64String) {
return BaseEncoding.base64().decode(base64String);
}
diff --git a/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java b/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
index 162654d..7d4e0c7 100644
--- a/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
+++ b/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
@@ -14,55 +14,35 @@
package com.google.gerrit.elasticsearch;
-import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableListMultimap;
import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.ListMultimap;
-import com.google.common.collect.MultimapBuilder;
-import com.google.common.collect.Sets;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.elasticsearch.bulk.BulkRequest;
import com.google.gerrit.elasticsearch.bulk.IndexRequest;
import com.google.gerrit.elasticsearch.bulk.UpdateRequest;
-import com.google.gerrit.entities.Account;
import com.google.gerrit.entities.Change;
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.converter.ChangeProtoConverter;
-import com.google.gerrit.entities.converter.PatchSetApprovalProtoConverter;
-import com.google.gerrit.entities.converter.PatchSetProtoConverter;
import com.google.gerrit.exceptions.StorageException;
import com.google.gerrit.index.FieldDef;
import com.google.gerrit.index.QueryOptions;
-import com.google.gerrit.index.RefState;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.query.DataSource;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
-import com.google.gerrit.server.ReviewerByEmailSet;
-import com.google.gerrit.server.ReviewerSet;
-import com.google.gerrit.server.StarredChangesUtil;
import com.google.gerrit.server.change.MergeabilityComputationBehavior;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.change.ChangeField;
import com.google.gerrit.server.index.change.ChangeIndex;
-import com.google.gerrit.server.project.SubmitRuleOptions;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
-import java.sql.Timestamp;
-import java.time.Instant;
-import java.time.format.DateTimeFormatter;
-import java.util.Collections;
-import java.util.Optional;
import java.util.Set;
import org.apache.http.HttpStatus;
import org.eclipse.jgit.lib.Config;
@@ -190,242 +170,12 @@
changeDataFactory.create(
parseProtoFrom(decodeBase64(c.getAsString()), ChangeProtoConverter.INSTANCE));
- // Any decoding that is done here must also be done in {@link LuceneChangeIndex}.
-
- // Patch sets.
- cd.setPatchSets(
- decodeProtos(source, ChangeField.PATCH_SET.getName(), PatchSetProtoConverter.INSTANCE));
-
- // Approvals.
- if (source.get(ChangeField.APPROVAL.getName()) != null) {
- cd.setCurrentApprovals(
- decodeProtos(
- source, ChangeField.APPROVAL.getName(), PatchSetApprovalProtoConverter.INSTANCE));
- } else if (fields.contains(ChangeField.APPROVAL.getName())) {
- cd.setCurrentApprovals(Collections.emptyList());
- }
-
- // Added & Deleted.
- JsonElement addedElement = source.get(ChangeField.ADDED.getName());
- JsonElement deletedElement = source.get(ChangeField.DELETED.getName());
- if (addedElement != null && deletedElement != null) {
- // Changed lines.
- int added = addedElement.getAsInt();
- int deleted = deletedElement.getAsInt();
- cd.setChangedLines(added, deleted);
- }
-
- // Star.
- JsonElement starredElement = source.get(ChangeField.STAR.getName());
- if (starredElement != null) {
- ListMultimap<Account.Id, String> stars = MultimapBuilder.hashKeys().arrayListValues().build();
- JsonArray starBy = starredElement.getAsJsonArray();
- if (starBy.size() > 0) {
- for (int i = 0; i < starBy.size(); i++) {
- String[] indexableFields = starBy.get(i).getAsString().split(":");
- Optional<Account.Id> id = Account.Id.tryParse(indexableFields[0]);
- if (id.isPresent()) {
- stars.put(id.get(), indexableFields[1]);
- }
- }
+ for (FieldDef<ChangeData, ?> field : getSchema().getFields().values()) {
+ if (fields.contains(field.getName()) && source.get(field.getName()) != null) {
+ field.setIfPossible(cd, new ElasticStoredValue(source.get(field.getName())));
}
- cd.setStars(stars);
- }
-
- // Mergeable.
- JsonElement mergeableElement = source.get(ChangeField.MERGEABLE.getName());
- if (mergeableElement != null && !skipFields.contains(ChangeField.MERGEABLE.getName())) {
- String mergeable = mergeableElement.getAsString();
- if ("1".equals(mergeable)) {
- cd.setMergeable(true);
- } else if ("0".equals(mergeable)) {
- cd.setMergeable(false);
- }
- }
-
- // Reviewed-by.
- if (source.get(ChangeField.REVIEWEDBY.getName()) != null) {
- JsonArray reviewedBy = source.get(ChangeField.REVIEWEDBY.getName()).getAsJsonArray();
- if (reviewedBy.size() > 0) {
- Set<Account.Id> accounts = Sets.newHashSetWithExpectedSize(reviewedBy.size());
- for (int i = 0; i < reviewedBy.size(); i++) {
- int aId = reviewedBy.get(i).getAsInt();
- if (reviewedBy.size() == 1 && aId == ChangeField.NOT_REVIEWED) {
- break;
- }
- accounts.add(Account.id(aId));
- }
- cd.setReviewedBy(accounts);
- }
- } else if (fields.contains(ChangeField.REVIEWEDBY.getName())) {
- cd.setReviewedBy(Collections.emptySet());
- }
-
- // Hashtag.
- if (source.get(ChangeField.HASHTAG.getName()) != null) {
- JsonArray hashtagArray = source.get(ChangeField.HASHTAG.getName()).getAsJsonArray();
- if (hashtagArray.size() > 0) {
- Set<String> hashtags = Sets.newHashSetWithExpectedSize(hashtagArray.size());
- for (int i = 0; i < hashtagArray.size(); i++) {
- hashtags.add(hashtagArray.get(i).getAsString());
- }
- cd.setHashtags(hashtags);
- }
- } else if (fields.contains(ChangeField.HASHTAG.getName())) {
- cd.setHashtags(Collections.emptySet());
- }
-
- // Star.
- if (source.get(ChangeField.STAR.getName()) != null) {
- JsonArray starArray = source.get(ChangeField.STAR.getName()).getAsJsonArray();
- if (starArray.size() > 0) {
- ListMultimap<Account.Id, String> stars =
- MultimapBuilder.hashKeys().arrayListValues().build();
- for (int i = 0; i < starArray.size(); i++) {
- StarredChangesUtil.StarField starField =
- StarredChangesUtil.StarField.parse(starArray.get(i).getAsString());
- stars.put(starField.accountId(), starField.label());
- }
- cd.setStars(stars);
- }
- } else if (fields.contains(ChangeField.STAR.getName())) {
- cd.setStars(ImmutableListMultimap.of());
- }
-
- // Reviewer.
- if (source.get(ChangeField.REVIEWER.getName()) != null) {
- cd.setReviewers(
- ChangeField.parseReviewerFieldValues(
- cd.getId(),
- FluentIterable.from(source.get(ChangeField.REVIEWER.getName()).getAsJsonArray())
- .transform(JsonElement::getAsString)));
- } else if (fields.contains(ChangeField.REVIEWER.getName())) {
- cd.setReviewers(ReviewerSet.empty());
- }
-
- // Reviewer-by-email.
- if (source.get(ChangeField.REVIEWER_BY_EMAIL.getName()) != null) {
- cd.setReviewersByEmail(
- ChangeField.parseReviewerByEmailFieldValues(
- cd.getId(),
- FluentIterable.from(
- source.get(ChangeField.REVIEWER_BY_EMAIL.getName()).getAsJsonArray())
- .transform(JsonElement::getAsString)));
- } else if (fields.contains(ChangeField.REVIEWER_BY_EMAIL.getName())) {
- cd.setReviewersByEmail(ReviewerByEmailSet.empty());
- }
-
- // Pending-reviewer.
- if (source.get(ChangeField.PENDING_REVIEWER.getName()) != null) {
- cd.setPendingReviewers(
- ChangeField.parseReviewerFieldValues(
- cd.getId(),
- FluentIterable.from(
- source.get(ChangeField.PENDING_REVIEWER.getName()).getAsJsonArray())
- .transform(JsonElement::getAsString)));
- } else if (fields.contains(ChangeField.PENDING_REVIEWER.getName())) {
- cd.setPendingReviewers(ReviewerSet.empty());
- }
-
- // Pending-reviewer-by-email.
- if (source.get(ChangeField.PENDING_REVIEWER_BY_EMAIL.getName()) != null) {
- cd.setPendingReviewersByEmail(
- ChangeField.parseReviewerByEmailFieldValues(
- cd.getId(),
- FluentIterable.from(
- source.get(ChangeField.PENDING_REVIEWER_BY_EMAIL.getName()).getAsJsonArray())
- .transform(JsonElement::getAsString)));
- } else if (fields.contains(ChangeField.PENDING_REVIEWER_BY_EMAIL.getName())) {
- cd.setPendingReviewersByEmail(ReviewerByEmailSet.empty());
- }
-
- // Stored-submit-record-strict.
- decodeSubmitRecords(
- source,
- ChangeField.STORED_SUBMIT_RECORD_STRICT.getName(),
- ChangeField.SUBMIT_RULE_OPTIONS_STRICT,
- cd);
-
- // Stored-submit-record-lenient.
- decodeSubmitRecords(
- source,
- ChangeField.STORED_SUBMIT_RECORD_LENIENT.getName(),
- ChangeField.SUBMIT_RULE_OPTIONS_LENIENT,
- cd);
-
- // Ref-state.
- if (fields.contains(ChangeField.REF_STATE.getName())) {
- cd.setRefStates(RefState.parseStates(getByteArray(source, ChangeField.REF_STATE.getName())));
- }
-
- // Ref-state-pattern.
- if (fields.contains(ChangeField.REF_STATE_PATTERN.getName())) {
- cd.setRefStatePatterns(getByteArray(source, ChangeField.REF_STATE_PATTERN.getName()));
- }
-
- // Unresolved-comment-count.
- decodeUnresolvedCommentCount(source, ChangeField.UNRESOLVED_COMMENT_COUNT.getName(), cd);
-
- // Attention set.
- if (fields.contains(ChangeField.ATTENTION_SET_FULL.getName())) {
- ChangeField.parseAttentionSet(
- FluentIterable.from(source.getAsJsonArray(ChangeField.ATTENTION_SET_FULL.getName()))
- .transform(ElasticChangeIndex::decodeBase64JsonElement)
- .toSet(),
- cd);
- }
-
- if (fields.contains(ChangeField.MERGED_ON.getName())) {
- decodeMergedOn(source, cd);
}
return cd;
}
-
- private Iterable<byte[]> getByteArray(JsonObject source, String name) {
- JsonElement element = source.get(name);
- return element != null
- ? Iterables.transform(element.getAsJsonArray(), e -> decodeBase64(e.getAsString()))
- : Collections.emptyList();
- }
-
- private void decodeSubmitRecords(
- JsonObject doc, String fieldName, SubmitRuleOptions opts, ChangeData out) {
- JsonArray records = doc.getAsJsonArray(fieldName);
- if (records == null) {
- return;
- }
- ChangeField.parseSubmitRecords(
- FluentIterable.from(records)
- .transform(ElasticChangeIndex::decodeBase64JsonElement)
- .toList(),
- opts,
- out);
- }
-
- private static String decodeBase64JsonElement(JsonElement input) {
- return new String(decodeBase64(input.getAsString()), UTF_8);
- }
-
- private void decodeUnresolvedCommentCount(JsonObject doc, String fieldName, ChangeData out) {
- JsonElement count = doc.get(fieldName);
- if (count == null) {
- return;
- }
- out.setUnresolvedCommentCount(count.getAsInt());
- }
-
- private void decodeMergedOn(JsonObject doc, ChangeData out) {
- JsonElement mergedOnField = doc.get(ChangeField.MERGED_ON.getName());
-
- Timestamp mergedOn = null;
- if (mergedOnField != null) {
- // Parse from ElasticMapping.TIMESTAMP_FIELD_FORMAT.
- // We currently use built-in ISO-based dateOptionalTime.
- // https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html#built-in-date-formats
- DateTimeFormatter isoFormatter = DateTimeFormatter.ISO_INSTANT;
- mergedOn = Timestamp.from(Instant.from(isoFormatter.parse(mergedOnField.getAsString())));
- }
- out.setMergedOn(mergedOn);
- }
}
diff --git a/java/com/google/gerrit/elasticsearch/ElasticStoredValue.java b/java/com/google/gerrit/elasticsearch/ElasticStoredValue.java
new file mode 100644
index 0000000..a02a715
--- /dev/null
+++ b/java/com/google/gerrit/elasticsearch/ElasticStoredValue.java
@@ -0,0 +1,86 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.elasticsearch;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+
+import com.google.gerrit.index.StoredValue;
+import com.google.gson.JsonElement;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
+import java.util.stream.StreamSupport;
+
+/** Bridge to recover fields from the elastic index. */
+public class ElasticStoredValue implements StoredValue {
+ private final JsonElement field;
+
+ ElasticStoredValue(JsonElement field) {
+ this.field = field;
+ }
+
+ @Override
+ public String asString() {
+ return field.getAsString();
+ }
+
+ @Override
+ public Iterable<String> asStrings() {
+ return StreamSupport.stream(field.getAsJsonArray().spliterator(), false)
+ .map(f -> f.getAsString())
+ .collect(toImmutableList());
+ }
+
+ @Override
+ public Integer asInteger() {
+ return field.getAsInt();
+ }
+
+ @Override
+ public Iterable<Integer> asIntegers() {
+ return StreamSupport.stream(field.getAsJsonArray().spliterator(), false)
+ .map(f -> f.getAsInt())
+ .collect(toImmutableList());
+ }
+
+ @Override
+ public Long asLong() {
+ return field.getAsLong();
+ }
+
+ @Override
+ public Iterable<Long> asLongs() {
+ return StreamSupport.stream(field.getAsJsonArray().spliterator(), false)
+ .map(f -> f.getAsLong())
+ .collect(toImmutableList());
+ }
+
+ @Override
+ public Timestamp asTimestamp() {
+ return Timestamp.from(Instant.from(DateTimeFormatter.ISO_INSTANT.parse(field.getAsString())));
+ }
+
+ @Override
+ public byte[] asByteArray() {
+ return AbstractElasticIndex.decodeBase64(field.getAsString());
+ }
+
+ @Override
+ public Iterable<byte[]> asByteArrays() {
+ return StreamSupport.stream(field.getAsJsonArray().spliterator(), false)
+ .map(f -> AbstractElasticIndex.decodeBase64(f.getAsString()))
+ .collect(toImmutableList());
+ }
+}
