Merge branch 'stable-3.6' into stable-3.7
* stable-3.6: (23 commits)
Prune _source to only contain stored fields
Introduce 'elasticsearch.codec' config
Disable tracking total number of hits
Use bool 'filter' queries instead of 'match'
Replace camelCase format name dateOptionalTime
Return cardinality from predicate when available
Update testcontainers to 1.17.5
Update testcontainers to 1.16.3
Elasticsearch tests: Recreate container for each test suite
Paginate no-limit queries
Introduce a SEARCH_AFTER index pagination type
Remove support for vulnerable ES versions
Add 7.16.* to supported versions
tests: Print container logs of startup failure
Use official elasticsearch docker image for tests
Add 7.10.* to supported versions
Use errors output field to determine _bulk API failure
elasticsearch-rest-client: Update to latest 8.3.2
docs: Split README content into more files
Do not wait for refresh when AutoFlush.DISABLED
...
Release-Notes: skip
Change-Id: I9e52753f1edf4557ed7b8dc68123e2368bd475ef
diff --git a/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java b/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
index fce86b9..408b582 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
@@ -36,7 +36,6 @@
import com.google.gerrit.elasticsearch.bulk.DeleteRequest;
import com.google.gerrit.entities.converter.ProtoConverter;
import com.google.gerrit.exceptions.StorageException;
-import com.google.gerrit.index.FieldDef;
import com.google.gerrit.index.FieldType;
import com.google.gerrit.index.Index;
import com.google.gerrit.index.QueryOptions;
@@ -244,13 +243,12 @@
protected abstract V fromDocument(JsonObject doc, Set<String> fields);
protected FieldBundle toFieldBundle(JsonObject doc) {
- Map<String, FieldDef<V, ?>> allFields = getSchema().getFields();
ListMultimap<String, Object> rawFields = ArrayListMultimap.create();
for (Map.Entry<String, JsonElement> element :
doc.get(client.adapter().rawFieldsKey()).getAsJsonObject().entrySet()) {
checkArgument(
- allFields.containsKey(element.getKey()), "Unrecognized field " + element.getKey());
- FieldType<?> type = allFields.get(element.getKey()).getType();
+ getSchema().hasField(element.getKey()), "Unrecognized field " + element.getKey());
+ FieldType<?> type = getSchema().getSchemaField(element.getKey()).getType();
Iterable<JsonElement> innerItems =
element.getValue().isJsonArray()
? element.getValue().getAsJsonArray()
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
index 1e94148..012e97c 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
@@ -93,15 +93,14 @@
@Override
public DataSource<AccountState> getSource(Predicate<AccountState> p, QueryOptions opts)
throws QueryParseException {
+ boolean useLegacyNumericFields = schema.hasField(AccountField.ID_FIELD_SPEC);
JsonArray sortArray =
getSortArray(
- schema.useLegacyNumericFields()
- ? AccountField.ID.getName()
- : AccountField.ID_STR.getName());
+ useLegacyNumericFields
+ ? AccountField.ID_FIELD_SPEC.getName()
+ : AccountField.ID_STR_FIELD_SPEC.getName());
return new ElasticQuerySource(
- p,
- opts.filterFields(o -> IndexUtils.accountFields(o, schema.useLegacyNumericFields())),
- sortArray);
+ p, opts.filterFields(o -> IndexUtils.accountFields(o, useLegacyNumericFields)), sortArray);
}
@Override
@@ -131,9 +130,9 @@
source
.getAsJsonObject()
.get(
- schema.useLegacyNumericFields()
- ? AccountField.ID.getName()
- : AccountField.ID_STR.getName())
+ schema.hasField(AccountField.ID_FIELD_SPEC)
+ ? AccountField.ID_FIELD_SPEC.getName()
+ : AccountField.ID_STR_FIELD_SPEC.getName())
.getAsInt());
// Use the AccountCache rather than depending on any stored fields in the document (of which
// there shouldn't be any). The most expensive part to compute anyway is the effective group
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
index 8504e16..7b97d8a 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
@@ -25,9 +25,9 @@
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.converter.ChangeProtoConverter;
import com.google.gerrit.exceptions.StorageException;
-import com.google.gerrit.index.FieldDef;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
+import com.google.gerrit.index.SchemaFieldDefs.SchemaField;
import com.google.gerrit.index.query.DataSource;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
@@ -70,7 +70,6 @@
private final ChangeMapping mapping;
private final ChangeData.Factory changeDataFactory;
private final Schema<ChangeData> schema;
- private final FieldDef<ChangeData, ?> idField;
private final ImmutableSet<String> skipFields;
@Inject
@@ -86,8 +85,6 @@
this.changeDataFactory = changeDataFactory;
this.schema = schema;
this.mapping = new ChangeMapping(schema, client.adapter());
- this.idField =
- this.schema.useLegacyNumericFields() ? ChangeField.LEGACY_ID : ChangeField.LEGACY_ID_STR;
this.skipFields =
MergeabilityComputationBehavior.fromConfig(gerritConfig).includeInIndex()
? ImmutableSet.of()
@@ -112,8 +109,7 @@
@Override
public DataSource<ChangeData> getSource(Predicate<ChangeData> p, QueryOptions opts)
throws QueryParseException {
- QueryOptions filteredOpts =
- opts.filterFields(o -> IndexUtils.changeFields(o, schema.useLegacyNumericFields()));
+ QueryOptions filteredOpts = opts.filterFields(o -> IndexUtils.changeFields(o));
return new ElasticQuerySource(p, filteredOpts, getSortArray());
}
@@ -124,7 +120,7 @@
JsonArray sortArray = new JsonArray();
addNamedElement(ChangeField.UPDATED.getName(), properties, sortArray);
addNamedElement(ChangeField.MERGED_ON.getName(), getMergedOnSortOptions(), sortArray);
- addNamedElement(idField.getName(), properties, sortArray);
+ addNamedElement(ChangeField.LEGACY_ID_STR.getName(), properties, sortArray);
return sortArray;
}
@@ -162,7 +158,7 @@
JsonElement c = source.get(ChangeField.CHANGE.getName());
if (c == null) {
- int id = source.get(idField.getName()).getAsInt();
+ int id = source.get(ChangeField.LEGACY_ID_STR.getName()).getAsInt();
// IndexUtils#changeFields ensures either CHANGE or PROJECT is always present.
String projectName = requireNonNull(source.get(ChangeField.PROJECT.getName()).getAsString());
return changeDataFactory.create(Project.nameKey(projectName), Change.id(id));
@@ -172,7 +168,7 @@
changeDataFactory.create(
parseProtoFrom(decodeBase64(c.getAsString()), ChangeProtoConverter.INSTANCE));
- for (FieldDef<ChangeData, ?> field : getSchema().getFields().values()) {
+ for (SchemaField<ChangeData, ?> field : getSchema().getSchemaFields().values()) {
if (fields.contains(field.getName()) && source.get(field.getName()) != null) {
field.setIfPossible(cd, new ElasticStoredValue(source.get(field.getName())));
}
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
index e0b337e..863e893 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
@@ -93,7 +93,7 @@
@Override
public DataSource<InternalGroup> getSource(Predicate<InternalGroup> p, QueryOptions opts)
throws QueryParseException {
- JsonArray sortArray = getSortArray(GroupField.UUID.getName());
+ JsonArray sortArray = getSortArray(GroupField.UUID_FIELD_SPEC.getName());
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::groupFields), sortArray);
}
@@ -120,7 +120,8 @@
}
AccountGroup.UUID uuid =
- AccountGroup.uuid(source.getAsJsonObject().get(GroupField.UUID.getName()).getAsString());
+ AccountGroup.uuid(
+ source.getAsJsonObject().get(GroupField.UUID_FIELD_SPEC.getName()).getAsString());
// Use the GroupCache rather than depending on any stored fields in the
// document (of which there shouldn't be any).
return groupCache.get().get(uuid).orElse(null);
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java
index 1668450..b2ad015 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java
@@ -15,9 +15,9 @@
package com.google.gerrit.elasticsearch;
import com.google.common.collect.ImmutableMap;
-import com.google.gerrit.index.FieldDef;
import com.google.gerrit.index.FieldType;
import com.google.gerrit.index.Schema;
+import com.google.gerrit.index.SchemaFieldDefs.SchemaField;
import com.google.gson.annotations.SerializedName;
import java.util.Map;
@@ -28,7 +28,7 @@
static Mapping createMapping(Schema<?> schema, ElasticQueryAdapter adapter) {
ElasticMapping.Builder mapping = new ElasticMapping.Builder(adapter);
- for (FieldDef<?, ?> field : schema.getFields().values()) {
+ for (SchemaField<?, ?> field : schema.getSchemaFields().values()) {
String name = field.getName();
FieldType<?> fieldType = field.getType();
if (fieldType == FieldType.EXACT) {
@@ -48,7 +48,7 @@
}
}
mapping.addSourceIncludes(
- schema.getFields().values().stream()
+ schema.getSchemaFields().values().stream()
.filter(f -> f.isStored())
.map(f -> f.getName())
.toArray(String[]::new));
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
index 3580089..df28dcc 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
@@ -17,8 +17,8 @@
import com.google.gerrit.elasticsearch.builders.BoolQueryBuilder;
import com.google.gerrit.elasticsearch.builders.QueryBuilder;
import com.google.gerrit.elasticsearch.builders.QueryBuilders;
-import com.google.gerrit.index.FieldDef;
import com.google.gerrit.index.FieldType;
+import com.google.gerrit.index.SchemaFieldDefs.SchemaField;
import com.google.gerrit.index.query.AndPredicate;
import com.google.gerrit.index.query.IndexPredicate;
import com.google.gerrit.index.query.IntegerRangePredicate;
@@ -79,7 +79,7 @@
private <T> QueryBuilder fieldQuery(IndexPredicate<T> p) throws QueryParseException {
FieldType<?> type = p.getType();
- FieldDef<?, ?> field = p.getField();
+ SchemaField<?, ?> field = p.getField();
String name = field.getName();
String value = p.getValue();
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticStoredValue.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticStoredValue.java
index a02a715..bd7c1c3 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticStoredValue.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticStoredValue.java
@@ -18,6 +18,7 @@
import com.google.gerrit.index.StoredValue;
import com.google.gson.JsonElement;
+import com.google.protobuf.MessageLite;
import java.sql.Timestamp;
import java.time.Instant;
import java.time.format.DateTimeFormatter;
@@ -83,4 +84,16 @@
.map(f -> AbstractElasticIndex.decodeBase64(f.getAsString()))
.collect(toImmutableList());
}
+
+ @Override
+ public MessageLite asProto() {
+ // Elasticsearch does not store protos
+ return null;
+ }
+
+ @Override
+ public Iterable<MessageLite> asProtos() {
+ // Elasticsearch does not store protos
+ return null;
+ }
}
