Merge branch 'stable-3.7'
* stable-3.7: (24 commits)
Add an analyzer with tokenizer:keyword to prefix fields
Prune _source to only contain stored fields
Introduce 'elasticsearch.codec' config
Disable tracking total number of hits
Use bool 'filter' queries instead of 'match'
Replace camelCase format name dateOptionalTime
Return cardinality from predicate when available
Update testcontainers to 1.17.5
Update testcontainers to 1.16.3
Elasticsearch tests: Recreate container for each test suite
Paginate no-limit queries
Introduce a SEARCH_AFTER index pagination type
Remove support for vulnerable ES versions
Add 7.16.* to supported versions
tests: Print container logs of startup failure
Use official elasticsearch docker image for tests
Add 7.10.* to supported versions
Use errors output field to determine _bulk API failure
elasticsearch-rest-client: Update to latest 8.3.2
docs: Split README content into more files
...
Release-Notes: skip
Change-Id: I8361cdc33f7331b8c106d9817061d55050373b1a
diff --git a/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java b/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
index 408b582..2309dad 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
@@ -137,6 +137,7 @@
protected final String indexName;
protected final Gson gson;
protected final ElasticQueryBuilder queryBuilder;
+ private final Function<V, K> valueToKeyFunction;
AbstractElasticIndex(
ElasticConfiguration config,
@@ -144,7 +145,8 @@
Schema<V> schema,
ElasticRestClientProvider client,
String indexName,
- AutoFlush autoFlush) {
+ AutoFlush autoFlush,
+ Function<V, K> valueToKeyFunction) {
this.config = config;
this.sitePaths = sitePaths;
this.schema = schema;
@@ -157,6 +159,12 @@
Map.of(
"refresh",
autoFlush == AutoFlush.ENABLED ? Boolean.TRUE.toString() : Boolean.FALSE.toString());
+ this.valueToKeyFunction = valueToKeyFunction;
+ }
+
+ @Override
+ public void deleteByValue(V value) {
+ delete(valueToKeyFunction.apply(value));
}
@Override
@@ -269,7 +277,7 @@
}
}
}
- return new FieldBundle(rawFields);
+ return new FieldBundle(rawFields, /* storesIndexedFields= */ false);
}
protected boolean hasErrors(Response response) {
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
index 012e97c..d1c5416 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
@@ -67,7 +67,7 @@
ElasticRestClientProvider client,
AutoFlush autoFlush,
@Assisted Schema<AccountState> schema) {
- super(cfg, sitePaths, schema, client, ACCOUNTS, autoFlush);
+ super(cfg, sitePaths, schema, client, ACCOUNTS, autoFlush, AccountIndex.ENTITY_TO_KEY);
this.accountCache = accountCache;
this.mapping = new AccountMapping(schema, client.adapter());
this.schema = schema;
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
index 7b97d8a..1cc00cd 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
@@ -81,14 +81,14 @@
@GerritServerConfig Config gerritConfig,
AutoFlush autoFlush,
@Assisted Schema<ChangeData> schema) {
- super(cfg, sitePaths, schema, clientBuilder, CHANGES, autoFlush);
+ super(cfg, sitePaths, schema, clientBuilder, CHANGES, autoFlush, ChangeIndex.ENTITY_TO_KEY);
this.changeDataFactory = changeDataFactory;
this.schema = schema;
this.mapping = new ChangeMapping(schema, client.adapter());
this.skipFields =
MergeabilityComputationBehavior.fromConfig(gerritConfig).includeInIndex()
? ImmutableSet.of()
- : ImmutableSet.of(ChangeField.MERGEABLE.getName());
+ : ImmutableSet.of(ChangeField.MERGEABLE_SPEC.getName());
}
@Override
@@ -118,9 +118,9 @@
properties.addProperty(ORDER, DESC_SORT_ORDER);
JsonArray sortArray = new JsonArray();
- addNamedElement(ChangeField.UPDATED.getName(), properties, sortArray);
- addNamedElement(ChangeField.MERGED_ON.getName(), getMergedOnSortOptions(), sortArray);
- addNamedElement(ChangeField.LEGACY_ID_STR.getName(), properties, sortArray);
+ addNamedElement(ChangeField.UPDATED_SPEC.getName(), properties, sortArray);
+ addNamedElement(ChangeField.MERGED_ON_SPEC.getName(), getMergedOnSortOptions(), sortArray);
+ addNamedElement(ChangeField.NUMERIC_ID_STR_SPEC.getName(), properties, sortArray);
return sortArray;
}
@@ -155,12 +155,13 @@
sourceElement = json.getAsJsonObject().get("fields");
}
JsonObject source = sourceElement.getAsJsonObject();
- JsonElement c = source.get(ChangeField.CHANGE.getName());
+ JsonElement c = source.get(ChangeField.CHANGE_SPEC.getName());
if (c == null) {
- int id = source.get(ChangeField.LEGACY_ID_STR.getName()).getAsInt();
+ int id = source.get(ChangeField.NUMERIC_ID_STR_SPEC.getName()).getAsInt();
// IndexUtils#changeFields ensures either CHANGE or PROJECT is always present.
- String projectName = requireNonNull(source.get(ChangeField.PROJECT.getName()).getAsString());
+ String projectName =
+ requireNonNull(source.get(ChangeField.PROJECT_SPEC.getName()).getAsString());
return changeDataFactory.create(Project.nameKey(projectName), Change.id(id));
}
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
index 863e893..d2b9228 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
@@ -67,7 +67,7 @@
ElasticRestClientProvider client,
AutoFlush autoFlush,
@Assisted Schema<InternalGroup> schema) {
- super(cfg, sitePaths, schema, client, GROUPS, autoFlush);
+ super(cfg, sitePaths, schema, client, GROUPS, autoFlush, GroupIndex.ENTITY_TO_KEY);
this.groupCache = groupCache;
this.mapping = new GroupMapping(schema, client.adapter());
this.schema = schema;
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java
index 4703f1f..2fd8e7a 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java
@@ -69,7 +69,7 @@
ElasticRestClientProvider client,
AutoFlush autoFlush,
@Assisted Schema<ProjectData> schema) {
- super(cfg, sitePaths, schema, client, PROJECTS, autoFlush);
+ super(cfg, sitePaths, schema, client, PROJECTS, autoFlush, ProjectIndex.ENTITY_TO_KEY);
this.projectCache = projectCache;
this.schema = schema;
this.mapping = new ProjectMapping(schema, client.adapter());
@@ -95,7 +95,7 @@
@Override
public DataSource<ProjectData> getSource(Predicate<ProjectData> p, QueryOptions opts)
throws QueryParseException {
- JsonArray sortArray = getSortArray(ProjectField.NAME.getName());
+ JsonArray sortArray = getSortArray(ProjectField.NAME_SPEC.getName());
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::projectFields), sortArray);
}
@@ -122,7 +122,8 @@
}
Project.NameKey nameKey =
- Project.nameKey(source.getAsJsonObject().get(ProjectField.NAME.getName()).getAsString());
+ Project.nameKey(
+ source.getAsJsonObject().get(ProjectField.NAME_SPEC.getName()).getAsString());
Optional<ProjectState> state = projectCache.get().get(nameKey);
if (!state.isPresent()) {
return null;
diff --git a/src/main/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java b/src/main/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java
index 196b8d6..1ff5b69 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java
@@ -22,6 +22,9 @@
import com.google.gerrit.elasticsearch.builders.XContentBuilder;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.Schema.Values;
+import com.google.gerrit.index.SchemaFieldDefs;
+import com.google.gerrit.proto.Protos;
+import com.google.protobuf.MessageLite;
import java.io.IOException;
public class UpdateRequest<V> extends BulkRequest {
@@ -40,12 +43,18 @@
protected String getRequest() {
try (XContentBuilder closeable = new XContentBuilder()) {
XContentBuilder builder = closeable.startObject();
- for (Values<V> values : schema.buildFields(v, skipFields)) {
- String name = values.getField().getName();
- if (values.getField().isRepeatable()) {
- builder.field(name, Streams.stream(values.getValues()).collect(toList()));
+ for (Values<V> schemaValues : schema.buildFields(v, skipFields)) {
+ String name = schemaValues.getField().getName();
+ Iterable<?> values = schemaValues.getValues();
+ if (SchemaFieldDefs.isProtoField(schemaValues.getField())) {
+ values =
+ Iterables.transform(
+ schemaValues.getValues(), v -> Protos.toByteArray((MessageLite) v));
+ }
+ if (schemaValues.getField().isRepeatable()) {
+ builder.field(name, Streams.stream(values).collect(toList()));
} else {
- Object element = Iterables.getOnlyElement(values.getValues(), "");
+ Object element = Iterables.getOnlyElement(values, "");
if (shouldAddElement(element)) {
builder.field(name, element);
}
diff --git a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java
index 9160b9e..7151e70 100644
--- a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java
+++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java
@@ -22,12 +22,12 @@
import com.google.gerrit.server.query.change.AbstractQueryChangesTest;
import com.google.gerrit.testing.ConfigSuite;
import com.google.gerrit.testing.GerritTestName;
-import com.google.gerrit.testing.InMemoryRepositoryManager;
import com.google.inject.Injector;
import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
import org.apache.http.impl.nio.client.HttpAsyncClients;
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.lib.Repository;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -86,8 +86,9 @@
@Test
public void testErrorResponseFromChangeIndex() throws Exception {
- TestRepository<InMemoryRepositoryManager.Repo> repo = createProject("repo");
- Change c = insert(repo, newChangeWithStatus(repo, Change.Status.NEW));
+ String repository = "repo";
+ TestRepository<Repository> repo = createAndOpenProject(repository);
+ Change c = insert(repository, newChangeWithStatus(repo, Change.Status.NEW));
gApi.changes().id(c.getChangeId()).index();
ElasticTestUtils.closeIndex(client, container, testName);
