Merge branch 'stable-3.7' into stable-3.8
* stable-3.7:
test: Simplify SSL setup
Add 8.9.* to supported versions
test: Always enable SSL for ES containers
Bump testcontainers to 1.19.7
Remove unused build var
test: Add assert for closing indexes
test: Use the 'withTag' helper to get DockerImageName
Include an 'Accept' header for Content-Type in requests
Add a debug log while insert/replace change index operation
Release-Notes: skip
Change-Id: I6ca7330ee220281b84ca248d6a4b91971467aea5
diff --git a/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java b/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
index 2e5aaaa..c022978 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
@@ -139,6 +139,7 @@
protected final String indexName;
protected final Gson gson;
protected final ElasticQueryBuilder queryBuilder;
+ private final Function<V, K> valueToKeyFunction;
AbstractElasticIndex(
ElasticConfiguration config,
@@ -146,7 +147,8 @@
Schema<V> schema,
ElasticRestClientProvider client,
String indexName,
- AutoFlush autoFlush) {
+ AutoFlush autoFlush,
+ Function<V, K> valueToKeyFunction) {
this.config = config;
this.sitePaths = sitePaths;
this.schema = schema;
@@ -159,6 +161,12 @@
Map.of(
"refresh",
autoFlush == AutoFlush.ENABLED ? Boolean.TRUE.toString() : Boolean.FALSE.toString());
+ this.valueToKeyFunction = valueToKeyFunction;
+ }
+
+ @Override
+ public void deleteByValue(V value) {
+ delete(valueToKeyFunction.apply(value));
}
@Override
@@ -271,7 +279,7 @@
}
}
}
- return new FieldBundle(rawFields);
+ return new FieldBundle(rawFields, /* storesIndexedFields= */ false);
}
protected boolean hasErrors(Response response) {
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
index 012e97c..d1c5416 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
@@ -67,7 +67,7 @@
ElasticRestClientProvider client,
AutoFlush autoFlush,
@Assisted Schema<AccountState> schema) {
- super(cfg, sitePaths, schema, client, ACCOUNTS, autoFlush);
+ super(cfg, sitePaths, schema, client, ACCOUNTS, autoFlush, AccountIndex.ENTITY_TO_KEY);
this.accountCache = accountCache;
this.mapping = new AccountMapping(schema, client.adapter());
this.schema = schema;
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
index 90fddf6..ec12a7a 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
@@ -84,14 +84,14 @@
@GerritServerConfig Config gerritConfig,
AutoFlush autoFlush,
@Assisted Schema<ChangeData> schema) {
- super(cfg, sitePaths, schema, clientBuilder, CHANGES, autoFlush);
+ super(cfg, sitePaths, schema, clientBuilder, CHANGES, autoFlush, ChangeIndex.ENTITY_TO_KEY);
this.changeDataFactory = changeDataFactory;
this.schema = schema;
this.mapping = new ChangeMapping(schema, client.adapter());
this.skipFields =
MergeabilityComputationBehavior.fromConfig(gerritConfig).includeInIndex()
? ImmutableSet.of()
- : ImmutableSet.of(ChangeField.MERGEABLE.getName());
+ : ImmutableSet.of(ChangeField.MERGEABLE_SPEC.getName());
}
@Override
@@ -132,9 +132,9 @@
properties.addProperty(ORDER, DESC_SORT_ORDER);
JsonArray sortArray = new JsonArray();
- addNamedElement(ChangeField.UPDATED.getName(), properties, sortArray);
- addNamedElement(ChangeField.MERGED_ON.getName(), getMergedOnSortOptions(), sortArray);
- addNamedElement(ChangeField.LEGACY_ID_STR.getName(), properties, sortArray);
+ addNamedElement(ChangeField.UPDATED_SPEC.getName(), properties, sortArray);
+ addNamedElement(ChangeField.MERGED_ON_SPEC.getName(), getMergedOnSortOptions(), sortArray);
+ addNamedElement(ChangeField.NUMERIC_ID_STR_SPEC.getName(), properties, sortArray);
return sortArray;
}
@@ -169,12 +169,13 @@
sourceElement = json.getAsJsonObject().get("fields");
}
JsonObject source = sourceElement.getAsJsonObject();
- JsonElement c = source.get(ChangeField.CHANGE.getName());
+ JsonElement c = source.get(ChangeField.CHANGE_SPEC.getName());
if (c == null) {
- int id = source.get(ChangeField.LEGACY_ID_STR.getName()).getAsInt();
+ int id = source.get(ChangeField.NUMERIC_ID_STR_SPEC.getName()).getAsInt();
// IndexUtils#changeFields ensures either CHANGE or PROJECT is always present.
- String projectName = requireNonNull(source.get(ChangeField.PROJECT.getName()).getAsString());
+ String projectName =
+ requireNonNull(source.get(ChangeField.PROJECT_SPEC.getName()).getAsString());
return changeDataFactory.create(Project.nameKey(projectName), Change.id(id));
}
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
index 863e893..d2b9228 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
@@ -67,7 +67,7 @@
ElasticRestClientProvider client,
AutoFlush autoFlush,
@Assisted Schema<InternalGroup> schema) {
- super(cfg, sitePaths, schema, client, GROUPS, autoFlush);
+ super(cfg, sitePaths, schema, client, GROUPS, autoFlush, GroupIndex.ENTITY_TO_KEY);
this.groupCache = groupCache;
this.mapping = new GroupMapping(schema, client.adapter());
this.schema = schema;
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java
index 48cb10a..daf2cf9 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java
@@ -76,12 +76,7 @@
}
Builder addExactField(String name) {
- FieldProperties key = new FieldProperties(adapter.exactFieldType());
- key.index = adapter.indexProperty();
- FieldProperties properties;
- properties = new FieldProperties(adapter.exactFieldType());
- properties.fields = ImmutableMap.of("key", key);
- fields.put(name, properties);
+ fields.put(name, new FieldProperties(adapter.exactFieldType()));
return this;
}
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java
index 4703f1f..2fd8e7a 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java
@@ -69,7 +69,7 @@
ElasticRestClientProvider client,
AutoFlush autoFlush,
@Assisted Schema<ProjectData> schema) {
- super(cfg, sitePaths, schema, client, PROJECTS, autoFlush);
+ super(cfg, sitePaths, schema, client, PROJECTS, autoFlush, ProjectIndex.ENTITY_TO_KEY);
this.projectCache = projectCache;
this.schema = schema;
this.mapping = new ProjectMapping(schema, client.adapter());
@@ -95,7 +95,7 @@
@Override
public DataSource<ProjectData> getSource(Predicate<ProjectData> p, QueryOptions opts)
throws QueryParseException {
- JsonArray sortArray = getSortArray(ProjectField.NAME.getName());
+ JsonArray sortArray = getSortArray(ProjectField.NAME_SPEC.getName());
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::projectFields), sortArray);
}
@@ -122,7 +122,8 @@
}
Project.NameKey nameKey =
- Project.nameKey(source.getAsJsonObject().get(ProjectField.NAME.getName()).getAsString());
+ Project.nameKey(
+ source.getAsJsonObject().get(ProjectField.NAME_SPEC.getName()).getAsString());
Optional<ProjectState> state = projectCache.get().get(nameKey);
if (!state.isPresent()) {
return null;
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
index df28dcc..b0dd2ad 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
@@ -152,9 +152,9 @@
if (value.endsWith("$") && !value.endsWith("\\$") && !value.endsWith("\\\\$")) {
value = value.substring(0, value.length() - 1);
}
- return QueryBuilders.regexpQuery(name + ".key", value);
+ return QueryBuilders.regexpQuery(name, value);
} else {
- return QueryBuilders.termQuery(name + ".key", value);
+ return QueryBuilders.termQuery(name, value);
}
}
}
diff --git a/src/main/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java b/src/main/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java
index 196b8d6..1ff5b69 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java
@@ -22,6 +22,9 @@
import com.google.gerrit.elasticsearch.builders.XContentBuilder;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.Schema.Values;
+import com.google.gerrit.index.SchemaFieldDefs;
+import com.google.gerrit.proto.Protos;
+import com.google.protobuf.MessageLite;
import java.io.IOException;
public class UpdateRequest<V> extends BulkRequest {
@@ -40,12 +43,18 @@
protected String getRequest() {
try (XContentBuilder closeable = new XContentBuilder()) {
XContentBuilder builder = closeable.startObject();
- for (Values<V> values : schema.buildFields(v, skipFields)) {
- String name = values.getField().getName();
- if (values.getField().isRepeatable()) {
- builder.field(name, Streams.stream(values.getValues()).collect(toList()));
+ for (Values<V> schemaValues : schema.buildFields(v, skipFields)) {
+ String name = schemaValues.getField().getName();
+ Iterable<?> values = schemaValues.getValues();
+ if (SchemaFieldDefs.isProtoField(schemaValues.getField())) {
+ values =
+ Iterables.transform(
+ schemaValues.getValues(), v -> Protos.toByteArray((MessageLite) v));
+ }
+ if (schemaValues.getField().isRepeatable()) {
+ builder.field(name, Streams.stream(values).collect(toList()));
} else {
- Object element = Iterables.getOnlyElement(values.getValues(), "");
+ Object element = Iterables.getOnlyElement(values, "");
if (shouldAddElement(element)) {
builder.field(name, element);
}
diff --git a/src/test/java/com/google/gerrit/elasticsearch/ElasticAbstractQueryChangesTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticAbstractQueryChangesTest.java
index d37e626..d5c60ad 100644
--- a/src/test/java/com/google/gerrit/elasticsearch/ElasticAbstractQueryChangesTest.java
+++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticAbstractQueryChangesTest.java
@@ -22,11 +22,11 @@
import com.google.gerrit.server.query.change.AbstractQueryChangesTest;
import com.google.gerrit.testing.ConfigSuite;
import com.google.gerrit.testing.GerritTestName;
-import com.google.gerrit.testing.InMemoryRepositoryManager;
import com.google.inject.Injector;
import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.lib.Repository;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Rule;
@@ -83,8 +83,9 @@
@Test
public void testErrorResponseFromChangeIndex() throws Exception {
- TestRepository<InMemoryRepositoryManager.Repo> repo = createProject("repo");
- Change c = insert(repo, newChangeWithStatus(repo, Change.Status.NEW));
+ String repository = "repo";
+ TestRepository<Repository> repo = createAndOpenProject(repository);
+ Change c = insert(repository, newChangeWithStatus(repo, Change.Status.NEW));
gApi.changes().id(c.getChangeId()).index();
ElasticTestUtils.closeIndex(client, container, testName);
