Merge branch 'stable-3.7' into stable-3.8 * stable-3.7: Index.replace: Log responses containing errors Remove warning stating that Elasticsearch is not production ready Add trace timer around Elasticsearch's performRequest Fix Flogger issues flagged by error prone Fix incorrect symlink in build docs Fix Flogger compile time errors Release-Notes: skip Change-Id: Ifd24fd0842e22433532a6169599368dfc0869b63
diff --git a/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java b/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java index 47de0b6..65eaaf0 100644 --- a/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java +++ b/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
@@ -139,6 +139,7 @@ protected final String indexName; protected final Gson gson; protected final ElasticQueryBuilder queryBuilder; + private final Function<V, K> valueToKeyFunction; AbstractElasticIndex( ElasticConfiguration config, @@ -146,7 +147,8 @@ Schema<V> schema, ElasticRestClientProvider client, String indexName, - AutoFlush autoFlush) { + AutoFlush autoFlush, + Function<V, K> valueToKeyFunction) { this.config = config; this.sitePaths = sitePaths; this.schema = schema; @@ -159,6 +161,12 @@ Map.of( "refresh", autoFlush == AutoFlush.ENABLED ? Boolean.TRUE.toString() : Boolean.FALSE.toString()); + this.valueToKeyFunction = valueToKeyFunction; + } + + @Override + public void deleteByValue(V value) { + delete(valueToKeyFunction.apply(value)); } @Override @@ -271,7 +279,7 @@ } } } - return new FieldBundle(rawFields); + return new FieldBundle(rawFields, /* storesIndexedFields= */ false); } protected boolean hasErrors(Response response) {
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java index 012e97c..d1c5416 100644 --- a/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java +++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
@@ -67,7 +67,7 @@ ElasticRestClientProvider client, AutoFlush autoFlush, @Assisted Schema<AccountState> schema) { - super(cfg, sitePaths, schema, client, ACCOUNTS, autoFlush); + super(cfg, sitePaths, schema, client, ACCOUNTS, autoFlush, AccountIndex.ENTITY_TO_KEY); this.accountCache = accountCache; this.mapping = new AccountMapping(schema, client.adapter()); this.schema = schema;
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java index 7b97d8a..1cc00cd 100644 --- a/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java +++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
@@ -81,14 +81,14 @@ @GerritServerConfig Config gerritConfig, AutoFlush autoFlush, @Assisted Schema<ChangeData> schema) { - super(cfg, sitePaths, schema, clientBuilder, CHANGES, autoFlush); + super(cfg, sitePaths, schema, clientBuilder, CHANGES, autoFlush, ChangeIndex.ENTITY_TO_KEY); this.changeDataFactory = changeDataFactory; this.schema = schema; this.mapping = new ChangeMapping(schema, client.adapter()); this.skipFields = MergeabilityComputationBehavior.fromConfig(gerritConfig).includeInIndex() ? ImmutableSet.of() - : ImmutableSet.of(ChangeField.MERGEABLE.getName()); + : ImmutableSet.of(ChangeField.MERGEABLE_SPEC.getName()); } @Override @@ -118,9 +118,9 @@ properties.addProperty(ORDER, DESC_SORT_ORDER); JsonArray sortArray = new JsonArray(); - addNamedElement(ChangeField.UPDATED.getName(), properties, sortArray); - addNamedElement(ChangeField.MERGED_ON.getName(), getMergedOnSortOptions(), sortArray); - addNamedElement(ChangeField.LEGACY_ID_STR.getName(), properties, sortArray); + addNamedElement(ChangeField.UPDATED_SPEC.getName(), properties, sortArray); + addNamedElement(ChangeField.MERGED_ON_SPEC.getName(), getMergedOnSortOptions(), sortArray); + addNamedElement(ChangeField.NUMERIC_ID_STR_SPEC.getName(), properties, sortArray); return sortArray; } @@ -155,12 +155,13 @@ sourceElement = json.getAsJsonObject().get("fields"); } JsonObject source = sourceElement.getAsJsonObject(); - JsonElement c = source.get(ChangeField.CHANGE.getName()); + JsonElement c = source.get(ChangeField.CHANGE_SPEC.getName()); if (c == null) { - int id = source.get(ChangeField.LEGACY_ID_STR.getName()).getAsInt(); + int id = source.get(ChangeField.NUMERIC_ID_STR_SPEC.getName()).getAsInt(); // IndexUtils#changeFields ensures either CHANGE or PROJECT is always present. - String projectName = requireNonNull(source.get(ChangeField.PROJECT.getName()).getAsString()); + String projectName = + requireNonNull(source.get(ChangeField.PROJECT_SPEC.getName()).getAsString()); return changeDataFactory.create(Project.nameKey(projectName), Change.id(id)); }
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java index 863e893..d2b9228 100644 --- a/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java +++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
@@ -67,7 +67,7 @@ ElasticRestClientProvider client, AutoFlush autoFlush, @Assisted Schema<InternalGroup> schema) { - super(cfg, sitePaths, schema, client, GROUPS, autoFlush); + super(cfg, sitePaths, schema, client, GROUPS, autoFlush, GroupIndex.ENTITY_TO_KEY); this.groupCache = groupCache; this.mapping = new GroupMapping(schema, client.adapter()); this.schema = schema;
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java index 48cb10a..daf2cf9 100644 --- a/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java +++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java
@@ -76,12 +76,7 @@ } Builder addExactField(String name) { - FieldProperties key = new FieldProperties(adapter.exactFieldType()); - key.index = adapter.indexProperty(); - FieldProperties properties; - properties = new FieldProperties(adapter.exactFieldType()); - properties.fields = ImmutableMap.of("key", key); - fields.put(name, properties); + fields.put(name, new FieldProperties(adapter.exactFieldType())); return this; }
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java index 4703f1f..2fd8e7a 100644 --- a/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java +++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java
@@ -69,7 +69,7 @@ ElasticRestClientProvider client, AutoFlush autoFlush, @Assisted Schema<ProjectData> schema) { - super(cfg, sitePaths, schema, client, PROJECTS, autoFlush); + super(cfg, sitePaths, schema, client, PROJECTS, autoFlush, ProjectIndex.ENTITY_TO_KEY); this.projectCache = projectCache; this.schema = schema; this.mapping = new ProjectMapping(schema, client.adapter()); @@ -95,7 +95,7 @@ @Override public DataSource<ProjectData> getSource(Predicate<ProjectData> p, QueryOptions opts) throws QueryParseException { - JsonArray sortArray = getSortArray(ProjectField.NAME.getName()); + JsonArray sortArray = getSortArray(ProjectField.NAME_SPEC.getName()); return new ElasticQuerySource(p, opts.filterFields(IndexUtils::projectFields), sortArray); } @@ -122,7 +122,8 @@ } Project.NameKey nameKey = - Project.nameKey(source.getAsJsonObject().get(ProjectField.NAME.getName()).getAsString()); + Project.nameKey( + source.getAsJsonObject().get(ProjectField.NAME_SPEC.getName()).getAsString()); Optional<ProjectState> state = projectCache.get().get(nameKey); if (!state.isPresent()) { return null;
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java index df28dcc..b0dd2ad 100644 --- a/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java +++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
@@ -152,9 +152,9 @@ if (value.endsWith("$") && !value.endsWith("\\$") && !value.endsWith("\\\\$")) { value = value.substring(0, value.length() - 1); } - return QueryBuilders.regexpQuery(name + ".key", value); + return QueryBuilders.regexpQuery(name, value); } else { - return QueryBuilders.termQuery(name + ".key", value); + return QueryBuilders.termQuery(name, value); } } }
diff --git a/src/main/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java b/src/main/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java index 196b8d6..1ff5b69 100644 --- a/src/main/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java +++ b/src/main/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java
@@ -22,6 +22,9 @@ import com.google.gerrit.elasticsearch.builders.XContentBuilder; import com.google.gerrit.index.Schema; import com.google.gerrit.index.Schema.Values; +import com.google.gerrit.index.SchemaFieldDefs; +import com.google.gerrit.proto.Protos; +import com.google.protobuf.MessageLite; import java.io.IOException; public class UpdateRequest<V> extends BulkRequest { @@ -40,12 +43,18 @@ protected String getRequest() { try (XContentBuilder closeable = new XContentBuilder()) { XContentBuilder builder = closeable.startObject(); - for (Values<V> values : schema.buildFields(v, skipFields)) { - String name = values.getField().getName(); - if (values.getField().isRepeatable()) { - builder.field(name, Streams.stream(values.getValues()).collect(toList())); + for (Values<V> schemaValues : schema.buildFields(v, skipFields)) { + String name = schemaValues.getField().getName(); + Iterable<?> values = schemaValues.getValues(); + if (SchemaFieldDefs.isProtoField(schemaValues.getField())) { + values = + Iterables.transform( + schemaValues.getValues(), v -> Protos.toByteArray((MessageLite) v)); + } + if (schemaValues.getField().isRepeatable()) { + builder.field(name, Streams.stream(values).collect(toList())); } else { - Object element = Iterables.getOnlyElement(values.getValues(), ""); + Object element = Iterables.getOnlyElement(values, ""); if (shouldAddElement(element)) { builder.field(name, element); }
diff --git a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java index 9160b9e..7151e70 100644 --- a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java +++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java
@@ -22,12 +22,12 @@ import com.google.gerrit.server.query.change.AbstractQueryChangesTest; import com.google.gerrit.testing.ConfigSuite; import com.google.gerrit.testing.GerritTestName; -import com.google.gerrit.testing.InMemoryRepositoryManager; import com.google.inject.Injector; import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; import org.apache.http.impl.nio.client.HttpAsyncClients; import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.Repository; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -86,8 +86,9 @@ @Test public void testErrorResponseFromChangeIndex() throws Exception { - TestRepository<InMemoryRepositoryManager.Repo> repo = createProject("repo"); - Change c = insert(repo, newChangeWithStatus(repo, Change.Status.NEW)); + String repository = "repo"; + TestRepository<Repository> repo = createAndOpenProject(repository); + Change c = insert(repository, newChangeWithStatus(repo, Change.Status.NEW)); gApi.changes().id(c.getChangeId()).index(); ElasticTestUtils.closeIndex(client, container, testName);