Merge branch 'stable-3.7'
* stable-3.7: (24 commits)
Add an analyzer with tokenizer:keyword to prefix fields
Prune _source to only contain stored fields
Introduce 'elasticsearch.codec' config
Disable tracking total number of hits
Use bool 'filter' queries instead of 'match'
Replace camelCase format name dateOptionalTime
Return cardinality from predicate when available
Update testcontainers to 1.17.5
Update testcontainers to 1.16.3
Elasticsearch tests: Recreate container for each test suite
Paginate no-limit queries
Introduce a SEARCH_AFTER index pagination type
Remove support for vulnerable ES versions
Add 7.16.* to supported versions
tests: Print container logs of startup failure
Use official elasticsearch docker image for tests
Add 7.10.* to supported versions
Use errors output field to determine _bulk API failure
elasticsearch-rest-client: Update to latest 8.3.2
docs: Split README content into more files
...
Release-Notes: skip
Change-Id: I8361cdc33f7331b8c106d9817061d55050373b1a
diff --git a/BUILD b/BUILD
index 4750354..4070a77 100644
--- a/BUILD
+++ b/BUILD
@@ -21,10 +21,10 @@
ELASTICSEARCH_DEPS = [
"@docker-java-api//jar",
"@docker-java-transport//jar",
+ "@docker-java-transport-zerodep//jar",
"@duct-tape//jar",
"@httpasyncclient//jar",
"@jackson-annotations//jar",
- "@jackson-core//jar",
"@jna//jar",
"@testcontainers-elasticsearch//jar",
"@testcontainers//jar",
@@ -73,6 +73,7 @@
tags = [
"docker",
"elastic",
+ "exclusive",
],
deps = ELASTICSEARCH_DEPS + PLUGIN_TEST_DEPS + [
QUERY_TESTS_DEP % name,
diff --git a/README.md b/README.md
index 6565f33..41d4279 100644
--- a/README.md
+++ b/README.md
@@ -3,9 +3,13 @@
Indexing backend libModule for [Gerrit Code Review](https://gerritcodereview.com)
based on [ElasticSearch](https://www.elastic.co/elasticsearch/).
-This module was original part of Gerrit core and then extracted into a separate
+This module was originally part of Gerrit core and then extracted into a separate
component from v3.5.0-rc3 as part of [Change-Id: Ib7b5167ce](https://gerrit-review.googlesource.com/c/gerrit/+/323676).
+Note that, ElasticSearch source code is no longer Apache 2.0-licensed for versions
+7.11 and newer. See ElasticSearch [2021 license change](https://www.elastic.co/pricing/faq/licensing)
+for more information.
+
## How to build
This libModule is built like a Gerrit in-tree plugin, using Bazelisk. See the
@@ -13,32 +17,13 @@
## Setup
-* Install index-elasticsearch module
+See the [setup instructions](src/main/resources/Documentation/setup.md) for how to install the
+index-elasticsearch module.
-Install the index-elasticsearch.jar into the `$GERRIT_SITE/lib` directory.
-
-Add the index-elasticsearch module to `$GERRIT_SITE/etc/gerrit.config` as follows:
-
-```ini
-[gerrit]
- installIndexModule = com.google.gerrit.elasticsearch.ElasticIndexModule
-```
-
-When installing the module on Gerrit replicas, use following example:
-
-```ini
-[gerrit]
- installIndexModule = com.google.gerrit.elasticsearch.ReplicaElasticIndexModule
-```
-
-For further information and supported options, refer to [config](src/main/resources/Documentation/config.md)
+For further information and supported options, refer to the [config](src/main/resources/Documentation/config.md)
documentation.
## Integration test
-Gerrit acceptance tests allow the execution with an alternate implementation of
-the indexing backend using the `GERRIT_INDEX_MODULE` environment variable.
-
-```sh
-bazel test --test_env=GERRIT_INDEX_MODULE=com.google.gerrit.elasticsearch.ElasticIndexModule //...
-```
+This libModule runs tests like a Gerrit in-tree plugin, using Bazelisk. See the
+[test instructions](src/main/resources/Documentation/build.md#Integration-test) for more details.
diff --git a/external_plugin_deps.bzl b/external_plugin_deps.bzl
index fc1420c..7ab2c29 100644
--- a/external_plugin_deps.bzl
+++ b/external_plugin_deps.bzl
@@ -1,30 +1,26 @@
load("//tools/bzl:maven_jar.bzl", "maven_jar")
-TESTCONTAINERS_VERSION = "1.15.3"
-
-# Ensure artifacts compatibility by selecting them from the Bill Of Materials
-# https://search.maven.org/artifact/net.openhft/chronicle-bom/2.20.191/pom
def external_plugin_deps():
maven_jar(
+ name = "jackson-core",
+ artifact = "com.fasterxml.jackson.core:jackson-core:2.11.3",
+ sha1 = "c2351800432bdbdd8284c3f5a7f0782a352aa84a",
+ )
+
+ # Ensure artifacts compatibility by selecting them from the Bill Of Materials
+ # https://search.maven.org/artifact/org.testcontainers/testcontainers/1.17.5/pom
+ TESTCONTAINERS_VERSION = "1.17.5"
+
+ maven_jar(
name = "testcontainers",
artifact = "org.testcontainers:testcontainers:" + TESTCONTAINERS_VERSION,
- sha1 = "95c6cfde71c2209f0c29cb14e432471e0b111880",
- )
-
- # When upgrading elasticsearch-rest-client, also upgrade httpcore-nio
- # and httpasyncclient as necessary in tools/nongoogle.bzl. Consider
- # also the other org.apache.httpcomponents dependencies in
- # WORKSPACE.
- maven_jar(
- name = "elasticsearch-rest-client",
- artifact = "org.elasticsearch.client:elasticsearch-rest-client:7.8.1",
- sha1 = "59feefe006a96a39f83b0dfb6780847e06c1d0a8",
+ sha1 = "7c5ad975fb789ecd09b1ee5f72907f48a300bc61",
)
maven_jar(
name = "testcontainers-elasticsearch",
artifact = "org.testcontainers:elasticsearch:" + TESTCONTAINERS_VERSION,
- sha1 = "595e3a50f59cd3c1d281ca6c1bc4037e277a1353",
+ sha1 = "060895f2fc6640ab4a6c383bc98c5c39ef644fbb",
)
maven_jar(
@@ -33,30 +29,49 @@
sha1 = "92edc22a9ab2f3e17c9bf700aaee377d50e8b530",
)
- maven_jar(
- name = "visible-assertions",
- artifact = "org.rnorth.visible-assertions:visible-assertions:2.1.2",
- sha1 = "20d31a578030ec8e941888537267d3123c2ad1c1",
- )
-
- maven_jar(
- name = "jna",
- artifact = "net.java.dev.jna:jna:5.5.0",
- sha1 = "0e0845217c4907822403912ad6828d8e0b256208",
- )
-
- DOCKER_JAVA_VERS = "3.2.8"
+ DOCKER_JAVA_VERS = "3.2.13"
maven_jar(
name = "docker-java-api",
artifact = "com.github.docker-java:docker-java-api:" + DOCKER_JAVA_VERS,
- sha1 = "4ac22a72d546a9f3523cd4b5fabffa77c4a6ec7c",
+ sha1 = "5817ef8f770cb7e740d590090bf352df9491f3c1",
)
maven_jar(
name = "docker-java-transport",
artifact = "com.github.docker-java:docker-java-transport:" + DOCKER_JAVA_VERS,
- sha1 = "c3b5598c67d0a5e2e780bf48f520da26b9915eab",
+ sha1 = "e9d308d1822181a9d48c99739f5eca014ec89199",
+ )
+
+ maven_jar(
+ name = "docker-java-transport-zerodep",
+ artifact = "com.github.docker-java:docker-java-transport-zerodep:" + DOCKER_JAVA_VERS,
+ sha1 = "4cbc2c09d6c264767a39624066987ed4a152bc68",
+ )
+
+ # Match version used in docker-java-transport
+ # https://search.maven.org/artifact/com.github.docker-java/docker-java-transport/3.2.12/pom
+ maven_jar(
+ name = "jna",
+ artifact = "net.java.dev.jna:jna:5.8.0",
+ sha1 = "3551d8d827e54858214107541d3aff9c615cb615",
+ )
+
+ # Match jackson.version from docker-java
+ # https://search.maven.org/artifact/com.github.docker-java/docker-java-parent/3.2.12/pom
+ maven_jar(
+ name = "jackson-annotations",
+ artifact = "com.fasterxml.jackson.core:jackson-annotations:2.10.3",
+ sha1 = "0f63b3b1da563767d04d2e4d3fc1ae0cdeffebe7",
+ )
+
+ # When upgrading elasticsearch-rest-client, also upgrade httpcore-nio
+ # and httpasyncclient as necessary. Consider also the other
+ # org.apache.httpcomponents dependencies in core.
+ maven_jar(
+ name = "elasticsearch-rest-client",
+ artifact = "org.elasticsearch.client:elasticsearch-rest-client:8.3.2",
+ sha1 = "bb5cb3dbd82ea75a6d49b9011ca5b1d125b30f00",
)
# elasticsearch-rest-client explicitly depends on this version
@@ -72,21 +87,3 @@
artifact = "org.apache.httpcomponents:httpcore-nio:4.4.12",
sha1 = "84cd29eca842f31db02987cfedea245af020198b",
)
-
- maven_jar(
- name = "jackson-core",
- artifact = "com.fasterxml.jackson.core:jackson-core:2.11.3",
- sha1 = "c2351800432bdbdd8284c3f5a7f0782a352aa84a",
- )
-
- maven_jar(
- name = "jackson-annotations",
- artifact = "com.fasterxml.jackson.core:jackson-annotations:2.10.3",
- sha1 = "0f63b3b1da563767d04d2e4d3fc1ae0cdeffebe7",
- )
-
- maven_jar(
- name = "httpasyncclient",
- artifact = "org.apache.httpcomponents:httpasyncclient:4.1.4",
- sha1 = "f3a3240681faae3fa46b573a4c7e50cec9db0d86",
- )
diff --git a/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java b/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
index 90ce18b..2309dad 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
@@ -19,6 +19,7 @@
import static com.google.gson.FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES;
import static java.nio.charset.StandardCharsets.UTF_8;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@@ -29,19 +30,19 @@
import com.google.common.io.BaseEncoding;
import com.google.common.io.CharStreams;
import com.google.gerrit.common.Nullable;
-import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
+import com.google.gerrit.elasticsearch.ElasticMapping.Mapping;
import com.google.gerrit.elasticsearch.builders.QueryBuilder;
import com.google.gerrit.elasticsearch.builders.SearchSourceBuilder;
import com.google.gerrit.elasticsearch.bulk.DeleteRequest;
import com.google.gerrit.entities.converter.ProtoConverter;
import com.google.gerrit.exceptions.StorageException;
-import com.google.gerrit.index.FieldDef;
import com.google.gerrit.index.FieldType;
import com.google.gerrit.index.Index;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.query.DataSource;
import com.google.gerrit.index.query.FieldBundle;
+import com.google.gerrit.index.query.HasCardinality;
import com.google.gerrit.index.query.ListResultSet;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
@@ -49,6 +50,7 @@
import com.google.gerrit.proto.Protos;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
+import com.google.gerrit.server.index.options.AutoFlush;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonArray;
@@ -64,7 +66,6 @@
import java.net.URLEncoder;
import java.sql.Timestamp;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -75,6 +76,7 @@
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.nio.entity.NStringEntity;
+import org.apache.http.util.EntityUtils;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
@@ -129,18 +131,22 @@
private final Schema<V> schema;
private final SitePaths sitePaths;
private final String indexNameRaw;
+ private final Map<String, String> refreshParam;
protected final ElasticRestClientProvider client;
protected final String indexName;
protected final Gson gson;
protected final ElasticQueryBuilder queryBuilder;
+ private final Function<V, K> valueToKeyFunction;
AbstractElasticIndex(
ElasticConfiguration config,
SitePaths sitePaths,
Schema<V> schema,
ElasticRestClientProvider client,
- String indexName) {
+ String indexName,
+ AutoFlush autoFlush,
+ Function<V, K> valueToKeyFunction) {
this.config = config;
this.sitePaths = sitePaths;
this.schema = schema;
@@ -149,6 +155,16 @@
this.indexName = config.getIndexName(indexName, schema.getVersion());
this.indexNameRaw = indexName;
this.client = client;
+ this.refreshParam =
+ Map.of(
+ "refresh",
+ autoFlush == AutoFlush.ENABLED ? Boolean.TRUE.toString() : Boolean.FALSE.toString());
+ this.valueToKeyFunction = valueToKeyFunction;
+ }
+
+ @Override
+ public void deleteByValue(V value) {
+ delete(valueToKeyFunction.apply(value));
}
@Override
@@ -174,7 +190,7 @@
@Override
public void delete(K id) {
String uri = getURI(BULK);
- Response response = postRequest(uri, getDeleteActions(id), getRefreshParam());
+ Response response = postRequestWithRefreshParam(uri, getDeleteActions(id));
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new StorageException(
@@ -217,14 +233,14 @@
protected abstract String getId(V v);
- protected String getMappingsForSingleType(MappingProperties properties) {
- return getMappingsFor(properties);
+ protected String getMappingsForSingleType(Mapping mapping) {
+ return getMappingsFor(mapping);
}
- protected String getMappingsFor(MappingProperties properties) {
+ protected String getMappingsFor(Mapping mapping) {
JsonObject mappings = new JsonObject();
- mappings.add(MAPPINGS, gson.toJsonTree(properties));
+ mappings.add(MAPPINGS, gson.toJsonTree(mapping));
return gson.toJson(mappings);
}
@@ -235,13 +251,12 @@
protected abstract V fromDocument(JsonObject doc, Set<String> fields);
protected FieldBundle toFieldBundle(JsonObject doc) {
- Map<String, FieldDef<V, ?>> allFields = getSchema().getFields();
ListMultimap<String, Object> rawFields = ArrayListMultimap.create();
for (Map.Entry<String, JsonElement> element :
doc.get(client.adapter().rawFieldsKey()).getAsJsonObject().entrySet()) {
checkArgument(
- allFields.containsKey(element.getKey()), "Unrecognized field " + element.getKey());
- FieldType<?> type = allFields.get(element.getKey()).getType();
+ getSchema().hasField(element.getKey()), "Unrecognized field " + element.getKey());
+ FieldType<?> type = getSchema().getSchemaField(element.getKey()).getType();
Iterable<JsonElement> innerItems =
element.getValue().isJsonArray()
? element.getValue().getAsJsonArray()
@@ -262,7 +277,21 @@
}
}
}
- return new FieldBundle(rawFields);
+ return new FieldBundle(rawFields, /* storesIndexedFields= */ false);
+ }
+
+ protected boolean hasErrors(Response response) {
+ try {
+ String contentType = response.getEntity().getContentType().getValue();
+ Preconditions.checkState(
+ contentType.equals(ContentType.APPLICATION_JSON.toString()),
+ String.format("Expected %s, but was: %s", ContentType.APPLICATION_JSON, contentType));
+ String responseStr = EntityUtils.toString(response.getEntity());
+ JsonObject responseJson = (JsonObject) new JsonParser().parse(responseStr);
+ return responseJson.get("errors").getAsBoolean();
+ } catch (IOException e) {
+ throw new StorageException(e);
+ }
}
protected String toAction(String type, String id, String action) {
@@ -282,12 +311,6 @@
array.add(arrayElement);
}
- protected Map<String, String> getRefreshParam() {
- Map<String, String> params = new HashMap<>();
- params.put("refresh", "true");
- return params;
- }
-
protected String getSearch(SearchSourceBuilder searchSource, JsonArray sortArray) {
JsonObject search = new JsonParser().parse(searchSource.toString()).getAsJsonObject();
search.add("sort", sortArray);
@@ -311,12 +334,8 @@
}
}
- protected Response postRequest(String uri, Object payload) {
- return performRequest("POST", uri, payload);
- }
-
- protected Response postRequest(String uri, Object payload, Map<String, String> params) {
- return performRequest("POST", uri, payload, params);
+ protected Response postRequestWithRefreshParam(String uri, Object payload) {
+ return performRequest("POST", uri, payload, refreshParam);
}
private String concatJsonString(String target, String addition) {
@@ -350,23 +369,32 @@
protected class ElasticQuerySource implements DataSource<V> {
private final QueryOptions opts;
+ private final Predicate<V> predicate;
private final String search;
ElasticQuerySource(Predicate<V> p, QueryOptions opts, JsonArray sortArray)
throws QueryParseException {
this.opts = opts;
+ this.predicate = p;
QueryBuilder qb = queryBuilder.toQueryBuilder(p);
SearchSourceBuilder searchSource =
new SearchSourceBuilder(client.adapter())
.query(qb)
- .from(opts.start())
- .size(opts.limit())
- .fields(Lists.newArrayList(opts.fields()));
+ .size(opts.pageSize())
+ .fields(Lists.newArrayList(opts.fields()))
+ .trackTotalHits(false);
+ searchSource =
+ opts.searchAfter() != null
+ ? searchSource.searchAfter((JsonArray) opts.searchAfter())
+ : searchSource.from(opts.start());
search = getSearch(searchSource, sortArray);
}
@Override
public int getCardinality() {
+ if (predicate instanceof HasCardinality) {
+ return ((HasCardinality) predicate).getCardinality();
+ }
return 10;
}
@@ -383,6 +411,7 @@
private <T> ResultSet<T> readImpl(Function<JsonObject, T> mapper) {
try {
String uri = getURI(SEARCH);
+ JsonArray searchAfter = null;
Response response =
performRequest(HttpPost.METHOD_NAME, uri, search, Collections.emptyMap());
StatusLine statusLine = response.getStatusLine();
@@ -393,16 +422,27 @@
if (obj.get("hits") != null) {
JsonArray json = obj.getAsJsonArray("hits");
ImmutableList.Builder<T> results = ImmutableList.builderWithExpectedSize(json.size());
+ JsonObject hit = null;
for (int i = 0; i < json.size(); i++) {
- T mapperResult = mapper.apply(json.get(i).getAsJsonObject());
+ hit = json.get(i).getAsJsonObject();
+ T mapperResult = mapper.apply(hit);
if (mapperResult != null) {
results.add(mapperResult);
}
}
- return new ListResultSet<>(results.build());
+ if (hit != null && hit.get("sort") != null) {
+ searchAfter = hit.getAsJsonArray("sort");
+ }
+ JsonArray finalSearchAfter = searchAfter;
+ return new ListResultSet<T>(results.build()) {
+ @Override
+ public Object searchAfter() {
+ return finalSearchAfter;
+ }
+ };
}
} else {
- logger.atSevere().log(statusLine.getReasonPhrase());
+ logger.atSevere().log("%s", statusLine.getReasonPhrase());
}
return new ListResultSet<>(ImmutableList.of());
} catch (IOException e) {
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
index beb57bd..d1c5416 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
@@ -15,7 +15,7 @@
package com.google.gerrit.elasticsearch;
import com.google.common.collect.ImmutableSet;
-import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
+import com.google.gerrit.elasticsearch.ElasticMapping.Mapping;
import com.google.gerrit.elasticsearch.bulk.BulkRequest;
import com.google.gerrit.elasticsearch.bulk.IndexRequest;
import com.google.gerrit.elasticsearch.bulk.UpdateRequest;
@@ -32,6 +32,7 @@
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.account.AccountField;
import com.google.gerrit.server.index.account.AccountIndex;
+import com.google.gerrit.server.index.options.AutoFlush;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
@@ -45,7 +46,7 @@
public class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, AccountState>
implements AccountIndex {
static class AccountMapping {
- final MappingProperties accounts;
+ final Mapping accounts;
AccountMapping(Schema<AccountState> schema, ElasticQueryAdapter adapter) {
this.accounts = ElasticMapping.createMapping(schema, adapter);
@@ -64,8 +65,9 @@
SitePaths sitePaths,
Provider<AccountCache> accountCache,
ElasticRestClientProvider client,
+ AutoFlush autoFlush,
@Assisted Schema<AccountState> schema) {
- super(cfg, sitePaths, schema, client, ACCOUNTS);
+ super(cfg, sitePaths, schema, client, ACCOUNTS, autoFlush, AccountIndex.ENTITY_TO_KEY);
this.accountCache = accountCache;
this.mapping = new AccountMapping(schema, client.adapter());
this.schema = schema;
@@ -78,9 +80,9 @@
.add(new UpdateRequest<>(schema, as, ImmutableSet.of()));
String uri = getURI(BULK);
- Response response = postRequest(uri, bulk, getRefreshParam());
+ Response response = postRequestWithRefreshParam(uri, bulk);
int statusCode = response.getStatusLine().getStatusCode();
- if (statusCode != HttpStatus.SC_OK) {
+ if (hasErrors(response) || statusCode != HttpStatus.SC_OK) {
throw new StorageException(
String.format(
"Failed to replace account %s in index %s: %s",
@@ -91,10 +93,12 @@
@Override
public DataSource<AccountState> getSource(Predicate<AccountState> p, QueryOptions opts)
throws QueryParseException {
- boolean useLegacyNumericFields = schema.hasField(AccountField.ID);
+ boolean useLegacyNumericFields = schema.hasField(AccountField.ID_FIELD_SPEC);
JsonArray sortArray =
getSortArray(
- useLegacyNumericFields ? AccountField.ID.getName() : AccountField.ID_STR.getName());
+ useLegacyNumericFields
+ ? AccountField.ID_FIELD_SPEC.getName()
+ : AccountField.ID_STR_FIELD_SPEC.getName());
return new ElasticQuerySource(
p, opts.filterFields(o -> IndexUtils.accountFields(o, useLegacyNumericFields)), sortArray);
}
@@ -126,9 +130,9 @@
source
.getAsJsonObject()
.get(
- schema.hasField(AccountField.ID)
- ? AccountField.ID.getName()
- : AccountField.ID_STR.getName())
+ schema.hasField(AccountField.ID_FIELD_SPEC)
+ ? AccountField.ID_FIELD_SPEC.getName()
+ : AccountField.ID_STR_FIELD_SPEC.getName())
.getAsInt());
// Use the AccountCache rather than depending on any stored fields in the document (of which
// there shouldn't be any). The most expensive part to compute anyway is the effective group
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
index 83be597..1cc00cd 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
@@ -17,7 +17,7 @@
import static java.util.Objects.requireNonNull;
import com.google.common.collect.ImmutableSet;
-import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
+import com.google.gerrit.elasticsearch.ElasticMapping.Mapping;
import com.google.gerrit.elasticsearch.bulk.BulkRequest;
import com.google.gerrit.elasticsearch.bulk.IndexRequest;
import com.google.gerrit.elasticsearch.bulk.UpdateRequest;
@@ -25,9 +25,9 @@
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.converter.ChangeProtoConverter;
import com.google.gerrit.exceptions.StorageException;
-import com.google.gerrit.index.FieldDef;
import com.google.gerrit.index.QueryOptions;
import com.google.gerrit.index.Schema;
+import com.google.gerrit.index.SchemaFieldDefs.SchemaField;
import com.google.gerrit.index.query.DataSource;
import com.google.gerrit.index.query.Predicate;
import com.google.gerrit.index.query.QueryParseException;
@@ -37,6 +37,7 @@
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.change.ChangeField;
import com.google.gerrit.server.index.change.ChangeIndex;
+import com.google.gerrit.server.index.options.AutoFlush;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
@@ -52,12 +53,12 @@
class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
implements ChangeIndex {
static class ChangeMapping {
- final MappingProperties changes;
- final MappingProperties openChanges;
- final MappingProperties closedChanges;
+ final Mapping changes;
+ final Mapping openChanges;
+ final Mapping closedChanges;
ChangeMapping(Schema<ChangeData> schema, ElasticQueryAdapter adapter) {
- MappingProperties mapping = ElasticMapping.createMapping(schema, adapter);
+ Mapping mapping = ElasticMapping.createMapping(schema, adapter);
this.changes = mapping;
this.openChanges = mapping;
this.closedChanges = mapping;
@@ -78,15 +79,16 @@
SitePaths sitePaths,
ElasticRestClientProvider clientBuilder,
@GerritServerConfig Config gerritConfig,
+ AutoFlush autoFlush,
@Assisted Schema<ChangeData> schema) {
- super(cfg, sitePaths, schema, clientBuilder, CHANGES);
+ super(cfg, sitePaths, schema, clientBuilder, CHANGES, autoFlush, ChangeIndex.ENTITY_TO_KEY);
this.changeDataFactory = changeDataFactory;
this.schema = schema;
this.mapping = new ChangeMapping(schema, client.adapter());
this.skipFields =
MergeabilityComputationBehavior.fromConfig(gerritConfig).includeInIndex()
? ImmutableSet.of()
- : ImmutableSet.of(ChangeField.MERGEABLE.getName());
+ : ImmutableSet.of(ChangeField.MERGEABLE_SPEC.getName());
}
@Override
@@ -95,9 +97,9 @@
new IndexRequest(getId(cd), indexName).add(new UpdateRequest<>(schema, cd, skipFields));
String uri = getURI(BULK);
- Response response = postRequest(uri, bulk, getRefreshParam());
+ Response response = postRequestWithRefreshParam(uri, bulk);
int statusCode = response.getStatusLine().getStatusCode();
- if (statusCode != HttpStatus.SC_OK) {
+ if (hasErrors(response) || statusCode != HttpStatus.SC_OK) {
throw new StorageException(
String.format(
"Failed to replace change %s in index %s: %s", cd.getId(), indexName, statusCode));
@@ -116,9 +118,9 @@
properties.addProperty(ORDER, DESC_SORT_ORDER);
JsonArray sortArray = new JsonArray();
- addNamedElement(ChangeField.UPDATED.getName(), properties, sortArray);
- addNamedElement(ChangeField.MERGED_ON.getName(), getMergedOnSortOptions(), sortArray);
- addNamedElement(ChangeField.LEGACY_ID_STR.getName(), properties, sortArray);
+ addNamedElement(ChangeField.UPDATED_SPEC.getName(), properties, sortArray);
+ addNamedElement(ChangeField.MERGED_ON_SPEC.getName(), getMergedOnSortOptions(), sortArray);
+ addNamedElement(ChangeField.NUMERIC_ID_STR_SPEC.getName(), properties, sortArray);
return sortArray;
}
@@ -153,12 +155,13 @@
sourceElement = json.getAsJsonObject().get("fields");
}
JsonObject source = sourceElement.getAsJsonObject();
- JsonElement c = source.get(ChangeField.CHANGE.getName());
+ JsonElement c = source.get(ChangeField.CHANGE_SPEC.getName());
if (c == null) {
- int id = source.get(ChangeField.LEGACY_ID_STR.getName()).getAsInt();
+ int id = source.get(ChangeField.NUMERIC_ID_STR_SPEC.getName()).getAsInt();
// IndexUtils#changeFields ensures either CHANGE or PROJECT is always present.
- String projectName = requireNonNull(source.get(ChangeField.PROJECT.getName()).getAsString());
+ String projectName =
+ requireNonNull(source.get(ChangeField.PROJECT_SPEC.getName()).getAsString());
return changeDataFactory.create(Project.nameKey(projectName), Change.id(id));
}
@@ -166,7 +169,7 @@
changeDataFactory.create(
parseProtoFrom(decodeBase64(c.getAsString()), ChangeProtoConverter.INSTANCE));
- for (FieldDef<ChangeData, ?> field : getSchema().getFields().values()) {
+ for (SchemaField<ChangeData, ?> field : getSchema().getSchemaFields().values()) {
if (fields.contains(field.getName()) && source.get(field.getName()) != null) {
field.setIfPossible(cd, new ElasticStoredValue(source.get(field.getName())));
}
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java
index e544d9f..2a5fcde 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java
@@ -43,9 +43,11 @@
static final String KEY_NUMBER_OF_SHARDS = "numberOfShards";
static final String KEY_NUMBER_OF_REPLICAS = "numberOfReplicas";
static final String KEY_MAX_RESULT_WINDOW = "maxResultWindow";
+ static final String KEY_CODEC = "codec";
static final String KEY_CONNECT_TIMEOUT = "connectTimeout";
static final String KEY_SOCKET_TIMEOUT = "socketTimeout";
+ static final String DEFAULT_CODEC = "default";
static final String DEFAULT_PORT = "9200";
static final String DEFAULT_USERNAME = "elastic";
static final int DEFAULT_NUMBER_OF_SHARDS = 1;
@@ -62,6 +64,7 @@
final int numberOfShards;
final int numberOfReplicas;
final int maxResultWindow;
+ final String codec;
final int connectTimeout;
final int socketTimeout;
final String prefix;
@@ -82,6 +85,7 @@
cfg.getInt(SECTION_ELASTICSEARCH, null, KEY_NUMBER_OF_REPLICAS, DEFAULT_NUMBER_OF_REPLICAS);
this.maxResultWindow =
cfg.getInt(SECTION_ELASTICSEARCH, null, KEY_MAX_RESULT_WINDOW, DEFAULT_MAX_RESULT_WINDOW);
+ this.codec = firstNonNull(cfg.getString(SECTION_ELASTICSEARCH, null, KEY_CODEC), DEFAULT_CODEC);
this.connectTimeout =
(int)
cfg.getTimeUnit(
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
index 781ed43..d2b9228 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
@@ -15,7 +15,7 @@
package com.google.gerrit.elasticsearch;
import com.google.common.collect.ImmutableSet;
-import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
+import com.google.gerrit.elasticsearch.ElasticMapping.Mapping;
import com.google.gerrit.elasticsearch.bulk.BulkRequest;
import com.google.gerrit.elasticsearch.bulk.IndexRequest;
import com.google.gerrit.elasticsearch.bulk.UpdateRequest;
@@ -32,6 +32,7 @@
import com.google.gerrit.server.index.IndexUtils;
import com.google.gerrit.server.index.group.GroupField;
import com.google.gerrit.server.index.group.GroupIndex;
+import com.google.gerrit.server.index.options.AutoFlush;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
@@ -45,7 +46,7 @@
public class ElasticGroupIndex extends AbstractElasticIndex<AccountGroup.UUID, InternalGroup>
implements GroupIndex {
static class GroupMapping {
- final MappingProperties groups;
+ final Mapping groups;
GroupMapping(Schema<InternalGroup> schema, ElasticQueryAdapter adapter) {
this.groups = ElasticMapping.createMapping(schema, adapter);
@@ -64,8 +65,9 @@
SitePaths sitePaths,
Provider<GroupCache> groupCache,
ElasticRestClientProvider client,
+ AutoFlush autoFlush,
@Assisted Schema<InternalGroup> schema) {
- super(cfg, sitePaths, schema, client, GROUPS);
+ super(cfg, sitePaths, schema, client, GROUPS, autoFlush, GroupIndex.ENTITY_TO_KEY);
this.groupCache = groupCache;
this.mapping = new GroupMapping(schema, client.adapter());
this.schema = schema;
@@ -78,9 +80,9 @@
.add(new UpdateRequest<>(schema, group, ImmutableSet.of()));
String uri = getURI(BULK);
- Response response = postRequest(uri, bulk, getRefreshParam());
+ Response response = postRequestWithRefreshParam(uri, bulk);
int statusCode = response.getStatusLine().getStatusCode();
- if (statusCode != HttpStatus.SC_OK) {
+ if (hasErrors(response) || statusCode != HttpStatus.SC_OK) {
throw new StorageException(
String.format(
"Failed to replace group %s in index %s: %s",
@@ -91,7 +93,7 @@
@Override
public DataSource<InternalGroup> getSource(Predicate<InternalGroup> p, QueryOptions opts)
throws QueryParseException {
- JsonArray sortArray = getSortArray(GroupField.UUID.getName());
+ JsonArray sortArray = getSortArray(GroupField.UUID_FIELD_SPEC.getName());
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::groupFields), sortArray);
}
@@ -118,7 +120,8 @@
}
AccountGroup.UUID uuid =
- AccountGroup.uuid(source.getAsJsonObject().get(GroupField.UUID.getName()).getAsString());
+ AccountGroup.uuid(
+ source.getAsJsonObject().get(GroupField.UUID_FIELD_SPEC.getName()).getAsString());
// Use the GroupCache rather than depending on any stored fields in the
// document (of which there shouldn't be any).
return groupCache.get().get(uuid).orElse(null);
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java
index 671e9f4..b1bb7b1 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java
@@ -14,6 +14,7 @@
package com.google.gerrit.elasticsearch;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.index.project.ProjectIndex;
import com.google.gerrit.server.ModuleImpl;
@@ -22,6 +23,7 @@
import com.google.gerrit.server.index.account.AccountIndex;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.group.GroupIndex;
+import com.google.gerrit.server.index.options.AutoFlush;
import com.google.inject.Inject;
import java.util.Map;
@@ -29,22 +31,28 @@
public class ElasticIndexModule extends AbstractIndexModule {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+ private final AutoFlush autoFlush;
+
+ @VisibleForTesting
public static ElasticIndexModule singleVersionWithExplicitVersions(
Map<String, Integer> versions, int threads, boolean slave) {
- return new ElasticIndexModule(versions, threads, slave);
+ return new ElasticIndexModule(versions, threads, slave, AutoFlush.ENABLED);
}
- public static ElasticIndexModule latestVersion(boolean slave) {
- return new ElasticIndexModule(null, 0, slave);
- }
-
- protected ElasticIndexModule(Map<String, Integer> singleVersions, int threads, boolean slave) {
- super(singleVersions, threads, slave);
+ public static ElasticIndexModule singleVersionWithExplicitVersions(
+ Map<String, Integer> versions, int threads, boolean slave, AutoFlush autoFlush) {
+ return new ElasticIndexModule(versions, threads, slave, autoFlush);
}
@Inject
public ElasticIndexModule() {
- this(null, 0, false);
+ this(null, 0, false, AutoFlush.ENABLED);
+ }
+
+ protected ElasticIndexModule(
+ Map<String, Integer> singleVersions, int threads, boolean slave, AutoFlush autoFlush) {
+ super(singleVersions, threads, slave);
+ this.autoFlush = autoFlush;
}
@Override
@@ -52,6 +60,7 @@
logger.atInfo().log("Gerrit index backend set to ElasticSearch");
super.configure();
install(ElasticRestClientProvider.module());
+ bind(AutoFlush.class).toInstance(autoFlush);
}
@Override
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexVersionDiscovery.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexVersionDiscovery.java
index 100022a..4217f45 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexVersionDiscovery.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexVersionDiscovery.java
@@ -49,7 +49,7 @@
String.format(
"Failed to discover index versions for %s: %d: %s",
name, statusLine.getStatusCode(), statusLine.getReasonPhrase());
- logger.atSevere().log(message);
+ logger.atSevere().log("%s", message);
throw new IOException(message);
}
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java
index edd05c9..48cb10a 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java
@@ -15,19 +15,20 @@
package com.google.gerrit.elasticsearch;
import com.google.common.collect.ImmutableMap;
-import com.google.gerrit.index.FieldDef;
import com.google.gerrit.index.FieldType;
import com.google.gerrit.index.Schema;
+import com.google.gerrit.index.SchemaFieldDefs.SchemaField;
+import com.google.gson.annotations.SerializedName;
import java.util.Map;
class ElasticMapping {
protected static final String TIMESTAMP_FIELD_TYPE = "date";
- protected static final String TIMESTAMP_FIELD_FORMAT = "dateOptionalTime";
+ protected static final String TIMESTAMP_FIELD_FORMAT = "date_optional_time";
- static MappingProperties createMapping(Schema<?> schema, ElasticQueryAdapter adapter) {
+ static Mapping createMapping(Schema<?> schema, ElasticQueryAdapter adapter) {
ElasticMapping.Builder mapping = new ElasticMapping.Builder(adapter);
- for (FieldDef<?, ?> field : schema.getFields().values()) {
+ for (SchemaField<?, ?> field : schema.getSchemaFields().values()) {
String name = field.getName();
FieldType<?> fieldType = field.getType();
if (fieldType == FieldType.EXACT) {
@@ -39,13 +40,20 @@
|| fieldType == FieldType.LONG) {
mapping.addNumber(name);
} else if (fieldType == FieldType.FULL_TEXT) {
- mapping.addStringWithAnalyzer(name);
- } else if (fieldType == FieldType.PREFIX || fieldType == FieldType.STORED_ONLY) {
+ mapping.addStringWithAnalyzer(name, "custom_with_char_filter");
+ } else if (fieldType == FieldType.PREFIX) {
+ mapping.addStringWithAnalyzer(name, "keyword_tokenizer");
+ } else if (fieldType == FieldType.STORED_ONLY) {
mapping.addString(name);
} else {
throw new IllegalStateException("Unsupported field type: " + fieldType.getName());
}
}
+ mapping.addSourceIncludes(
+ schema.getSchemaFields().values().stream()
+ .filter(f -> f.isStored())
+ .map(f -> f.getName())
+ .toArray(String[]::new));
return mapping.build();
}
@@ -53,15 +61,18 @@
private final ElasticQueryAdapter adapter;
private final ImmutableMap.Builder<String, FieldProperties> fields =
new ImmutableMap.Builder<>();
+ private final ImmutableMap.Builder<String, String[]> sourceIncludes =
+ new ImmutableMap.Builder<>();
Builder(ElasticQueryAdapter adapter) {
this.adapter = adapter;
}
- MappingProperties build() {
- MappingProperties properties = new MappingProperties();
- properties.properties = fields.build();
- return properties;
+ Mapping build() {
+ Mapping mapping = new Mapping();
+ mapping.properties = fields.build();
+ mapping.source = sourceIncludes.build();
+ return mapping;
}
Builder addExactField(String name) {
@@ -92,20 +103,28 @@
return this;
}
- Builder addStringWithAnalyzer(String name) {
+ Builder addStringWithAnalyzer(String name, String analyzer) {
FieldProperties key = new FieldProperties(adapter.stringFieldType());
- key.analyzer = "custom_with_char_filter";
+ key.analyzer = analyzer;
fields.put(name, key);
return this;
}
+ Builder addSourceIncludes(String[] includes) {
+ sourceIncludes.put("includes", includes);
+ return this;
+ }
+
Builder add(String name, String type) {
fields.put(name, new FieldProperties(type));
return this;
}
}
- static class MappingProperties {
+ static class Mapping {
+ @SerializedName("_source")
+ Map<String, String[]> source;
+
Map<String, FieldProperties> properties;
}
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java
index b8bfc38..2fd8e7a 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java
@@ -15,7 +15,7 @@
package com.google.gerrit.elasticsearch;
import com.google.common.collect.ImmutableSet;
-import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
+import com.google.gerrit.elasticsearch.ElasticMapping.Mapping;
import com.google.gerrit.elasticsearch.bulk.BulkRequest;
import com.google.gerrit.elasticsearch.bulk.IndexRequest;
import com.google.gerrit.elasticsearch.bulk.UpdateRequest;
@@ -31,6 +31,7 @@
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.IndexUtils;
+import com.google.gerrit.server.index.options.AutoFlush;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectState;
import com.google.gson.JsonArray;
@@ -47,7 +48,7 @@
public class ElasticProjectIndex extends AbstractElasticIndex<Project.NameKey, ProjectData>
implements ProjectIndex {
static class ProjectMapping {
- MappingProperties projects;
+ Mapping projects;
ProjectMapping(Schema<ProjectData> schema, ElasticQueryAdapter adapter) {
this.projects = ElasticMapping.createMapping(schema, adapter);
@@ -66,8 +67,9 @@
SitePaths sitePaths,
Provider<ProjectCache> projectCache,
ElasticRestClientProvider client,
+ AutoFlush autoFlush,
@Assisted Schema<ProjectData> schema) {
- super(cfg, sitePaths, schema, client, PROJECTS);
+ super(cfg, sitePaths, schema, client, PROJECTS, autoFlush, ProjectIndex.ENTITY_TO_KEY);
this.projectCache = projectCache;
this.schema = schema;
this.mapping = new ProjectMapping(schema, client.adapter());
@@ -80,9 +82,9 @@
.add(new UpdateRequest<>(schema, projectState, ImmutableSet.of()));
String uri = getURI(BULK);
- Response response = postRequest(uri, bulk, getRefreshParam());
+ Response response = postRequestWithRefreshParam(uri, bulk);
int statusCode = response.getStatusLine().getStatusCode();
- if (statusCode != HttpStatus.SC_OK) {
+ if (hasErrors(response) || statusCode != HttpStatus.SC_OK) {
throw new StorageException(
String.format(
"Failed to replace project %s in index %s: %s",
@@ -93,7 +95,7 @@
@Override
public DataSource<ProjectData> getSource(Predicate<ProjectData> p, QueryOptions opts)
throws QueryParseException {
- JsonArray sortArray = getSortArray(ProjectField.NAME.getName());
+ JsonArray sortArray = getSortArray(ProjectField.NAME_SPEC.getName());
return new ElasticQuerySource(p, opts.filterFields(IndexUtils::projectFields), sortArray);
}
@@ -120,7 +122,8 @@
}
Project.NameKey nameKey =
- Project.nameKey(source.getAsJsonObject().get(ProjectField.NAME.getName()).getAsString());
+ Project.nameKey(
+ source.getAsJsonObject().get(ProjectField.NAME_SPEC.getName()).getAsString());
Optional<ProjectState> state = projectCache.get().get(nameKey);
if (!state.isPresent()) {
return null;
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
index 9fc070a..df28dcc 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
@@ -17,8 +17,8 @@
import com.google.gerrit.elasticsearch.builders.BoolQueryBuilder;
import com.google.gerrit.elasticsearch.builders.QueryBuilder;
import com.google.gerrit.elasticsearch.builders.QueryBuilders;
-import com.google.gerrit.index.FieldDef;
import com.google.gerrit.index.FieldType;
+import com.google.gerrit.index.SchemaFieldDefs.SchemaField;
import com.google.gerrit.index.query.AndPredicate;
import com.google.gerrit.index.query.IndexPredicate;
import com.google.gerrit.index.query.IntegerRangePredicate;
@@ -51,7 +51,7 @@
private <T> BoolQueryBuilder and(Predicate<T> p) throws QueryParseException {
BoolQueryBuilder b = QueryBuilders.boolQuery();
for (Predicate<T> c : p.getChildren()) {
- b.must(toQueryBuilder(c));
+ b.filter(toQueryBuilder(c));
}
return b;
}
@@ -72,14 +72,14 @@
// Lucene does not support negation, start with all and subtract.
BoolQueryBuilder q = QueryBuilders.boolQuery();
- q.must(QueryBuilders.matchAllQuery());
+ q.filter(QueryBuilders.matchAllQuery());
q.mustNot(toQueryBuilder(n));
return q;
}
private <T> QueryBuilder fieldQuery(IndexPredicate<T> p) throws QueryParseException {
FieldType<?> type = p.getType();
- FieldDef<?, ?> field = p.getField();
+ SchemaField<?, ?> field = p.getField();
String name = field.getName();
String value = p.getValue();
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticSetting.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticSetting.java
index 7ec0566..c054da6 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticSetting.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticSetting.java
@@ -15,6 +15,8 @@
package com.google.gerrit.elasticsearch;
import com.google.common.collect.ImmutableMap;
+import com.google.gson.annotations.SerializedName;
+import java.util.HashMap;
import java.util.Map;
class ElasticSetting {
@@ -36,6 +38,7 @@
properties.numberOfShards = config.getNumberOfShards();
properties.numberOfReplicas = config.numberOfReplicas;
properties.maxResultWindow = config.maxResultWindow;
+ properties.codec = config.codec;
return properties;
}
@@ -57,6 +60,12 @@
FieldProperties analyzer = new FieldProperties();
analyzer.customWithCharFilter = customAnalyzer;
+ analyzer.keywordTokenizer =
+ new HashMap<>() {
+ {
+ put("tokenizer", "keyword");
+ }
+ };
fields.put("analyzer", analyzer);
return this;
}
@@ -73,6 +82,9 @@
}
static class SettingProperties {
+ @SerializedName("index.codec")
+ String codec;
+
Map<String, FieldProperties> analysis;
Integer numberOfShards;
Integer numberOfReplicas;
@@ -87,6 +99,7 @@
String[] mappings;
FieldProperties customMapping;
FieldProperties customWithCharFilter;
+ Map<String, String> keywordTokenizer;
FieldProperties() {}
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticVersion.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticVersion.java
index b5bf44b..dffdf3e 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticVersion.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticVersion.java
@@ -18,9 +18,7 @@
import java.util.regex.Pattern;
public enum ElasticVersion {
- V7_6("7.6.*"),
- V7_7("7.7.*"),
- V7_8("7.8.*");
+ V7_16("7.16.*");
private final String version;
private final Pattern pattern;
diff --git a/src/main/java/com/google/gerrit/elasticsearch/PrimaryElasticIndexModule.java b/src/main/java/com/google/gerrit/elasticsearch/PrimaryElasticIndexModule.java
index 8500844..823c657 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/PrimaryElasticIndexModule.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/PrimaryElasticIndexModule.java
@@ -16,11 +16,12 @@
import com.google.gerrit.server.ModuleImpl;
import com.google.gerrit.server.index.AbstractIndexModule;
+import com.google.gerrit.server.index.options.AutoFlush;
@ModuleImpl(name = AbstractIndexModule.INDEX_MODULE)
public class PrimaryElasticIndexModule extends ElasticIndexModule {
public PrimaryElasticIndexModule() {
- super(null, 0, false);
+ super(null, 0, false, AutoFlush.ENABLED);
}
}
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ReplicaElasticIndexModule.java b/src/main/java/com/google/gerrit/elasticsearch/ReplicaElasticIndexModule.java
index 8845cbe..86ccac6 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ReplicaElasticIndexModule.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ReplicaElasticIndexModule.java
@@ -16,11 +16,12 @@
import com.google.gerrit.server.ModuleImpl;
import com.google.gerrit.server.index.AbstractIndexModule;
+import com.google.gerrit.server.index.options.AutoFlush;
@ModuleImpl(name = AbstractIndexModule.INDEX_MODULE)
public class ReplicaElasticIndexModule extends ElasticIndexModule {
public ReplicaElasticIndexModule() {
- super(null, 0, true);
+ super(null, 0, true, AutoFlush.ENABLED);
}
}
diff --git a/src/main/java/com/google/gerrit/elasticsearch/builders/BoolQueryBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/builders/BoolQueryBuilder.java
index a204919..741aec7 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/builders/BoolQueryBuilder.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/builders/BoolQueryBuilder.java
@@ -42,6 +42,15 @@
}
/**
+ * Adds a query that <b>must</b> appear in the matching documents and will not contribute to
+ * scoring.
+ */
+ public BoolQueryBuilder filter(QueryBuilder queryBuilder) {
+ filterClauses.add(queryBuilder);
+ return this;
+ }
+
+ /**
* Adds a query that <b>must not</b> appear in the matching documents and will not contribute to
* scoring.
*/
diff --git a/src/main/java/com/google/gerrit/elasticsearch/builders/SearchAfterBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/builders/SearchAfterBuilder.java
new file mode 100644
index 0000000..0951217
--- /dev/null
+++ b/src/main/java/com/google/gerrit/elasticsearch/builders/SearchAfterBuilder.java
@@ -0,0 +1,45 @@
+// Copyright (C) 2022 The Android Open Source Project, 2009-2015 Elasticsearch
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.elasticsearch.builders;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonPrimitive;
+import java.io.IOException;
+
+/**
+ * A trimmed down and modified version of org.elasticsearch.search.searchafter.SearchAfterBuilder.
+ */
+public final class SearchAfterBuilder {
+ private JsonArray sortValues;
+
+ public SearchAfterBuilder(JsonArray sortValues) {
+ this.sortValues = sortValues;
+ }
+
+ public void innerToXContent(XContentBuilder builder) throws IOException {
+ builder.startArray("search_after");
+ for (int i = 0; i < sortValues.size(); i++) {
+ JsonPrimitive value = sortValues.get(i).getAsJsonPrimitive();
+ if (value.isNumber()) {
+ builder.value(value.getAsLong());
+ } else if (value.isBoolean()) {
+ builder.value(value.getAsBoolean());
+ } else {
+ builder.value(value.getAsString());
+ }
+ }
+ builder.endArray();
+ }
+}
diff --git a/src/main/java/com/google/gerrit/elasticsearch/builders/SearchSourceBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/builders/SearchSourceBuilder.java
index 35cbea9..7e4ea93 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/builders/SearchSourceBuilder.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/builders/SearchSourceBuilder.java
@@ -15,6 +15,7 @@
package com.google.gerrit.elasticsearch.builders;
import com.google.gerrit.elasticsearch.ElasticQueryAdapter;
+import com.google.gson.JsonArray;
import java.io.IOException;
import java.util.List;
@@ -28,10 +29,14 @@
private QuerySourceBuilder querySourceBuilder;
+ private SearchAfterBuilder searchAfterBuilder;
+
private int from = -1;
private int size = -1;
+ private boolean trackTotalHits = true;
+
private List<String> fieldNames;
/** Constructs a new search source builder. */
@@ -53,12 +58,22 @@
return this;
}
+ public SearchSourceBuilder searchAfter(JsonArray sortValues) {
+ this.searchAfterBuilder = new SearchAfterBuilder(sortValues);
+ return this;
+ }
+
/** The number of search hits to return. Defaults to <tt>10</tt>. */
public SearchSourceBuilder size(int size) {
this.size = size;
return this;
}
+ public SearchSourceBuilder trackTotalHits(boolean track) {
+ this.trackTotalHits = track;
+ return this;
+ }
+
/**
* Sets the fields to load and return as part of the search request. If none are specified, the
* source of the document will be returned.
@@ -93,6 +108,10 @@
builder.field("size", size);
}
+ if (!trackTotalHits) {
+ builder.field("track_total_hits", false);
+ }
+
if (querySourceBuilder != null) {
querySourceBuilder.innerToXContent(builder);
}
@@ -108,5 +127,9 @@
builder.endArray();
}
}
+
+ if (searchAfterBuilder != null) {
+ searchAfterBuilder.innerToXContent(builder);
+ }
}
}
diff --git a/src/main/java/com/google/gerrit/elasticsearch/builders/XContentBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/builders/XContentBuilder.java
index 9c44583..853596d 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/builders/XContentBuilder.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/builders/XContentBuilder.java
@@ -152,6 +152,8 @@
generator.writeString((String) value);
} else if (type == Integer.class) {
generator.writeNumber(((Integer) value));
+ } else if (type == Long.class) {
+ generator.writeNumber(((Long) value));
} else if (type == byte[].class) {
generator.writeBinary((byte[]) value);
} else if (value instanceof Date) {
diff --git a/src/main/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java b/src/main/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java
index 196b8d6..1ff5b69 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java
@@ -22,6 +22,9 @@
import com.google.gerrit.elasticsearch.builders.XContentBuilder;
import com.google.gerrit.index.Schema;
import com.google.gerrit.index.Schema.Values;
+import com.google.gerrit.index.SchemaFieldDefs;
+import com.google.gerrit.proto.Protos;
+import com.google.protobuf.MessageLite;
import java.io.IOException;
public class UpdateRequest<V> extends BulkRequest {
@@ -40,12 +43,18 @@
protected String getRequest() {
try (XContentBuilder closeable = new XContentBuilder()) {
XContentBuilder builder = closeable.startObject();
- for (Values<V> values : schema.buildFields(v, skipFields)) {
- String name = values.getField().getName();
- if (values.getField().isRepeatable()) {
- builder.field(name, Streams.stream(values.getValues()).collect(toList()));
+ for (Values<V> schemaValues : schema.buildFields(v, skipFields)) {
+ String name = schemaValues.getField().getName();
+ Iterable<?> values = schemaValues.getValues();
+ if (SchemaFieldDefs.isProtoField(schemaValues.getField())) {
+ values =
+ Iterables.transform(
+ schemaValues.getValues(), v -> Protos.toByteArray((MessageLite) v));
+ }
+ if (schemaValues.getField().isRepeatable()) {
+ builder.field(name, Streams.stream(values).collect(toList()));
} else {
- Object element = Iterables.getOnlyElement(values.getValues(), "");
+ Object element = Iterables.getOnlyElement(values, "");
if (shouldAddElement(element)) {
builder.field(name, element);
}
diff --git a/src/main/resources/Documentation/about.md b/src/main/resources/Documentation/about.md
new file mode 100644
index 0000000..59fe0a3
--- /dev/null
+++ b/src/main/resources/Documentation/about.md
@@ -0,0 +1,7 @@
+# Index backend for Gerrit, based on ElasticSearch
+
+Indexing backend libModule for [Gerrit Code Review](https://gerritcodereview.com)
+based on [ElasticSearch](https://www.elastic.co/elasticsearch/).
+
+This module was originally part of Gerrit core and then extracted into a separate
+component from v3.5.0-rc3 as part of [Change-Id: Ib7b5167ce](https://gerrit-review.googlesource.com/c/gerrit/+/323676).
diff --git a/src/main/resources/Documentation/build.md b/src/main/resources/Documentation/build.md
index 1720f52..acb8d85 100644
--- a/src/main/resources/Documentation/build.md
+++ b/src/main/resources/Documentation/build.md
@@ -1,12 +1,17 @@
# Build
-This plugin is built with Bazel in-tree build.
+This plugin is built with Bazel in-tree build. This plugin depends on the Elasticsearch Java Low
+Level REST Client (abbreviated as LLRC by the ES dev team) for integration with an Elasticsearch
+cluster. The LLRC is licensed as Apache v2 (even after ES itself has moved to SSPL) and is
+compatible with all ES versions. See the [LLRC
+docs](https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/8.3/java-rest-low-usage-maven.html)
+for more information.
## Build in Gerrit tree
-Create a symbolic link of the repsotiory source to the Gerrit source
-tree /plugins/index-elasticsearch directory, and the external_plugin_deps.bzl
-dependencies linked to /plugins/external_plugin_deps.bzl.
+Create a symbolic link of the repository source to the Gerrit source
+tree plugins/index-elasticsearch directory, and the external_plugin_deps.bzl
+dependencies linked to plugins/external_plugin_deps.bzl.
Example:
@@ -18,7 +23,7 @@
ln -sf ../../external_plugin_deps.bzl .
```
-From the Gerrit source tree issue the command `bazelsk build plugins/index-elasticsearch`.
+From the Gerrit source tree issue the command `bazelisk build plugins/index-elasticsearch`.
Example:
@@ -26,16 +31,29 @@
bazelisk build plugins/index-elasticsearch
```
-The libModule jar file is created under `basel-bin/plugins/index-elasticsearch/index-elasticsearch.jar`
+The libModule jar file is created under `bazel-bin/plugins/index-elasticsearch/index-elasticsearch.jar`
-To execute the tests run `bazelisk test plugins/index-elasticsearch/...` from the Gerrit source tree.
+## Integration test
-Example:
+There are two different ways to run tests for this module. You can either run only the tests
+provided by the module or you can run all Gerrit core acceptance tests with the indexing backend set
+to this module.
+
+To run only the tests provided by this plugin:
```sh
bazelisk test plugins/index-elasticsearch/...
```
+Gerrit acceptance tests allow the execution with an alternate implementation of
+the indexing backend using the `GERRIT_INDEX_MODULE` environment variable.
+
+```sh
+bazelisk test --test_env=GERRIT_INDEX_MODULE=com.google.gerrit.elasticsearch.ElasticIndexModule //...
+```
+
+## IDE setup
+
This project can be imported into the Eclipse IDE.
Add the plugin name to the `CUSTOM_PLUGINS` and to the
`CUSTOM_PLUGINS_TEST_DEPS` set in Gerrit core in
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 2164d22..927d229 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -1,104 +1,104 @@
-Configuration
-=============
+# Configuration
-[[index]]
-=== Section index
+## Section index
-[[index.maxLimit]]index.maxLimit::
-+
-Maximum limit to allow for search queries. Requesting results above this
-limit will truncate the list (but will still set `_more_changes` on
-result lists). Set to 0 for no limit.
-This value should not exceed the `index.max_result_window` value configured
-on the Elasticsearch server.
-If a value is not configured during site initialization, defaults to
-10000, which is the default value of `index.max_result_window` in Elasticsearch.
+### index.maxLimit
-[[elasticsearch]]
-=== Section elasticsearch
+Maximum limit to allow for search queries. Requesting results above this limit will truncate the
+list (but will still set `_more_changes` on result lists). Set to 0 for no limit. This value
+should not exceed the `index.max_result_window` value configured on the Elasticsearch server. If a
+value is not configured during site initialization, defaults to 10000, which is the default value
+of `index.max_result_window` in Elasticsearch.
-WARNING: Support for Elasticsearch is still experimental and is not recommended
-for production use. For compatibility information, please refer to the
-link:https://www.gerritcodereview.com/elasticsearch.html[project homepage,role=external,window=_blank].
+## Section elasticsearch
+
+WARNING: Support for Elasticsearch is still experimental and is not recommended for production
+use. For compatibility information, please refer to the [project homepage](https://www.gerritcodereview.com/elasticsearch.html).
Note that when Gerrit is configured to use Elasticsearch, the Elasticsearch
server(s) must be reachable during the site initialization.
-[[elasticsearch.prefix]]elasticsearch.prefix::
-+
-This setting can be used to prefix index names to allow multiple Gerrit
-instances in a single Elasticsearch cluster. Prefix `gerrit1_` would result in a
-change index named `gerrit1_changes_0001`.
-+
+### elasticsearch.prefix
+
+This setting can be used to prefix index names to allow multiple Gerrit instances in a single
+Elasticsearch cluster. Prefix `gerrit1_` would result in a change index named
+`gerrit1_changes_0001`.
+
Not set by default.
-[[elasticsearch.server]]elasticsearch.server::
-+
-Elasticsearch server URI in the form `http[s]://hostname:port`. The `port` is
-optional and defaults to `9200` if not specified.
-+
-At least one server must be specified. May be specified multiple times to
-configure multiple Elasticsearch servers.
-+
+### elasticsearch.server
+
+Elasticsearch server URI in the form `http[s]://hostname:port`. The `port` is optional and defaults
+to `9200` if not specified.
+
+At least one server must be specified. May be specified multiple times to configure multiple
+Elasticsearch servers.
+
Note that the site initialization program only allows to configure a single
server. To configure multiple servers the `gerrit.config` file must be edited
manually.
-[[elasticsearch.numberOfShards]]elasticsearch.numberOfShards::
-+
+### elasticsearch.numberOfShards
+
Sets the number of shards to use per index. Refer to the
-link:https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html#_static_index_settings[
-Elasticsearch documentation,role=external,window=_blank] for details.
-+
+[Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html#_static_index_settings) for details.
+
Defaults to 1.
-[[elasticsearch.numberOfReplicas]]elasticsearch.numberOfReplicas::
-+
+### elasticsearch.numberOfReplicas
+
Sets the number of replicas to use per index. Refer to the
-link:https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html#dynamic-index-settings[
-Elasticsearch documentation,role=external,window=_blank] for details.
-+
+[Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html#dynamic-index-settings) for details.
+
Defaults to 1.
-[[elasticsearch.maxResultWindow]]elasticsearch.maxResultWindow::
-+
+### elasticsearch.maxResultWindow
+
Sets the maximum value of `from + size` for searches to use per index. Refer to the
-link:https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html#dynamic-index-settings[
-Elasticsearch documentation,role=external,window=_blank] for details.
-+
+[Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html#dynamic-index-settings) for details.
+
Defaults to 10000.
-[[elasticsearch.connectTimeout]]elasticsearch.connectTimeout::
-+
+### elasticsearch.connectTimeout
+
Sets the timeout for connecting to elasticsearch.
-+
+
Defaults to `1 second`.
-[[elasticsearch.socketTimeout]]elasticsearch.socketTimeout::
-+
+### elasticsearch.socketTimeout
+
Sets the timeout for the underlying connection. For more information, refer to
-link:#httpd.idleTimeout[`httpd.idleTimeout`].
-+
+[`httpd.idleTimeout`](https://gerrit-documentation.storage.googleapis.com/Documentation/3.5.2/config-gerrit.html#httpd.idleTimeout).
+
Defaults to `30 seconds`.
-==== Elasticsearch Security
+## Elasticsearch Security
-When security is enabled in Elasticsearch, the username and password must be provided.
-Note that the same username and password are used for all servers.
+When security is enabled in Elasticsearch, the username and password must be provided. Note that
+the same username and password are used for all servers.
For further information about Elasticsearch security, please refer to
-link:https://www.elastic.co/guide/en/elasticsearch/reference/current/security-getting-started.html[the documentation,role=external,window=_blank].
-This is the current documentation link. Select another Elasticsearch version
-from the dropdown menu available on that page if need be.
+[the documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-getting-started.html). This is the current documentation link. Select another Elasticsearch version from the dropdown menu available on that page if need be.
-[[elasticsearch.username]]elasticsearch.username::
-+
+### elasticsearch.username
+
Username used to connect to Elasticsearch.
-+
+
If a password is set, defaults to `elastic`, otherwise not set by default.
-[[elasticsearch.password]]elasticsearch.password::
-+
+### elasticsearch.password
+
Password used to connect to Elasticsearch.
-+
+
Not set by default.
+
+### elasticsearch.codec
+
+Sets the codec to be used for the index data. For further information about supported codecs, please refer to the static index setting
+[index.codec](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html#index-codec).
+
+Defaults to `default`.
+
+[Back to @PLUGIN@ documentation index][index]
+
+[index]: index.html
diff --git a/src/main/resources/Documentation/setup.md b/src/main/resources/Documentation/setup.md
new file mode 100644
index 0000000..7877b29
--- /dev/null
+++ b/src/main/resources/Documentation/setup.md
@@ -0,0 +1,26 @@
+# Setup
+
+* Install index-elasticsearch module
+
+Install the index-elasticsearch.jar into the `$GERRIT_SITE/lib` directory.
+
+Add the index-elasticsearch module to `$GERRIT_SITE/etc/gerrit.config` as follows:
+
+```ini
+[gerrit]
+ installIndexModule = com.google.gerrit.elasticsearch.ElasticIndexModule
+```
+
+When installing the module on Gerrit replicas, use following example:
+
+```ini
+[gerrit]
+ installIndexModule = com.google.gerrit.elasticsearch.ReplicaElasticIndexModule
+```
+
+For further information and supported options, refer to [config](config.html)
+documentation.
+
+[Back to @PLUGIN@ documentation index][index]
+
+[index]: index.html
diff --git a/src/test/java/com/google/gerrit/elasticsearch/ElasticContainer.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticContainer.java
index 503852b..86b1bac 100644
--- a/src/test/java/com/google/gerrit/elasticsearch/ElasticContainer.java
+++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticContainer.java
@@ -14,38 +14,41 @@
package com.google.gerrit.elasticsearch;
+import com.google.common.flogger.FluentLogger;
import org.apache.http.HttpHost;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.ContainerLaunchException;
import org.testcontainers.elasticsearch.ElasticsearchContainer;
import org.testcontainers.utility.DockerImageName;
/* Helper class for running ES integration tests in docker container */
public class ElasticContainer extends ElasticsearchContainer {
+ private static FluentLogger logger = FluentLogger.forEnclosingClass();
private static final int ELASTICSEARCH_DEFAULT_PORT = 9200;
public static ElasticContainer createAndStart(ElasticVersion version) {
ElasticContainer container = new ElasticContainer(version);
- container.start();
+ try {
+ container.start();
+ } catch (ContainerLaunchException e) {
+ logger.atSevere().log(
+ "Failed to launch elastic container. Logs from container :\n%s", container.getLogs());
+ throw e;
+ }
return container;
}
private static String getImageName(ElasticVersion version) {
switch (version) {
- case V7_6:
- return "blacktop/elasticsearch:7.6.2";
- case V7_7:
- return "blacktop/elasticsearch:7.7.1";
- case V7_8:
- return "blacktop/elasticsearch:7.8.1";
+ case V7_16:
+ return "docker.elastic.co/elasticsearch/elasticsearch:7.16.2";
}
throw new IllegalStateException("No tests for version: " + version.name());
}
private ElasticContainer(ElasticVersion version) {
- super(
- DockerImageName.parse(getImageName(version))
- .asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch"));
+ super(DockerImageName.parse(getImageName(version)));
}
@Override
diff --git a/src/test/java/com/google/gerrit/elasticsearch/ElasticTestUtils.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticTestUtils.java
index c3ca595..f5ba9db 100644
--- a/src/test/java/com/google/gerrit/elasticsearch/ElasticTestUtils.java
+++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticTestUtils.java
@@ -14,6 +14,8 @@
package com.google.gerrit.elasticsearch;
+import static java.util.concurrent.TimeUnit.MINUTES;
+
import com.google.gerrit.index.IndexDefinition;
import com.google.gerrit.server.LibModuleType;
import com.google.gerrit.testing.GerritTestName;
@@ -25,6 +27,9 @@
import com.google.inject.TypeLiteral;
import java.util.Collection;
import java.util.UUID;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
import org.eclipse.jgit.lib.Config;
public final class ElasticTestUtils {
@@ -81,6 +86,22 @@
return Guice.createInjector(new InMemoryModule(elasticsearchConfig));
}
+ public static void closeIndex(
+ CloseableHttpAsyncClient client, ElasticContainer container, GerritTestName testName)
+ throws Exception {
+ client
+ .execute(
+ new HttpPost(
+ String.format(
+ "http://%s:%d/%s*/_close",
+ container.getHttpHost().getHostName(),
+ container.getHttpHost().getPort(),
+ testName.getSanitizedMethodName())),
+ HttpClientContext.create(),
+ null)
+ .get(5, MINUTES);
+ }
+
private ElasticTestUtils() {
// hide default constructor
}
diff --git a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryAccountsTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryAccountsTest.java
index 4ee5a16..7f6a585 100644
--- a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryAccountsTest.java
+++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryAccountsTest.java
@@ -14,12 +14,19 @@
package com.google.gerrit.elasticsearch;
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.gerrit.testing.GerritJUnit.assertThrows;
+
+import com.google.gerrit.exceptions.StorageException;
import com.google.gerrit.server.query.account.AbstractQueryAccountsTest;
import com.google.gerrit.testing.ConfigSuite;
import com.google.inject.Injector;
+import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
+import org.apache.http.impl.nio.client.HttpAsyncClients;
import org.eclipse.jgit.lib.Config;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Test;
public class ElasticV7QueryAccountsTest extends AbstractQueryAccountsTest {
@ConfigSuite.Default
@@ -27,14 +34,21 @@
return ElasticTestUtils.createConfig();
}
+ @ConfigSuite.Config
+ public static Config searchAfterPaginationType() {
+ Config config = defaultConfig();
+ config.setString("index", null, "paginationType", "SEARCH_AFTER");
+ return config;
+ }
+
private static ElasticContainer container;
+ private static CloseableHttpAsyncClient client;
@BeforeClass
public static void startIndexService() {
- if (container == null) {
- // Only start Elasticsearch once
- container = ElasticContainer.createAndStart(ElasticVersion.V7_8);
- }
+ container = ElasticContainer.createAndStart(ElasticVersion.V7_16);
+ client = HttpAsyncClients.createDefault();
+ client.start();
}
@AfterClass
@@ -54,4 +68,14 @@
protected Injector createInjector() {
return ElasticTestUtils.createInjector(config, testName, container);
}
+
+ @Test
+ public void testErrorResponseFromAccountIndex() throws Exception {
+ gApi.accounts().self().index();
+
+ ElasticTestUtils.closeIndex(client, container, testName);
+ StorageException thrown =
+ assertThrows(StorageException.class, () -> gApi.accounts().self().index());
+ assertThat(thrown).hasMessageThat().contains("Failed to replace account");
+ }
}
diff --git a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java
index d704e40..7151e70 100644
--- a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java
+++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java
@@ -14,23 +14,20 @@
package com.google.gerrit.elasticsearch;
-import static com.google.common.truth.TruthJUnit.assume;
-import static java.util.concurrent.TimeUnit.MINUTES;
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.gerrit.testing.GerritJUnit.assertThrows;
import com.google.gerrit.entities.Change;
-import com.google.gerrit.server.change.ChangeInserter;
-import com.google.gerrit.server.index.change.ChangeField;
+import com.google.gerrit.exceptions.StorageException;
import com.google.gerrit.server.query.change.AbstractQueryChangesTest;
import com.google.gerrit.testing.ConfigSuite;
import com.google.gerrit.testing.GerritTestName;
-import com.google.gerrit.testing.InMemoryRepositoryManager.Repo;
import com.google.inject.Injector;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
import org.apache.http.impl.nio.client.HttpAsyncClients;
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.lib.Repository;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -43,17 +40,21 @@
return ElasticTestUtils.createConfig();
}
+ @ConfigSuite.Config
+ public static Config searchAfterPaginationType() {
+ Config config = defaultConfig();
+ config.setString("index", null, "paginationType", "SEARCH_AFTER");
+ return config;
+ }
+
private static ElasticContainer container;
private static CloseableHttpAsyncClient client;
@BeforeClass
public static void startIndexService() {
- if (container == null) {
- // Only start Elasticsearch once
- container = ElasticContainer.createAndStart(ElasticVersion.V7_8);
- client = HttpAsyncClients.createDefault();
- client.start();
- }
+ container = ElasticContainer.createAndStart(ElasticVersion.V7_16);
+ client = HttpAsyncClients.createDefault();
+ client.start();
}
@AfterClass
@@ -69,17 +70,7 @@
public void closeIndex() throws Exception {
// Close the index after each test to prevent exceeding Elasticsearch's
// shard limit (see Issue 10120).
- client
- .execute(
- new HttpPost(
- String.format(
- "http://%s:%d/%s*/_close",
- container.getHttpHost().getHostName(),
- container.getHttpHost().getPort(),
- testName.getSanitizedMethodName())),
- HttpClientContext.create(),
- null)
- .get(5, MINUTES);
+ ElasticTestUtils.closeIndex(client, container, testName);
}
@Override
@@ -88,54 +79,21 @@
ElasticTestUtils.createAllIndexes(injector);
}
- @Test
- @Override
- // TODO(davido): overrides byTopic() method to adjust to ES behaviour for
- // "prefixtopic" predicate. This should be fixed in a follow-up change.
- public void byTopic() throws Exception {
-
- TestRepository<Repo> repo = createProject("repo");
- ChangeInserter ins1 = newChangeWithTopic(repo, "feature1");
- Change change1 = insert(repo, ins1);
-
- ChangeInserter ins2 = newChangeWithTopic(repo, "feature2");
- Change change2 = insert(repo, ins2);
-
- ChangeInserter ins3 = newChangeWithTopic(repo, "Cherrypick-feature2");
- Change change3 = insert(repo, ins3);
-
- ChangeInserter ins4 = newChangeWithTopic(repo, "feature2-fixup");
- Change change4 = insert(repo, ins4);
-
- ChangeInserter ins5 = newChangeWithTopic(repo, "https://gerrit.local");
- Change change5 = insert(repo, ins5);
-
- ChangeInserter ins6 = newChangeWithTopic(repo, "git_gerrit_training");
- Change change6 = insert(repo, ins6);
-
- Change change_no_topic = insert(repo, newChange(repo));
-
- assertQuery("intopic:foo");
- assertQuery("intopic:feature1", change1);
- assertQuery("intopic:feature2", change4, change3, change2);
- assertQuery("topic:feature2", change2);
- assertQuery("intopic:feature2", change4, change3, change2);
- assertQuery("intopic:fixup", change4);
- assertQuery("intopic:gerrit", change6, change5);
- assertQuery("topic:\"\"", change_no_topic);
- assertQuery("intopic:\"\"", change_no_topic);
-
- assume().that(getSchema().hasField(ChangeField.PREFIX_TOPIC)).isTrue();
- // change3 is considered by ES in prefixtopic:feature query, see
- // https://www.elastic.co/guide/en/elasticsearch/reference/8.2/query-dsl-match-query-phrase-prefix.html
- // assertQuery("prefixtopic:feature", change4, change2, change1);
- assertQuery("prefixtopic:feature", change4, change3, change2, change1);
- assertQuery("prefixtopic:Cher", change3);
- assertQuery("prefixtopic:feature22");
- }
-
@Override
protected Injector createInjector() {
return ElasticTestUtils.createInjector(config, testName, container);
}
+
+ @Test
+ public void testErrorResponseFromChangeIndex() throws Exception {
+ String repository = "repo";
+ TestRepository<Repository> repo = createAndOpenProject(repository);
+ Change c = insert(repository, newChangeWithStatus(repo, Change.Status.NEW));
+ gApi.changes().id(c.getChangeId()).index();
+
+ ElasticTestUtils.closeIndex(client, container, testName);
+ StorageException thrown =
+ assertThrows(StorageException.class, () -> gApi.changes().id(c.getChangeId()).index());
+ assertThat(thrown).hasMessageThat().contains("Failed to reindex change");
+ }
}
diff --git a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryGroupsTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryGroupsTest.java
index 649c0bc..271ee19 100644
--- a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryGroupsTest.java
+++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryGroupsTest.java
@@ -14,12 +14,20 @@
package com.google.gerrit.elasticsearch;
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.gerrit.testing.GerritJUnit.assertThrows;
+
+import com.google.gerrit.exceptions.StorageException;
+import com.google.gerrit.extensions.api.groups.GroupApi;
import com.google.gerrit.server.query.group.AbstractQueryGroupsTest;
import com.google.gerrit.testing.ConfigSuite;
import com.google.inject.Injector;
+import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
+import org.apache.http.impl.nio.client.HttpAsyncClients;
import org.eclipse.jgit.lib.Config;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Test;
public class ElasticV7QueryGroupsTest extends AbstractQueryGroupsTest {
@ConfigSuite.Default
@@ -27,14 +35,21 @@
return ElasticTestUtils.createConfig();
}
+ @ConfigSuite.Config
+ public static Config searchAfterPaginationType() {
+ Config config = defaultConfig();
+ config.setString("index", null, "paginationType", "SEARCH_AFTER");
+ return config;
+ }
+
private static ElasticContainer container;
+ private static CloseableHttpAsyncClient client;
@BeforeClass
public static void startIndexService() {
- if (container == null) {
- // Only start Elasticsearch once
- container = ElasticContainer.createAndStart(ElasticVersion.V7_8);
- }
+ container = ElasticContainer.createAndStart(ElasticVersion.V7_16);
+ client = HttpAsyncClients.createDefault();
+ client.start();
}
@AfterClass
@@ -54,4 +69,14 @@
protected Injector createInjector() {
return ElasticTestUtils.createInjector(config, testName, container);
}
+
+ @Test
+ public void testErrorResponseFromGroupIndex() throws Exception {
+ GroupApi group = gApi.groups().create("test");
+ group.index();
+
+ ElasticTestUtils.closeIndex(client, container, testName);
+ StorageException thrown = assertThrows(StorageException.class, () -> group.index());
+ assertThat(thrown).hasMessageThat().contains("Failed to replace group");
+ }
}
diff --git a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryProjectsTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryProjectsTest.java
index d3b3d44..38eef22 100644
--- a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryProjectsTest.java
+++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryProjectsTest.java
@@ -14,12 +14,20 @@
package com.google.gerrit.elasticsearch;
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.gerrit.testing.GerritJUnit.assertThrows;
+
+import com.google.gerrit.exceptions.StorageException;
+import com.google.gerrit.extensions.api.projects.ProjectApi;
import com.google.gerrit.server.query.project.AbstractQueryProjectsTest;
import com.google.gerrit.testing.ConfigSuite;
import com.google.inject.Injector;
+import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
+import org.apache.http.impl.nio.client.HttpAsyncClients;
import org.eclipse.jgit.lib.Config;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Test;
public class ElasticV7QueryProjectsTest extends AbstractQueryProjectsTest {
@ConfigSuite.Default
@@ -27,14 +35,21 @@
return ElasticTestUtils.createConfig();
}
+ @ConfigSuite.Config
+ public static Config searchAfterPaginationType() {
+ Config config = defaultConfig();
+ config.setString("index", null, "paginationType", "SEARCH_AFTER");
+ return config;
+ }
+
private static ElasticContainer container;
+ private static CloseableHttpAsyncClient client;
@BeforeClass
public static void startIndexService() {
- if (container == null) {
- // Only start Elasticsearch once
- container = ElasticContainer.createAndStart(ElasticVersion.V7_8);
- }
+ container = ElasticContainer.createAndStart(ElasticVersion.V7_16);
+ client = HttpAsyncClients.createDefault();
+ client.start();
}
@AfterClass
@@ -54,4 +69,14 @@
protected Injector createInjector() {
return ElasticTestUtils.createInjector(config, testName, container);
}
+
+ @Test
+ public void testErrorResponseFromProjectIndex() throws Exception {
+ ProjectApi project = gApi.projects().create("test");
+ project.index(false);
+
+ ElasticTestUtils.closeIndex(client, container, testName);
+ StorageException thrown = assertThrows(StorageException.class, () -> project.index(false));
+ assertThat(thrown).hasMessageThat().contains("Failed to replace project");
+ }
}
diff --git a/src/test/java/com/google/gerrit/elasticsearch/ElasticVersionTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticVersionTest.java
index 2ce3a2c..ea7782b 100644
--- a/src/test/java/com/google/gerrit/elasticsearch/ElasticVersionTest.java
+++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticVersionTest.java
@@ -22,14 +22,8 @@
public class ElasticVersionTest {
@Test
public void supportedVersion() throws Exception {
- assertThat(ElasticVersion.forVersion("7.6.0")).isEqualTo(ElasticVersion.V7_6);
- assertThat(ElasticVersion.forVersion("7.6.1")).isEqualTo(ElasticVersion.V7_6);
-
- assertThat(ElasticVersion.forVersion("7.7.0")).isEqualTo(ElasticVersion.V7_7);
- assertThat(ElasticVersion.forVersion("7.7.1")).isEqualTo(ElasticVersion.V7_7);
-
- assertThat(ElasticVersion.forVersion("7.8.0")).isEqualTo(ElasticVersion.V7_8);
- assertThat(ElasticVersion.forVersion("7.8.1")).isEqualTo(ElasticVersion.V7_8);
+ assertThat(ElasticVersion.forVersion("7.16.0")).isEqualTo(ElasticVersion.V7_16);
+ assertThat(ElasticVersion.forVersion("7.16.1")).isEqualTo(ElasticVersion.V7_16);
}
@Test
