Merge branch 'stable-3.5' into stable-3.6 * stable-3.5: test: Simplify SSL setup Add 8.9.* to supported versions test: Always enable SSL for ES containers Bump testcontainers to 1.19.7 Remove unused build var test: Add assert for closing indexes test: Use the 'withTag' helper to get DockerImageName Include an 'Accept' header for Content-Type in requests Add a debug log while insert/replace change index operation Release-Notes: skip Change-Id: I1426e2fd38b21d64c81804560b96b34a07ccd466
diff --git a/BUILD b/BUILD index 0f6f594..858c03a 100644 --- a/BUILD +++ b/BUILD
@@ -122,7 +122,8 @@ exclude = ["src/test/java/**/Elastic*Query*" + SUFFIX], ), tags = ["elastic"], - deps = PLUGIN_TEST_DEPS + [ + deps = ELASTICSEARCH_DEPS + PLUGIN_TEST_DEPS + [ + ":elasticsearch_test_utils", ":index-elasticsearch__plugin", ], )
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java index 2a5fcde..0379ed9 100644 --- a/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java +++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java
@@ -18,6 +18,8 @@ import com.google.common.base.Strings; import com.google.common.flogger.FluentLogger; +import com.google.gerrit.index.IndexConfig; +import com.google.gerrit.index.PaginationType; import com.google.gerrit.server.config.GerritServerConfig; import com.google.inject.Inject; import com.google.inject.ProvisionException; @@ -70,7 +72,12 @@ final String prefix; @Inject - ElasticConfiguration(@GerritServerConfig Config cfg) { + ElasticConfiguration(@GerritServerConfig Config cfg, IndexConfig indexConfig) { + if (PaginationType.NONE == indexConfig.paginationType()) { + throw new ProvisionException( + "The 'index.paginationType = NONE' configuration is not supported by Elasticsearch"); + } + this.cfg = cfg; this.password = cfg.getString(SECTION_ELASTICSEARCH, null, KEY_PASSWORD); this.username =
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java index 1668450..9ea9fcb 100644 --- a/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java +++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java
@@ -40,8 +40,10 @@ || fieldType == FieldType.LONG) { mapping.addNumber(name); } else if (fieldType == FieldType.FULL_TEXT) { - mapping.addStringWithAnalyzer(name); - } else if (fieldType == FieldType.PREFIX || fieldType == FieldType.STORED_ONLY) { + mapping.addStringWithAnalyzer(name, "custom_with_char_filter"); + } else if (fieldType == FieldType.PREFIX) { + mapping.addStringWithAnalyzer(name, "keyword_tokenizer"); + } else if (fieldType == FieldType.STORED_ONLY) { mapping.addString(name); } else { throw new IllegalStateException("Unsupported field type: " + fieldType.getName()); @@ -101,9 +103,9 @@ return this; } - Builder addStringWithAnalyzer(String name) { + Builder addStringWithAnalyzer(String name, String analyzer) { FieldProperties key = new FieldProperties(adapter.stringFieldType()); - key.analyzer = "custom_with_char_filter"; + key.analyzer = analyzer; fields.put(name, key); return this; }
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java index cde5560..3580089 100644 --- a/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java +++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
@@ -29,7 +29,6 @@ import com.google.gerrit.index.query.QueryParseException; import com.google.gerrit.index.query.RegexPredicate; import com.google.gerrit.index.query.TimestampRangePredicate; -import java.time.Instant; public class ElasticQueryBuilder { @@ -119,9 +118,8 @@ } private <T> QueryBuilder notTimestamp(TimestampRangePredicate<T> r) throws QueryParseException { - if (r.getMinTimestamp().getTime() == 0) { - return QueryBuilders.rangeQuery(r.getField().getName()) - .gt(Instant.ofEpochMilli(r.getMaxTimestamp().getTime())); + if (r.getMinTimestamp().toEpochMilli() == 0) { + return QueryBuilders.rangeQuery(r.getField().getName()).gt(r.getMaxTimestamp()); } throw new QueryParseException("cannot negate: " + r); } @@ -129,15 +127,14 @@ private <T> QueryBuilder timestampQuery(IndexPredicate<T> p) throws QueryParseException { if (p instanceof TimestampRangePredicate) { TimestampRangePredicate<T> r = (TimestampRangePredicate<T>) p; - if (r.getMaxTimestamp().getTime() == Long.MAX_VALUE) { + if (r.getMaxTimestamp().toEpochMilli() == Long.MAX_VALUE) { // The time range only has the start value, search from the start to the max supported value // Long.MAX_VALUE - return QueryBuilders.rangeQuery(r.getField().getName()) - .gte(Instant.ofEpochMilli(r.getMinTimestamp().getTime())); + return QueryBuilders.rangeQuery(r.getField().getName()).gte(r.getMinTimestamp()); } return QueryBuilders.rangeQuery(r.getField().getName()) - .gte(Instant.ofEpochMilli(r.getMinTimestamp().getTime())) - .lte(Instant.ofEpochMilli(r.getMaxTimestamp().getTime())); + .gte(r.getMinTimestamp()) + .lte(r.getMaxTimestamp()); } throw new QueryParseException("not a timestamp: " + p); }
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticSetting.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticSetting.java index 0059574..2fcfc90 100644 --- a/src/main/java/com/google/gerrit/elasticsearch/ElasticSetting.java +++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticSetting.java
@@ -59,6 +59,7 @@ FieldProperties analyzer = new FieldProperties(); analyzer.customWithCharFilter = customAnalyzer; + analyzer.keywordTokenizer = ImmutableMap.of("tokenizer", "keyword"); fields.put("analyzer", analyzer); return this; } @@ -92,6 +93,7 @@ String[] mappings; FieldProperties customMapping; FieldProperties customWithCharFilter; + Map<String, String> keywordTokenizer; FieldProperties() {}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index 74fe50c..5d0dead 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md
@@ -10,6 +10,16 @@ value is not configured during site initialization, defaults to 10000, which is the default value of `index.max_result_window` in Elasticsearch. +### index.paginationType + +The pagination type to use when index queries are repeated to obtain the next set of results. +Supported values are: `OFFSET` and `SEARCH_AFTER`. For more information, refer to +[`index.paginationType`](https://gerrit-review.googlesource.com/Documentation/config-gerrit.html#index.paginationType). + +Defaults to `OFFSET`. +Note: paginationType `NONE` is not supported and Gerrit will not start if it is configured (results +in `ProvisionException`). + ## Section elasticsearch For compatibility information, please refer to the [project homepage](https://www.gerritcodereview.com/elasticsearch.html).
diff --git a/src/test/java/com/google/gerrit/elasticsearch/ElasticConfigurationTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticConfigurationTest.java index 7e044c3..9d20edc 100644 --- a/src/test/java/com/google/gerrit/elasticsearch/ElasticConfigurationTest.java +++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticConfigurationTest.java
@@ -25,6 +25,7 @@ import static java.util.stream.Collectors.toList; import com.google.common.collect.ImmutableList; +import com.google.gerrit.index.IndexConfig; import com.google.inject.ProvisionException; import java.util.Arrays; import org.apache.http.HttpHost; @@ -35,7 +36,7 @@ @Test public void singleServerNoOtherConfig() throws Exception { Config cfg = newConfig(); - ElasticConfiguration esCfg = new ElasticConfiguration(cfg); + ElasticConfiguration esCfg = newElasticConfig(cfg); assertHosts(esCfg, "http://elastic:1234"); assertThat(esCfg.username).isNull(); assertThat(esCfg.password).isNull(); @@ -46,7 +47,7 @@ public void serverWithoutPortSpecified() throws Exception { Config cfg = new Config(); cfg.setString(SECTION_ELASTICSEARCH, null, KEY_SERVER, "http://elastic"); - ElasticConfiguration esCfg = new ElasticConfiguration(cfg); + ElasticConfiguration esCfg = newElasticConfig(cfg); assertHosts(esCfg, "http://elastic:9200"); } @@ -54,7 +55,7 @@ public void prefix() throws Exception { Config cfg = newConfig(); cfg.setString(SECTION_ELASTICSEARCH, null, KEY_PREFIX, "myprefix"); - ElasticConfiguration esCfg = new ElasticConfiguration(cfg); + ElasticConfiguration esCfg = newElasticConfig(cfg); assertThat(esCfg.prefix).isEqualTo("myprefix"); } @@ -63,7 +64,7 @@ Config cfg = newConfig(); cfg.setString(SECTION_ELASTICSEARCH, null, KEY_USERNAME, "myself"); cfg.setString(SECTION_ELASTICSEARCH, null, KEY_PASSWORD, "s3kr3t"); - ElasticConfiguration esCfg = new ElasticConfiguration(cfg); + ElasticConfiguration esCfg = newElasticConfig(cfg); assertThat(esCfg.username).isEqualTo("myself"); assertThat(esCfg.password).isEqualTo("s3kr3t"); } @@ -72,7 +73,7 @@ public void withAuthenticationPasswordOnlyUsesDefaultUsername() throws Exception { Config cfg = newConfig(); cfg.setString(SECTION_ELASTICSEARCH, null, KEY_PASSWORD, "s3kr3t"); - ElasticConfiguration esCfg = new ElasticConfiguration(cfg); + ElasticConfiguration esCfg = newElasticConfig(cfg); assertThat(esCfg.username).isEqualTo(DEFAULT_USERNAME); assertThat(esCfg.password).isEqualTo("s3kr3t"); } @@ -85,20 +86,20 @@ null, KEY_SERVER, ImmutableList.of("http://elastic1:1234", "http://elastic2:1234")); - ElasticConfiguration esCfg = new ElasticConfiguration(cfg); + ElasticConfiguration esCfg = newElasticConfig(cfg); assertHosts(esCfg, "http://elastic1:1234", "http://elastic2:1234"); } @Test public void noServers() throws Exception { - assertProvisionException(new Config()); + assertProvisionException(new Config(), "No valid Elasticsearch servers configured"); } @Test public void singleServerInvalid() throws Exception { Config cfg = new Config(); cfg.setString(SECTION_ELASTICSEARCH, null, KEY_SERVER, "foo"); - assertProvisionException(cfg); + assertProvisionException(cfg, "No valid Elasticsearch servers configured"); } @Test @@ -106,24 +107,35 @@ Config cfg = new Config(); cfg.setStringList( SECTION_ELASTICSEARCH, null, KEY_SERVER, ImmutableList.of("http://elastic1:1234", "foo")); - ElasticConfiguration esCfg = new ElasticConfiguration(cfg); + ElasticConfiguration esCfg = newElasticConfig(cfg); assertHosts(esCfg, "http://elastic1:1234"); } + @Test + public void unsupportedPaginationTypeNone() { + Config cfg = new Config(); + cfg.setString("index", null, "paginationType", "NONE"); + assertProvisionException( + cfg, "The 'index.paginationType = NONE' configuration is not supported by Elasticsearch"); + } + private static Config newConfig() { Config config = new Config(); config.setString(SECTION_ELASTICSEARCH, null, KEY_SERVER, "http://elastic:1234"); return config; } + private static ElasticConfiguration newElasticConfig(Config cfg) { + return new ElasticConfiguration(cfg, IndexConfig.fromConfig(cfg).build()); + } + private void assertHosts(ElasticConfiguration cfg, Object... hostURIs) throws Exception { assertThat(Arrays.asList(cfg.getHosts()).stream().map(HttpHost::toURI).collect(toList())) .containsExactly(hostURIs); } - private void assertProvisionException(Config cfg) { - ProvisionException thrown = - assertThrows(ProvisionException.class, () -> new ElasticConfiguration(cfg)); - assertThat(thrown).hasMessageThat().contains("No valid Elasticsearch servers configured"); + private void assertProvisionException(Config cfg, String msg) { + ProvisionException thrown = assertThrows(ProvisionException.class, () -> newElasticConfig(cfg)); + assertThat(thrown).hasMessageThat().contains(msg); } }