Merge branch 'stable-3.5' into stable-3.6
* stable-3.5:
test: Simplify SSL setup
Add 8.9.* to supported versions
test: Always enable SSL for ES containers
Bump testcontainers to 1.19.7
Remove unused build var
test: Add assert for closing indexes
test: Use the 'withTag' helper to get DockerImageName
Include an 'Accept' header for Content-Type in requests
Add a debug log while insert/replace change index operation
Release-Notes: skip
Change-Id: I1426e2fd38b21d64c81804560b96b34a07ccd466
diff --git a/BUILD b/BUILD
index 0f6f594..858c03a 100644
--- a/BUILD
+++ b/BUILD
@@ -122,7 +122,8 @@
exclude = ["src/test/java/**/Elastic*Query*" + SUFFIX],
),
tags = ["elastic"],
- deps = PLUGIN_TEST_DEPS + [
+ deps = ELASTICSEARCH_DEPS + PLUGIN_TEST_DEPS + [
+ ":elasticsearch_test_utils",
":index-elasticsearch__plugin",
],
)
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java
index 2a5fcde..0379ed9 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java
@@ -18,6 +18,8 @@
import com.google.common.base.Strings;
import com.google.common.flogger.FluentLogger;
+import com.google.gerrit.index.IndexConfig;
+import com.google.gerrit.index.PaginationType;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.inject.Inject;
import com.google.inject.ProvisionException;
@@ -70,7 +72,12 @@
final String prefix;
@Inject
- ElasticConfiguration(@GerritServerConfig Config cfg) {
+ ElasticConfiguration(@GerritServerConfig Config cfg, IndexConfig indexConfig) {
+ if (PaginationType.NONE == indexConfig.paginationType()) {
+ throw new ProvisionException(
+ "The 'index.paginationType = NONE' configuration is not supported by Elasticsearch");
+ }
+
this.cfg = cfg;
this.password = cfg.getString(SECTION_ELASTICSEARCH, null, KEY_PASSWORD);
this.username =
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java
index 1668450..9ea9fcb 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java
@@ -40,8 +40,10 @@
|| fieldType == FieldType.LONG) {
mapping.addNumber(name);
} else if (fieldType == FieldType.FULL_TEXT) {
- mapping.addStringWithAnalyzer(name);
- } else if (fieldType == FieldType.PREFIX || fieldType == FieldType.STORED_ONLY) {
+ mapping.addStringWithAnalyzer(name, "custom_with_char_filter");
+ } else if (fieldType == FieldType.PREFIX) {
+ mapping.addStringWithAnalyzer(name, "keyword_tokenizer");
+ } else if (fieldType == FieldType.STORED_ONLY) {
mapping.addString(name);
} else {
throw new IllegalStateException("Unsupported field type: " + fieldType.getName());
@@ -101,9 +103,9 @@
return this;
}
- Builder addStringWithAnalyzer(String name) {
+ Builder addStringWithAnalyzer(String name, String analyzer) {
FieldProperties key = new FieldProperties(adapter.stringFieldType());
- key.analyzer = "custom_with_char_filter";
+ key.analyzer = analyzer;
fields.put(name, key);
return this;
}
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
index cde5560..3580089 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
@@ -29,7 +29,6 @@
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.index.query.RegexPredicate;
import com.google.gerrit.index.query.TimestampRangePredicate;
-import java.time.Instant;
public class ElasticQueryBuilder {
@@ -119,9 +118,8 @@
}
private <T> QueryBuilder notTimestamp(TimestampRangePredicate<T> r) throws QueryParseException {
- if (r.getMinTimestamp().getTime() == 0) {
- return QueryBuilders.rangeQuery(r.getField().getName())
- .gt(Instant.ofEpochMilli(r.getMaxTimestamp().getTime()));
+ if (r.getMinTimestamp().toEpochMilli() == 0) {
+ return QueryBuilders.rangeQuery(r.getField().getName()).gt(r.getMaxTimestamp());
}
throw new QueryParseException("cannot negate: " + r);
}
@@ -129,15 +127,14 @@
private <T> QueryBuilder timestampQuery(IndexPredicate<T> p) throws QueryParseException {
if (p instanceof TimestampRangePredicate) {
TimestampRangePredicate<T> r = (TimestampRangePredicate<T>) p;
- if (r.getMaxTimestamp().getTime() == Long.MAX_VALUE) {
+ if (r.getMaxTimestamp().toEpochMilli() == Long.MAX_VALUE) {
// The time range only has the start value, search from the start to the max supported value
// Long.MAX_VALUE
- return QueryBuilders.rangeQuery(r.getField().getName())
- .gte(Instant.ofEpochMilli(r.getMinTimestamp().getTime()));
+ return QueryBuilders.rangeQuery(r.getField().getName()).gte(r.getMinTimestamp());
}
return QueryBuilders.rangeQuery(r.getField().getName())
- .gte(Instant.ofEpochMilli(r.getMinTimestamp().getTime()))
- .lte(Instant.ofEpochMilli(r.getMaxTimestamp().getTime()));
+ .gte(r.getMinTimestamp())
+ .lte(r.getMaxTimestamp());
}
throw new QueryParseException("not a timestamp: " + p);
}
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticSetting.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticSetting.java
index 0059574..2fcfc90 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/ElasticSetting.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticSetting.java
@@ -59,6 +59,7 @@
FieldProperties analyzer = new FieldProperties();
analyzer.customWithCharFilter = customAnalyzer;
+ analyzer.keywordTokenizer = ImmutableMap.of("tokenizer", "keyword");
fields.put("analyzer", analyzer);
return this;
}
@@ -92,6 +93,7 @@
String[] mappings;
FieldProperties customMapping;
FieldProperties customWithCharFilter;
+ Map<String, String> keywordTokenizer;
FieldProperties() {}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 74fe50c..5d0dead 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -10,6 +10,16 @@
value is not configured during site initialization, defaults to 10000, which is the default value
of `index.max_result_window` in Elasticsearch.
+### index.paginationType
+
+The pagination type to use when index queries are repeated to obtain the next set of results.
+Supported values are: `OFFSET` and `SEARCH_AFTER`. For more information, refer to
+[`index.paginationType`](https://gerrit-review.googlesource.com/Documentation/config-gerrit.html#index.paginationType).
+
+Defaults to `OFFSET`.
+Note: paginationType `NONE` is not supported and Gerrit will not start if it is configured (results
+in `ProvisionException`).
+
## Section elasticsearch
For compatibility information, please refer to the [project homepage](https://www.gerritcodereview.com/elasticsearch.html).
diff --git a/src/test/java/com/google/gerrit/elasticsearch/ElasticConfigurationTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticConfigurationTest.java
index 7e044c3..9d20edc 100644
--- a/src/test/java/com/google/gerrit/elasticsearch/ElasticConfigurationTest.java
+++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticConfigurationTest.java
@@ -25,6 +25,7 @@
import static java.util.stream.Collectors.toList;
import com.google.common.collect.ImmutableList;
+import com.google.gerrit.index.IndexConfig;
import com.google.inject.ProvisionException;
import java.util.Arrays;
import org.apache.http.HttpHost;
@@ -35,7 +36,7 @@
@Test
public void singleServerNoOtherConfig() throws Exception {
Config cfg = newConfig();
- ElasticConfiguration esCfg = new ElasticConfiguration(cfg);
+ ElasticConfiguration esCfg = newElasticConfig(cfg);
assertHosts(esCfg, "http://elastic:1234");
assertThat(esCfg.username).isNull();
assertThat(esCfg.password).isNull();
@@ -46,7 +47,7 @@
public void serverWithoutPortSpecified() throws Exception {
Config cfg = new Config();
cfg.setString(SECTION_ELASTICSEARCH, null, KEY_SERVER, "http://elastic");
- ElasticConfiguration esCfg = new ElasticConfiguration(cfg);
+ ElasticConfiguration esCfg = newElasticConfig(cfg);
assertHosts(esCfg, "http://elastic:9200");
}
@@ -54,7 +55,7 @@
public void prefix() throws Exception {
Config cfg = newConfig();
cfg.setString(SECTION_ELASTICSEARCH, null, KEY_PREFIX, "myprefix");
- ElasticConfiguration esCfg = new ElasticConfiguration(cfg);
+ ElasticConfiguration esCfg = newElasticConfig(cfg);
assertThat(esCfg.prefix).isEqualTo("myprefix");
}
@@ -63,7 +64,7 @@
Config cfg = newConfig();
cfg.setString(SECTION_ELASTICSEARCH, null, KEY_USERNAME, "myself");
cfg.setString(SECTION_ELASTICSEARCH, null, KEY_PASSWORD, "s3kr3t");
- ElasticConfiguration esCfg = new ElasticConfiguration(cfg);
+ ElasticConfiguration esCfg = newElasticConfig(cfg);
assertThat(esCfg.username).isEqualTo("myself");
assertThat(esCfg.password).isEqualTo("s3kr3t");
}
@@ -72,7 +73,7 @@
public void withAuthenticationPasswordOnlyUsesDefaultUsername() throws Exception {
Config cfg = newConfig();
cfg.setString(SECTION_ELASTICSEARCH, null, KEY_PASSWORD, "s3kr3t");
- ElasticConfiguration esCfg = new ElasticConfiguration(cfg);
+ ElasticConfiguration esCfg = newElasticConfig(cfg);
assertThat(esCfg.username).isEqualTo(DEFAULT_USERNAME);
assertThat(esCfg.password).isEqualTo("s3kr3t");
}
@@ -85,20 +86,20 @@
null,
KEY_SERVER,
ImmutableList.of("http://elastic1:1234", "http://elastic2:1234"));
- ElasticConfiguration esCfg = new ElasticConfiguration(cfg);
+ ElasticConfiguration esCfg = newElasticConfig(cfg);
assertHosts(esCfg, "http://elastic1:1234", "http://elastic2:1234");
}
@Test
public void noServers() throws Exception {
- assertProvisionException(new Config());
+ assertProvisionException(new Config(), "No valid Elasticsearch servers configured");
}
@Test
public void singleServerInvalid() throws Exception {
Config cfg = new Config();
cfg.setString(SECTION_ELASTICSEARCH, null, KEY_SERVER, "foo");
- assertProvisionException(cfg);
+ assertProvisionException(cfg, "No valid Elasticsearch servers configured");
}
@Test
@@ -106,24 +107,35 @@
Config cfg = new Config();
cfg.setStringList(
SECTION_ELASTICSEARCH, null, KEY_SERVER, ImmutableList.of("http://elastic1:1234", "foo"));
- ElasticConfiguration esCfg = new ElasticConfiguration(cfg);
+ ElasticConfiguration esCfg = newElasticConfig(cfg);
assertHosts(esCfg, "http://elastic1:1234");
}
+ @Test
+ public void unsupportedPaginationTypeNone() {
+ Config cfg = new Config();
+ cfg.setString("index", null, "paginationType", "NONE");
+ assertProvisionException(
+ cfg, "The 'index.paginationType = NONE' configuration is not supported by Elasticsearch");
+ }
+
private static Config newConfig() {
Config config = new Config();
config.setString(SECTION_ELASTICSEARCH, null, KEY_SERVER, "http://elastic:1234");
return config;
}
+ private static ElasticConfiguration newElasticConfig(Config cfg) {
+ return new ElasticConfiguration(cfg, IndexConfig.fromConfig(cfg).build());
+ }
+
private void assertHosts(ElasticConfiguration cfg, Object... hostURIs) throws Exception {
assertThat(Arrays.asList(cfg.getHosts()).stream().map(HttpHost::toURI).collect(toList()))
.containsExactly(hostURIs);
}
- private void assertProvisionException(Config cfg) {
- ProvisionException thrown =
- assertThrows(ProvisionException.class, () -> new ElasticConfiguration(cfg));
- assertThat(thrown).hasMessageThat().contains("No valid Elasticsearch servers configured");
+ private void assertProvisionException(Config cfg, String msg) {
+ ProvisionException thrown = assertThrows(ProvisionException.class, () -> newElasticConfig(cfg));
+ assertThat(thrown).hasMessageThat().contains(msg);
}
}
