Merge branch 'stable-3.3' into stable-3.4 * stable-3.3: Elasticsearch: Discontinue EOL version 7.4 support Change-Id: I5d5669898d56e977ec6a00b2766b66ecfa773bb5
diff --git a/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java b/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java index e56f470..44a377a 100644 --- a/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java +++ b/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
@@ -84,6 +84,9 @@ protected static final String BULK = "_bulk"; protected static final String MAPPINGS = "mappings"; protected static final String ORDER = "order"; + protected static final String DESC_SORT_ORDER = "desc"; + protected static final String ASC_SORT_ORDER = "asc"; + protected static final String UNMAPPED_TYPE = "unmapped_type"; protected static final String SEARCH = "_search"; protected static final String SETTINGS = "settings"; @@ -288,7 +291,7 @@ protected JsonArray getSortArray(String idFieldName) { JsonObject properties = new JsonObject(); - properties.addProperty(ORDER, "asc"); + properties.addProperty(ORDER, ASC_SORT_ORDER); JsonArray sortArray = new JsonArray(); addNamedElement(idFieldName, properties, sortArray);
diff --git a/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java b/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java index 625a598..162654d 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java +++ b/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
@@ -37,6 +37,7 @@ import com.google.gerrit.exceptions.StorageException; import com.google.gerrit.index.FieldDef; import com.google.gerrit.index.QueryOptions; +import com.google.gerrit.index.RefState; import com.google.gerrit.index.Schema; import com.google.gerrit.index.query.DataSource; import com.google.gerrit.index.query.Predicate; @@ -57,6 +58,9 @@ import com.google.gson.JsonObject; import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; +import java.sql.Timestamp; +import java.time.Instant; +import java.time.format.DateTimeFormatter; import java.util.Collections; import java.util.Optional; import java.util.Set; @@ -133,14 +137,24 @@ private JsonArray getSortArray() { JsonObject properties = new JsonObject(); - properties.addProperty(ORDER, "desc"); + properties.addProperty(ORDER, DESC_SORT_ORDER); JsonArray sortArray = new JsonArray(); addNamedElement(ChangeField.UPDATED.getName(), properties, sortArray); + addNamedElement(ChangeField.MERGED_ON.getName(), getMergedOnSortOptions(), sortArray); addNamedElement(idField.getName(), properties, sortArray); return sortArray; } + private JsonObject getMergedOnSortOptions() { + JsonObject sortOptions = new JsonObject(); + sortOptions.addProperty(ORDER, DESC_SORT_ORDER); + // Ignore the sort field if it does not exist in index. Otherwise the search would fail on open + // changes, because the corresponding documents do not have mergedOn field. + sortOptions.addProperty(UNMAPPED_TYPE, ElasticMapping.TIMESTAMP_FIELD_TYPE); + return sortOptions; + } + @Override protected String getDeleteActions(Change.Id c) { return getDeleteRequest(c); @@ -341,7 +355,7 @@ // Ref-state. if (fields.contains(ChangeField.REF_STATE.getName())) { - cd.setRefStates(getByteArray(source, ChangeField.REF_STATE.getName())); + cd.setRefStates(RefState.parseStates(getByteArray(source, ChangeField.REF_STATE.getName()))); } // Ref-state-pattern. @@ -361,6 +375,10 @@ cd); } + if (fields.contains(ChangeField.MERGED_ON.getName())) { + decodeMergedOn(source, cd); + } + return cd; } @@ -396,4 +414,18 @@ } out.setUnresolvedCommentCount(count.getAsInt()); } + + private void decodeMergedOn(JsonObject doc, ChangeData out) { + JsonElement mergedOnField = doc.get(ChangeField.MERGED_ON.getName()); + + Timestamp mergedOn = null; + if (mergedOnField != null) { + // Parse from ElasticMapping.TIMESTAMP_FIELD_FORMAT. + // We currently use built-in ISO-based dateOptionalTime. + // https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html#built-in-date-formats + DateTimeFormatter isoFormatter = DateTimeFormatter.ISO_INSTANT; + mergedOn = Timestamp.from(Instant.from(isoFormatter.parse(mergedOnField.getAsString()))); + } + out.setMergedOn(mergedOn); + } }
diff --git a/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java b/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java index 06b128c..c443529 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java +++ b/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java
@@ -26,8 +26,10 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; import org.apache.http.HttpHost; import org.eclipse.jgit.lib.Config; +import org.elasticsearch.client.RestClientBuilder; @Singleton class ElasticConfiguration { @@ -41,12 +43,16 @@ static final String KEY_NUMBER_OF_SHARDS = "numberOfShards"; static final String KEY_NUMBER_OF_REPLICAS = "numberOfReplicas"; static final String KEY_MAX_RESULT_WINDOW = "maxResultWindow"; + static final String KEY_CONNECT_TIMEOUT = "connectTimeout"; + static final String KEY_SOCKET_TIMEOUT = "socketTimeout"; static final String DEFAULT_PORT = "9200"; static final String DEFAULT_USERNAME = "elastic"; static final int DEFAULT_NUMBER_OF_SHARDS = 1; static final int DEFAULT_NUMBER_OF_REPLICAS = 1; static final int DEFAULT_MAX_RESULT_WINDOW = 10000; + static final int DEFAULT_CONNECT_TIMEOUT = RestClientBuilder.DEFAULT_CONNECT_TIMEOUT_MILLIS; + static final int DEFAULT_SOCKET_TIMEOUT = RestClientBuilder.DEFAULT_SOCKET_TIMEOUT_MILLIS; private final Config cfg; private final List<HttpHost> hosts; @@ -56,6 +62,8 @@ final int numberOfShards; final int numberOfReplicas; final int maxResultWindow; + final int connectTimeout; + final int socketTimeout; final String prefix; @Inject @@ -74,6 +82,22 @@ cfg.getInt(SECTION_ELASTICSEARCH, null, KEY_NUMBER_OF_REPLICAS, DEFAULT_NUMBER_OF_REPLICAS); this.maxResultWindow = cfg.getInt(SECTION_ELASTICSEARCH, null, KEY_MAX_RESULT_WINDOW, DEFAULT_MAX_RESULT_WINDOW); + this.connectTimeout = + (int) + cfg.getTimeUnit( + SECTION_ELASTICSEARCH, + null, + KEY_CONNECT_TIMEOUT, + DEFAULT_CONNECT_TIMEOUT, + TimeUnit.MILLISECONDS); + this.socketTimeout = + (int) + cfg.getTimeUnit( + SECTION_ELASTICSEARCH, + null, + KEY_SOCKET_TIMEOUT, + DEFAULT_SOCKET_TIMEOUT, + TimeUnit.MILLISECONDS); this.hosts = new ArrayList<>(); for (String server : cfg.getStringList(SECTION_ELASTICSEARCH, null, KEY_SERVER)) { try {
diff --git a/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java b/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java index f8c2ec5..781ed43 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java +++ b/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
@@ -20,6 +20,7 @@ import com.google.gerrit.elasticsearch.bulk.IndexRequest; import com.google.gerrit.elasticsearch.bulk.UpdateRequest; import com.google.gerrit.entities.AccountGroup; +import com.google.gerrit.entities.InternalGroup; import com.google.gerrit.exceptions.StorageException; import com.google.gerrit.index.QueryOptions; import com.google.gerrit.index.Schema; @@ -28,7 +29,6 @@ import com.google.gerrit.index.query.QueryParseException; import com.google.gerrit.server.account.GroupCache; import com.google.gerrit.server.config.SitePaths; -import com.google.gerrit.server.group.InternalGroup; import com.google.gerrit.server.index.IndexUtils; import com.google.gerrit.server.index.group.GroupField; import com.google.gerrit.server.index.group.GroupIndex;
diff --git a/java/com/google/gerrit/elasticsearch/ElasticMapping.java b/java/com/google/gerrit/elasticsearch/ElasticMapping.java index f8c4168..edd05c9 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticMapping.java +++ b/java/com/google/gerrit/elasticsearch/ElasticMapping.java
@@ -21,6 +21,10 @@ import java.util.Map; class ElasticMapping { + + protected static final String TIMESTAMP_FIELD_TYPE = "date"; + protected static final String TIMESTAMP_FIELD_FORMAT = "dateOptionalTime"; + static MappingProperties createMapping(Schema<?> schema, ElasticQueryAdapter adapter) { ElasticMapping.Builder mapping = new ElasticMapping.Builder(adapter); for (FieldDef<?, ?> field : schema.getFields().values()) { @@ -71,9 +75,9 @@ } Builder addTimestamp(String name) { - FieldProperties properties = new FieldProperties("date"); - properties.type = "date"; - properties.format = "dateOptionalTime"; + FieldProperties properties = new FieldProperties(TIMESTAMP_FIELD_TYPE); + properties.type = TIMESTAMP_FIELD_TYPE; + properties.format = TIMESTAMP_FIELD_FORMAT; fields.put(name, properties); return this; }
diff --git a/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java b/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java index d05e91c..40ac603 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java +++ b/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
@@ -29,7 +29,6 @@ import com.google.gerrit.index.query.QueryParseException; import com.google.gerrit.index.query.RegexPredicate; import com.google.gerrit.index.query.TimestampRangePredicate; -import com.google.gerrit.server.query.change.AfterPredicate; import java.time.Instant; public class ElasticQueryBuilder { @@ -130,7 +129,9 @@ private <T> QueryBuilder timestampQuery(IndexPredicate<T> p) throws QueryParseException { if (p instanceof TimestampRangePredicate) { TimestampRangePredicate<T> r = (TimestampRangePredicate<T>) p; - if (p instanceof AfterPredicate) { + if (r.getMaxTimestamp().getTime() == Long.MAX_VALUE) { + // The time range only has the start value, search from the start to the max supported value + // Long.MAX_VALUE return QueryBuilders.rangeQuery(r.getField().getName()) .gte(Instant.ofEpochMilli(r.getMinTimestamp().getTime())); }
diff --git a/java/com/google/gerrit/elasticsearch/ElasticRestClientProvider.java b/java/com/google/gerrit/elasticsearch/ElasticRestClientProvider.java index f635b23..b41f365 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticRestClientProvider.java +++ b/java/com/google/gerrit/elasticsearch/ElasticRestClientProvider.java
@@ -27,6 +27,7 @@ import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.elasticsearch.client.Request; @@ -128,10 +129,19 @@ private RestClient build() { RestClientBuilder builder = RestClient.builder(cfg.getHosts()); + setConfiguredTimeouts(builder); setConfiguredCredentialsIfAny(builder); return builder.build(); } + private void setConfiguredTimeouts(RestClientBuilder builder) { + builder.setRequestConfigCallback( + (RequestConfig.Builder requestConfigBuilder) -> + requestConfigBuilder + .setConnectTimeout(cfg.connectTimeout) + .setSocketTimeout(cfg.socketTimeout)); + } + private void setConfiguredCredentialsIfAny(RestClientBuilder builder) { String username = cfg.username; String password = cfg.password;
diff --git a/javatests/com/google/gerrit/elasticsearch/BUILD b/javatests/com/google/gerrit/elasticsearch/BUILD index be35d5a..3036811 100644 --- a/javatests/com/google/gerrit/elasticsearch/BUILD +++ b/javatests/com/google/gerrit/elasticsearch/BUILD
@@ -55,7 +55,6 @@ ELASTICSEARCH_TAGS = [ "docker", "elastic", - "exclusive", ] [junit_tests(