Merge pagination improvements from Gerrit 3.4
The below changes were cherry-picked from core stable-3.4 onto
the rewritten module history where stable-3.4 would have been.
* 3.4:
Paginate no-limit queries
Introduce a SEARCH_AFTER index pagination type
Change-Id: I11600ad4c11c2c546a23058b1606056d76825cd7
diff --git a/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java b/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
index d87f993..3b300cf 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
@@ -371,9 +371,12 @@
SearchSourceBuilder searchSource =
new SearchSourceBuilder(client.adapter())
.query(qb)
- .from(opts.start())
- .size(opts.limit())
+ .size(opts.pageSize())
.fields(Lists.newArrayList(opts.fields()));
+ searchSource =
+ opts.searchAfter() != null
+ ? searchSource.searchAfter((JsonArray) opts.searchAfter()).trackTotalHits(false)
+ : searchSource.from(opts.start());
search = getSearch(searchSource, sortArray);
}
@@ -395,6 +398,7 @@
private <T> ResultSet<T> readImpl(Function<JsonObject, T> mapper) {
try {
String uri = getURI(SEARCH);
+ JsonArray searchAfter = null;
Response response =
performRequest(HttpPost.METHOD_NAME, uri, search, Collections.emptyMap());
StatusLine statusLine = response.getStatusLine();
@@ -405,13 +409,24 @@
if (obj.get("hits") != null) {
JsonArray json = obj.getAsJsonArray("hits");
ImmutableList.Builder<T> results = ImmutableList.builderWithExpectedSize(json.size());
+ JsonObject hit = null;
for (int i = 0; i < json.size(); i++) {
- T mapperResult = mapper.apply(json.get(i).getAsJsonObject());
+ hit = json.get(i).getAsJsonObject();
+ T mapperResult = mapper.apply(hit);
if (mapperResult != null) {
results.add(mapperResult);
}
}
- return new ListResultSet<>(results.build());
+ if (hit != null && hit.get("sort") != null) {
+ searchAfter = hit.getAsJsonArray("sort");
+ }
+ JsonArray finalSearchAfter = searchAfter;
+ return new ListResultSet<T>(results.build()) {
+ @Override
+ public Object searchAfter() {
+ return finalSearchAfter;
+ }
+ };
}
} else {
logger.atSevere().log(statusLine.getReasonPhrase());
diff --git a/src/main/java/com/google/gerrit/elasticsearch/builders/SearchAfterBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/builders/SearchAfterBuilder.java
new file mode 100644
index 0000000..0951217
--- /dev/null
+++ b/src/main/java/com/google/gerrit/elasticsearch/builders/SearchAfterBuilder.java
@@ -0,0 +1,45 @@
+// Copyright (C) 2022 The Android Open Source Project, 2009-2015 Elasticsearch
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.elasticsearch.builders;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonPrimitive;
+import java.io.IOException;
+
+/**
+ * A trimmed down and modified version of org.elasticsearch.search.searchafter.SearchAfterBuilder.
+ */
+public final class SearchAfterBuilder {
+ private JsonArray sortValues;
+
+ public SearchAfterBuilder(JsonArray sortValues) {
+ this.sortValues = sortValues;
+ }
+
+ public void innerToXContent(XContentBuilder builder) throws IOException {
+ builder.startArray("search_after");
+ for (int i = 0; i < sortValues.size(); i++) {
+ JsonPrimitive value = sortValues.get(i).getAsJsonPrimitive();
+ if (value.isNumber()) {
+ builder.value(value.getAsLong());
+ } else if (value.isBoolean()) {
+ builder.value(value.getAsBoolean());
+ } else {
+ builder.value(value.getAsString());
+ }
+ }
+ builder.endArray();
+ }
+}
diff --git a/src/main/java/com/google/gerrit/elasticsearch/builders/SearchSourceBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/builders/SearchSourceBuilder.java
index 35cbea9..7e4ea93 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/builders/SearchSourceBuilder.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/builders/SearchSourceBuilder.java
@@ -15,6 +15,7 @@
package com.google.gerrit.elasticsearch.builders;
import com.google.gerrit.elasticsearch.ElasticQueryAdapter;
+import com.google.gson.JsonArray;
import java.io.IOException;
import java.util.List;
@@ -28,10 +29,14 @@
private QuerySourceBuilder querySourceBuilder;
+ private SearchAfterBuilder searchAfterBuilder;
+
private int from = -1;
private int size = -1;
+ private boolean trackTotalHits = true;
+
private List<String> fieldNames;
/** Constructs a new search source builder. */
@@ -53,12 +58,22 @@
return this;
}
+ public SearchSourceBuilder searchAfter(JsonArray sortValues) {
+ this.searchAfterBuilder = new SearchAfterBuilder(sortValues);
+ return this;
+ }
+
/** The number of search hits to return. Defaults to <tt>10</tt>. */
public SearchSourceBuilder size(int size) {
this.size = size;
return this;
}
+ public SearchSourceBuilder trackTotalHits(boolean track) {
+ this.trackTotalHits = track;
+ return this;
+ }
+
/**
* Sets the fields to load and return as part of the search request. If none are specified, the
* source of the document will be returned.
@@ -93,6 +108,10 @@
builder.field("size", size);
}
+ if (!trackTotalHits) {
+ builder.field("track_total_hits", false);
+ }
+
if (querySourceBuilder != null) {
querySourceBuilder.innerToXContent(builder);
}
@@ -108,5 +127,9 @@
builder.endArray();
}
}
+
+ if (searchAfterBuilder != null) {
+ searchAfterBuilder.innerToXContent(builder);
+ }
}
}
diff --git a/src/main/java/com/google/gerrit/elasticsearch/builders/XContentBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/builders/XContentBuilder.java
index 9c44583..853596d 100644
--- a/src/main/java/com/google/gerrit/elasticsearch/builders/XContentBuilder.java
+++ b/src/main/java/com/google/gerrit/elasticsearch/builders/XContentBuilder.java
@@ -152,6 +152,8 @@
generator.writeString((String) value);
} else if (type == Integer.class) {
generator.writeNumber(((Integer) value));
+ } else if (type == Long.class) {
+ generator.writeNumber(((Long) value));
} else if (type == byte[].class) {
generator.writeBinary((byte[]) value);
} else if (value instanceof Date) {
diff --git a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryAccountsTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryAccountsTest.java
index 6bdcf3c..752a1e7 100644
--- a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryAccountsTest.java
+++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryAccountsTest.java
@@ -34,6 +34,13 @@
return ElasticTestUtils.createConfig();
}
+ @ConfigSuite.Config
+ public static Config searchAfterPaginationType() {
+ Config config = defaultConfig();
+ config.setString("index", null, "paginationType", "SEARCH_AFTER");
+ return config;
+ }
+
private static ElasticContainer container;
private static CloseableHttpAsyncClient client;
diff --git a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java
index 4929970..9a85129 100644
--- a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java
+++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java
@@ -40,6 +40,13 @@
return ElasticTestUtils.createConfig();
}
+ @ConfigSuite.Config
+ public static Config searchAfterPaginationType() {
+ Config config = defaultConfig();
+ config.setString("index", null, "paginationType", "SEARCH_AFTER");
+ return config;
+ }
+
private static ElasticContainer container;
private static CloseableHttpAsyncClient client;
diff --git a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryGroupsTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryGroupsTest.java
index c9c1e10..6eef24c 100644
--- a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryGroupsTest.java
+++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryGroupsTest.java
@@ -35,6 +35,13 @@
return ElasticTestUtils.createConfig();
}
+ @ConfigSuite.Config
+ public static Config searchAfterPaginationType() {
+ Config config = defaultConfig();
+ config.setString("index", null, "paginationType", "SEARCH_AFTER");
+ return config;
+ }
+
private static ElasticContainer container;
private static CloseableHttpAsyncClient client;
diff --git a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryProjectsTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryProjectsTest.java
index e0383b0..70cd7de 100644
--- a/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryProjectsTest.java
+++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryProjectsTest.java
@@ -35,6 +35,13 @@
return ElasticTestUtils.createConfig();
}
+ @ConfigSuite.Config
+ public static Config searchAfterPaginationType() {
+ Config config = defaultConfig();
+ config.setString("index", null, "paginationType", "SEARCH_AFTER");
+ return config;
+ }
+
private static ElasticContainer container;
private static CloseableHttpAsyncClient client;
