Merge pagination improvements from Gerrit 3.4 The below changes were cherry-picked from core stable-3.4 onto the rewritten module history where stable-3.4 would have been. * 3.4: Paginate no-limit queries Introduce a SEARCH_AFTER index pagination type Change-Id: I11600ad4c11c2c546a23058b1606056d76825cd7
diff --git a/BUILD b/BUILD new file mode 100644 index 0000000..b16fb0d --- /dev/null +++ b/BUILD
@@ -0,0 +1,96 @@ +load("@rules_java//java:defs.bzl", "java_library") +load("//tools/bzl:junit.bzl", "junit_tests") +load( + "//tools/bzl:plugin.bzl", + "PLUGIN_DEPS", + "PLUGIN_TEST_DEPS", + "gerrit_plugin", +) + +gerrit_plugin( + name = "index-elasticsearch", + srcs = glob(["src/main/java/**/*.java"]), + deps = [ + "@elasticsearch-rest-client//jar", + "@httpasyncclient//jar", + "@httpcore-nio//jar", + "@jackson-core//jar", + ], +) + +ELASTICSEARCH_DEPS = [ + "@docker-java-api//jar", + "@docker-java-transport//jar", + "@duct-tape//jar", + "@httpasyncclient//jar", + "@jackson-annotations//jar", + "@jackson-core//jar", + "@jna//jar", + "@testcontainers-elasticsearch//jar", + "@testcontainers//jar", +] + +java_library( + name = "index-elasticsearch__plugin_test_deps", + testonly = True, + srcs = [], + visibility = ["//visibility:public"], + exports = ELASTICSEARCH_DEPS, +) + +java_library( + name = "elasticsearch_test_utils", + testonly = True, + srcs = glob( + ["src/test/java/**/*.java"], + exclude = ["src/test/java/**/*Test.java"], + ), + visibility = ["//visibility:public"], + deps = ELASTICSEARCH_DEPS + PLUGIN_DEPS + PLUGIN_TEST_DEPS + [ + ":index-elasticsearch__plugin", + ], +) + +QUERY_TESTS_DEP = "//javatests/com/google/gerrit/server/query/%s:abstract_query_tests" + +ACCOUNT_QUERY_TESTS_DEP = "//javatests/com/google/gerrit/server/query/account:abstract_query_tests" + +TYPES = [ + "account", + "change", + "group", + "project", +] + +SUFFIX = "sTest.java" + +ELASTICSEARCH_TESTS_V7 = {i: "ElasticV7Query" + i.capitalize() + SUFFIX for i in TYPES} + +[junit_tests( + name = "elasticsearch_query_%ss_test_V7" % name, + size = "enormous", + srcs = ["src/test/java/com/google/gerrit/elasticsearch/" + src], + tags = [ + "docker", + "elastic", + "exclusive", + ], + deps = ELASTICSEARCH_DEPS + PLUGIN_TEST_DEPS + [ + QUERY_TESTS_DEP % name, + ":elasticsearch_test_utils", + ":index-elasticsearch__plugin", + ], +) for name, src in ELASTICSEARCH_TESTS_V7.items()] + +junit_tests( + name = "index-elasticsearch_tests", + size = "small", + srcs = glob( + ["src/test/java/**/*Test.java"], + exclude = ["src/test/java/**/Elastic*Query*" + SUFFIX], + ), + tags = ["elastic"], + deps = PLUGIN_TEST_DEPS + [ + ":index-elasticsearch__plugin", + ], +)
diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..a8708e7 --- /dev/null +++ b/Jenkinsfile
@@ -0,0 +1,2 @@ +pluginPipeline(formatCheckId: 'gerritforge:index-elasticsearch-code-style', + buildCheckId: 'gerritforge:index-elasticsearch-build-test')
diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE
@@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License.
diff --git a/README.md b/README.md new file mode 100644 index 0000000..41d4279 --- /dev/null +++ b/README.md
@@ -0,0 +1,29 @@ +# Index backend for Gerrit, based on ElasticSearch + +Indexing backend libModule for [Gerrit Code Review](https://gerritcodereview.com) +based on [ElasticSearch](https://www.elastic.co/elasticsearch/). + +This module was originally part of Gerrit core and then extracted into a separate +component from v3.5.0-rc3 as part of [Change-Id: Ib7b5167ce](https://gerrit-review.googlesource.com/c/gerrit/+/323676). + +Note that, ElasticSearch source code is no longer Apache 2.0-licensed for versions +7.11 and newer. See ElasticSearch [2021 license change](https://www.elastic.co/pricing/faq/licensing) +for more information. + +## How to build + +This libModule is built like a Gerrit in-tree plugin, using Bazelisk. See the +[build instructions](src/main/resources/Documentation/build.md) for more details. + +## Setup + +See the [setup instructions](src/main/resources/Documentation/setup.md) for how to install the +index-elasticsearch module. + +For further information and supported options, refer to the [config](src/main/resources/Documentation/config.md) +documentation. + +## Integration test + +This libModule runs tests like a Gerrit in-tree plugin, using Bazelisk. See the +[test instructions](src/main/resources/Documentation/build.md#Integration-test) for more details.
diff --git a/external_plugin_deps.bzl b/external_plugin_deps.bzl new file mode 100644 index 0000000..8b44d57 --- /dev/null +++ b/external_plugin_deps.bzl
@@ -0,0 +1,93 @@ +load("//tools/bzl:maven_jar.bzl", "maven_jar") + +TESTCONTAINERS_VERSION = "1.15.3" + +# Ensure artifacts compatibility by selecting them from the Bill Of Materials +# https://search.maven.org/artifact/net.openhft/chronicle-bom/2.20.191/pom +def external_plugin_deps(): + maven_jar( + name = "testcontainers", + artifact = "org.testcontainers:testcontainers:" + TESTCONTAINERS_VERSION, + sha1 = "95c6cfde71c2209f0c29cb14e432471e0b111880", + ) + + # When upgrading elasticsearch-rest-client, also upgrade httpcore-nio + # and httpasyncclient as necessary in tools/nongoogle.bzl. Consider + # also the other org.apache.httpcomponents dependencies in + # WORKSPACE. + maven_jar( + name = "elasticsearch-rest-client", + artifact = "org.elasticsearch.client:elasticsearch-rest-client:8.3.2", + sha1 = "bb5cb3dbd82ea75a6d49b9011ca5b1d125b30f00", + ) + + maven_jar( + name = "testcontainers-elasticsearch", + artifact = "org.testcontainers:elasticsearch:" + TESTCONTAINERS_VERSION, + sha1 = "595e3a50f59cd3c1d281ca6c1bc4037e277a1353", + ) + + maven_jar( + name = "duct-tape", + artifact = "org.rnorth.duct-tape:duct-tape:1.0.8", + sha1 = "92edc22a9ab2f3e17c9bf700aaee377d50e8b530", + ) + + maven_jar( + name = "visible-assertions", + artifact = "org.rnorth.visible-assertions:visible-assertions:2.1.2", + sha1 = "20d31a578030ec8e941888537267d3123c2ad1c1", + ) + + maven_jar( + name = "jna", + artifact = "net.java.dev.jna:jna:5.5.0", + sha1 = "0e0845217c4907822403912ad6828d8e0b256208", + ) + + DOCKER_JAVA_VERS = "3.2.8" + + maven_jar( + name = "docker-java-api", + artifact = "com.github.docker-java:docker-java-api:" + DOCKER_JAVA_VERS, + sha1 = "4ac22a72d546a9f3523cd4b5fabffa77c4a6ec7c", + ) + + maven_jar( + name = "docker-java-transport", + artifact = "com.github.docker-java:docker-java-transport:" + DOCKER_JAVA_VERS, + sha1 = "c3b5598c67d0a5e2e780bf48f520da26b9915eab", + ) + + # elasticsearch-rest-client explicitly depends on this version + maven_jar( + name = "httpasyncclient", + artifact = "org.apache.httpcomponents:httpasyncclient:4.1.4", + sha1 = "f3a3240681faae3fa46b573a4c7e50cec9db0d86", + ) + + # elasticsearch-rest-client explicitly depends on this version + maven_jar( + name = "httpcore-nio", + artifact = "org.apache.httpcomponents:httpcore-nio:4.4.12", + sha1 = "84cd29eca842f31db02987cfedea245af020198b", + ) + + maven_jar( + name = "jackson-core", + artifact = "com.fasterxml.jackson.core:jackson-core:2.11.3", + sha1 = "c2351800432bdbdd8284c3f5a7f0782a352aa84a", + ) + + maven_jar( + name = "jackson-annotations", + artifact = "com.fasterxml.jackson.core:jackson-annotations:2.10.3", + sha1 = "0f63b3b1da563767d04d2e4d3fc1ae0cdeffebe7", + ) + + # elasticsearch-rest-client explicitly depends on this version + maven_jar( + name = "httpasyncclient", + artifact = "org.apache.httpcomponents:httpasyncclient:4.1.4", + sha1 = "f3a3240681faae3fa46b573a4c7e50cec9db0d86", + )
diff --git a/java/com/google/gerrit/elasticsearch/BUILD b/java/com/google/gerrit/elasticsearch/BUILD deleted file mode 100644 index 8bab80b..0000000 --- a/java/com/google/gerrit/elasticsearch/BUILD +++ /dev/null
@@ -1,33 +0,0 @@ -load("@rules_java//java:defs.bzl", "java_library") - -java_library( - name = "elasticsearch", - srcs = glob(["**/*.java"]), - visibility = ["//visibility:public"], - deps = [ - "//java/com/google/gerrit/common:annotations", - "//java/com/google/gerrit/entities", - "//java/com/google/gerrit/exceptions", - "//java/com/google/gerrit/extensions:api", - "//java/com/google/gerrit/index", - "//java/com/google/gerrit/index:query_exception", - "//java/com/google/gerrit/index/project", - "//java/com/google/gerrit/lifecycle", - "//java/com/google/gerrit/proto", - "//java/com/google/gerrit/server", - "//lib:gson", - "//lib:guava", - "//lib:jgit", - "//lib:protobuf", - "//lib/commons:lang", - "//lib/elasticsearch-rest-client", - "//lib/flogger:api", - "//lib/guice", - "//lib/guice:guice-assistedinject", - "//lib/httpcomponents:httpasyncclient", - "//lib/httpcomponents:httpclient", - "//lib/httpcomponents:httpcore", - "//lib/httpcomponents:httpcore-nio", - "//lib/jackson:jackson-core", - ], -)
diff --git a/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java b/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java deleted file mode 100644 index 162654d..0000000 --- a/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java +++ /dev/null
@@ -1,431 +0,0 @@ -// Copyright (C) 2014 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.gerrit.elasticsearch; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Objects.requireNonNull; - -import com.google.common.collect.FluentIterable; -import com.google.common.collect.ImmutableListMultimap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.google.common.collect.ListMultimap; -import com.google.common.collect.MultimapBuilder; -import com.google.common.collect.Sets; -import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties; -import com.google.gerrit.elasticsearch.bulk.BulkRequest; -import com.google.gerrit.elasticsearch.bulk.IndexRequest; -import com.google.gerrit.elasticsearch.bulk.UpdateRequest; -import com.google.gerrit.entities.Account; -import com.google.gerrit.entities.Change; -import com.google.gerrit.entities.Project; -import com.google.gerrit.entities.converter.ChangeProtoConverter; -import com.google.gerrit.entities.converter.PatchSetApprovalProtoConverter; -import com.google.gerrit.entities.converter.PatchSetProtoConverter; -import com.google.gerrit.exceptions.StorageException; -import com.google.gerrit.index.FieldDef; -import com.google.gerrit.index.QueryOptions; -import com.google.gerrit.index.RefState; -import com.google.gerrit.index.Schema; -import com.google.gerrit.index.query.DataSource; -import com.google.gerrit.index.query.Predicate; -import com.google.gerrit.index.query.QueryParseException; -import com.google.gerrit.server.ReviewerByEmailSet; -import com.google.gerrit.server.ReviewerSet; -import com.google.gerrit.server.StarredChangesUtil; -import com.google.gerrit.server.change.MergeabilityComputationBehavior; -import com.google.gerrit.server.config.GerritServerConfig; -import com.google.gerrit.server.config.SitePaths; -import com.google.gerrit.server.index.IndexUtils; -import com.google.gerrit.server.index.change.ChangeField; -import com.google.gerrit.server.index.change.ChangeIndex; -import com.google.gerrit.server.project.SubmitRuleOptions; -import com.google.gerrit.server.query.change.ChangeData; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.inject.Inject; -import com.google.inject.assistedinject.Assisted; -import java.sql.Timestamp; -import java.time.Instant; -import java.time.format.DateTimeFormatter; -import java.util.Collections; -import java.util.Optional; -import java.util.Set; -import org.apache.http.HttpStatus; -import org.eclipse.jgit.lib.Config; -import org.elasticsearch.client.Response; - -/** Secondary index implementation using Elasticsearch. */ -class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData> - implements ChangeIndex { - static class ChangeMapping { - final MappingProperties changes; - final MappingProperties openChanges; - final MappingProperties closedChanges; - - ChangeMapping(Schema<ChangeData> schema, ElasticQueryAdapter adapter) { - MappingProperties mapping = ElasticMapping.createMapping(schema, adapter); - this.changes = mapping; - this.openChanges = mapping; - this.closedChanges = mapping; - } - } - - private static final String CHANGES = "changes"; - - private final ChangeMapping mapping; - private final ChangeData.Factory changeDataFactory; - private final Schema<ChangeData> schema; - private final FieldDef<ChangeData, ?> idField; - private final ImmutableSet<String> skipFields; - - @Inject - ElasticChangeIndex( - ElasticConfiguration cfg, - ChangeData.Factory changeDataFactory, - SitePaths sitePaths, - ElasticRestClientProvider clientBuilder, - @GerritServerConfig Config gerritConfig, - @Assisted Schema<ChangeData> schema) { - super(cfg, sitePaths, schema, clientBuilder, CHANGES); - this.changeDataFactory = changeDataFactory; - this.schema = schema; - this.mapping = new ChangeMapping(schema, client.adapter()); - this.idField = - this.schema.useLegacyNumericFields() ? ChangeField.LEGACY_ID : ChangeField.LEGACY_ID_STR; - this.skipFields = - MergeabilityComputationBehavior.fromConfig(gerritConfig).includeInIndex() - ? ImmutableSet.of() - : ImmutableSet.of(ChangeField.MERGEABLE.getName()); - } - - @Override - public void replace(ChangeData cd) { - BulkRequest bulk = - new IndexRequest(getId(cd), indexName).add(new UpdateRequest<>(schema, cd, skipFields)); - - String uri = getURI(BULK); - Response response = postRequest(uri, bulk, getRefreshParam()); - int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode != HttpStatus.SC_OK) { - throw new StorageException( - String.format( - "Failed to replace change %s in index %s: %s", cd.getId(), indexName, statusCode)); - } - } - - @Override - public DataSource<ChangeData> getSource(Predicate<ChangeData> p, QueryOptions opts) - throws QueryParseException { - QueryOptions filteredOpts = - opts.filterFields(o -> IndexUtils.changeFields(o, schema.useLegacyNumericFields())); - return new ElasticQuerySource(p, filteredOpts, getSortArray()); - } - - private JsonArray getSortArray() { - JsonObject properties = new JsonObject(); - properties.addProperty(ORDER, DESC_SORT_ORDER); - - JsonArray sortArray = new JsonArray(); - addNamedElement(ChangeField.UPDATED.getName(), properties, sortArray); - addNamedElement(ChangeField.MERGED_ON.getName(), getMergedOnSortOptions(), sortArray); - addNamedElement(idField.getName(), properties, sortArray); - return sortArray; - } - - private JsonObject getMergedOnSortOptions() { - JsonObject sortOptions = new JsonObject(); - sortOptions.addProperty(ORDER, DESC_SORT_ORDER); - // Ignore the sort field if it does not exist in index. Otherwise the search would fail on open - // changes, because the corresponding documents do not have mergedOn field. - sortOptions.addProperty(UNMAPPED_TYPE, ElasticMapping.TIMESTAMP_FIELD_TYPE); - return sortOptions; - } - - @Override - protected String getDeleteActions(Change.Id c) { - return getDeleteRequest(c); - } - - @Override - protected String getMappings() { - return getMappingsFor(mapping.changes); - } - - @Override - protected String getId(ChangeData cd) { - return cd.getId().toString(); - } - - @Override - protected ChangeData fromDocument(JsonObject json, Set<String> fields) { - JsonElement sourceElement = json.get("_source"); - if (sourceElement == null) { - sourceElement = json.getAsJsonObject().get("fields"); - } - JsonObject source = sourceElement.getAsJsonObject(); - JsonElement c = source.get(ChangeField.CHANGE.getName()); - - if (c == null) { - int id = source.get(idField.getName()).getAsInt(); - // IndexUtils#changeFields ensures either CHANGE or PROJECT is always present. - String projectName = requireNonNull(source.get(ChangeField.PROJECT.getName()).getAsString()); - return changeDataFactory.create(Project.nameKey(projectName), Change.id(id)); - } - - ChangeData cd = - changeDataFactory.create( - parseProtoFrom(decodeBase64(c.getAsString()), ChangeProtoConverter.INSTANCE)); - - // Any decoding that is done here must also be done in {@link LuceneChangeIndex}. - - // Patch sets. - cd.setPatchSets( - decodeProtos(source, ChangeField.PATCH_SET.getName(), PatchSetProtoConverter.INSTANCE)); - - // Approvals. - if (source.get(ChangeField.APPROVAL.getName()) != null) { - cd.setCurrentApprovals( - decodeProtos( - source, ChangeField.APPROVAL.getName(), PatchSetApprovalProtoConverter.INSTANCE)); - } else if (fields.contains(ChangeField.APPROVAL.getName())) { - cd.setCurrentApprovals(Collections.emptyList()); - } - - // Added & Deleted. - JsonElement addedElement = source.get(ChangeField.ADDED.getName()); - JsonElement deletedElement = source.get(ChangeField.DELETED.getName()); - if (addedElement != null && deletedElement != null) { - // Changed lines. - int added = addedElement.getAsInt(); - int deleted = deletedElement.getAsInt(); - cd.setChangedLines(added, deleted); - } - - // Star. - JsonElement starredElement = source.get(ChangeField.STAR.getName()); - if (starredElement != null) { - ListMultimap<Account.Id, String> stars = MultimapBuilder.hashKeys().arrayListValues().build(); - JsonArray starBy = starredElement.getAsJsonArray(); - if (starBy.size() > 0) { - for (int i = 0; i < starBy.size(); i++) { - String[] indexableFields = starBy.get(i).getAsString().split(":"); - Optional<Account.Id> id = Account.Id.tryParse(indexableFields[0]); - if (id.isPresent()) { - stars.put(id.get(), indexableFields[1]); - } - } - } - cd.setStars(stars); - } - - // Mergeable. - JsonElement mergeableElement = source.get(ChangeField.MERGEABLE.getName()); - if (mergeableElement != null && !skipFields.contains(ChangeField.MERGEABLE.getName())) { - String mergeable = mergeableElement.getAsString(); - if ("1".equals(mergeable)) { - cd.setMergeable(true); - } else if ("0".equals(mergeable)) { - cd.setMergeable(false); - } - } - - // Reviewed-by. - if (source.get(ChangeField.REVIEWEDBY.getName()) != null) { - JsonArray reviewedBy = source.get(ChangeField.REVIEWEDBY.getName()).getAsJsonArray(); - if (reviewedBy.size() > 0) { - Set<Account.Id> accounts = Sets.newHashSetWithExpectedSize(reviewedBy.size()); - for (int i = 0; i < reviewedBy.size(); i++) { - int aId = reviewedBy.get(i).getAsInt(); - if (reviewedBy.size() == 1 && aId == ChangeField.NOT_REVIEWED) { - break; - } - accounts.add(Account.id(aId)); - } - cd.setReviewedBy(accounts); - } - } else if (fields.contains(ChangeField.REVIEWEDBY.getName())) { - cd.setReviewedBy(Collections.emptySet()); - } - - // Hashtag. - if (source.get(ChangeField.HASHTAG.getName()) != null) { - JsonArray hashtagArray = source.get(ChangeField.HASHTAG.getName()).getAsJsonArray(); - if (hashtagArray.size() > 0) { - Set<String> hashtags = Sets.newHashSetWithExpectedSize(hashtagArray.size()); - for (int i = 0; i < hashtagArray.size(); i++) { - hashtags.add(hashtagArray.get(i).getAsString()); - } - cd.setHashtags(hashtags); - } - } else if (fields.contains(ChangeField.HASHTAG.getName())) { - cd.setHashtags(Collections.emptySet()); - } - - // Star. - if (source.get(ChangeField.STAR.getName()) != null) { - JsonArray starArray = source.get(ChangeField.STAR.getName()).getAsJsonArray(); - if (starArray.size() > 0) { - ListMultimap<Account.Id, String> stars = - MultimapBuilder.hashKeys().arrayListValues().build(); - for (int i = 0; i < starArray.size(); i++) { - StarredChangesUtil.StarField starField = - StarredChangesUtil.StarField.parse(starArray.get(i).getAsString()); - stars.put(starField.accountId(), starField.label()); - } - cd.setStars(stars); - } - } else if (fields.contains(ChangeField.STAR.getName())) { - cd.setStars(ImmutableListMultimap.of()); - } - - // Reviewer. - if (source.get(ChangeField.REVIEWER.getName()) != null) { - cd.setReviewers( - ChangeField.parseReviewerFieldValues( - cd.getId(), - FluentIterable.from(source.get(ChangeField.REVIEWER.getName()).getAsJsonArray()) - .transform(JsonElement::getAsString))); - } else if (fields.contains(ChangeField.REVIEWER.getName())) { - cd.setReviewers(ReviewerSet.empty()); - } - - // Reviewer-by-email. - if (source.get(ChangeField.REVIEWER_BY_EMAIL.getName()) != null) { - cd.setReviewersByEmail( - ChangeField.parseReviewerByEmailFieldValues( - cd.getId(), - FluentIterable.from( - source.get(ChangeField.REVIEWER_BY_EMAIL.getName()).getAsJsonArray()) - .transform(JsonElement::getAsString))); - } else if (fields.contains(ChangeField.REVIEWER_BY_EMAIL.getName())) { - cd.setReviewersByEmail(ReviewerByEmailSet.empty()); - } - - // Pending-reviewer. - if (source.get(ChangeField.PENDING_REVIEWER.getName()) != null) { - cd.setPendingReviewers( - ChangeField.parseReviewerFieldValues( - cd.getId(), - FluentIterable.from( - source.get(ChangeField.PENDING_REVIEWER.getName()).getAsJsonArray()) - .transform(JsonElement::getAsString))); - } else if (fields.contains(ChangeField.PENDING_REVIEWER.getName())) { - cd.setPendingReviewers(ReviewerSet.empty()); - } - - // Pending-reviewer-by-email. - if (source.get(ChangeField.PENDING_REVIEWER_BY_EMAIL.getName()) != null) { - cd.setPendingReviewersByEmail( - ChangeField.parseReviewerByEmailFieldValues( - cd.getId(), - FluentIterable.from( - source.get(ChangeField.PENDING_REVIEWER_BY_EMAIL.getName()).getAsJsonArray()) - .transform(JsonElement::getAsString))); - } else if (fields.contains(ChangeField.PENDING_REVIEWER_BY_EMAIL.getName())) { - cd.setPendingReviewersByEmail(ReviewerByEmailSet.empty()); - } - - // Stored-submit-record-strict. - decodeSubmitRecords( - source, - ChangeField.STORED_SUBMIT_RECORD_STRICT.getName(), - ChangeField.SUBMIT_RULE_OPTIONS_STRICT, - cd); - - // Stored-submit-record-lenient. - decodeSubmitRecords( - source, - ChangeField.STORED_SUBMIT_RECORD_LENIENT.getName(), - ChangeField.SUBMIT_RULE_OPTIONS_LENIENT, - cd); - - // Ref-state. - if (fields.contains(ChangeField.REF_STATE.getName())) { - cd.setRefStates(RefState.parseStates(getByteArray(source, ChangeField.REF_STATE.getName()))); - } - - // Ref-state-pattern. - if (fields.contains(ChangeField.REF_STATE_PATTERN.getName())) { - cd.setRefStatePatterns(getByteArray(source, ChangeField.REF_STATE_PATTERN.getName())); - } - - // Unresolved-comment-count. - decodeUnresolvedCommentCount(source, ChangeField.UNRESOLVED_COMMENT_COUNT.getName(), cd); - - // Attention set. - if (fields.contains(ChangeField.ATTENTION_SET_FULL.getName())) { - ChangeField.parseAttentionSet( - FluentIterable.from(source.getAsJsonArray(ChangeField.ATTENTION_SET_FULL.getName())) - .transform(ElasticChangeIndex::decodeBase64JsonElement) - .toSet(), - cd); - } - - if (fields.contains(ChangeField.MERGED_ON.getName())) { - decodeMergedOn(source, cd); - } - - return cd; - } - - private Iterable<byte[]> getByteArray(JsonObject source, String name) { - JsonElement element = source.get(name); - return element != null - ? Iterables.transform(element.getAsJsonArray(), e -> decodeBase64(e.getAsString())) - : Collections.emptyList(); - } - - private void decodeSubmitRecords( - JsonObject doc, String fieldName, SubmitRuleOptions opts, ChangeData out) { - JsonArray records = doc.getAsJsonArray(fieldName); - if (records == null) { - return; - } - ChangeField.parseSubmitRecords( - FluentIterable.from(records) - .transform(ElasticChangeIndex::decodeBase64JsonElement) - .toList(), - opts, - out); - } - - private static String decodeBase64JsonElement(JsonElement input) { - return new String(decodeBase64(input.getAsString()), UTF_8); - } - - private void decodeUnresolvedCommentCount(JsonObject doc, String fieldName, ChangeData out) { - JsonElement count = doc.get(fieldName); - if (count == null) { - return; - } - out.setUnresolvedCommentCount(count.getAsInt()); - } - - private void decodeMergedOn(JsonObject doc, ChangeData out) { - JsonElement mergedOnField = doc.get(ChangeField.MERGED_ON.getName()); - - Timestamp mergedOn = null; - if (mergedOnField != null) { - // Parse from ElasticMapping.TIMESTAMP_FIELD_FORMAT. - // We currently use built-in ISO-based dateOptionalTime. - // https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html#built-in-date-formats - DateTimeFormatter isoFormatter = DateTimeFormatter.ISO_INSTANT; - mergedOn = Timestamp.from(Instant.from(isoFormatter.parse(mergedOnField.getAsString()))); - } - out.setMergedOn(mergedOn); - } -}
diff --git a/java/com/google/gerrit/pgm/init/index/elasticsearch/ElasticIndexModuleOnInit.java b/java/com/google/gerrit/pgm/init/index/elasticsearch/ElasticIndexModuleOnInit.java deleted file mode 100644 index f086ab1..0000000 --- a/java/com/google/gerrit/pgm/init/index/elasticsearch/ElasticIndexModuleOnInit.java +++ /dev/null
@@ -1,41 +0,0 @@ -// Copyright (C) 2017 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.gerrit.pgm.init.index.elasticsearch; - -import com.google.gerrit.elasticsearch.ElasticAccountIndex; -import com.google.gerrit.elasticsearch.ElasticGroupIndex; -import com.google.gerrit.pgm.init.index.IndexModuleOnInit; -import com.google.gerrit.server.index.account.AccountIndex; -import com.google.gerrit.server.index.group.GroupIndex; -import com.google.inject.AbstractModule; -import com.google.inject.assistedinject.FactoryModuleBuilder; - -public class ElasticIndexModuleOnInit extends AbstractModule { - - @Override - protected void configure() { - install( - new FactoryModuleBuilder() - .implement(AccountIndex.class, ElasticAccountIndex.class) - .build(AccountIndex.Factory.class)); - - install( - new FactoryModuleBuilder() - .implement(GroupIndex.class, ElasticGroupIndex.class) - .build(GroupIndex.Factory.class)); - - install(new IndexModuleOnInit()); - } -}
diff --git a/javatests/com/google/gerrit/acceptance/pgm/ElasticReindexIT.java b/javatests/com/google/gerrit/acceptance/pgm/ElasticReindexIT.java deleted file mode 100644 index f23cc10..0000000 --- a/javatests/com/google/gerrit/acceptance/pgm/ElasticReindexIT.java +++ /dev/null
@@ -1,36 +0,0 @@ -// Copyright (C) 2018 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.gerrit.acceptance.pgm; - -import static com.google.gerrit.elasticsearch.ElasticTestUtils.createAllIndexes; -import static com.google.gerrit.elasticsearch.ElasticTestUtils.getConfig; - -import com.google.gerrit.elasticsearch.ElasticVersion; -import com.google.gerrit.testing.ConfigSuite; -import com.google.inject.Injector; -import org.eclipse.jgit.lib.Config; - -public class ElasticReindexIT extends AbstractReindexTests { - - @ConfigSuite.Default - public static Config elasticsearchV7() { - return getConfig(ElasticVersion.V7_8); - } - - @Override - public void configureIndex(Injector injector) { - createAllIndexes(injector); - } -}
diff --git a/javatests/com/google/gerrit/acceptance/ssh/ElasticIndexIT.java b/javatests/com/google/gerrit/acceptance/ssh/ElasticIndexIT.java deleted file mode 100644 index f35bcb7..0000000 --- a/javatests/com/google/gerrit/acceptance/ssh/ElasticIndexIT.java +++ /dev/null
@@ -1,38 +0,0 @@ -// Copyright (C) 2018 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.gerrit.acceptance.ssh; - -import static com.google.gerrit.elasticsearch.ElasticTestUtils.createAllIndexes; -import static com.google.gerrit.elasticsearch.ElasticTestUtils.getConfig; - -import com.google.gerrit.elasticsearch.ElasticVersion; -import com.google.gerrit.index.IndexType; -import com.google.gerrit.testing.ConfigSuite; -import com.google.inject.Injector; -import org.eclipse.jgit.lib.Config; - -/** Tests for every supported {@link IndexType#isElasticsearch()} most recent index version. */ -public class ElasticIndexIT extends AbstractIndexTests { - - @ConfigSuite.Default - public static Config elasticsearchV7() { - return getConfig(ElasticVersion.V7_8); - } - - @Override - public void configureIndex(Injector injector) { - createAllIndexes(injector); - } -}
diff --git a/javatests/com/google/gerrit/elasticsearch/BUILD b/javatests/com/google/gerrit/elasticsearch/BUILD deleted file mode 100644 index 3036811..0000000 --- a/javatests/com/google/gerrit/elasticsearch/BUILD +++ /dev/null
@@ -1,85 +0,0 @@ -load("@rules_java//java:defs.bzl", "java_library") -load("//tools/bzl:junit.bzl", "junit_tests") - -java_library( - name = "elasticsearch_test_utils", - testonly = True, - srcs = [ - "ElasticContainer.java", - "ElasticTestUtils.java", - ], - visibility = ["//visibility:public"], - deps = [ - "//java/com/google/gerrit/elasticsearch", - "//java/com/google/gerrit/index", - "//lib:guava", - "//lib:jgit", - "//lib:junit", - "//lib/guice", - "//lib/httpcomponents:httpcore", - "//lib/jackson:jackson-annotations", - "//lib/log:api", - "//lib/testcontainers", - "//lib/testcontainers:docker-java-api", - "//lib/testcontainers:docker-java-transport", - "//lib/testcontainers:testcontainers-elasticsearch", - ], -) - -ELASTICSEARCH_DEPS = [ - ":elasticsearch_test_utils", - "//java/com/google/gerrit/elasticsearch", - "//java/com/google/gerrit/testing:gerrit-test-util", - "//lib/guice", - "//lib:jgit", -] - -HTTP_TEST_DEPS = [ - "//lib/httpcomponents:httpasyncclient", - "//lib/httpcomponents:httpclient", -] - -QUERY_TESTS_DEP = "//javatests/com/google/gerrit/server/query/%s:abstract_query_tests" - -TYPES = [ - "account", - "change", - "group", - "project", -] - -SUFFIX = "sTest.java" - -ELASTICSEARCH_TESTS_V7 = {i: "ElasticV7Query" + i.capitalize() + SUFFIX for i in TYPES} - -ELASTICSEARCH_TAGS = [ - "docker", - "elastic", -] - -[junit_tests( - name = "elasticsearch_query_%ss_test_V7" % name, - size = "large", - srcs = [src], - tags = ELASTICSEARCH_TAGS, - deps = ELASTICSEARCH_DEPS + [QUERY_TESTS_DEP % name] + HTTP_TEST_DEPS, -) for name, src in ELASTICSEARCH_TESTS_V7.items()] - -junit_tests( - name = "elasticsearch_tests", - size = "small", - srcs = glob( - ["*Test.java"], - exclude = ["Elastic*Query*" + SUFFIX], - ), - tags = ["elastic"], - deps = [ - "//java/com/google/gerrit/elasticsearch", - "//java/com/google/gerrit/testing:gerrit-test-util", - "//lib:guava", - "//lib:jgit", - "//lib/guice", - "//lib/httpcomponents:httpcore", - "//lib/truth", - ], -)
diff --git a/javatests/com/google/gerrit/elasticsearch/ElasticTestUtils.java b/javatests/com/google/gerrit/elasticsearch/ElasticTestUtils.java deleted file mode 100644 index dcc6880..0000000 --- a/javatests/com/google/gerrit/elasticsearch/ElasticTestUtils.java +++ /dev/null
@@ -1,54 +0,0 @@ -// Copyright (C) 2016 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.gerrit.elasticsearch; - -import com.google.gerrit.index.IndexDefinition; -import com.google.inject.Injector; -import com.google.inject.Key; -import com.google.inject.TypeLiteral; -import java.util.Collection; -import java.util.UUID; -import org.eclipse.jgit.lib.Config; - -public final class ElasticTestUtils { - public static void configure(Config config, ElasticContainer container, String prefix) { - String hostname = container.getHttpHost().getHostName(); - int port = container.getHttpHost().getPort(); - config.setString("index", null, "type", "elasticsearch"); - config.setString("elasticsearch", null, "server", "http://" + hostname + ":" + port); - config.setString("elasticsearch", null, "prefix", prefix); - config.setInt("index", null, "maxLimit", 10000); - } - - public static void createAllIndexes(Injector injector) { - Collection<IndexDefinition<?, ?, ?>> indexDefs = - injector.getInstance(Key.get(new TypeLiteral<Collection<IndexDefinition<?, ?, ?>>>() {})); - for (IndexDefinition<?, ?, ?> indexDef : indexDefs) { - indexDef.getIndexCollection().getSearchIndex().deleteAll(); - } - } - - public static Config getConfig(ElasticVersion version) { - ElasticContainer container = ElasticContainer.createAndStart(version); - String indicesPrefix = UUID.randomUUID().toString(); - Config cfg = new Config(); - configure(cfg, container, indicesPrefix); - return cfg; - } - - private ElasticTestUtils() { - // hide default constructor - } -}
diff --git a/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java b/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java similarity index 91% rename from java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java rename to src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java index a564272..3b300cf 100644 --- a/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java +++ b/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
@@ -19,6 +19,7 @@ import static com.google.gson.FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES; import static java.nio.charset.StandardCharsets.UTF_8; +import com.google.common.base.Preconditions; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -49,6 +50,7 @@ import com.google.gerrit.proto.Protos; import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.index.IndexUtils; +import com.google.gerrit.server.index.options.AutoFlush; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonArray; @@ -64,7 +66,6 @@ import java.net.URLEncoder; import java.sql.Timestamp; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -75,6 +76,7 @@ import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.ContentType; import org.apache.http.nio.entity.NStringEntity; +import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; @@ -90,7 +92,7 @@ protected static final String SEARCH = "_search"; protected static final String SETTINGS = "settings"; - protected static byte[] decodeBase64(String base64String) { + static byte[] decodeBase64(String base64String) { return BaseEncoding.base64().decode(base64String); } @@ -129,6 +131,7 @@ private final Schema<V> schema; private final SitePaths sitePaths; private final String indexNameRaw; + private final Map<String, String> refreshParam; protected final ElasticRestClientProvider client; protected final String indexName; @@ -140,7 +143,8 @@ SitePaths sitePaths, Schema<V> schema, ElasticRestClientProvider client, - String indexName) { + String indexName, + AutoFlush autoFlush) { this.config = config; this.sitePaths = sitePaths; this.schema = schema; @@ -149,6 +153,15 @@ this.indexName = config.getIndexName(indexName, schema.getVersion()); this.indexNameRaw = indexName; this.client = client; + this.refreshParam = + Map.of( + "refresh", + autoFlush == AutoFlush.ENABLED ? Boolean.TRUE.toString() : Boolean.FALSE.toString()); + } + + @Override + public void insert(V obj) { + replace(obj); } @Override @@ -169,7 +182,7 @@ @Override public void delete(K id) { String uri = getURI(BULK); - Response response = postRequest(uri, getDeleteActions(id), getRefreshParam()); + Response response = postRequestWithRefreshParam(uri, getDeleteActions(id)); int statusCode = response.getStatusLine().getStatusCode(); if (statusCode != HttpStatus.SC_OK) { throw new StorageException( @@ -260,6 +273,20 @@ return new FieldBundle(rawFields); } + protected boolean hasErrors(Response response) { + try { + String contentType = response.getEntity().getContentType().getValue(); + Preconditions.checkState( + contentType.equals(ContentType.APPLICATION_JSON.toString()), + String.format("Expected %s, but was: %s", ContentType.APPLICATION_JSON, contentType)); + String responseStr = EntityUtils.toString(response.getEntity()); + JsonObject responseJson = (JsonObject) new JsonParser().parse(responseStr); + return responseJson.get("errors").getAsBoolean(); + } catch (IOException e) { + throw new StorageException(e); + } + } + protected String toAction(String type, String id, String action) { JsonObject properties = new JsonObject(); properties.addProperty("_id", id); @@ -277,12 +304,6 @@ array.add(arrayElement); } - protected Map<String, String> getRefreshParam() { - Map<String, String> params = new HashMap<>(); - params.put("refresh", "true"); - return params; - } - protected String getSearch(SearchSourceBuilder searchSource, JsonArray sortArray) { JsonObject search = new JsonParser().parse(searchSource.toString()).getAsJsonObject(); search.add("sort", sortArray); @@ -306,12 +327,8 @@ } } - protected Response postRequest(String uri, Object payload) { - return performRequest("POST", uri, payload); - } - - protected Response postRequest(String uri, Object payload, Map<String, String> params) { - return performRequest("POST", uri, payload, params); + protected Response postRequestWithRefreshParam(String uri, Object payload) { + return performRequest("POST", uri, payload, refreshParam); } private String concatJsonString(String target, String addition) {
diff --git a/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java similarity index 94% rename from java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java rename to src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java index 8967789..1da80ae 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java +++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
@@ -32,6 +32,7 @@ import com.google.gerrit.server.index.IndexUtils; import com.google.gerrit.server.index.account.AccountField; import com.google.gerrit.server.index.account.AccountIndex; +import com.google.gerrit.server.index.options.AutoFlush; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -64,8 +65,9 @@ SitePaths sitePaths, Provider<AccountCache> accountCache, ElasticRestClientProvider client, + AutoFlush autoFlush, @Assisted Schema<AccountState> schema) { - super(cfg, sitePaths, schema, client, ACCOUNTS); + super(cfg, sitePaths, schema, client, ACCOUNTS, autoFlush); this.accountCache = accountCache; this.mapping = new AccountMapping(schema, client.adapter()); this.schema = schema; @@ -78,9 +80,9 @@ .add(new UpdateRequest<>(schema, as, ImmutableSet.of())); String uri = getURI(BULK); - Response response = postRequest(uri, bulk, getRefreshParam()); + Response response = postRequestWithRefreshParam(uri, bulk); int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode != HttpStatus.SC_OK) { + if (hasErrors(response) || statusCode != HttpStatus.SC_OK) { throw new StorageException( String.format( "Failed to replace account %s in index %s: %s",
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java new file mode 100644 index 0000000..a4674bf --- /dev/null +++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
@@ -0,0 +1,183 @@ +// Copyright (C) 2014 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch; + +import static java.util.Objects.requireNonNull; + +import com.google.common.collect.ImmutableSet; +import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties; +import com.google.gerrit.elasticsearch.bulk.BulkRequest; +import com.google.gerrit.elasticsearch.bulk.IndexRequest; +import com.google.gerrit.elasticsearch.bulk.UpdateRequest; +import com.google.gerrit.entities.Change; +import com.google.gerrit.entities.Project; +import com.google.gerrit.entities.converter.ChangeProtoConverter; +import com.google.gerrit.exceptions.StorageException; +import com.google.gerrit.index.FieldDef; +import com.google.gerrit.index.QueryOptions; +import com.google.gerrit.index.Schema; +import com.google.gerrit.index.query.DataSource; +import com.google.gerrit.index.query.Predicate; +import com.google.gerrit.index.query.QueryParseException; +import com.google.gerrit.server.change.MergeabilityComputationBehavior; +import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.IndexUtils; +import com.google.gerrit.server.index.change.ChangeField; +import com.google.gerrit.server.index.change.ChangeIndex; +import com.google.gerrit.server.index.options.AutoFlush; +import com.google.gerrit.server.query.change.ChangeData; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.inject.Inject; +import com.google.inject.assistedinject.Assisted; +import java.util.Set; +import org.apache.http.HttpStatus; +import org.eclipse.jgit.lib.Config; +import org.elasticsearch.client.Response; + +/** Secondary index implementation using Elasticsearch. */ +class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData> + implements ChangeIndex { + static class ChangeMapping { + final MappingProperties changes; + final MappingProperties openChanges; + final MappingProperties closedChanges; + + ChangeMapping(Schema<ChangeData> schema, ElasticQueryAdapter adapter) { + MappingProperties mapping = ElasticMapping.createMapping(schema, adapter); + this.changes = mapping; + this.openChanges = mapping; + this.closedChanges = mapping; + } + } + + private static final String CHANGES = "changes"; + + private final ChangeMapping mapping; + private final ChangeData.Factory changeDataFactory; + private final Schema<ChangeData> schema; + private final FieldDef<ChangeData, ?> idField; + private final ImmutableSet<String> skipFields; + + @Inject + ElasticChangeIndex( + ElasticConfiguration cfg, + ChangeData.Factory changeDataFactory, + SitePaths sitePaths, + ElasticRestClientProvider clientBuilder, + @GerritServerConfig Config gerritConfig, + AutoFlush autoFlush, + @Assisted Schema<ChangeData> schema) { + super(cfg, sitePaths, schema, clientBuilder, CHANGES, autoFlush); + this.changeDataFactory = changeDataFactory; + this.schema = schema; + this.mapping = new ChangeMapping(schema, client.adapter()); + this.idField = + this.schema.useLegacyNumericFields() ? ChangeField.LEGACY_ID : ChangeField.LEGACY_ID_STR; + this.skipFields = + MergeabilityComputationBehavior.fromConfig(gerritConfig).includeInIndex() + ? ImmutableSet.of() + : ImmutableSet.of(ChangeField.MERGEABLE.getName()); + } + + @Override + public void replace(ChangeData cd) { + BulkRequest bulk = + new IndexRequest(getId(cd), indexName).add(new UpdateRequest<>(schema, cd, skipFields)); + + String uri = getURI(BULK); + Response response = postRequestWithRefreshParam(uri, bulk); + int statusCode = response.getStatusLine().getStatusCode(); + if (hasErrors(response) || statusCode != HttpStatus.SC_OK) { + throw new StorageException( + String.format( + "Failed to replace change %s in index %s: %s", cd.getId(), indexName, statusCode)); + } + } + + @Override + public DataSource<ChangeData> getSource(Predicate<ChangeData> p, QueryOptions opts) + throws QueryParseException { + QueryOptions filteredOpts = + opts.filterFields(o -> IndexUtils.changeFields(o, schema.useLegacyNumericFields())); + return new ElasticQuerySource(p, filteredOpts, getSortArray()); + } + + private JsonArray getSortArray() { + JsonObject properties = new JsonObject(); + properties.addProperty(ORDER, DESC_SORT_ORDER); + + JsonArray sortArray = new JsonArray(); + addNamedElement(ChangeField.UPDATED.getName(), properties, sortArray); + addNamedElement(ChangeField.MERGED_ON.getName(), getMergedOnSortOptions(), sortArray); + addNamedElement(idField.getName(), properties, sortArray); + return sortArray; + } + + private JsonObject getMergedOnSortOptions() { + JsonObject sortOptions = new JsonObject(); + sortOptions.addProperty(ORDER, DESC_SORT_ORDER); + // Ignore the sort field if it does not exist in index. Otherwise the search would fail on open + // changes, because the corresponding documents do not have mergedOn field. + sortOptions.addProperty(UNMAPPED_TYPE, ElasticMapping.TIMESTAMP_FIELD_TYPE); + return sortOptions; + } + + @Override + protected String getDeleteActions(Change.Id c) { + return getDeleteRequest(c); + } + + @Override + protected String getMappings() { + return getMappingsFor(mapping.changes); + } + + @Override + protected String getId(ChangeData cd) { + return cd.getId().toString(); + } + + @Override + protected ChangeData fromDocument(JsonObject json, Set<String> fields) { + JsonElement sourceElement = json.get("_source"); + if (sourceElement == null) { + sourceElement = json.getAsJsonObject().get("fields"); + } + JsonObject source = sourceElement.getAsJsonObject(); + JsonElement c = source.get(ChangeField.CHANGE.getName()); + + if (c == null) { + int id = source.get(idField.getName()).getAsInt(); + // IndexUtils#changeFields ensures either CHANGE or PROJECT is always present. + String projectName = requireNonNull(source.get(ChangeField.PROJECT.getName()).getAsString()); + return changeDataFactory.create(Project.nameKey(projectName), Change.id(id)); + } + + ChangeData cd = + changeDataFactory.create( + parseProtoFrom(decodeBase64(c.getAsString()), ChangeProtoConverter.INSTANCE)); + + for (FieldDef<ChangeData, ?> field : getSchema().getFields().values()) { + if (fields.contains(field.getName()) && source.get(field.getName()) != null) { + field.setIfPossible(cd, new ElasticStoredValue(source.get(field.getName()))); + } + } + + return cd; + } +}
diff --git a/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java similarity index 97% rename from java/com/google/gerrit/elasticsearch/ElasticConfiguration.java rename to src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java index c443529..e544d9f 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java +++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticConfiguration.java
@@ -32,7 +32,7 @@ import org.elasticsearch.client.RestClientBuilder; @Singleton -class ElasticConfiguration { +public class ElasticConfiguration { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); static final String SECTION_ELASTICSEARCH = "elasticsearch"; @@ -50,7 +50,7 @@ static final String DEFAULT_USERNAME = "elastic"; static final int DEFAULT_NUMBER_OF_SHARDS = 1; static final int DEFAULT_NUMBER_OF_REPLICAS = 1; - static final int DEFAULT_MAX_RESULT_WINDOW = 10000; + static final int DEFAULT_MAX_RESULT_WINDOW = Integer.MAX_VALUE; static final int DEFAULT_CONNECT_TIMEOUT = RestClientBuilder.DEFAULT_CONNECT_TIMEOUT_MILLIS; static final int DEFAULT_SOCKET_TIMEOUT = RestClientBuilder.DEFAULT_SOCKET_TIMEOUT_MILLIS;
diff --git a/java/com/google/gerrit/elasticsearch/ElasticException.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticException.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/ElasticException.java rename to src/main/java/com/google/gerrit/elasticsearch/ElasticException.java
diff --git a/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java similarity index 93% rename from java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java rename to src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java index 781ed43..626203a 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java +++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
@@ -32,6 +32,7 @@ import com.google.gerrit.server.index.IndexUtils; import com.google.gerrit.server.index.group.GroupField; import com.google.gerrit.server.index.group.GroupIndex; +import com.google.gerrit.server.index.options.AutoFlush; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -64,8 +65,9 @@ SitePaths sitePaths, Provider<GroupCache> groupCache, ElasticRestClientProvider client, + AutoFlush autoFlush, @Assisted Schema<InternalGroup> schema) { - super(cfg, sitePaths, schema, client, GROUPS); + super(cfg, sitePaths, schema, client, GROUPS, autoFlush); this.groupCache = groupCache; this.mapping = new GroupMapping(schema, client.adapter()); this.schema = schema; @@ -78,9 +80,9 @@ .add(new UpdateRequest<>(schema, group, ImmutableSet.of())); String uri = getURI(BULK); - Response response = postRequest(uri, bulk, getRefreshParam()); + Response response = postRequestWithRefreshParam(uri, bulk); int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode != HttpStatus.SC_OK) { + if (hasErrors(response) || statusCode != HttpStatus.SC_OK) { throw new StorageException( String.format( "Failed to replace group %s in index %s: %s",
diff --git a/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java similarity index 64% rename from java/com/google/gerrit/elasticsearch/ElasticIndexModule.java rename to src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java index 15d6126..b1bb7b1 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java +++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java
@@ -14,32 +14,53 @@ package com.google.gerrit.elasticsearch; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.flogger.FluentLogger; import com.google.gerrit.index.project.ProjectIndex; +import com.google.gerrit.server.ModuleImpl; import com.google.gerrit.server.index.AbstractIndexModule; import com.google.gerrit.server.index.VersionManager; import com.google.gerrit.server.index.account.AccountIndex; import com.google.gerrit.server.index.change.ChangeIndex; import com.google.gerrit.server.index.group.GroupIndex; +import com.google.gerrit.server.index.options.AutoFlush; +import com.google.inject.Inject; import java.util.Map; +@ModuleImpl(name = AbstractIndexModule.INDEX_MODULE) public class ElasticIndexModule extends AbstractIndexModule { + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + private final AutoFlush autoFlush; + + @VisibleForTesting public static ElasticIndexModule singleVersionWithExplicitVersions( Map<String, Integer> versions, int threads, boolean slave) { - return new ElasticIndexModule(versions, threads, slave); + return new ElasticIndexModule(versions, threads, slave, AutoFlush.ENABLED); } - public static ElasticIndexModule latestVersion(boolean slave) { - return new ElasticIndexModule(null, 0, slave); + public static ElasticIndexModule singleVersionWithExplicitVersions( + Map<String, Integer> versions, int threads, boolean slave, AutoFlush autoFlush) { + return new ElasticIndexModule(versions, threads, slave, autoFlush); } - private ElasticIndexModule(Map<String, Integer> singleVersions, int threads, boolean slave) { + @Inject + public ElasticIndexModule() { + this(null, 0, false, AutoFlush.ENABLED); + } + + protected ElasticIndexModule( + Map<String, Integer> singleVersions, int threads, boolean slave, AutoFlush autoFlush) { super(singleVersions, threads, slave); + this.autoFlush = autoFlush; } @Override public void configure() { + logger.atInfo().log("Gerrit index backend set to ElasticSearch"); super.configure(); install(ElasticRestClientProvider.module()); + bind(AutoFlush.class).toInstance(autoFlush); } @Override
diff --git a/java/com/google/gerrit/elasticsearch/ElasticIndexVersionDiscovery.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexVersionDiscovery.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/ElasticIndexVersionDiscovery.java rename to src/main/java/com/google/gerrit/elasticsearch/ElasticIndexVersionDiscovery.java
diff --git a/java/com/google/gerrit/elasticsearch/ElasticIndexVersionManager.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexVersionManager.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/ElasticIndexVersionManager.java rename to src/main/java/com/google/gerrit/elasticsearch/ElasticIndexVersionManager.java
diff --git a/java/com/google/gerrit/elasticsearch/ElasticMapping.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/ElasticMapping.java rename to src/main/java/com/google/gerrit/elasticsearch/ElasticMapping.java
diff --git a/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java similarity index 94% rename from java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java rename to src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java index b8bfc38..99b202d 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java +++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticProjectIndex.java
@@ -31,6 +31,7 @@ import com.google.gerrit.index.query.QueryParseException; import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.index.IndexUtils; +import com.google.gerrit.server.index.options.AutoFlush; import com.google.gerrit.server.project.ProjectCache; import com.google.gerrit.server.project.ProjectState; import com.google.gson.JsonArray; @@ -66,8 +67,9 @@ SitePaths sitePaths, Provider<ProjectCache> projectCache, ElasticRestClientProvider client, + AutoFlush autoFlush, @Assisted Schema<ProjectData> schema) { - super(cfg, sitePaths, schema, client, PROJECTS); + super(cfg, sitePaths, schema, client, PROJECTS, autoFlush); this.projectCache = projectCache; this.schema = schema; this.mapping = new ProjectMapping(schema, client.adapter()); @@ -80,9 +82,9 @@ .add(new UpdateRequest<>(schema, projectState, ImmutableSet.of())); String uri = getURI(BULK); - Response response = postRequest(uri, bulk, getRefreshParam()); + Response response = postRequestWithRefreshParam(uri, bulk); int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode != HttpStatus.SC_OK) { + if (hasErrors(response) || statusCode != HttpStatus.SC_OK) { throw new StorageException( String.format( "Failed to replace project %s in index %s: %s",
diff --git a/java/com/google/gerrit/elasticsearch/ElasticQueryAdapter.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryAdapter.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/ElasticQueryAdapter.java rename to src/main/java/com/google/gerrit/elasticsearch/ElasticQueryAdapter.java
diff --git a/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java rename to src/main/java/com/google/gerrit/elasticsearch/ElasticQueryBuilder.java
diff --git a/java/com/google/gerrit/elasticsearch/ElasticRestClientProvider.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticRestClientProvider.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/ElasticRestClientProvider.java rename to src/main/java/com/google/gerrit/elasticsearch/ElasticRestClientProvider.java
diff --git a/java/com/google/gerrit/elasticsearch/ElasticSetting.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticSetting.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/ElasticSetting.java rename to src/main/java/com/google/gerrit/elasticsearch/ElasticSetting.java
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ElasticStoredValue.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticStoredValue.java new file mode 100644 index 0000000..a02a715 --- /dev/null +++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticStoredValue.java
@@ -0,0 +1,86 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch; + +import static com.google.common.collect.ImmutableList.toImmutableList; + +import com.google.gerrit.index.StoredValue; +import com.google.gson.JsonElement; +import java.sql.Timestamp; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.util.stream.StreamSupport; + +/** Bridge to recover fields from the elastic index. */ +public class ElasticStoredValue implements StoredValue { + private final JsonElement field; + + ElasticStoredValue(JsonElement field) { + this.field = field; + } + + @Override + public String asString() { + return field.getAsString(); + } + + @Override + public Iterable<String> asStrings() { + return StreamSupport.stream(field.getAsJsonArray().spliterator(), false) + .map(f -> f.getAsString()) + .collect(toImmutableList()); + } + + @Override + public Integer asInteger() { + return field.getAsInt(); + } + + @Override + public Iterable<Integer> asIntegers() { + return StreamSupport.stream(field.getAsJsonArray().spliterator(), false) + .map(f -> f.getAsInt()) + .collect(toImmutableList()); + } + + @Override + public Long asLong() { + return field.getAsLong(); + } + + @Override + public Iterable<Long> asLongs() { + return StreamSupport.stream(field.getAsJsonArray().spliterator(), false) + .map(f -> f.getAsLong()) + .collect(toImmutableList()); + } + + @Override + public Timestamp asTimestamp() { + return Timestamp.from(Instant.from(DateTimeFormatter.ISO_INSTANT.parse(field.getAsString()))); + } + + @Override + public byte[] asByteArray() { + return AbstractElasticIndex.decodeBase64(field.getAsString()); + } + + @Override + public Iterable<byte[]> asByteArrays() { + return StreamSupport.stream(field.getAsJsonArray().spliterator(), false) + .map(f -> AbstractElasticIndex.decodeBase64(f.getAsString())) + .collect(toImmutableList()); + } +}
diff --git a/java/com/google/gerrit/elasticsearch/ElasticVersion.java b/src/main/java/com/google/gerrit/elasticsearch/ElasticVersion.java similarity index 95% rename from java/com/google/gerrit/elasticsearch/ElasticVersion.java rename to src/main/java/com/google/gerrit/elasticsearch/ElasticVersion.java index c6400df..dffdf3e 100644 --- a/java/com/google/gerrit/elasticsearch/ElasticVersion.java +++ b/src/main/java/com/google/gerrit/elasticsearch/ElasticVersion.java
@@ -18,9 +18,7 @@ import java.util.regex.Pattern; public enum ElasticVersion { - V7_6("7.6.*"), - V7_7("7.7.*"), - V7_8("7.8.*"); + V7_16("7.16.*"); private final String version; private final Pattern pattern; @@ -45,7 +43,6 @@ * * @param version for which to return an ElasticVersion * @return the corresponding ElasticVersion if supported - * @throws UnsupportedVersion */ public static ElasticVersion forVersion(String version) { for (ElasticVersion value : ElasticVersion.values()) {
diff --git a/src/main/java/com/google/gerrit/elasticsearch/PrimaryElasticIndexModule.java b/src/main/java/com/google/gerrit/elasticsearch/PrimaryElasticIndexModule.java new file mode 100644 index 0000000..823c657 --- /dev/null +++ b/src/main/java/com/google/gerrit/elasticsearch/PrimaryElasticIndexModule.java
@@ -0,0 +1,27 @@ +// Copyright (C) 2014 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch; + +import com.google.gerrit.server.ModuleImpl; +import com.google.gerrit.server.index.AbstractIndexModule; +import com.google.gerrit.server.index.options.AutoFlush; + +@ModuleImpl(name = AbstractIndexModule.INDEX_MODULE) +public class PrimaryElasticIndexModule extends ElasticIndexModule { + + public PrimaryElasticIndexModule() { + super(null, 0, false, AutoFlush.ENABLED); + } +}
diff --git a/src/main/java/com/google/gerrit/elasticsearch/ReplicaElasticIndexModule.java b/src/main/java/com/google/gerrit/elasticsearch/ReplicaElasticIndexModule.java new file mode 100644 index 0000000..86ccac6 --- /dev/null +++ b/src/main/java/com/google/gerrit/elasticsearch/ReplicaElasticIndexModule.java
@@ -0,0 +1,27 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch; + +import com.google.gerrit.server.ModuleImpl; +import com.google.gerrit.server.index.AbstractIndexModule; +import com.google.gerrit.server.index.options.AutoFlush; + +@ModuleImpl(name = AbstractIndexModule.INDEX_MODULE) +public class ReplicaElasticIndexModule extends ElasticIndexModule { + + public ReplicaElasticIndexModule() { + super(null, 0, true, AutoFlush.ENABLED); + } +}
diff --git a/java/com/google/gerrit/elasticsearch/builders/BoolQueryBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/builders/BoolQueryBuilder.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/builders/BoolQueryBuilder.java rename to src/main/java/com/google/gerrit/elasticsearch/builders/BoolQueryBuilder.java
diff --git a/java/com/google/gerrit/elasticsearch/builders/ExistsQueryBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/builders/ExistsQueryBuilder.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/builders/ExistsQueryBuilder.java rename to src/main/java/com/google/gerrit/elasticsearch/builders/ExistsQueryBuilder.java
diff --git a/java/com/google/gerrit/elasticsearch/builders/MatchAllQueryBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/builders/MatchAllQueryBuilder.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/builders/MatchAllQueryBuilder.java rename to src/main/java/com/google/gerrit/elasticsearch/builders/MatchAllQueryBuilder.java
diff --git a/java/com/google/gerrit/elasticsearch/builders/MatchQueryBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/builders/MatchQueryBuilder.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/builders/MatchQueryBuilder.java rename to src/main/java/com/google/gerrit/elasticsearch/builders/MatchQueryBuilder.java
diff --git a/java/com/google/gerrit/elasticsearch/builders/QueryBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/builders/QueryBuilder.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/builders/QueryBuilder.java rename to src/main/java/com/google/gerrit/elasticsearch/builders/QueryBuilder.java
diff --git a/java/com/google/gerrit/elasticsearch/builders/QueryBuilders.java b/src/main/java/com/google/gerrit/elasticsearch/builders/QueryBuilders.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/builders/QueryBuilders.java rename to src/main/java/com/google/gerrit/elasticsearch/builders/QueryBuilders.java
diff --git a/java/com/google/gerrit/elasticsearch/builders/QuerySourceBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/builders/QuerySourceBuilder.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/builders/QuerySourceBuilder.java rename to src/main/java/com/google/gerrit/elasticsearch/builders/QuerySourceBuilder.java
diff --git a/java/com/google/gerrit/elasticsearch/builders/RangeQueryBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/builders/RangeQueryBuilder.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/builders/RangeQueryBuilder.java rename to src/main/java/com/google/gerrit/elasticsearch/builders/RangeQueryBuilder.java
diff --git a/java/com/google/gerrit/elasticsearch/builders/RegexpQueryBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/builders/RegexpQueryBuilder.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/builders/RegexpQueryBuilder.java rename to src/main/java/com/google/gerrit/elasticsearch/builders/RegexpQueryBuilder.java
diff --git a/java/com/google/gerrit/elasticsearch/builders/SearchAfterBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/builders/SearchAfterBuilder.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/builders/SearchAfterBuilder.java rename to src/main/java/com/google/gerrit/elasticsearch/builders/SearchAfterBuilder.java
diff --git a/java/com/google/gerrit/elasticsearch/builders/SearchSourceBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/builders/SearchSourceBuilder.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/builders/SearchSourceBuilder.java rename to src/main/java/com/google/gerrit/elasticsearch/builders/SearchSourceBuilder.java
diff --git a/java/com/google/gerrit/elasticsearch/builders/TermQueryBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/builders/TermQueryBuilder.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/builders/TermQueryBuilder.java rename to src/main/java/com/google/gerrit/elasticsearch/builders/TermQueryBuilder.java
diff --git a/java/com/google/gerrit/elasticsearch/builders/XContentBuilder.java b/src/main/java/com/google/gerrit/elasticsearch/builders/XContentBuilder.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/builders/XContentBuilder.java rename to src/main/java/com/google/gerrit/elasticsearch/builders/XContentBuilder.java
diff --git a/java/com/google/gerrit/elasticsearch/bulk/ActionRequest.java b/src/main/java/com/google/gerrit/elasticsearch/bulk/ActionRequest.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/bulk/ActionRequest.java rename to src/main/java/com/google/gerrit/elasticsearch/bulk/ActionRequest.java
diff --git a/java/com/google/gerrit/elasticsearch/bulk/BulkRequest.java b/src/main/java/com/google/gerrit/elasticsearch/bulk/BulkRequest.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/bulk/BulkRequest.java rename to src/main/java/com/google/gerrit/elasticsearch/bulk/BulkRequest.java
diff --git a/java/com/google/gerrit/elasticsearch/bulk/DeleteRequest.java b/src/main/java/com/google/gerrit/elasticsearch/bulk/DeleteRequest.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/bulk/DeleteRequest.java rename to src/main/java/com/google/gerrit/elasticsearch/bulk/DeleteRequest.java
diff --git a/java/com/google/gerrit/elasticsearch/bulk/IndexRequest.java b/src/main/java/com/google/gerrit/elasticsearch/bulk/IndexRequest.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/bulk/IndexRequest.java rename to src/main/java/com/google/gerrit/elasticsearch/bulk/IndexRequest.java
diff --git a/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java b/src/main/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java similarity index 100% rename from java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java rename to src/main/java/com/google/gerrit/elasticsearch/bulk/UpdateRequest.java
diff --git a/src/main/java/com/google/gerrit/elasticsearch/init/InitElasticSearchIndex.java b/src/main/java/com/google/gerrit/elasticsearch/init/InitElasticSearchIndex.java new file mode 100644 index 0000000..76b2d3b --- /dev/null +++ b/src/main/java/com/google/gerrit/elasticsearch/init/InitElasticSearchIndex.java
@@ -0,0 +1,73 @@ +package com.google.gerrit.elasticsearch.init; + +import com.google.common.collect.Iterables; +import com.google.gerrit.index.IndexType; +import com.google.gerrit.index.SchemaDefinitions; +import com.google.gerrit.pgm.init.api.ConsoleUI; +import com.google.gerrit.pgm.init.api.InitFlags; +import com.google.gerrit.pgm.init.api.InitStep; +import com.google.gerrit.pgm.init.api.Section; +import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.IndexModule; +import com.google.gerrit.server.index.IndexUtils; +import com.google.inject.Inject; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; + +public class InitElasticSearchIndex implements InitStep { + private final ConsoleUI ui; + private final Section index; + private final SitePaths site; + private final InitFlags initFlags; + private final Section gerrit; + private final Section.Factory sections; + + @Inject + InitElasticSearchIndex( + ConsoleUI ui, Section.Factory sections, SitePaths site, InitFlags initFlags) { + this.ui = ui; + this.index = sections.get("index", null); + this.gerrit = sections.get("gerrit", null); + this.site = site; + this.initFlags = initFlags; + this.sections = sections; + } + + @Override + public void run() throws IOException { + ui.header("Index"); + IndexType type = + new IndexType( + index.select("Type", "type", IndexType.getDefault(), IndexType.getKnownTypes())); + + Section elasticsearch = sections.get("elasticsearch", null); + elasticsearch.string("Index Prefix", "prefix", "gerrit_"); + elasticsearch.string("Server", "server", "http://localhost:9200"); + index.string("Result window size", "maxLimit", "10000"); + + if ((site.isNew || isEmptySite()) && type.isLucene()) { + for (SchemaDefinitions<?> def : IndexModule.ALL_SCHEMA_DEFS) { + IndexUtils.setReady(site, def.getName(), def.getLatest().getVersion(), true); + } + } else { + String message = + String.format( + "\nThe index must be %sbuilt before starting Gerrit:\n" + + " java -jar gerrit.war reindex -d site_path\n", + site.isNew ? "" : "re"); + ui.message(message); + initFlags.autoStart = false; + } + } + + private boolean isEmptySite() { + try (DirectoryStream<Path> files = + Files.newDirectoryStream(site.resolve(gerrit.get("basePath")))) { + return Iterables.isEmpty(files); + } catch (IOException e) { + return true; + } + } +}
diff --git a/src/main/main.iml b/src/main/main.iml new file mode 100644 index 0000000..908ad4f --- /dev/null +++ b/src/main/main.iml
@@ -0,0 +1,11 @@ +<?xml version="1.0" encoding="UTF-8"?> +<module type="JAVA_MODULE" version="4"> + <component name="NewModuleRootManager" inherit-compiler-output="true"> + <exclude-output /> + <content url="file://$MODULE_DIR$"> + <sourceFolder url="file://$MODULE_DIR$/java" isTestSource="false" /> + </content> + <orderEntry type="inheritedJdk" /> + <orderEntry type="sourceFolder" forTests="false" /> + </component> +</module> \ No newline at end of file
diff --git a/src/main/resources/Documentation/about.md b/src/main/resources/Documentation/about.md new file mode 100644 index 0000000..59fe0a3 --- /dev/null +++ b/src/main/resources/Documentation/about.md
@@ -0,0 +1,7 @@ +# Index backend for Gerrit, based on ElasticSearch + +Indexing backend libModule for [Gerrit Code Review](https://gerritcodereview.com) +based on [ElasticSearch](https://www.elastic.co/elasticsearch/). + +This module was originally part of Gerrit core and then extracted into a separate +component from v3.5.0-rc3 as part of [Change-Id: Ib7b5167ce](https://gerrit-review.googlesource.com/c/gerrit/+/323676).
diff --git a/src/main/resources/Documentation/build.md b/src/main/resources/Documentation/build.md new file mode 100644 index 0000000..acb8d85 --- /dev/null +++ b/src/main/resources/Documentation/build.md
@@ -0,0 +1,71 @@ +# Build + +This plugin is built with Bazel in-tree build. This plugin depends on the Elasticsearch Java Low +Level REST Client (abbreviated as LLRC by the ES dev team) for integration with an Elasticsearch +cluster. The LLRC is licensed as Apache v2 (even after ES itself has moved to SSPL) and is +compatible with all ES versions. See the [LLRC +docs](https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/8.3/java-rest-low-usage-maven.html) +for more information. + +## Build in Gerrit tree + +Create a symbolic link of the repository source to the Gerrit source +tree plugins/index-elasticsearch directory, and the external_plugin_deps.bzl +dependencies linked to plugins/external_plugin_deps.bzl. + +Example: + +```sh +git clone https://gerrit.googlesource.com/gerrit +git clone https://gerrit.googlesource.com/modules/index-elasticsearch +cd gerrit/plugins +ln -s ../../index-elasticsearch index-elasticsearch +ln -sf ../../external_plugin_deps.bzl . +``` + +From the Gerrit source tree issue the command `bazelisk build plugins/index-elasticsearch`. + +Example: + +```sh +bazelisk build plugins/index-elasticsearch +``` + +The libModule jar file is created under `bazel-bin/plugins/index-elasticsearch/index-elasticsearch.jar` + +## Integration test + +There are two different ways to run tests for this module. You can either run only the tests +provided by the module or you can run all Gerrit core acceptance tests with the indexing backend set +to this module. + +To run only the tests provided by this plugin: + +```sh +bazelisk test plugins/index-elasticsearch/... +``` + +Gerrit acceptance tests allow the execution with an alternate implementation of +the indexing backend using the `GERRIT_INDEX_MODULE` environment variable. + +```sh +bazelisk test --test_env=GERRIT_INDEX_MODULE=com.google.gerrit.elasticsearch.ElasticIndexModule //... +``` + +## IDE setup + +This project can be imported into the Eclipse IDE. +Add the plugin name to the `CUSTOM_PLUGINS` and to the +`CUSTOM_PLUGINS_TEST_DEPS` set in Gerrit core in +`tools/bzl/plugins.bzl`, and execute: + +``` + ./tools/eclipse/project.py +``` + +More information about Bazel can be found in the [Gerrit +documentation](../../../Documentation/dev-bazel.html). + +[Back to @PLUGIN@ documentation index][index] + +[index]: index.html
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md new file mode 100644 index 0000000..ad5d5d0 --- /dev/null +++ b/src/main/resources/Documentation/config.md
@@ -0,0 +1,97 @@ +# Configuration + +## Section index + +### index.maxLimit + +Maximum limit to allow for search queries. Requesting results above this limit will truncate the +list (but will still set `_more_changes` on result lists). Set to 0 for no limit. This value +should not exceed the `index.max_result_window` value configured on the Elasticsearch server. If a +value is not configured during site initialization, defaults to 10000, which is the default value +of `index.max_result_window` in Elasticsearch. + +## Section elasticsearch + +WARNING: Support for Elasticsearch is still experimental and is not recommended for production +use. For compatibility information, please refer to the [project homepage](https://www.gerritcodereview.com/elasticsearch.html). + +Note that when Gerrit is configured to use Elasticsearch, the Elasticsearch +server(s) must be reachable during the site initialization. + +### elasticsearch.prefix + +This setting can be used to prefix index names to allow multiple Gerrit instances in a single +Elasticsearch cluster. Prefix `gerrit1_` would result in a change index named +`gerrit1_changes_0001`. + +Not set by default. + +### elasticsearch.server + +Elasticsearch server URI in the form `http[s]://hostname:port`. The `port` is optional and defaults +to `9200` if not specified. + +At least one server must be specified. May be specified multiple times to configure multiple +Elasticsearch servers. + +Note that the site initialization program only allows to configure a single +server. To configure multiple servers the `gerrit.config` file must be edited +manually. + +### elasticsearch.numberOfShards + +Sets the number of shards to use per index. Refer to the +[Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html#_static_index_settings) for details. + +Defaults to 1. + +### elasticsearch.numberOfReplicas + +Sets the number of replicas to use per index. Refer to the +[Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html#dynamic-index-settings) for details. + +Defaults to 1. + +### elasticsearch.maxResultWindow + +Sets the maximum value of `from + size` for searches to use per index. Refer to the +[Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html#dynamic-index-settings) for details. + +Defaults to 10000. + +### elasticsearch.connectTimeout + +Sets the timeout for connecting to elasticsearch. + +Defaults to `1 second`. + +### elasticsearch.socketTimeout + +Sets the timeout for the underlying connection. For more information, refer to +[`httpd.idleTimeout`](https://gerrit-documentation.storage.googleapis.com/Documentation/3.5.2/config-gerrit.html#httpd.idleTimeout). + +Defaults to `30 seconds`. + +## Elasticsearch Security + +When security is enabled in Elasticsearch, the username and password must be provided. Note that +the same username and password are used for all servers. + +For further information about Elasticsearch security, please refer to +[the documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-getting-started.html). This is the current documentation link. Select another Elasticsearch version from the dropdown menu available on that page if need be. + +### elasticsearch.username + +Username used to connect to Elasticsearch. + +If a password is set, defaults to `elastic`, otherwise not set by default. + +### elasticsearch.password + +Password used to connect to Elasticsearch. + +Not set by default. + +[Back to @PLUGIN@ documentation index][index] + +[index]: index.html
diff --git a/src/main/resources/Documentation/setup.md b/src/main/resources/Documentation/setup.md new file mode 100644 index 0000000..7877b29 --- /dev/null +++ b/src/main/resources/Documentation/setup.md
@@ -0,0 +1,26 @@ +# Setup + +* Install index-elasticsearch module + +Install the index-elasticsearch.jar into the `$GERRIT_SITE/lib` directory. + +Add the index-elasticsearch module to `$GERRIT_SITE/etc/gerrit.config` as follows: + +```ini +[gerrit] + installIndexModule = com.google.gerrit.elasticsearch.ElasticIndexModule +``` + +When installing the module on Gerrit replicas, use following example: + +```ini +[gerrit] + installIndexModule = com.google.gerrit.elasticsearch.ReplicaElasticIndexModule +``` + +For further information and supported options, refer to [config](config.html) +documentation. + +[Back to @PLUGIN@ documentation index][index] + +[index]: index.html
diff --git a/javatests/com/google/gerrit/elasticsearch/ElasticConfigurationTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticConfigurationTest.java similarity index 100% rename from javatests/com/google/gerrit/elasticsearch/ElasticConfigurationTest.java rename to src/test/java/com/google/gerrit/elasticsearch/ElasticConfigurationTest.java
diff --git a/javatests/com/google/gerrit/elasticsearch/ElasticContainer.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticContainer.java similarity index 69% rename from javatests/com/google/gerrit/elasticsearch/ElasticContainer.java rename to src/test/java/com/google/gerrit/elasticsearch/ElasticContainer.java index c330961..4fddc72 100644 --- a/javatests/com/google/gerrit/elasticsearch/ElasticContainer.java +++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticContainer.java
@@ -14,45 +14,41 @@ package com.google.gerrit.elasticsearch; +import com.google.common.flogger.FluentLogger; import org.apache.http.HttpHost; -import org.junit.AssumptionViolatedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testcontainers.containers.ContainerLaunchException; import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.utility.DockerImageName; /* Helper class for running ES integration tests in docker container */ public class ElasticContainer extends ElasticsearchContainer { + private static FluentLogger logger = FluentLogger.forEnclosingClass(); private static final int ELASTICSEARCH_DEFAULT_PORT = 9200; public static ElasticContainer createAndStart(ElasticVersion version) { - // Assumption violation is not natively supported by Testcontainers. - // See https://github.com/testcontainers/testcontainers-java/issues/343 + ElasticContainer container = new ElasticContainer(version); try { - ElasticContainer container = new ElasticContainer(version); container.start(); - return container; - } catch (Throwable t) { - throw new AssumptionViolatedException("Unable to start container", t); + } catch (ContainerLaunchException e) { + logger.atSevere().log( + "Failed to launch elastic container. Logs from container :\n" + container.getLogs()); + throw e; } + return container; } private static String getImageName(ElasticVersion version) { switch (version) { - case V7_6: - return "blacktop/elasticsearch:7.6.2"; - case V7_7: - return "blacktop/elasticsearch:7.7.1"; - case V7_8: - return "blacktop/elasticsearch:7.8.1"; + case V7_16: + return "docker.elastic.co/elasticsearch/elasticsearch:7.16.2"; } throw new IllegalStateException("No tests for version: " + version.name()); } private ElasticContainer(ElasticVersion version) { - super( - DockerImageName.parse(getImageName(version)) - .asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch")); + super(DockerImageName.parse(getImageName(version))); } @Override
diff --git a/src/test/java/com/google/gerrit/elasticsearch/ElasticTestUtils.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticTestUtils.java new file mode 100644 index 0000000..f5ba9db --- /dev/null +++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticTestUtils.java
@@ -0,0 +1,108 @@ +// Copyright (C) 2016 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.elasticsearch; + +import static java.util.concurrent.TimeUnit.MINUTES; + +import com.google.gerrit.index.IndexDefinition; +import com.google.gerrit.server.LibModuleType; +import com.google.gerrit.testing.GerritTestName; +import com.google.gerrit.testing.InMemoryModule; +import com.google.gerrit.testing.IndexConfig; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.TypeLiteral; +import java.util.Collection; +import java.util.UUID; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; +import org.eclipse.jgit.lib.Config; + +public final class ElasticTestUtils { + public static void configure(Config config, ElasticContainer container, String prefix) { + String hostname = container.getHttpHost().getHostName(); + int port = container.getHttpHost().getPort(); + config.setString("index", null, "type", "elasticsearch"); + config.setString("elasticsearch", null, "server", "http://" + hostname + ":" + port); + config.setString("elasticsearch", null, "prefix", prefix); + config.setInt("index", null, "maxLimit", 10000); + } + + public static void createAllIndexes(Injector injector) { + Collection<IndexDefinition<?, ?, ?>> indexDefs = + injector.getInstance(Key.get(new TypeLiteral<Collection<IndexDefinition<?, ?, ?>>>() {})); + for (IndexDefinition<?, ?, ?> indexDef : indexDefs) { + indexDef.getIndexCollection().getSearchIndex().deleteAll(); + } + } + + public static Config getConfig(ElasticVersion version) { + ElasticContainer container = ElasticContainer.createAndStart(version); + String indicesPrefix = UUID.randomUUID().toString(); + Config cfg = new Config(); + configure(cfg, container, indicesPrefix); + return cfg; + } + + public static Config createConfig() { + Config cfg = IndexConfig.create(); + + // For some reason enabling the staleness checker increases the flakiness of the Elasticsearch + // tests. Hence disable the staleness checker. + cfg.setBoolean("index", null, "autoReindexIfStale", false); + + return cfg; + } + + public static void configureElasticModule(Config elasticsearchConfig) { + elasticsearchConfig.setString( + "index", + null, + "install" + LibModuleType.INDEX_MODULE_TYPE.getConfigKey(), + "com.google.gerrit.elasticsearch.ElasticIndexModule"); + } + + public static Injector createInjector( + Config config, GerritTestName testName, ElasticContainer container) { + Config elasticsearchConfig = new Config(config); + ElasticTestUtils.configureElasticModule(elasticsearchConfig); + InMemoryModule.setDefaults(elasticsearchConfig); + String indicesPrefix = testName.getSanitizedMethodName(); + ElasticTestUtils.configure(elasticsearchConfig, container, indicesPrefix); + return Guice.createInjector(new InMemoryModule(elasticsearchConfig)); + } + + public static void closeIndex( + CloseableHttpAsyncClient client, ElasticContainer container, GerritTestName testName) + throws Exception { + client + .execute( + new HttpPost( + String.format( + "http://%s:%d/%s*/_close", + container.getHttpHost().getHostName(), + container.getHttpHost().getPort(), + testName.getSanitizedMethodName())), + HttpClientContext.create(), + null) + .get(5, MINUTES); + } + + private ElasticTestUtils() { + // hide default constructor + } +}
diff --git a/javatests/com/google/gerrit/elasticsearch/ElasticV7QueryAccountsTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryAccountsTest.java similarity index 66% rename from javatests/com/google/gerrit/elasticsearch/ElasticV7QueryAccountsTest.java rename to src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryAccountsTest.java index 628fe2f..752a1e7 100644 --- a/javatests/com/google/gerrit/elasticsearch/ElasticV7QueryAccountsTest.java +++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryAccountsTest.java
@@ -14,20 +14,24 @@ package com.google.gerrit.elasticsearch; +import static com.google.common.truth.Truth.assertThat; +import static com.google.gerrit.testing.GerritJUnit.assertThrows; + +import com.google.gerrit.exceptions.StorageException; import com.google.gerrit.server.query.account.AbstractQueryAccountsTest; import com.google.gerrit.testing.ConfigSuite; -import com.google.gerrit.testing.InMemoryModule; -import com.google.gerrit.testing.IndexConfig; -import com.google.inject.Guice; import com.google.inject.Injector; +import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; +import org.apache.http.impl.nio.client.HttpAsyncClients; import org.eclipse.jgit.lib.Config; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Test; public class ElasticV7QueryAccountsTest extends AbstractQueryAccountsTest { @ConfigSuite.Default public static Config defaultConfig() { - return IndexConfig.createForElasticsearch(); + return ElasticTestUtils.createConfig(); } @ConfigSuite.Config @@ -38,12 +42,15 @@ } private static ElasticContainer container; + private static CloseableHttpAsyncClient client; @BeforeClass public static void startIndexService() { if (container == null) { // Only start Elasticsearch once - container = ElasticContainer.createAndStart(ElasticVersion.V7_8); + container = ElasticContainer.createAndStart(ElasticVersion.V7_16); + client = HttpAsyncClients.createDefault(); + client.start(); } } @@ -62,10 +69,16 @@ @Override protected Injector createInjector() { - Config elasticsearchConfig = new Config(config); - InMemoryModule.setDefaults(elasticsearchConfig); - String indicesPrefix = testName.getSanitizedMethodName(); - ElasticTestUtils.configure(elasticsearchConfig, container, indicesPrefix); - return Guice.createInjector(new InMemoryModule(elasticsearchConfig)); + return ElasticTestUtils.createInjector(config, testName, container); + } + + @Test + public void testErrorResponseFromAccountIndex() throws Exception { + gApi.accounts().self().index(); + + ElasticTestUtils.closeIndex(client, container, testName); + StorageException thrown = + assertThrows(StorageException.class, () -> gApi.accounts().self().index()); + assertThat(thrown).hasMessageThat().contains("Failed to replace account"); } }
diff --git a/javatests/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java similarity index 69% rename from javatests/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java rename to src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java index ea172b0..9a85129 100644 --- a/javatests/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java +++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java
@@ -14,29 +14,30 @@ package com.google.gerrit.elasticsearch; -import static java.util.concurrent.TimeUnit.MINUTES; +import static com.google.common.truth.Truth.assertThat; +import static com.google.gerrit.testing.GerritJUnit.assertThrows; +import com.google.gerrit.entities.Change; +import com.google.gerrit.exceptions.StorageException; import com.google.gerrit.server.query.change.AbstractQueryChangesTest; import com.google.gerrit.testing.ConfigSuite; import com.google.gerrit.testing.GerritTestName; -import com.google.gerrit.testing.InMemoryModule; -import com.google.gerrit.testing.IndexConfig; -import com.google.inject.Guice; +import com.google.gerrit.testing.InMemoryRepositoryManager; import com.google.inject.Injector; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; import org.apache.http.impl.nio.client.HttpAsyncClients; +import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.lib.Config; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Rule; +import org.junit.Test; public class ElasticV7QueryChangesTest extends AbstractQueryChangesTest { @ConfigSuite.Default public static Config defaultConfig() { - return IndexConfig.createForElasticsearch(); + return ElasticTestUtils.createConfig(); } @ConfigSuite.Config @@ -53,7 +54,7 @@ public static void startIndexService() { if (container == null) { // Only start Elasticsearch once - container = ElasticContainer.createAndStart(ElasticVersion.V7_8); + container = ElasticContainer.createAndStart(ElasticVersion.V7_16); client = HttpAsyncClients.createDefault(); client.start(); } @@ -72,17 +73,7 @@ public void closeIndex() throws Exception { // Close the index after each test to prevent exceeding Elasticsearch's // shard limit (see Issue 10120). - client - .execute( - new HttpPost( - String.format( - "http://%s:%d/%s*/_close", - container.getHttpHost().getHostName(), - container.getHttpHost().getPort(), - testName.getSanitizedMethodName())), - HttpClientContext.create(), - null) - .get(5, MINUTES); + ElasticTestUtils.closeIndex(client, container, testName); } @Override @@ -93,10 +84,18 @@ @Override protected Injector createInjector() { - Config elasticsearchConfig = new Config(config); - InMemoryModule.setDefaults(elasticsearchConfig); - String indicesPrefix = testName.getSanitizedMethodName(); - ElasticTestUtils.configure(elasticsearchConfig, container, indicesPrefix); - return Guice.createInjector(new InMemoryModule(elasticsearchConfig)); + return ElasticTestUtils.createInjector(config, testName, container); + } + + @Test + public void testErrorResponseFromChangeIndex() throws Exception { + TestRepository<InMemoryRepositoryManager.Repo> repo = createProject("repo"); + Change c = insert(repo, newChangeWithStatus(repo, Change.Status.NEW)); + gApi.changes().id(c.getChangeId()).index(); + + ElasticTestUtils.closeIndex(client, container, testName); + StorageException thrown = + assertThrows(StorageException.class, () -> gApi.changes().id(c.getChangeId()).index()); + assertThat(thrown).hasMessageThat().contains("Failed to reindex change"); } }
diff --git a/javatests/com/google/gerrit/elasticsearch/ElasticV7QueryGroupsTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryGroupsTest.java similarity index 64% rename from javatests/com/google/gerrit/elasticsearch/ElasticV7QueryGroupsTest.java rename to src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryGroupsTest.java index f54ed1d..6eef24c 100644 --- a/javatests/com/google/gerrit/elasticsearch/ElasticV7QueryGroupsTest.java +++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryGroupsTest.java
@@ -14,20 +14,25 @@ package com.google.gerrit.elasticsearch; +import static com.google.common.truth.Truth.assertThat; +import static com.google.gerrit.testing.GerritJUnit.assertThrows; + +import com.google.gerrit.exceptions.StorageException; +import com.google.gerrit.extensions.api.groups.GroupApi; import com.google.gerrit.server.query.group.AbstractQueryGroupsTest; import com.google.gerrit.testing.ConfigSuite; -import com.google.gerrit.testing.InMemoryModule; -import com.google.gerrit.testing.IndexConfig; -import com.google.inject.Guice; import com.google.inject.Injector; +import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; +import org.apache.http.impl.nio.client.HttpAsyncClients; import org.eclipse.jgit.lib.Config; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Test; public class ElasticV7QueryGroupsTest extends AbstractQueryGroupsTest { @ConfigSuite.Default public static Config defaultConfig() { - return IndexConfig.createForElasticsearch(); + return ElasticTestUtils.createConfig(); } @ConfigSuite.Config @@ -38,12 +43,15 @@ } private static ElasticContainer container; + private static CloseableHttpAsyncClient client; @BeforeClass public static void startIndexService() { if (container == null) { // Only start Elasticsearch once - container = ElasticContainer.createAndStart(ElasticVersion.V7_8); + container = ElasticContainer.createAndStart(ElasticVersion.V7_16); + client = HttpAsyncClients.createDefault(); + client.start(); } } @@ -62,10 +70,16 @@ @Override protected Injector createInjector() { - Config elasticsearchConfig = new Config(config); - InMemoryModule.setDefaults(elasticsearchConfig); - String indicesPrefix = testName.getSanitizedMethodName(); - ElasticTestUtils.configure(elasticsearchConfig, container, indicesPrefix); - return Guice.createInjector(new InMemoryModule(elasticsearchConfig)); + return ElasticTestUtils.createInjector(config, testName, container); + } + + @Test + public void testErrorResponseFromGroupIndex() throws Exception { + GroupApi group = gApi.groups().create("test"); + group.index(); + + ElasticTestUtils.closeIndex(client, container, testName); + StorageException thrown = assertThrows(StorageException.class, () -> group.index()); + assertThat(thrown).hasMessageThat().contains("Failed to replace group"); } }
diff --git a/javatests/com/google/gerrit/elasticsearch/ElasticV7QueryProjectsTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryProjectsTest.java similarity index 64% rename from javatests/com/google/gerrit/elasticsearch/ElasticV7QueryProjectsTest.java rename to src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryProjectsTest.java index 6a75f82..70cd7de 100644 --- a/javatests/com/google/gerrit/elasticsearch/ElasticV7QueryProjectsTest.java +++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticV7QueryProjectsTest.java
@@ -14,20 +14,25 @@ package com.google.gerrit.elasticsearch; +import static com.google.common.truth.Truth.assertThat; +import static com.google.gerrit.testing.GerritJUnit.assertThrows; + +import com.google.gerrit.exceptions.StorageException; +import com.google.gerrit.extensions.api.projects.ProjectApi; import com.google.gerrit.server.query.project.AbstractQueryProjectsTest; import com.google.gerrit.testing.ConfigSuite; -import com.google.gerrit.testing.InMemoryModule; -import com.google.gerrit.testing.IndexConfig; -import com.google.inject.Guice; import com.google.inject.Injector; +import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; +import org.apache.http.impl.nio.client.HttpAsyncClients; import org.eclipse.jgit.lib.Config; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Test; public class ElasticV7QueryProjectsTest extends AbstractQueryProjectsTest { @ConfigSuite.Default public static Config defaultConfig() { - return IndexConfig.createForElasticsearch(); + return ElasticTestUtils.createConfig(); } @ConfigSuite.Config @@ -38,12 +43,15 @@ } private static ElasticContainer container; + private static CloseableHttpAsyncClient client; @BeforeClass public static void startIndexService() { if (container == null) { // Only start Elasticsearch once - container = ElasticContainer.createAndStart(ElasticVersion.V7_8); + container = ElasticContainer.createAndStart(ElasticVersion.V7_16); + client = HttpAsyncClients.createDefault(); + client.start(); } } @@ -62,10 +70,16 @@ @Override protected Injector createInjector() { - Config elasticsearchConfig = new Config(config); - InMemoryModule.setDefaults(elasticsearchConfig); - String indicesPrefix = testName.getSanitizedMethodName(); - ElasticTestUtils.configure(elasticsearchConfig, container, indicesPrefix); - return Guice.createInjector(new InMemoryModule(elasticsearchConfig)); + return ElasticTestUtils.createInjector(config, testName, container); + } + + @Test + public void testErrorResponseFromProjectIndex() throws Exception { + ProjectApi project = gApi.projects().create("test"); + project.index(false); + + ElasticTestUtils.closeIndex(client, container, testName); + StorageException thrown = assertThrows(StorageException.class, () -> project.index(false)); + assertThat(thrown).hasMessageThat().contains("Failed to replace project"); } }
diff --git a/javatests/com/google/gerrit/elasticsearch/ElasticVersionTest.java b/src/test/java/com/google/gerrit/elasticsearch/ElasticVersionTest.java similarity index 72% rename from javatests/com/google/gerrit/elasticsearch/ElasticVersionTest.java rename to src/test/java/com/google/gerrit/elasticsearch/ElasticVersionTest.java index 2ce3a2c..ea7782b 100644 --- a/javatests/com/google/gerrit/elasticsearch/ElasticVersionTest.java +++ b/src/test/java/com/google/gerrit/elasticsearch/ElasticVersionTest.java
@@ -22,14 +22,8 @@ public class ElasticVersionTest { @Test public void supportedVersion() throws Exception { - assertThat(ElasticVersion.forVersion("7.6.0")).isEqualTo(ElasticVersion.V7_6); - assertThat(ElasticVersion.forVersion("7.6.1")).isEqualTo(ElasticVersion.V7_6); - - assertThat(ElasticVersion.forVersion("7.7.0")).isEqualTo(ElasticVersion.V7_7); - assertThat(ElasticVersion.forVersion("7.7.1")).isEqualTo(ElasticVersion.V7_7); - - assertThat(ElasticVersion.forVersion("7.8.0")).isEqualTo(ElasticVersion.V7_8); - assertThat(ElasticVersion.forVersion("7.8.1")).isEqualTo(ElasticVersion.V7_8); + assertThat(ElasticVersion.forVersion("7.16.0")).isEqualTo(ElasticVersion.V7_16); + assertThat(ElasticVersion.forVersion("7.16.1")).isEqualTo(ElasticVersion.V7_16); } @Test
diff --git a/src/test/test.iml b/src/test/test.iml new file mode 100644 index 0000000..a0e49a3 --- /dev/null +++ b/src/test/test.iml
@@ -0,0 +1,11 @@ +<?xml version="1.0" encoding="UTF-8"?> +<module type="JAVA_MODULE" version="4"> + <component name="NewModuleRootManager" inherit-compiler-output="true"> + <exclude-output /> + <content url="file://$MODULE_DIR$"> + <sourceFolder url="file://$MODULE_DIR$/java" isTestSource="true" /> + </content> + <orderEntry type="inheritedJdk" /> + <orderEntry type="sourceFolder" forTests="false" /> + </component> +</module> \ No newline at end of file