Elasticsearch: Add support for V6 / one index type
Starting with ES 6.0.0, multiple mapping types within an index are no
longer supported. Replace the open and closed changes index types with
just one (type), for that V6 case. Name that single type '_doc', to help
prepare V6 indices in Gerrit for a potential yet smoother migration to
V7. -Choose such a name as it becomes the default one starting with V7.
Consistently do so as well for the accounts and groups indices. Doing so
has no negative impacts on the trivial way index types are used today.
Refer to [1] for all these matters considered herein. Stick to the V2
index type names for the V5 case, as the latter does not support names
starting with an underscore (such as '_doc' for V6+).
Set the _type field only for Elasticsearch V2 and V5 usage, as it is
deprecated starting with V6; cf. [2]. This is consistent with the above.
These were the sole breaking changes between Elasticsearch V5 and V6 for
the Gerrit ES client implementation. This change is also meant to
preserve potentially existing V2 /default integrations or deployments.
[1] https://www.elastic.co/guide/en/elasticsearch/reference/current/removal-of-types.html#_schedule_for_removal_of_mapping_types
[2] https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-type-field.html
Bug: Issue 9112
Change-Id: I92e6c74741976ef002aadded4e1915927aef46e5
diff --git a/Documentation/config-gerrit.txt b/Documentation/config-gerrit.txt
index 03e06e2..8f1fd59 100644
--- a/Documentation/config-gerrit.txt
+++ b/Documentation/config-gerrit.txt
@@ -2792,10 +2792,13 @@
=== Section elasticsearch
WARNING: The Elasticsearch support has only been tested with Elasticsearch
-version 2.4.x. Support for other versions is not guaranteed.
+versions 2.4, 5.6 and 6.2. Support for other versions is not guaranteed.
-Open and closed changes are indexed in a single index, separated
-into types `open_changes` and `closed_changes` respectively.
+Open and closed changes are indexed in a single index, separated into types
+`open_changes` and `closed_changes` respectively, if using Elasticsearch
+versions 2.4 or 5.6. Open and closed changes are merged into the default `_doc`
+type otherwise. The latter is also used for accounts and groups indices starting
+with Elasticsearch 6.2.
[[elasticsearch.prefix]]elasticsearch.prefix::
+
diff --git a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
index 45bd2b9..cbc3937 100644
--- a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
+++ b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java
@@ -20,6 +20,7 @@
import com.google.common.collect.FluentIterable;
import com.google.common.io.CharStreams;
+import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.elasticsearch.builders.SearchSourceBuilder;
import com.google.gerrit.elasticsearch.bulk.DeleteRequest;
import com.google.gerrit.server.config.SitePaths;
@@ -50,6 +51,7 @@
abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
protected static final String BULK = "_bulk";
+ protected static final String MAPPINGS = "mappings";
protected static final String ORDER = "order";
protected static final String SEARCH = "_search";
@@ -79,6 +81,7 @@
private final Schema<V> schema;
private final SitePaths sitePaths;
private final String indexNameRaw;
+ private final String type;
protected final ElasticRestClientProvider client;
protected final String indexName;
@@ -98,6 +101,7 @@
this.indexName = cfg.getIndexName(indexName, schema.getVersion());
this.indexNameRaw = indexName;
this.client = client;
+ this.type = client.adapter().getType(indexName);
}
@Override
@@ -117,7 +121,7 @@
@Override
public void delete(K id) throws IOException {
- String uri = getURI(indexNameRaw, BULK);
+ String uri = getURI(type, BULK);
Response response = postRequest(getDeleteActions(id), uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
@@ -156,8 +160,20 @@
protected abstract String getId(V v);
+ protected String getMappingsForSingleType(String candidateType, MappingProperties properties) {
+ return getMappingsFor(client.adapter().getType(candidateType), properties);
+ }
+
+ protected String getMappingsFor(String type, MappingProperties properties) {
+ JsonObject mappingType = new JsonObject();
+ mappingType.add(type, gson.toJsonTree(properties));
+ JsonObject mappings = new JsonObject();
+ mappings.add(MAPPINGS, gson.toJsonTree(mappingType));
+ return gson.toJson(mappings);
+ }
+
protected String delete(String type, K id) {
- return new DeleteRequest(id.toString(), indexName, type).toString();
+ return new DeleteRequest(id.toString(), indexName, type, client.adapter()).toString();
}
protected void addNamedElement(String name, JsonObject element, JsonArray array) {
diff --git a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
index 178014c..6ea24dd 100644
--- a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
+++ b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java
@@ -16,7 +16,6 @@
import static com.google.gerrit.server.index.account.AccountField.ID;
-import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.elasticsearch.builders.QueryBuilder;
@@ -73,6 +72,7 @@
private final AccountMapping mapping;
private final Provider<AccountCache> accountCache;
private final Schema<AccountState> schema;
+ private final String type;
@AssistedInject
ElasticAccountIndex(
@@ -85,14 +85,16 @@
this.accountCache = accountCache;
this.mapping = new AccountMapping(schema, client.adapter());
this.schema = schema;
+ this.type = client.adapter().getType(ACCOUNTS);
}
@Override
public void replace(AccountState as) throws IOException {
BulkRequest bulk =
- new IndexRequest(getId(as), indexName, ACCOUNTS).add(new UpdateRequest<>(schema, as));
+ new IndexRequest(getId(as), indexName, type, client.adapter())
+ .add(new UpdateRequest<>(schema, as));
- String uri = getURI(ACCOUNTS, BULK);
+ String uri = getURI(type, BULK);
Response response = postRequest(bulk, uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
@@ -111,13 +113,12 @@
@Override
protected String getDeleteActions(Account.Id a) {
- return delete(ACCOUNTS, a);
+ return delete(type, a);
}
@Override
protected String getMappings() {
- ImmutableMap<String, AccountMapping> mappings = ImmutableMap.of("mappings", mapping);
- return gson.toJson(mappings);
+ return getMappingsForSingleType(ACCOUNTS, mapping.accounts);
}
@Override
@@ -152,7 +153,7 @@
public ResultSet<AccountState> read() throws OrmException {
try {
List<AccountState> results = Collections.emptyList();
- String uri = getURI(ACCOUNTS, SEARCH);
+ String uri = getURI(type, SEARCH);
Response response = postRequest(search, uri, Collections.emptyMap());
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() == HttpStatus.SC_OK) {
diff --git a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
index bf1d615..9a1b399 100644
--- a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
+++ b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java
@@ -83,11 +83,13 @@
private static final Logger log = LoggerFactory.getLogger(ElasticChangeIndex.class);
public static class ChangeMapping {
+ public MappingProperties changes;
public MappingProperties openChanges;
public MappingProperties closedChanges;
public ChangeMapping(Schema<ChangeData> schema, ElasticQueryAdapter adapter) {
MappingProperties mapping = ElasticMapping.createMapping(schema, adapter);
+ this.changes = mapping;
this.openChanges = mapping;
this.closedChanges = mapping;
}
@@ -102,6 +104,7 @@
private final ChangeData.Factory changeDataFactory;
private final FillArgs fillArgs;
private final Schema<ChangeData> schema;
+ private final String type;
@AssistedInject
ElasticChangeIndex(
@@ -118,6 +121,7 @@
this.fillArgs = fillArgs;
this.schema = schema;
this.mapping = new ChangeMapping(schema, client.adapter());
+ this.type = client.adapter().getType(CHANGES);
}
@Override
@@ -137,12 +141,15 @@
throw new IOException(e);
}
+ ElasticQueryAdapter adapter = client.adapter();
BulkRequest bulk =
- new IndexRequest(getId(cd), indexName, insertIndex)
- .add(new UpdateRequest<>(fillArgs, schema, cd))
- .add(new DeleteRequest(cd.getId().toString(), indexName, deleteIndex));
+ new IndexRequest(getId(cd), indexName, adapter.getType(insertIndex), adapter)
+ .add(new UpdateRequest<>(fillArgs, schema, cd));
+ if (!adapter.usePostV5Type()) {
+ bulk.add(new DeleteRequest(cd.getId().toString(), indexName, deleteIndex, adapter));
+ }
- String uri = getURI(CHANGES, BULK);
+ String uri = getURI(type, BULK);
Response response = postRequest(bulk, uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
@@ -157,23 +164,36 @@
throws QueryParseException {
Set<Change.Status> statuses = ChangeIndexRewriter.getPossibleStatus(p);
List<String> indexes = Lists.newArrayListWithCapacity(2);
- if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()) {
- indexes.add(OPEN_CHANGES);
- }
- if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
- indexes.add(CLOSED_CHANGES);
+ if (client.adapter().usePostV5Type()) {
+ if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()
+ || !Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
+ indexes.add(ElasticQueryAdapter.POST_V5_TYPE);
+ }
+ } else {
+ if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()) {
+ indexes.add(OPEN_CHANGES);
+ }
+ if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
+ indexes.add(CLOSED_CHANGES);
+ }
}
return new QuerySource(indexes, p, opts);
}
@Override
protected String getDeleteActions(Id c) {
+ if (client.adapter().usePostV5Type()) {
+ return delete(ElasticQueryAdapter.POST_V5_TYPE, c);
+ }
return delete(OPEN_CHANGES, c) + delete(CLOSED_CHANGES, c);
}
@Override
protected String getMappings() {
- return gson.toJson(ImmutableMap.of("mappings", mapping));
+ if (client.adapter().usePostV5Type()) {
+ return getMappingsFor(ElasticQueryAdapter.POST_V5_TYPE, mapping.changes);
+ }
+ return gson.toJson(ImmutableMap.of(MAPPINGS, mapping));
}
@Override
diff --git a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
index 86997cd..ff5827a 100644
--- a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
+++ b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java
@@ -14,7 +14,6 @@
package com.google.gerrit.elasticsearch;
-import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
import com.google.gerrit.elasticsearch.builders.QueryBuilder;
@@ -70,6 +69,7 @@
private final GroupMapping mapping;
private final Provider<GroupCache> groupCache;
private final Schema<AccountGroup> schema;
+ private final String type;
@AssistedInject
ElasticGroupIndex(
@@ -82,14 +82,16 @@
this.groupCache = groupCache;
this.mapping = new GroupMapping(schema, client.adapter());
this.schema = schema;
+ this.type = client.adapter().getType(GROUPS);
}
@Override
public void replace(AccountGroup group) throws IOException {
BulkRequest bulk =
- new IndexRequest(getId(group), indexName, GROUPS).add(new UpdateRequest<>(schema, group));
+ new IndexRequest(getId(group), indexName, type, client.adapter())
+ .add(new UpdateRequest<>(schema, group));
- String uri = getURI(GROUPS, BULK);
+ String uri = getURI(type, BULK);
Response response = postRequest(bulk, uri, getRefreshParam());
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
@@ -108,13 +110,12 @@
@Override
protected String getDeleteActions(AccountGroup.UUID g) {
- return delete(GROUPS, g);
+ return delete(type, g);
}
@Override
protected String getMappings() {
- ImmutableMap<String, GroupMapping> mappings = ImmutableMap.of("mappings", mapping);
- return gson.toJson(mappings);
+ return getMappingsForSingleType(GROUPS, mapping.groups);
}
@Override
@@ -149,7 +150,7 @@
public ResultSet<AccountGroup> read() throws OrmException {
try {
List<AccountGroup> results = Collections.emptyList();
- String uri = getURI(GROUPS, SEARCH);
+ String uri = getURI(type, SEARCH);
Response response = postRequest(search, uri, Collections.emptyMap());
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() == HttpStatus.SC_OK) {
diff --git a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryAdapter.java b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryAdapter.java
index 72af49a..6eb9384 100644
--- a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryAdapter.java
+++ b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticQueryAdapter.java
@@ -17,7 +17,11 @@
import com.google.gson.JsonObject;
public class ElasticQueryAdapter {
+ static final String POST_V5_TYPE = "_doc";
+
private final boolean ignoreUnmapped;
+ private final boolean usePostV5Type;
+
private final String searchFilteringName;
private final String indicesExistParam;
private final String exactFieldType;
@@ -26,6 +30,8 @@
ElasticQueryAdapter(ElasticVersion version) {
this.ignoreUnmapped = version == ElasticVersion.V2_4;
+ this.usePostV5Type = version == ElasticVersion.V6_2;
+
switch (version) {
case V5_6:
case V6_2:
@@ -52,6 +58,12 @@
}
}
+ public void setType(JsonObject properties, String type) {
+ if (!usePostV5Type) {
+ properties.addProperty("_type", type);
+ }
+ }
+
public String searchFilteringName() {
return searchFilteringName;
}
@@ -71,4 +83,12 @@
String indexProperty() {
return indexProperty;
}
+
+ boolean usePostV5Type() {
+ return usePostV5Type;
+ }
+
+ String getType(String preV6Type) {
+ return usePostV5Type() ? POST_V5_TYPE : preV6Type;
+ }
}
diff --git a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/bulk/ActionRequest.java b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/bulk/ActionRequest.java
index c7757b2..7392d09 100644
--- a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/bulk/ActionRequest.java
+++ b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/bulk/ActionRequest.java
@@ -14,6 +14,7 @@
package com.google.gerrit.elasticsearch.bulk;
+import com.google.gerrit.elasticsearch.ElasticQueryAdapter;
import com.google.gson.JsonObject;
abstract class ActionRequest extends BulkRequest {
@@ -22,12 +23,15 @@
private final String id;
private final String index;
private final String type;
+ private final ElasticQueryAdapter adapter;
- protected ActionRequest(String action, String id, String index, String type) {
+ protected ActionRequest(
+ String action, String id, String index, String type, ElasticQueryAdapter adapter) {
this.action = action;
this.id = id;
this.index = index;
this.type = type;
+ this.adapter = adapter;
}
@Override
@@ -35,7 +39,7 @@
JsonObject properties = new JsonObject();
properties.addProperty("_id", id);
properties.addProperty("_index", index);
- properties.addProperty("_type", type);
+ adapter.setType(properties, type);
JsonObject jsonAction = new JsonObject();
jsonAction.add(action, properties);
diff --git a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/bulk/DeleteRequest.java b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/bulk/DeleteRequest.java
index 7d549ca..570d5a0 100644
--- a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/bulk/DeleteRequest.java
+++ b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/bulk/DeleteRequest.java
@@ -14,9 +14,11 @@
package com.google.gerrit.elasticsearch.bulk;
+import com.google.gerrit.elasticsearch.ElasticQueryAdapter;
+
public class DeleteRequest extends ActionRequest {
- public DeleteRequest(String id, String index, String type) {
- super("delete", id, index, type);
+ public DeleteRequest(String id, String index, String type, ElasticQueryAdapter adapter) {
+ super("delete", id, index, type, adapter);
}
}
diff --git a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/bulk/IndexRequest.java b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/bulk/IndexRequest.java
index b131501..c571a0e 100644
--- a/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/bulk/IndexRequest.java
+++ b/gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/bulk/IndexRequest.java
@@ -14,9 +14,11 @@
package com.google.gerrit.elasticsearch.bulk;
+import com.google.gerrit.elasticsearch.ElasticQueryAdapter;
+
public class IndexRequest extends ActionRequest {
- public IndexRequest(String id, String index, String type) {
- super("index", id, index, type);
+ public IndexRequest(String id, String index, String type, ElasticQueryAdapter adapter) {
+ super("index", id, index, type, adapter);
}
}
