From 2be7e19cae6da0e922782917509f8b1a4091347b Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Thu, 23 Apr 2020 18:06:42 +0100 Subject: [PATCH 01/24] Add _index_template/_simulate_index/{name} api This adds a new api to simulate matching the given index name against the index templates in the system. --- .../indices.post_simulate_index_template.json | 35 +++++ .../elasticsearch/action/ActionModule.java | 17 ++- .../post/PostSimulateIndexTemplateAction.java | 33 +++++ .../post/SimulateIndexTemplateRequest.java | 87 ++++++++++++ .../post/SimulateIndexTemplateResponse.java | 122 +++++++++++++++++ .../TransportSimulateIndexTemplateAction.java | 128 ++++++++++++++++++ .../put/PutIndexTemplateV2Action.java | 6 +- .../metadata/MetadataCreateIndexService.java | 2 +- .../MetadataIndexTemplateService.java | 7 +- .../cluster/metadata/Template.java | 2 +- .../RestPostSimulateIndexTemplateAction.java | 63 +++++++++ 11 files changed, 489 insertions(+), 13 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/indices.post_simulate_index_template.json create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/template/post/PostSimulateIndexTemplateAction.java create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java create mode 100644 server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPostSimulateIndexTemplateAction.java diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.post_simulate_index_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.post_simulate_index_template.json new file mode 100644 index 0000000000000..ca72f8174d3ed --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.post_simulate_index_template.json @@ -0,0 +1,35 @@ +{ + "indices.post_simulate_index_template":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html", + "description": "Simulate matching the given index name against the index templates in the system" + }, + "stability":"stable", + "url":{ + "paths":[ + { + "path":"/_index_template/_simulate_index/{name}", + "methods":[ + "POST" + ], + "parts":{ + "name":{ + "type":"string", + "description":"The name of the index (it will have to be a concrete index name)" + } + } + } + ] + }, + "params":{ + "master_timeout":{ + "type":"time", + "description":"Specify timeout for connection to master" + } + }, + "body":{ + "description":"New index template definition, which will be included in the simulation, as if it already exists in the system", + "required":false + } + } +} diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index cb52f610607f8..9bd34cb3cb9eb 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -28,9 +28,6 @@ import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsAction; import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction; import org.elasticsearch.action.admin.cluster.configuration.TransportClearVotingConfigExclusionsAction; -import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction; -import org.elasticsearch.action.admin.indices.datastream.GetDataStreamsAction; -import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction; import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction; import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsAction; @@ -108,6 +105,9 @@ import org.elasticsearch.action.admin.indices.close.TransportVerifyShardBeforeCloseAction; import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; +import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction; +import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction; +import org.elasticsearch.action.admin.indices.datastream.GetDataStreamsAction; import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction; import org.elasticsearch.action.admin.indices.flush.FlushAction; @@ -158,6 +158,8 @@ import org.elasticsearch.action.admin.indices.template.get.TransportGetComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.get.TransportGetIndexTemplateV2Action; import org.elasticsearch.action.admin.indices.template.get.TransportGetIndexTemplatesAction; +import org.elasticsearch.action.admin.indices.template.post.PostSimulateIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.post.TransportSimulateIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateV2Action; @@ -260,11 +262,9 @@ import org.elasticsearch.rest.action.admin.cluster.RestClusterStatsAction; import org.elasticsearch.rest.action.admin.cluster.RestClusterUpdateSettingsAction; import org.elasticsearch.rest.action.admin.cluster.RestCreateSnapshotAction; -import org.elasticsearch.rest.action.admin.indices.RestDeleteDataStreamAction; import org.elasticsearch.rest.action.admin.cluster.RestDeleteRepositoryAction; import org.elasticsearch.rest.action.admin.cluster.RestDeleteSnapshotAction; import org.elasticsearch.rest.action.admin.cluster.RestDeleteStoredScriptAction; -import org.elasticsearch.rest.action.admin.indices.RestGetDataStreamsAction; import org.elasticsearch.rest.action.admin.cluster.RestGetRepositoriesAction; import org.elasticsearch.rest.action.admin.cluster.RestGetScriptContextAction; import org.elasticsearch.rest.action.admin.cluster.RestGetScriptLanguageAction; @@ -277,7 +277,6 @@ import org.elasticsearch.rest.action.admin.cluster.RestNodesStatsAction; import org.elasticsearch.rest.action.admin.cluster.RestNodesUsageAction; import org.elasticsearch.rest.action.admin.cluster.RestPendingClusterTasksAction; -import org.elasticsearch.rest.action.admin.indices.RestCreateDataStreamAction; import org.elasticsearch.rest.action.admin.cluster.RestPutRepositoryAction; import org.elasticsearch.rest.action.admin.cluster.RestPutStoredScriptAction; import org.elasticsearch.rest.action.admin.cluster.RestReloadSecureSettingsAction; @@ -288,8 +287,10 @@ import org.elasticsearch.rest.action.admin.indices.RestAnalyzeAction; import org.elasticsearch.rest.action.admin.indices.RestClearIndicesCacheAction; import org.elasticsearch.rest.action.admin.indices.RestCloseIndexAction; +import org.elasticsearch.rest.action.admin.indices.RestCreateDataStreamAction; import org.elasticsearch.rest.action.admin.indices.RestCreateIndexAction; import org.elasticsearch.rest.action.admin.indices.RestDeleteComponentTemplateAction; +import org.elasticsearch.rest.action.admin.indices.RestDeleteDataStreamAction; import org.elasticsearch.rest.action.admin.indices.RestDeleteIndexAction; import org.elasticsearch.rest.action.admin.indices.RestDeleteIndexTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestDeleteIndexTemplateV2Action; @@ -297,6 +298,7 @@ import org.elasticsearch.rest.action.admin.indices.RestForceMergeAction; import org.elasticsearch.rest.action.admin.indices.RestGetAliasesAction; import org.elasticsearch.rest.action.admin.indices.RestGetComponentTemplateAction; +import org.elasticsearch.rest.action.admin.indices.RestGetDataStreamsAction; import org.elasticsearch.rest.action.admin.indices.RestGetFieldMappingAction; import org.elasticsearch.rest.action.admin.indices.RestGetIndexTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestGetIndexTemplateV2Action; @@ -310,6 +312,7 @@ import org.elasticsearch.rest.action.admin.indices.RestIndicesShardStoresAction; import org.elasticsearch.rest.action.admin.indices.RestIndicesStatsAction; import org.elasticsearch.rest.action.admin.indices.RestOpenIndexAction; +import org.elasticsearch.rest.action.admin.indices.RestPostSimulateIndexTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestPutComponentTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateV2Action; @@ -550,6 +553,7 @@ public void reg actions.register(PutIndexTemplateV2Action.INSTANCE, TransportPutIndexTemplateV2Action.class); actions.register(GetIndexTemplateV2Action.INSTANCE, TransportGetIndexTemplateV2Action.class); actions.register(DeleteIndexTemplateV2Action.INSTANCE, TransportDeleteIndexTemplateV2Action.class); + actions.register(PostSimulateIndexTemplateAction.INSTANCE, TransportSimulateIndexTemplateAction.class); } actions.register(ValidateQueryAction.INSTANCE, TransportValidateQueryAction.class); actions.register(RefreshAction.INSTANCE, TransportRefreshAction.class); @@ -700,6 +704,7 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestPutIndexTemplateV2Action()); registerHandler.accept(new RestGetIndexTemplateV2Action()); registerHandler.accept(new RestDeleteIndexTemplateV2Action()); + registerHandler.accept(new RestPostSimulateIndexTemplateAction()); } registerHandler.accept(new RestPutMappingAction()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/PostSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/PostSimulateIndexTemplateAction.java new file mode 100644 index 0000000000000..77e8c1d8d223c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/PostSimulateIndexTemplateAction.java @@ -0,0 +1,33 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.template.post; + +import org.elasticsearch.action.ActionType; + +public class PostSimulateIndexTemplateAction extends ActionType { + + public static final PostSimulateIndexTemplateAction INSTANCE = new PostSimulateIndexTemplateAction(); + public static final String NAME = "indices:admin/index_template/simulate_index/post"; + + private PostSimulateIndexTemplateAction() { + super(NAME, SimulateIndexTemplateResponse::new); + } + +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java new file mode 100644 index 0000000000000..a0df587cf0334 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java @@ -0,0 +1,87 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.template.post; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateV2Action; +import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +public class SimulateIndexTemplateRequest extends MasterNodeReadRequest { + + private String indexName; + + @Nullable + private PutIndexTemplateV2Action.Request indexTemplateRequest; + + public SimulateIndexTemplateRequest(String indexName) { + if (Strings.isNullOrEmpty(indexName)) { + throw new IllegalArgumentException("index name cannot be null or empty"); + } + this.indexName = indexName; + } + + public SimulateIndexTemplateRequest(StreamInput in) throws IOException { + super(in); + indexName = in.readString(); + in.readOptionalWriteable(PutIndexTemplateRequest::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(indexName); + out.writeOptionalWriteable(indexTemplateRequest); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (indexTemplateRequest != null) { + validationException = indexTemplateRequest.validateIndexTemplate(validationException); + } + return validationException; + } + + public String getIndexName() { + return indexName; + } + + @Nullable + public PutIndexTemplateV2Action.Request getIndexTemplateRequest() { + return indexTemplateRequest; + } + + public SimulateIndexTemplateRequest indexName(String indexName) { + this.indexName = indexName; + return this; + } + + public SimulateIndexTemplateRequest indexTemplateRequest(PutIndexTemplateV2Action.Request indexTemplateRequest) { + this.indexTemplateRequest = indexTemplateRequest; + return this; + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java new file mode 100644 index 0000000000000..507f4d0c842c1 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java @@ -0,0 +1,122 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.template.post; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Contains the information on what V2 templates would match a given index. + */ +public class SimulateIndexTemplateResponse extends ActionResponse implements ToXContentObject { + + private static final ParseField TEMPLATE = new ParseField("template"); + private static final ParseField OVERLAPPING_V1_TEMPLATES = new ParseField("overlapping_v1_templates"); + + @Nullable + // the resolved settings, mappings and aliases for the matched templates, if any + private Template resolvedTemplate; + + @Nullable + // a map of v1 template names and their index patterns that would overlap when matching the given index name + private Map> overlappingV1Templates; + + public SimulateIndexTemplateResponse(@Nullable Template resolvedTemplate, @Nullable Map> overlappingV1Templates) { + this.resolvedTemplate = resolvedTemplate; + this.overlappingV1Templates = overlappingV1Templates; + } + + public SimulateIndexTemplateResponse(StreamInput in) throws IOException { + super(in); + resolvedTemplate = in.readOptionalWriteable(Template::new); + if (in.readBoolean()) { + int conflictingV1TemplatesCount = in.readInt(); + overlappingV1Templates = new HashMap<>(conflictingV1TemplatesCount, 1L); + for (int i = 0; i < conflictingV1TemplatesCount; i++) { + String templateName = in.readString(); + overlappingV1Templates.put(templateName, in.readStringList()); + } + } else { + this.overlappingV1Templates = null; + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalWriteable(resolvedTemplate); + if (overlappingV1Templates != null) { + out.writeBoolean(true); + out.writeInt(overlappingV1Templates.size()); + for (Map.Entry> entry : overlappingV1Templates.entrySet()) { + out.writeString(entry.getKey()); + out.writeStringCollection(entry.getValue()); + } + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (this.resolvedTemplate != null) { + builder.field(TEMPLATE.getPreferredName(), this.resolvedTemplate); + } + if (this.overlappingV1Templates != null) { + builder.field(OVERLAPPING_V1_TEMPLATES.getPreferredName(), overlappingV1Templates); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SimulateIndexTemplateResponse that = (SimulateIndexTemplateResponse) o; + return Objects.equals(resolvedTemplate, that.resolvedTemplate) + && Objects.deepEquals(overlappingV1Templates, that.overlappingV1Templates); + } + + @Override + public int hashCode() { + return Objects.hash(resolvedTemplate, overlappingV1Templates); + } + + @Override + public String toString() { + return "SimulateIndexTemplateResponse{" + "resolved template=" + resolvedTemplate + ", overlapping v1 templates=" + + String.join("|", overlappingV1Templates.keySet()) + "}"; + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java new file mode 100644 index 0000000000000..62507b195a7d5 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -0,0 +1,128 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.template.post; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexTemplateV2; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.cluster.metadata.MetadataCreateIndexService.resolveV2Mappings; +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template; +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveSettings; + +public class TransportSimulateIndexTemplateAction + extends TransportMasterNodeReadAction { + + private final MetadataIndexTemplateService indexTemplateService; + private final NamedXContentRegistry xContentRegistry; + + @Inject + public TransportSimulateIndexTemplateAction(TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, MetadataIndexTemplateService indexTemplateService, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + NamedXContentRegistry xContentRegistry) { + super(PostSimulateIndexTemplateAction.NAME, transportService, clusterService, threadPool, actionFilters, + SimulateIndexTemplateRequest::new, indexNameExpressionResolver); + this.indexTemplateService = indexTemplateService; + this.xContentRegistry = xContentRegistry; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected SimulateIndexTemplateResponse read(StreamInput in) throws IOException { + return new SimulateIndexTemplateResponse(in); + } + + @Override + protected void masterOperation(Task task, SimulateIndexTemplateRequest request, ClusterState state, + ActionListener listener) throws Exception { + ClusterState simulateOnClusterState = state; + if (request.getIndexTemplateRequest() != null) { + // we'll "locally" add the template defined by the user in the cluster state (as if it existed in the system) + String simulateTemplateToAdd = "simulate_new_template_" + UUIDs.randomBase64UUID(); + simulateOnClusterState = indexTemplateService.addIndexTemplateV2(state, request.getIndexTemplateRequest().create(), + simulateTemplateToAdd, request.getIndexTemplateRequest().indexTemplate()); + } + + String matchingTemplate = findV2Template(simulateOnClusterState.metadata(), request.getIndexName(), false); + Settings settings = resolveSettings(simulateOnClusterState.metadata(), matchingTemplate); + + // empty request mapping as the user can't specify any explicit mappings via the simulate api + Map mappings = resolveV2Mappings("{}", simulateOnClusterState, matchingTemplate, xContentRegistry); + String mappingsJson = Strings.toString(XContentFactory.jsonBuilder() + .startObject() + .field(MapperService.SINGLE_MAPPING_NAME, mappings) + .endObject()); + + List> resolvedAliases = MetadataIndexTemplateService.resolveAliases(simulateOnClusterState.metadata(), + matchingTemplate); + Map aliases = new HashMap<>(); + for (Map alias : resolvedAliases) { + for (Map.Entry aliasEntry : alias.entrySet()) { + if (aliases.containsKey(aliasEntry.getKey()) == false) { + aliases.put(aliasEntry.getKey(), aliasEntry.getValue()); + } + } + } + + IndexTemplateV2 templateV2 = simulateOnClusterState.metadata().templatesV2().get(matchingTemplate); + assert templateV2 != null : "the matched template must exist"; + + Map> conflictingV1Templates = MetadataIndexTemplateService.findConflictingV1Templates(simulateOnClusterState + , matchingTemplate, templateV2.indexPatterns()); + Template template = new Template(settings, mappingsJson == null ? null : new CompressedXContent(mappingsJson), aliases); + listener.onResponse(new SimulateIndexTemplateResponse(template, conflictingV1Templates)); + } + + @Override + protected ClusterBlockException checkBlock(SimulateIndexTemplateRequest request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateV2Action.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateV2Action.java index 19c6f9a0cabea..3cceeebac7f58 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateV2Action.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateV2Action.java @@ -87,6 +87,11 @@ public ActionRequestValidationException validate() { if (name == null || Strings.hasText(name) == false) { validationException = addValidationError("name is missing", validationException); } + validationException = validateIndexTemplate(validationException); + return validationException; + } + + public ActionRequestValidationException validateIndexTemplate(@Nullable ActionRequestValidationException validationException) { if (indexTemplate == null) { validationException = addValidationError("an index template is required", validationException); } else { @@ -99,7 +104,6 @@ public ActionRequestValidationException validate() { } } } - return validationException; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 4661cdb43602f..21636d7a96e6a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -502,7 +502,7 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu Collections.singletonList(templateName), metadataTransformer); } - static Map resolveV2Mappings(final String requestMappings, + public static Map resolveV2Mappings(final String requestMappings, final ClusterState currentState, final String templateName, final NamedXContentRegistry xContentRegistry) throws Exception { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 67c836e649c06..2d7cc67b3ce61 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -332,8 +332,7 @@ static void validateV2TemplateRequest(Metadata metadata, String name, IndexTempl } } - // Package visible for testing - ClusterState addIndexTemplateV2(final ClusterState currentState, final boolean create, + public ClusterState addIndexTemplateV2(final ClusterState currentState, final boolean create, final String name, final IndexTemplateV2 template) throws Exception { if (create && currentState.metadata().templatesV2().containsKey(name)) { throw new IllegalArgumentException("index template [" + name + "] already exists"); @@ -413,8 +412,8 @@ ClusterState addIndexTemplateV2(final ClusterState currentState, final boolean c * Return a map of v1 template names to their index patterns for v1 templates that would overlap * with the given v2 template's index patterns. */ - static Map> findConflictingV1Templates(final ClusterState state, final String candidateName, - final List indexPatterns) { + public static Map> findConflictingV1Templates(final ClusterState state, final String candidateName, + final List indexPatterns) { Automaton v2automaton = Regex.simpleMatchToAutomaton(indexPatterns.toArray(Strings.EMPTY_ARRAY)); Map> overlappingTemplates = new HashMap<>(); for (ObjectObjectCursor cursor : state.metadata().templates()) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java index f86952c65fde9..2497e986c97d9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java @@ -83,7 +83,7 @@ public Template(@Nullable Settings settings, @Nullable CompressedXContent mappin this.aliases = aliases; } - Template(StreamInput in) throws IOException { + public Template(StreamInput in) throws IOException { if (in.readBoolean()) { this.settings = Settings.readSettingsFromStream(in); } else { diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPostSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPostSimulateIndexTemplateAction.java new file mode 100644 index 0000000000000..3a18fe4aa361f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPostSimulateIndexTemplateAction.java @@ -0,0 +1,63 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.admin.indices; + +import org.elasticsearch.action.admin.indices.template.post.PostSimulateIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.post.SimulateIndexTemplateRequest; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateV2Action; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.metadata.IndexTemplateV2; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.POST; + +public class RestPostSimulateIndexTemplateAction extends BaseRestHandler { + + @Override + public List routes() { + return List.of( + new Route(POST, "/_index_template/_simulate_index/{name}")); + } + + @Override + public String getName() { + return "post_simulate_index_template_action"; + } + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + SimulateIndexTemplateRequest simulateIndexTemplateRequest = new SimulateIndexTemplateRequest(request.param("name")); + simulateIndexTemplateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", + simulateIndexTemplateRequest.masterNodeTimeout())); + if (request.hasContent()) { + PutIndexTemplateV2Action.Request indexTemplateRequest = new PutIndexTemplateV2Action.Request("simulating_template"); + indexTemplateRequest.indexTemplate(IndexTemplateV2.parse(request.contentParser())); + simulateIndexTemplateRequest.indexTemplateRequest(indexTemplateRequest); + } + + return channel -> client.execute(PostSimulateIndexTemplateAction.INSTANCE, simulateIndexTemplateRequest, + new RestToXContentListener<>(channel)); + } +} From 17a912ec388360b6fc2592fa98c1c74fb3307504 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Fri, 24 Apr 2020 11:03:50 +0100 Subject: [PATCH 02/24] Remove unused logger --- .../admin/indices/template/put/PutIndexTemplateRequest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index 1ad5cedb9332c..0be54db3b0f38 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.action.admin.indices.template.put; -import org.apache.logging.log4j.LogManager; import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; @@ -34,7 +33,6 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -65,8 +63,6 @@ */ public class PutIndexTemplateRequest extends MasterNodeRequest implements IndicesRequest { - private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(PutIndexTemplateRequest.class)); - private String name; private String cause = ""; From 490ca72dc0ad1c2d67f6c842a8c75ac93a767f40 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Fri, 24 Apr 2020 12:20:34 +0100 Subject: [PATCH 03/24] Tests --- .../client/RestHighLevelClientTests.java | 1 + .../10_basic.yml | 69 ++++++++++++++++ .../post/SimulateIndexTemplateRequest.java | 22 ++++- .../SimulateIndexTemplateRequestTests.java | 81 +++++++++++++++++++ 4 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml create mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index b067bd20c731c..8d188ad164a54 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -861,6 +861,7 @@ public void testApiNamingConventions() throws Exception { "indices.create_data_stream", "indices.get_data_streams", "indices.delete_data_stream", + "indices.post_simulate_index_template" }; //These API are not required for high-level client feature completeness String[] notRequiredApi = new String[] { diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml new file mode 100644 index 0000000000000..19e4ef32a7c4a --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml @@ -0,0 +1,69 @@ +--- +"Simulate index template without new template in the body": + - skip: + version: " - 7.7.99" + reason: "simulate index template API is unavailable before 7.8" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation" + indices.put_index_template: + name: test + body: + index_patterns: test-* + template: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + field: + type: keyword + + - do: + indices.post_simulate_index_template: + name: test + + - match: {template.settings.number_of_shards: 1} + - match: {template.settings.number_of_replicas: 0} + - match: {template.mappings._doc.properties.filed.type: "keyword"} + +--- +"Simulate index template specifying a new template": + - skip: + version: " - 7.7.99" + reason: "simulate index template API is unavailable before 7.8" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation" + indices.put_index_template: + name: test + body: + index_patterns: test-* + priority: 10 + template: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + field: + type: keyword + + - do: + indices.post_simulate_index_template: + name: test + body: + index_patterns: test-* + "priority": 15, + "template": { + "settings": { + "index.blocks.write": true + } + } + + + - match: {template.settings.index.blocks.write: true} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java index a0df587cf0334..d43e535e0f856 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.admin.indices.template.post; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateV2Action; import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.common.Nullable; @@ -29,6 +28,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; +import java.util.Objects; public class SimulateIndexTemplateRequest extends MasterNodeReadRequest { @@ -47,7 +47,7 @@ public SimulateIndexTemplateRequest(String indexName) { public SimulateIndexTemplateRequest(StreamInput in) throws IOException { super(in); indexName = in.readString(); - in.readOptionalWriteable(PutIndexTemplateRequest::new); + indexTemplateRequest = in.readOptionalWriteable(PutIndexTemplateV2Action.Request::new); } @Override @@ -84,4 +84,22 @@ public SimulateIndexTemplateRequest indexTemplateRequest(PutIndexTemplateV2Actio this.indexTemplateRequest = indexTemplateRequest; return this; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SimulateIndexTemplateRequest that = (SimulateIndexTemplateRequest) o; + return indexName.equals(that.indexName) && + Objects.equals(indexTemplateRequest, that.indexTemplateRequest); + } + + @Override + public int hashCode() { + return Objects.hash(indexName, indexTemplateRequest); + } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java new file mode 100644 index 0000000000000..ba0be33403fc0 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java @@ -0,0 +1,81 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.template.post; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateV2Action; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexTemplateV2; +import org.elasticsearch.cluster.metadata.IndexTemplateV2Tests; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; +import java.util.List; + +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +public class SimulateIndexTemplateRequestTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return SimulateIndexTemplateRequest::new; + } + + @Override + protected SimulateIndexTemplateRequest createTestInstance() { + SimulateIndexTemplateRequest req = new SimulateIndexTemplateRequest(randomAlphaOfLength(10)); + PutIndexTemplateV2Action.Request newTemplateRequest = new PutIndexTemplateV2Action.Request(randomAlphaOfLength(4)); + newTemplateRequest.indexTemplate(IndexTemplateV2Tests.randomInstance()); + req.indexTemplateRequest(newTemplateRequest); + return req; + } + + @Override + protected SimulateIndexTemplateRequest mutateInstance(SimulateIndexTemplateRequest instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + public void testIndexNameCannotBeNullOrEmpty() { + expectThrows(IllegalArgumentException.class, () -> new SimulateIndexTemplateRequest((String) null)); + expectThrows(IllegalArgumentException.class, () -> new SimulateIndexTemplateRequest("")); + } + + public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { + Template template = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(), null, null); + IndexTemplateV2 globalTemplate = new IndexTemplateV2(List.of("*"), template, null, null, null, null); + + PutIndexTemplateV2Action.Request request = new PutIndexTemplateV2Action.Request("test"); + request.indexTemplate(globalTemplate); + + SimulateIndexTemplateRequest simulateRequest = new SimulateIndexTemplateRequest("testing"); + simulateRequest.indexTemplateRequest(request); + + ActionRequestValidationException validationException = simulateRequest.validate(); + assertThat(validationException, is(notNullValue())); + List validationErrors = validationException.validationErrors(); + assertThat(validationErrors.size(), is(1)); + String error = validationErrors.get(0); + assertThat(error, is("global V2 templates may not specify the setting " + IndexMetadata.SETTING_INDEX_HIDDEN)); + } +} From 20eeea45f8d2d965cedbef03f19f0985ce935da2 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Fri, 24 Apr 2020 12:34:25 +0100 Subject: [PATCH 04/24] Yaml parsing and new line --- .../indices.post_simulate_index_template/10_basic.yml | 8 +++----- .../template/post/SimulateIndexTemplateRequestTests.java | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml index 19e4ef32a7c4a..db9c4afb97218 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml @@ -59,11 +59,9 @@ body: index_patterns: test-* "priority": 15, - "template": { - "settings": { - "index.blocks.write": true - } - } + "template": + "settings": + "index.blocks.write": true - match: {template.settings.index.blocks.write: true} diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java index ba0be33403fc0..e11a19bbc169e 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java @@ -36,7 +36,7 @@ import static org.hamcrest.Matchers.notNullValue; public class SimulateIndexTemplateRequestTests extends AbstractWireSerializingTestCase { - + @Override protected Writeable.Reader instanceReader() { return SimulateIndexTemplateRequest::new; From 871980a966bbbfb72d81e2658b9f507837c36fca Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Fri, 24 Apr 2020 14:18:54 +0100 Subject: [PATCH 05/24] Fix tests --- .../10_basic.yml | 99 ++++++++++++++++--- .../TransportSimulateIndexTemplateAction.java | 4 + 2 files changed, 90 insertions(+), 13 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml index db9c4afb97218..6742813bf6cff 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml @@ -7,11 +7,11 @@ - do: allowed_warnings: - - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation" + - "index template [test] has index patterns [te*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation" indices.put_index_template: name: test body: - index_patterns: test-* + index_patterns: te* template: settings: number_of_shards: 1 @@ -25,9 +25,10 @@ indices.post_simulate_index_template: name: test - - match: {template.settings.number_of_shards: 1} - - match: {template.settings.number_of_replicas: 0} - - match: {template.mappings._doc.properties.filed.type: "keyword"} + - match: {template.settings.index.number_of_shards: "1"} + - match: {template.settings.index.number_of_replicas: "0"} + - match: {template.mappings._doc.properties.field.type: "keyword"} + - match: {overlapping_v1_templates: {}} --- "Simulate index template specifying a new template": @@ -38,11 +39,11 @@ - do: allowed_warnings: - - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation" + - "index template [test] has index patterns [te*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation" indices.put_index_template: name: test body: - index_patterns: test-* + index_patterns: te* priority: 10 template: settings: @@ -57,11 +58,83 @@ indices.post_simulate_index_template: name: test body: - index_patterns: test-* - "priority": 15, - "template": - "settings": - "index.blocks.write": true + index_patterns: te* + priority: 15 + template: + settings: + index.blocks.write: true + + - match: {template.settings.index.blocks.write: "true"} + - match: {overlapping_v1_templates: {}} + +--- +"Simulate index template with index not matching any template": + - skip: + version: " - 7.7.99" + reason: "simulate index template API is unavailable before 7.8" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [test] has index patterns [te*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation" + indices.put_index_template: + name: test + body: + index_patterns: te* + priority: 10 + template: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + field: + type: keyword + + - do: + indices.post_simulate_index_template: + name: will_not_match + + - match: {body: null} +--- +"Simulate index matches overlapping V1 and V2 templates": + - skip: + version: " - 7.7.99" + reason: "simulate index template API is unavailable before 7.8" + features: allowed_warnings + + - do: + indices.put_template: + name: v1_template + body: + index_patterns: [t*, t1*] + settings: + number_of_shards: 5 + + - do: + allowed_warnings: + - "index template [test] has index patterns [te*] matching patterns from existing older templates [v1_template] with patterns + (v1_template => [t*, t1*]); this template [test] will take precedence during new index creation" + indices.put_index_template: + name: test + body: + index_patterns: te* + priority: 10 + template: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + field: + type: keyword + + - do: + indices.post_simulate_index_template: + name: test - - match: {template.settings.index.blocks.write: true} + - match: {template.settings.index.number_of_shards: "1"} + - match: {template.settings.index.number_of_replicas: "0"} + - match: {template.mappings._doc.properties.field.type: "keyword"} + - match: {overlapping_v1_templates: {v1_template: ["t*", "t1*"]}} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 62507b195a7d5..0a809ab5ea1cd 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -92,6 +92,10 @@ protected void masterOperation(Task task, SimulateIndexTemplateRequest request, } String matchingTemplate = findV2Template(simulateOnClusterState.metadata(), request.getIndexName(), false); + if (matchingTemplate == null) { + listener.onResponse(new SimulateIndexTemplateResponse(null, null)); + return; + } Settings settings = resolveSettings(simulateOnClusterState.metadata(), matchingTemplate); // empty request mapping as the user can't specify any explicit mappings via the simulate api From f34071d40356be7f21eeccaae6f1d654a0c8dbdb Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Fri, 24 Apr 2020 14:42:41 +0100 Subject: [PATCH 06/24] Fix version restriction --- .../10_basic.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml index 6742813bf6cff..4ee17aceaba31 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml @@ -1,8 +1,8 @@ --- "Simulate index template without new template in the body": - skip: - version: " - 7.7.99" - reason: "simulate index template API is unavailable before 7.8" + version: " - 7.99.99" + reason: "simulate index template API has not been backported" features: allowed_warnings - do: @@ -33,8 +33,8 @@ --- "Simulate index template specifying a new template": - skip: - version: " - 7.7.99" - reason: "simulate index template API is unavailable before 7.8" + version: " - 7.99.99" + reason: "simulate index template API has not been backported" features: allowed_warnings - do: @@ -70,8 +70,8 @@ --- "Simulate index template with index not matching any template": - skip: - version: " - 7.7.99" - reason: "simulate index template API is unavailable before 7.8" + version: " - 7.99.99" + reason: "simulate index template API has not been backported" features: allowed_warnings - do: @@ -100,8 +100,8 @@ --- "Simulate index matches overlapping V1 and V2 templates": - skip: - version: " - 7.7.99" - reason: "simulate index template API is unavailable before 7.8" + version: " - 7.99.99" + reason: "simulate index template API has not been backported" features: allowed_warnings - do: From 8ba793acb79d915f3cf849115981d4f4fd6cbb9c Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Fri, 24 Apr 2020 15:27:49 +0100 Subject: [PATCH 07/24] Fix response serialisation --- .../indices/template/post/SimulateIndexTemplateResponse.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java index 507f4d0c842c1..5f88829484f97 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java @@ -80,6 +80,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(entry.getKey()); out.writeStringCollection(entry.getValue()); } + } else { + out.writeBoolean(false); } } From 4b85534b3dbe6f18eddf4c02b2523ee89229edd5 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Fri, 24 Apr 2020 16:18:25 +0100 Subject: [PATCH 08/24] Rename PostSimulateIndexTemplateAction to SimulateIndexTemplateAction --- .../main/java/org/elasticsearch/action/ActionModule.java | 4 ++-- ...TemplateAction.java => SimulateIndexTemplateAction.java} | 6 +++--- .../template/post/TransportSimulateIndexTemplateAction.java | 2 +- .../admin/indices/RestPostSimulateIndexTemplateAction.java | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) rename server/src/main/java/org/elasticsearch/action/admin/indices/template/post/{PostSimulateIndexTemplateAction.java => SimulateIndexTemplateAction.java} (80%) diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 9bd34cb3cb9eb..6f03ec2185459 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -158,7 +158,7 @@ import org.elasticsearch.action.admin.indices.template.get.TransportGetComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.get.TransportGetIndexTemplateV2Action; import org.elasticsearch.action.admin.indices.template.get.TransportGetIndexTemplatesAction; -import org.elasticsearch.action.admin.indices.template.post.PostSimulateIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.post.SimulateIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.post.TransportSimulateIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction; @@ -553,7 +553,7 @@ public void reg actions.register(PutIndexTemplateV2Action.INSTANCE, TransportPutIndexTemplateV2Action.class); actions.register(GetIndexTemplateV2Action.INSTANCE, TransportGetIndexTemplateV2Action.class); actions.register(DeleteIndexTemplateV2Action.INSTANCE, TransportDeleteIndexTemplateV2Action.class); - actions.register(PostSimulateIndexTemplateAction.INSTANCE, TransportSimulateIndexTemplateAction.class); + actions.register(SimulateIndexTemplateAction.INSTANCE, TransportSimulateIndexTemplateAction.class); } actions.register(ValidateQueryAction.INSTANCE, TransportValidateQueryAction.class); actions.register(RefreshAction.INSTANCE, TransportRefreshAction.class); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/PostSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateAction.java similarity index 80% rename from server/src/main/java/org/elasticsearch/action/admin/indices/template/post/PostSimulateIndexTemplateAction.java rename to server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateAction.java index 77e8c1d8d223c..ae787c17259e4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/PostSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateAction.java @@ -21,12 +21,12 @@ import org.elasticsearch.action.ActionType; -public class PostSimulateIndexTemplateAction extends ActionType { +public class SimulateIndexTemplateAction extends ActionType { - public static final PostSimulateIndexTemplateAction INSTANCE = new PostSimulateIndexTemplateAction(); + public static final SimulateIndexTemplateAction INSTANCE = new SimulateIndexTemplateAction(); public static final String NAME = "indices:admin/index_template/simulate_index/post"; - private PostSimulateIndexTemplateAction() { + private SimulateIndexTemplateAction() { super(NAME, SimulateIndexTemplateResponse::new); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 0a809ab5ea1cd..a6ecef57f3318 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -64,7 +64,7 @@ public TransportSimulateIndexTemplateAction(TransportService transportService, C ThreadPool threadPool, MetadataIndexTemplateService indexTemplateService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, NamedXContentRegistry xContentRegistry) { - super(PostSimulateIndexTemplateAction.NAME, transportService, clusterService, threadPool, actionFilters, + super(SimulateIndexTemplateAction.NAME, transportService, clusterService, threadPool, actionFilters, SimulateIndexTemplateRequest::new, indexNameExpressionResolver); this.indexTemplateService = indexTemplateService; this.xContentRegistry = xContentRegistry; diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPostSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPostSimulateIndexTemplateAction.java index 3a18fe4aa361f..b43bfc9171d17 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPostSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPostSimulateIndexTemplateAction.java @@ -19,7 +19,7 @@ package org.elasticsearch.rest.action.admin.indices; -import org.elasticsearch.action.admin.indices.template.post.PostSimulateIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.post.SimulateIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.post.SimulateIndexTemplateRequest; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateV2Action; import org.elasticsearch.client.node.NodeClient; @@ -57,7 +57,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC simulateIndexTemplateRequest.indexTemplateRequest(indexTemplateRequest); } - return channel -> client.execute(PostSimulateIndexTemplateAction.INSTANCE, simulateIndexTemplateRequest, + return channel -> client.execute(SimulateIndexTemplateAction.INSTANCE, simulateIndexTemplateRequest, new RestToXContentListener<>(channel)); } } From b98a59bd6fc8a36a3f57f6eb07a5c0f4c5a942e6 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Fri, 24 Apr 2020 16:20:46 +0100 Subject: [PATCH 09/24] Rename RestPostSimulateIndexTemplateAction to RestSimulateIndexTemplateAction --- .../src/main/java/org/elasticsearch/action/ActionModule.java | 4 ++-- ...mplateAction.java => RestSimulateIndexTemplateAction.java} | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename server/src/main/java/org/elasticsearch/rest/action/admin/indices/{RestPostSimulateIndexTemplateAction.java => RestSimulateIndexTemplateAction.java} (97%) diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 6f03ec2185459..a49a6d7c313ac 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -312,7 +312,7 @@ import org.elasticsearch.rest.action.admin.indices.RestIndicesShardStoresAction; import org.elasticsearch.rest.action.admin.indices.RestIndicesStatsAction; import org.elasticsearch.rest.action.admin.indices.RestOpenIndexAction; -import org.elasticsearch.rest.action.admin.indices.RestPostSimulateIndexTemplateAction; +import org.elasticsearch.rest.action.admin.indices.RestSimulateIndexTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestPutComponentTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateV2Action; @@ -704,7 +704,7 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestPutIndexTemplateV2Action()); registerHandler.accept(new RestGetIndexTemplateV2Action()); registerHandler.accept(new RestDeleteIndexTemplateV2Action()); - registerHandler.accept(new RestPostSimulateIndexTemplateAction()); + registerHandler.accept(new RestSimulateIndexTemplateAction()); } registerHandler.accept(new RestPutMappingAction()); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPostSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java similarity index 97% rename from server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPostSimulateIndexTemplateAction.java rename to server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java index b43bfc9171d17..a2386aaa083f2 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPostSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java @@ -33,7 +33,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; -public class RestPostSimulateIndexTemplateAction extends BaseRestHandler { +public class RestSimulateIndexTemplateAction extends BaseRestHandler { @Override public List routes() { From cbcc9770decf5ed6932fd4471786eb2fa710947b Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Fri, 24 Apr 2020 17:28:50 +0100 Subject: [PATCH 10/24] Remove "post_" prefix --- .../elasticsearch/client/RestHighLevelClientTests.java | 2 +- ...template.json => indices.simulate_index_template.json} | 2 +- .../indices.post_simulate_index_template/10_basic.yml | 8 ++++---- .../admin/indices/RestSimulateIndexTemplateAction.java | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) rename rest-api-spec/src/main/resources/rest-api-spec/api/{indices.post_simulate_index_template.json => indices.simulate_index_template.json} (95%) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index 8d188ad164a54..2fd70731ab1d5 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -861,7 +861,7 @@ public void testApiNamingConventions() throws Exception { "indices.create_data_stream", "indices.get_data_streams", "indices.delete_data_stream", - "indices.post_simulate_index_template" + "indices.simulate_index_template" }; //These API are not required for high-level client feature completeness String[] notRequiredApi = new String[] { diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.post_simulate_index_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_index_template.json similarity index 95% rename from rest-api-spec/src/main/resources/rest-api-spec/api/indices.post_simulate_index_template.json rename to rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_index_template.json index ca72f8174d3ed..d87d7feb6beec 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.post_simulate_index_template.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_index_template.json @@ -1,5 +1,5 @@ { - "indices.post_simulate_index_template":{ + "indices.simulate_index_template":{ "documentation":{ "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html", "description": "Simulate matching the given index name against the index templates in the system" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml index 4ee17aceaba31..e706717e2034c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml @@ -22,7 +22,7 @@ type: keyword - do: - indices.post_simulate_index_template: + indices.simulate_index_template: name: test - match: {template.settings.index.number_of_shards: "1"} @@ -55,7 +55,7 @@ type: keyword - do: - indices.post_simulate_index_template: + indices.simulate_index_template: name: test body: index_patterns: te* @@ -92,7 +92,7 @@ type: keyword - do: - indices.post_simulate_index_template: + indices.simulate_index_template: name: will_not_match - match: {body: null} @@ -131,7 +131,7 @@ type: keyword - do: - indices.post_simulate_index_template: + indices.simulate_index_template: name: test - match: {template.settings.index.number_of_shards: "1"} diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java index a2386aaa083f2..a73af8019e781 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java @@ -43,7 +43,7 @@ public List routes() { @Override public String getName() { - return "post_simulate_index_template_action"; + return "simulate_index_template_action"; } @Override From 3ca7e204c195a86e2bce35ef6f234eb5af1308d9 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Mon, 27 Apr 2020 11:13:38 +0100 Subject: [PATCH 11/24] Drop /post from action name Co-Authored-By: Lee Hinman --- .../indices/template/post/SimulateIndexTemplateAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateAction.java index ae787c17259e4..370afe5af3748 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateAction.java @@ -24,7 +24,7 @@ public class SimulateIndexTemplateAction extends ActionType { public static final SimulateIndexTemplateAction INSTANCE = new SimulateIndexTemplateAction(); - public static final String NAME = "indices:admin/index_template/simulate_index/post"; + public static final String NAME = "indices:admin/index_template/simulate_index"; private SimulateIndexTemplateAction() { super(NAME, SimulateIndexTemplateResponse::new); From a436e39cfd973840e6b705ee07dc40203a8fb1c2 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Mon, 27 Apr 2020 11:16:56 +0100 Subject: [PATCH 12/24] Update rest api spec Co-Authored-By: Lee Hinman --- .../rest-api-spec/api/indices.simulate_index_template.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_index_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_index_template.json index d87d7feb6beec..c9a966e3fa137 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_index_template.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_index_template.json @@ -15,7 +15,7 @@ "parts":{ "name":{ "type":"string", - "description":"The name of the index (it will have to be a concrete index name)" + "description":"The name of the index (it must be a concrete index name)" } } } From 604936a60e8476be976327fb5ab1543e938bddc3 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Mon, 27 Apr 2020 15:06:57 +0100 Subject: [PATCH 13/24] Drop "post" from package name --- .../10_basic.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename rest-api-spec/src/main/resources/rest-api-spec/test/{indices.post_simulate_index_template => indices.simulate_index_template}/10_basic.yml (100%) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml similarity index 100% rename from rest-api-spec/src/main/resources/rest-api-spec/test/indices.post_simulate_index_template/10_basic.yml rename to rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml From edc86109550bba6903b89250161665d934943921 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Mon, 27 Apr 2020 16:03:33 +0100 Subject: [PATCH 14/24] Fix indentation --- .../cluster/metadata/MetadataCreateIndexService.java | 6 +++--- .../cluster/metadata/MetadataIndexTemplateService.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 21636d7a96e6a..0dcb29e48ae8c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -503,9 +503,9 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu } public static Map resolveV2Mappings(final String requestMappings, - final ClusterState currentState, - final String templateName, - final NamedXContentRegistry xContentRegistry) throws Exception { + final ClusterState currentState, + final String templateName, + final NamedXContentRegistry xContentRegistry) throws Exception { final Map mappings = Collections.unmodifiableMap(parseV2Mappings(requestMappings, MetadataIndexTemplateService.resolveMappings(currentState, templateName), xContentRegistry)); return mappings; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 2d7cc67b3ce61..121f5c12e9e7a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -333,7 +333,7 @@ static void validateV2TemplateRequest(Metadata metadata, String name, IndexTempl } public ClusterState addIndexTemplateV2(final ClusterState currentState, final boolean create, - final String name, final IndexTemplateV2 template) throws Exception { + final String name, final IndexTemplateV2 template) throws Exception { if (create && currentState.metadata().templatesV2().containsKey(name)) { throw new IllegalArgumentException("index template [" + name + "] already exists"); } From 34d7a75c92c8ba7d25cf62b5d3e65f13d0146621 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Mon, 27 Apr 2020 16:05:23 +0100 Subject: [PATCH 15/24] Use the resolveAndValidateAliases method to resolve aliases --- .../10_basic.yml | 4 + .../TransportSimulateIndexTemplateAction.java | 73 +++++++++++++++---- .../metadata/MetadataCreateIndexService.java | 7 +- 3 files changed, 65 insertions(+), 19 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml index e706717e2034c..748e086b39efa 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml @@ -63,9 +63,13 @@ template: settings: index.blocks.write: true + aliases: + test_alias: {} - match: {template.settings.index.blocks.write: "true"} - match: {overlapping_v1_templates: {}} + - length: {template.aliases: 1} + - is_true: template.aliases.test_alias --- "Simulate index template with index not matching any template": diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index a6ecef57f3318..9a932f51b78c2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.indices.template.post; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; @@ -26,8 +27,12 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.AliasValidator; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.IndexTemplateV2; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.service.ClusterService; @@ -39,35 +44,47 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.Closeable; import java.io.IOException; -import java.util.HashMap; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.MetadataCreateIndexService.resolveV2Mappings; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveSettings; +import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED; public class TransportSimulateIndexTemplateAction extends TransportMasterNodeReadAction { private final MetadataIndexTemplateService indexTemplateService; private final NamedXContentRegistry xContentRegistry; + private final IndicesService indicesService; + private AliasValidator aliasValidator; @Inject public TransportSimulateIndexTemplateAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool, MetadataIndexTemplateService indexTemplateService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - NamedXContentRegistry xContentRegistry) { + NamedXContentRegistry xContentRegistry, IndicesService indicesService) { super(SimulateIndexTemplateAction.NAME, transportService, clusterService, threadPool, actionFilters, SimulateIndexTemplateRequest::new, indexNameExpressionResolver); this.indexTemplateService = indexTemplateService; this.xContentRegistry = xContentRegistry; + this.indicesService = indicesService; + this.aliasValidator = new AliasValidator(); } @Override @@ -107,22 +124,46 @@ protected void masterOperation(Task task, SimulateIndexTemplateRequest request, List> resolvedAliases = MetadataIndexTemplateService.resolveAliases(simulateOnClusterState.metadata(), matchingTemplate); - Map aliases = new HashMap<>(); - for (Map alias : resolvedAliases) { - for (Map.Entry aliasEntry : alias.entrySet()) { - if (aliases.containsKey(aliasEntry.getKey()) == false) { - aliases.put(aliasEntry.getKey(), aliasEntry.getValue()); - } + + // create the index with dummy settings in the cluster state so we can parse and validate the aliases + Settings dummySettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(settings) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) + .build(); + final IndexMetadata indexMetadata = IndexMetadata.builder(request.getIndexName()).settings(dummySettings).build(); + + simulateOnClusterState = ClusterState.builder(simulateOnClusterState) + .metadata(Metadata.builder(simulateOnClusterState.metadata()) + .put(indexMetadata, true) + .build()) + .build(); + + IndexService tempIndexService = indicesService.createIndex(indexMetadata, Collections.emptyList(), false); + final Index index = tempIndexService.index(); + try (Closeable dummy = () -> tempIndexService.close("temp", false)) { + List aliases = MetadataCreateIndexService.resolveAndValidateAliases(request.getIndexName(), Set.of(), + resolvedAliases, simulateOnClusterState.metadata(), aliasValidator, xContentRegistry, + // the context is only used for validation so it's fine to pass fake values for the + // shard id and the current timestamp + tempIndexService.newQueryShardContext(0, null, () -> 0L, null)); + + IndexTemplateV2 templateV2 = simulateOnClusterState.metadata().templatesV2().get(matchingTemplate); + assert templateV2 != null : "the matched template must exist"; + + Map> conflictingV1Templates = + MetadataIndexTemplateService.findConflictingV1Templates(simulateOnClusterState + , matchingTemplate, templateV2.indexPatterns()); + Template template = new Template(settings, mappingsJson == null ? null : new CompressedXContent(mappingsJson), + aliases.stream().collect(Collectors.toMap(AliasMetadata::getAlias, Function.identity()))); + listener.onResponse(new SimulateIndexTemplateResponse(template, conflictingV1Templates)); + } finally { + if (index != null) { + indicesService.removeIndex(index, NO_LONGER_ASSIGNED, "created to validate template aliases"); } } - - IndexTemplateV2 templateV2 = simulateOnClusterState.metadata().templatesV2().get(matchingTemplate); - assert templateV2 != null : "the matched template must exist"; - - Map> conflictingV1Templates = MetadataIndexTemplateService.findConflictingV1Templates(simulateOnClusterState - , matchingTemplate, templateV2.indexPatterns()); - Template template = new Template(settings, mappingsJson == null ? null : new CompressedXContent(mappingsJson), aliases); - listener.onResponse(new SimulateIndexTemplateResponse(template, conflictingV1Templates)); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 0dcb29e48ae8c..badcefe4518fc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -735,9 +735,10 @@ static int getIndexNumberOfRoutingShards(Settings indexSettings, @Nullable Index * @return the list of resolved aliases, with the explicitly provided aliases occurring first (having a higher priority) followed by * the ones inherited from the templates */ - static List resolveAndValidateAliases(String index, Set aliases, List> templateAliases, - Metadata metadata, AliasValidator aliasValidator, - NamedXContentRegistry xContentRegistry, QueryShardContext queryShardContext) { + public static List resolveAndValidateAliases(String index, Set aliases, + List> templateAliases, Metadata metadata, + AliasValidator aliasValidator, NamedXContentRegistry xContentRegistry, + QueryShardContext queryShardContext) { List resolvedAliases = new ArrayList<>(); for (Alias alias : aliases) { aliasValidator.validateAlias(alias, index, metadata); From 6a319da3fc7bc0c290d07335125d694ef95ce679 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Mon, 27 Apr 2020 16:09:12 +0100 Subject: [PATCH 16/24] Rename overlapping_v1_templates to overlapping --- .../10_basic.yml | 6 ++-- .../post/SimulateIndexTemplateResponse.java | 30 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml index 748e086b39efa..52d9015a006d7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml @@ -28,7 +28,7 @@ - match: {template.settings.index.number_of_shards: "1"} - match: {template.settings.index.number_of_replicas: "0"} - match: {template.mappings._doc.properties.field.type: "keyword"} - - match: {overlapping_v1_templates: {}} + - match: {overlapping: {}} --- "Simulate index template specifying a new template": @@ -67,7 +67,7 @@ test_alias: {} - match: {template.settings.index.blocks.write: "true"} - - match: {overlapping_v1_templates: {}} + - match: {overlapping: {}} - length: {template.aliases: 1} - is_true: template.aliases.test_alias @@ -141,4 +141,4 @@ - match: {template.settings.index.number_of_shards: "1"} - match: {template.settings.index.number_of_replicas: "0"} - match: {template.mappings._doc.properties.field.type: "keyword"} - - match: {overlapping_v1_templates: {v1_template: ["t*", "t1*"]}} + - match: {overlapping: {v1_template: ["t*", "t1*"]}} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java index 5f88829484f97..b482659f212c7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java @@ -40,7 +40,7 @@ public class SimulateIndexTemplateResponse extends ActionResponse implements ToXContentObject { private static final ParseField TEMPLATE = new ParseField("template"); - private static final ParseField OVERLAPPING_V1_TEMPLATES = new ParseField("overlapping_v1_templates"); + private static final ParseField OVERLAPPING = new ParseField("overlapping"); @Nullable // the resolved settings, mappings and aliases for the matched templates, if any @@ -48,11 +48,11 @@ public class SimulateIndexTemplateResponse extends ActionResponse implements ToX @Nullable // a map of v1 template names and their index patterns that would overlap when matching the given index name - private Map> overlappingV1Templates; + private Map> overlappingTemplates; - public SimulateIndexTemplateResponse(@Nullable Template resolvedTemplate, @Nullable Map> overlappingV1Templates) { + public SimulateIndexTemplateResponse(@Nullable Template resolvedTemplate, @Nullable Map> overlappingTemplates) { this.resolvedTemplate = resolvedTemplate; - this.overlappingV1Templates = overlappingV1Templates; + this.overlappingTemplates = overlappingTemplates; } public SimulateIndexTemplateResponse(StreamInput in) throws IOException { @@ -60,23 +60,23 @@ public SimulateIndexTemplateResponse(StreamInput in) throws IOException { resolvedTemplate = in.readOptionalWriteable(Template::new); if (in.readBoolean()) { int conflictingV1TemplatesCount = in.readInt(); - overlappingV1Templates = new HashMap<>(conflictingV1TemplatesCount, 1L); + overlappingTemplates = new HashMap<>(conflictingV1TemplatesCount, 1L); for (int i = 0; i < conflictingV1TemplatesCount; i++) { String templateName = in.readString(); - overlappingV1Templates.put(templateName, in.readStringList()); + overlappingTemplates.put(templateName, in.readStringList()); } } else { - this.overlappingV1Templates = null; + this.overlappingTemplates = null; } } @Override public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(resolvedTemplate); - if (overlappingV1Templates != null) { + if (overlappingTemplates != null) { out.writeBoolean(true); - out.writeInt(overlappingV1Templates.size()); - for (Map.Entry> entry : overlappingV1Templates.entrySet()) { + out.writeInt(overlappingTemplates.size()); + for (Map.Entry> entry : overlappingTemplates.entrySet()) { out.writeString(entry.getKey()); out.writeStringCollection(entry.getValue()); } @@ -91,8 +91,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (this.resolvedTemplate != null) { builder.field(TEMPLATE.getPreferredName(), this.resolvedTemplate); } - if (this.overlappingV1Templates != null) { - builder.field(OVERLAPPING_V1_TEMPLATES.getPreferredName(), overlappingV1Templates); + if (this.overlappingTemplates != null) { + builder.field(OVERLAPPING.getPreferredName(), overlappingTemplates); } builder.endObject(); return builder; @@ -108,17 +108,17 @@ public boolean equals(Object o) { } SimulateIndexTemplateResponse that = (SimulateIndexTemplateResponse) o; return Objects.equals(resolvedTemplate, that.resolvedTemplate) - && Objects.deepEquals(overlappingV1Templates, that.overlappingV1Templates); + && Objects.deepEquals(overlappingTemplates, that.overlappingTemplates); } @Override public int hashCode() { - return Objects.hash(resolvedTemplate, overlappingV1Templates); + return Objects.hash(resolvedTemplate, overlappingTemplates); } @Override public String toString() { return "SimulateIndexTemplateResponse{" + "resolved template=" + resolvedTemplate + ", overlapping v1 templates=" - + String.join("|", overlappingV1Templates.keySet()) + "}"; + + String.join("|", overlappingTemplates.keySet()) + "}"; } } From 41342820266a7092b6a4ae88239ddcb0922456f1 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Tue, 28 Apr 2020 12:03:25 +0100 Subject: [PATCH 17/24] Update methods docs and fix use case where candidate exists --- .../metadata/MetadataIndexTemplateService.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 121f5c12e9e7a..3e451f3ca7641 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -431,7 +431,7 @@ public static Map> findConflictingV1Templates(final Cluster /** * Return a map of v2 template names to their index patterns for v2 templates that would overlap - * with the given v1 template's index patterns. + * with the given template's index patterns. */ static Map> findConflictingV2Templates(final ClusterState state, final String candidateName, final List indexPatterns) { @@ -441,6 +441,12 @@ static Map> findConflictingV2Templates(final ClusterState s /** * Return a map of v2 template names to their index patterns for v2 templates that would overlap * with the given template's index patterns. + * + * Based on the provided checkPriority and priority parameters this aims to report the overlapping + * index templates regardless of the priority (ie. checkPriority == false) or otherwise overlapping + * templates with the same priority as the given priority parameter (this is useful when trying to + * add a new template, as we don't support multiple overlapping, from an index pattern perspective, + * index templates with the same priority). */ static Map> findConflictingV2Templates(final ClusterState state, final String candidateName, final List indexPatterns, boolean checkPriority, Long priority) { @@ -458,6 +464,9 @@ static Map> findConflictingV2Templates(final ClusterState s } } } + // if the candidate was a V2 template that already exists in the cluster state it will "overlap" with itself so remove it from the + // results + overlappingTemplates.remove(candidateName); return overlappingTemplates; } From facb14adaa80db06dc0932688bf245c8906b661e Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Tue, 28 Apr 2020 12:50:37 +0100 Subject: [PATCH 18/24] Return overlapping V1 and V2 templates and their patterns --- .../10_basic.yml | 30 +++++++++++++++---- .../TransportSimulateIndexTemplateAction.java | 11 ++++--- .../MetadataIndexTemplateService.java | 4 +-- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml index 52d9015a006d7..161cf0677ab6c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml @@ -41,7 +41,7 @@ allowed_warnings: - "index template [test] has index patterns [te*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation" indices.put_index_template: - name: test + name: existing_test body: index_patterns: te* priority: 10 @@ -67,7 +67,7 @@ test_alias: {} - match: {template.settings.index.blocks.write: "true"} - - match: {overlapping: {}} + - match: {overlapping: {existing_test: ["te*"]}} - length: {template.aliases: 1} - is_true: template.aliases.test_alias @@ -118,13 +118,31 @@ - do: allowed_warnings: - - "index template [test] has index patterns [te*] matching patterns from existing older templates [v1_template] with patterns - (v1_template => [t*, t1*]); this template [test] will take precedence during new index creation" + - "index template [v2_template] has index patterns [te*] matching patterns from existing older templates [v1_template] with patterns + (v1_template => [t*, t1*]); this template [v2_template] will take precedence during new index creation" indices.put_index_template: - name: test + name: v2_template body: index_patterns: te* priority: 10 + template: + settings: + number_of_shards: 10 + number_of_replicas: 2 + mappings: + properties: + field: + type: text + + - do: + allowed_warnings: + - "index template [winning_v2_template] has index patterns [te*] matching patterns from existing older templates [v1_template] with patterns + (v1_template => [t*, t1*]); this template [winning_v2_template] will take precedence during new index creation" + indices.put_index_template: + name: winning_v2_template + body: + index_patterns: te* + priority: 20 template: settings: number_of_shards: 1 @@ -141,4 +159,4 @@ - match: {template.settings.index.number_of_shards: "1"} - match: {template.settings.index.number_of_replicas: "0"} - match: {template.mappings._doc.properties.field.type: "keyword"} - - match: {overlapping: {v1_template: ["t*", "t1*"]}} + - match: {overlapping: {v1_template: ["t*", "t1*"], v2_template: ["te*"]}} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 9a932f51b78c2..a5857ef8fd7a0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -62,6 +62,8 @@ import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.MetadataCreateIndexService.resolveV2Mappings; +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV1Templates; +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV2Templates; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveSettings; import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED; @@ -153,12 +155,13 @@ protected void masterOperation(Task task, SimulateIndexTemplateRequest request, IndexTemplateV2 templateV2 = simulateOnClusterState.metadata().templatesV2().get(matchingTemplate); assert templateV2 != null : "the matched template must exist"; - Map> conflictingV1Templates = - MetadataIndexTemplateService.findConflictingV1Templates(simulateOnClusterState - , matchingTemplate, templateV2.indexPatterns()); + Map> overlapping = findConflictingV1Templates(simulateOnClusterState, matchingTemplate, + templateV2.indexPatterns()); + overlapping.putAll(findConflictingV2Templates(simulateOnClusterState, matchingTemplate, templateV2.indexPatterns())); + Template template = new Template(settings, mappingsJson == null ? null : new CompressedXContent(mappingsJson), aliases.stream().collect(Collectors.toMap(AliasMetadata::getAlias, Function.identity()))); - listener.onResponse(new SimulateIndexTemplateResponse(template, conflictingV1Templates)); + listener.onResponse(new SimulateIndexTemplateResponse(template, overlapping)); } finally { if (index != null) { indicesService.removeIndex(index, NO_LONGER_ASSIGNED, "created to validate template aliases"); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 3e451f3ca7641..5a336d093732b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -433,8 +433,8 @@ public static Map> findConflictingV1Templates(final Cluster * Return a map of v2 template names to their index patterns for v2 templates that would overlap * with the given template's index patterns. */ - static Map> findConflictingV2Templates(final ClusterState state, final String candidateName, - final List indexPatterns) { + public static Map> findConflictingV2Templates(final ClusterState state, final String candidateName, + final List indexPatterns) { return findConflictingV2Templates(state, candidateName, indexPatterns, false, null); } From 651b578087625e222bd7c9d1a8825a8f2130a906 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Tue, 28 Apr 2020 13:00:18 +0100 Subject: [PATCH 19/24] Remove v1 mentions --- .../template/post/SimulateIndexTemplateResponse.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java index b482659f212c7..7c4d1bb7adace 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java @@ -47,7 +47,7 @@ public class SimulateIndexTemplateResponse extends ActionResponse implements ToX private Template resolvedTemplate; @Nullable - // a map of v1 template names and their index patterns that would overlap when matching the given index name + // a map of template names and their index patterns that would overlap when matching the given index name private Map> overlappingTemplates; public SimulateIndexTemplateResponse(@Nullable Template resolvedTemplate, @Nullable Map> overlappingTemplates) { @@ -59,9 +59,9 @@ public SimulateIndexTemplateResponse(StreamInput in) throws IOException { super(in); resolvedTemplate = in.readOptionalWriteable(Template::new); if (in.readBoolean()) { - int conflictingV1TemplatesCount = in.readInt(); - overlappingTemplates = new HashMap<>(conflictingV1TemplatesCount, 1L); - for (int i = 0; i < conflictingV1TemplatesCount; i++) { + int overlappingTemplatesCount = in.readInt(); + overlappingTemplates = new HashMap<>(overlappingTemplatesCount, 1L); + for (int i = 0; i < overlappingTemplatesCount; i++) { String templateName = in.readString(); overlappingTemplates.put(templateName, in.readStringList()); } @@ -118,7 +118,7 @@ public int hashCode() { @Override public String toString() { - return "SimulateIndexTemplateResponse{" + "resolved template=" + resolvedTemplate + ", overlapping v1 templates=" + return "SimulateIndexTemplateResponse{" + "resolved template=" + resolvedTemplate + ", overlapping templates=" + String.join("|", overlappingTemplates.keySet()) + "}"; } } From 03d9b0fc8f5e29a695dc95f3f68301029d37eb9f Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Tue, 28 Apr 2020 17:09:29 +0100 Subject: [PATCH 20/24] Drop dynamic keys from response --- .../test/indices.simulate_index_template/10_basic.yml | 8 ++++++-- .../template/post/SimulateIndexTemplateResponse.java | 11 ++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml index 161cf0677ab6c..22eea620a7a4f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml @@ -67,7 +67,8 @@ test_alias: {} - match: {template.settings.index.blocks.write: "true"} - - match: {overlapping: {existing_test: ["te*"]}} + - match: {overlapping.0.name: existing_test} + - match: {overlapping.0.index_patterns: ["te*"]} - length: {template.aliases: 1} - is_true: template.aliases.test_alias @@ -159,4 +160,7 @@ - match: {template.settings.index.number_of_shards: "1"} - match: {template.settings.index.number_of_replicas: "0"} - match: {template.mappings._doc.properties.field.type: "keyword"} - - match: {overlapping: {v1_template: ["t*", "t1*"], v2_template: ["te*"]}} + - match: {overlapping.0.name: v1_template} + - match: {overlapping.0.index_patterns: ["t*", "t1*"]} + - match: {overlapping.1.name: v2_template} + - match: {overlapping.1.index_patterns: ["te*"]} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java index 7c4d1bb7adace..a5d2756387038 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java @@ -41,6 +41,8 @@ public class SimulateIndexTemplateResponse extends ActionResponse implements ToX private static final ParseField TEMPLATE = new ParseField("template"); private static final ParseField OVERLAPPING = new ParseField("overlapping"); + private static final ParseField NAME = new ParseField("name"); + private static final ParseField INDEX_PATTERNS = new ParseField("index_patterns"); @Nullable // the resolved settings, mappings and aliases for the matched templates, if any @@ -92,7 +94,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(TEMPLATE.getPreferredName(), this.resolvedTemplate); } if (this.overlappingTemplates != null) { - builder.field(OVERLAPPING.getPreferredName(), overlappingTemplates); + builder.startArray(OVERLAPPING.getPreferredName()); + for (Map.Entry> entry : overlappingTemplates.entrySet()) { + builder.startObject(); + builder.field(NAME.getPreferredName(), entry.getKey()); + builder.field(INDEX_PATTERNS.getPreferredName(), entry.getValue()); + builder.endObject(); + } + builder.endArray(); } builder.endObject(); return builder; From 8863d1f0e347ee1896c4e6d0635a80c04462987b Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Tue, 28 Apr 2020 17:12:19 +0100 Subject: [PATCH 21/24] Use additional map --- .../template/post/TransportSimulateIndexTemplateAction.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index a5857ef8fd7a0..c0adc27394c53 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -55,6 +55,7 @@ import java.io.Closeable; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -155,8 +156,8 @@ protected void masterOperation(Task task, SimulateIndexTemplateRequest request, IndexTemplateV2 templateV2 = simulateOnClusterState.metadata().templatesV2().get(matchingTemplate); assert templateV2 != null : "the matched template must exist"; - Map> overlapping = findConflictingV1Templates(simulateOnClusterState, matchingTemplate, - templateV2.indexPatterns()); + Map> overlapping = new HashMap<>(); + overlapping.putAll(findConflictingV1Templates(simulateOnClusterState, matchingTemplate, templateV2.indexPatterns())); overlapping.putAll(findConflictingV2Templates(simulateOnClusterState, matchingTemplate, templateV2.indexPatterns())); Template template = new Template(settings, mappingsJson == null ? null : new CompressedXContent(mappingsJson), From db5782bca51e856de14f83aeaaa23c183a80f135 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Tue, 28 Apr 2020 17:16:05 +0100 Subject: [PATCH 22/24] Update remove indexService reason --- .../template/post/TransportSimulateIndexTemplateAction.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index c0adc27394c53..0bc91c9c0276a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -165,7 +165,8 @@ protected void masterOperation(Task task, SimulateIndexTemplateRequest request, listener.onResponse(new SimulateIndexTemplateResponse(template, overlapping)); } finally { if (index != null) { - indicesService.removeIndex(index, NO_LONGER_ASSIGNED, "created to validate template aliases"); + indicesService.removeIndex(index, NO_LONGER_ASSIGNED, + "created as part of a simulation for an index name matching the index templates in the system"); } } } From ab918fde8a8deeacf66dd85a5e269cecf1300aaf Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Tue, 28 Apr 2020 18:15:39 +0100 Subject: [PATCH 23/24] Use composed_of in the simulated template --- .../indices.simulate_index_template/10_basic.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml index 22eea620a7a4f..7eb888bee0d30 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml @@ -54,6 +54,18 @@ field: type: keyword + - do: + cluster.put_component_template: + name: ct + body: + template: + settings: + index.number_of_replicas: 2 + mappings: + properties: + ct_field: + type: keyword + - do: indices.simulate_index_template: name: test @@ -65,8 +77,11 @@ index.blocks.write: true aliases: test_alias: {} + composed_of: ["ct"] - match: {template.settings.index.blocks.write: "true"} + - match: {template.settings.index.number_of_replicas: "2"} + - match: {template.mappings._doc.properties.ct_field.type: "keyword"} - match: {overlapping.0.name: existing_test} - match: {overlapping.0.index_patterns: ["te*"]} - length: {template.aliases: 1} From e36864758147a5e90ffc6baef7b43a60318384c4 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Tue, 28 Apr 2020 18:16:23 +0100 Subject: [PATCH 24/24] Assert overlapping is array --- .../test/indices.simulate_index_template/10_basic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml index 7eb888bee0d30..1fbb64a77a641 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml @@ -28,7 +28,7 @@ - match: {template.settings.index.number_of_shards: "1"} - match: {template.settings.index.number_of_replicas: "0"} - match: {template.mappings._doc.properties.field.type: "keyword"} - - match: {overlapping: {}} + - match: {overlapping: []} --- "Simulate index template specifying a new template":