Adding new require_alias option to indexing requests

docs/reference/docs/index_.asciidoc

@@ -25,9 +25,9 @@ and <<update-delete-docs-in-a-backing-index>>.
 
 `POST /<target>/_create/<_id>`
 
-IMPORTANT: You cannot add new documents to a data stream using the 
-`PUT /<target>/_doc/<_id>` request format. To specify a document ID, use the 
-`PUT /<target>/_create/<_id>` format instead. See 
+IMPORTANT: You cannot add new documents to a data stream using the
+`PUT /<target>/_doc/<_id>` request format. To specify a document ID, use the
+`PUT /<target>/_create/<_id>` format instead. See
 <<add-documents-to-a-data-stream>>.
 
 [[docs-index-api-path-params]]
@@ -94,6 +94,8 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=version_type]
 
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards]
 
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=no_auto_create]
+
 [[docs-index-api-request-body]]
 ==== {api-request-body-title}
 

docs/reference/docs/update.asciidoc

@@ -53,6 +53,8 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=if_primary_term]
 `lang`::
 (Optional, string) The script language. Default: `painless`.
 
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=no_auto_create]
+
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=refresh]
 
 `retry_on_conflict`::

docs/reference/rest-api/common-parms.asciidoc

@@ -570,6 +570,12 @@ such as `1264`.
 A value of `-1` indicates {es} was unable to compute this number.
 end::memory[]
 
+tag::no_auto_create[]
+`no_auto_create`::
+(Optional, boolean) When true, this prevents the request from auto creating the
+index. When false, auto creation behavior defaults to the configured settings
+(See <<index-creation, Index Creation>>). Defaults to false.
+
 tag::node-filter[]
 `<node_filter>`::
 (Optional, string)

modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexValidator.java

@@ -111,7 +111,7 @@ static void validateAgainstAliases(SearchRequest source, IndexRequest destinatio
             return;
         }
         String target = destination.index();
-        if (false == autoCreateIndex.shouldAutoCreate(target, clusterState)) {
+        if (false == autoCreateIndex.shouldAutoCreate(target, destination.isNoAutoCreate(), clusterState)) {
             /*
              * If we're going to autocreate the index we don't need to resolve
              * it. This is the same sort of dance that TransportIndexRequest

rest-api-spec/src/main/resources/rest-api-spec/api/index.json

@@ -92,6 +92,10 @@
       "pipeline":{
         "type":"string",
         "description":"The pipeline id to preprocess incoming documents with"
+      },
+      "no_auto_create": {
+        "type": "boolean",
+        "description": "When true, disallows this specific request from creating the specified index. Default is false and relies on the configured auto creation settings"
       }
     },
     "body":{

rest-api-spec/src/main/resources/rest-api-spec/api/update.json

@@ -99,6 +99,10 @@
       "if_primary_term":{
         "type":"number",
         "description":"only perform the update operation if the last operation that has changed the document has the specified primary term"
+      },
+      "no_auto_create": {
+        "type": "boolean",
+        "description": "When true, disallows this specific request from creating the specified index. Default is false and relies on the configured auto creation settings"
       }
     },
     "body":{

rest-api-spec/src/main/resources/rest-api-spec/test/bulk/10_basic.yml

@@ -116,3 +116,36 @@
            {"index": {"_index": "test_index", "_id": "test_id"}}
            {"f1": "v1", "f2": 42}
            {}
+
+---
+"When setting no_auto_create flag":
+  - skip:
+      # TODO adjust after backport
+      version: " - 7.99.99"
+      reason: "no_auto_create flag was added in version 7.9"
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: new_index_not_created
+              no_auto_create: true
+          - f: 1
+          - index:
+              _index: new_index_created
+          - f: 2
+          - index:
+              _index: another_index_created
+              no_auto_create: true
+          - f: 3
+          - create:
+              _index: another_index_created
+          - f: 4
+  - match: { errors: true }
+  - match: { items.0.index.status: 404 }
+  - match: { items.0.index.error.type: index_not_found_exception }
+  - match: { items.0.index.error.reason: "no such index [new_index_not_created] and [no_auto_create] request flag is [true]" }
+  - match: { items.1.index.result: created }
+  - match: { items.2.index.result: created }
+  - match: { items.3.create.result: created }

rest-api-spec/src/main/resources/rest-api-spec/test/index/70_auto_create.yml

@@ -0,0 +1,18 @@
+---
+"Set no_auto_create flag":
+  - skip:
+      # TODO adjust after backport
+      version: " - 7.99.99"
+      reason: "no_auto_create flag added in 7.9+"
+  - do:
+      catch: missing
+      index:
+        index:   test_no_auto_create
+        no_auto_create: true
+        body:    { foo: bar }
+
+  - do:
+      index:
+        index:   test_no_auto_create
+        no_auto_create: false
+        body:    { foo: bar }

rest-api-spec/src/main/resources/rest-api-spec/test/update/95_auto_create.yml

@@ -0,0 +1,23 @@
+---
+"Set no_auto_create flag":
+  - skip:
+      # TODO adjust after backport
+      version: " - 7.99.99"
+      reason: "no_auto_create flag added in 7.9+"
+  - do:
+      catch: missing
+      update:
+        index:            test_1_autocreate
+        id:               1
+        no_auto_create:   true
+        body:
+          doc:            { foo: bar, count: 1 }
+          doc_as_upsert:  true
+  - do:
+      update:
+        index:            test_1_autocreate
+        id:               1
+        no_auto_create:   false
+        body:
+          doc:            { foo: bar, count: 1 }
+          doc_as_upsert:  true

server/src/main/java/org/elasticsearch/action/DocWriteRequest.java

@@ -43,6 +43,9 @@
  */
 public interface DocWriteRequest<T> extends IndicesRequest, Accountable {
 
+    // Flag set for disallowing index auto creation for an individual write request.
+    String NO_AUTO_CREATE = "no_auto_create";
+
     /**
      * Set the index for this request
      * @return the Request
@@ -142,6 +145,11 @@ public interface DocWriteRequest<T> extends IndicesRequest, Accountable {
      */
     OpType opType();
 
+    /**
+     * Should this request override autocreate settings and specifically disallow the index being auto created?
+     * @return boolean flag, when true specifically disallows index autocreation
+     */
+    boolean isNoAutoCreate();
     /**
      * Requested operation type to perform on the document
      */

server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.bulk;
 
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.update.UpdateRequest;
@@ -62,6 +63,7 @@ public final class BulkRequestParser {
     private static final ParseField SOURCE = new ParseField("_source");
     private static final ParseField IF_SEQ_NO = new ParseField("if_seq_no");
     private static final ParseField IF_PRIMARY_TERM = new ParseField("if_primary_term");
+    private static final ParseField NO_AUTO_CREATE = new ParseField(DocWriteRequest.NO_AUTO_CREATE);
 
     // TODO: Remove this parameter once the BulkMonitoring endpoint has been removed
     private final boolean errorOnType;
@@ -166,6 +168,7 @@ public void parse(
                 long ifPrimaryTerm = UNASSIGNED_PRIMARY_TERM;
                 int retryOnConflict = 0;
                 String pipeline = defaultPipeline;
+                boolean noAutoCreate = false;
 
                 // at this stage, next token can either be END_OBJECT (and use default index and type, with auto generated id)
                 // or START_OBJECT which will have another set of parameters
@@ -208,6 +211,8 @@ public void parse(
                                 pipeline = stringDeduplicator.computeIfAbsent(parser.text(), Function.identity());
                             } else if (SOURCE.match(currentFieldName, parser.getDeprecationHandler())) {
                                 fetchSourceContext = FetchSourceContext.fromXContent(parser);
+                            } else if (NO_AUTO_CREATE.match(currentFieldName, parser.getDeprecationHandler())) {
+                                noAutoCreate = parser.booleanValue();
                             } else {
                                 throw new IllegalArgumentException("Action/metadata line [" + line + "] contains an unknown parameter ["
                                         + currentFieldName + "]");
@@ -245,19 +250,22 @@ public void parse(
                             indexRequestConsumer.accept(new IndexRequest(index).id(id).routing(routing)
                                     .version(version).versionType(versionType)
                                     .setPipeline(pipeline).setIfSeqNo(ifSeqNo).setIfPrimaryTerm(ifPrimaryTerm)
-                                    .source(sliceTrimmingCarriageReturn(data, from, nextMarker,xContentType), xContentType), type);
+                                    .source(sliceTrimmingCarriageReturn(data, from, nextMarker,xContentType), xContentType)
+                                    .setNoAutoCreate(noAutoCreate), type);
                         } else {
                             indexRequestConsumer.accept(new IndexRequest(index).id(id).routing(routing)
                                     .version(version).versionType(versionType)
                                     .create("create".equals(opType)).setPipeline(pipeline)
                                     .setIfSeqNo(ifSeqNo).setIfPrimaryTerm(ifPrimaryTerm)
-                                    .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType), type);
+                                    .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType)
+                                    .setNoAutoCreate(noAutoCreate), type);
                         }
                     } else if ("create".equals(action)) {
                         indexRequestConsumer.accept(new IndexRequest(index).id(id).routing(routing)
                                 .version(version).versionType(versionType)
                                 .create(true).setPipeline(pipeline).setIfSeqNo(ifSeqNo).setIfPrimaryTerm(ifPrimaryTerm)
-                                .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType), type);
+                                .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType)
+                                .setNoAutoCreate(noAutoCreate), type);
                     } else if ("update".equals(action)) {
                         if (version != Versions.MATCH_ANY || versionType != VersionType.INTERNAL) {
                             throw new IllegalArgumentException("Update requests do not support versioning. " +
@@ -266,6 +274,7 @@ public void parse(
                         UpdateRequest updateRequest = new UpdateRequest().index(index).id(id).routing(routing)
                                 .retryOnConflict(retryOnConflict)
                                 .setIfSeqNo(ifSeqNo).setIfPrimaryTerm(ifPrimaryTerm)
+                                .setNoAutoCreate(noAutoCreate)
                                 .routing(routing);
                         // EMPTY is safe here because we never call namedObject
                         try (InputStream dataStream = sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType).streamInput();

server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java

@@ -221,24 +221,24 @@ protected void doInternalExecute(Task task, BulkRequest bulkRequest, ActionListe
 
         if (needToCheck()) {
             // Attempt to create all the indices that we're going to need during the bulk before we start.
-            // Step 1: collect all the indices in the request
-            final Set<String> indices = bulkRequest.requests.stream()
+            // Step 1: collect all the indices in the request and if the request is specifically disallowing index auto creation
+            final Map<String, Boolean> indices = bulkRequest.requests.stream()
                     // delete requests should not attempt to create the index (if the index does not
                     // exists), unless an external versioning is used
                 .filter(request -> request.opType() != DocWriteRequest.OpType.DELETE
                         || request.versionType() == VersionType.EXTERNAL
                         || request.versionType() == VersionType.EXTERNAL_GTE)
-                .map(DocWriteRequest::index)
-                .collect(Collectors.toSet());
+                .collect(Collectors.toMap(DocWriteRequest::index, DocWriteRequest::isNoAutoCreate, (v1, v2) -> v1 && v2));
             /* Step 2: filter that to indices that don't exist and we can create. At the same time build a map of indices we can't create
              * that we'll use when we try to run the requests. */
             final Map<String, IndexNotFoundException> indicesThatCannotBeCreated = new HashMap<>();
             Set<String> autoCreateIndices = new HashSet<>();
             ClusterState state = clusterService.state();
-            for (String index : indices) {
+            for (Map.Entry<String, Boolean> indexAndFlag : indices.entrySet()) {
+                final String index = indexAndFlag.getKey();
                 boolean shouldAutoCreate;
                 try {
-                    shouldAutoCreate = shouldAutoCreate(index, state);
+                    shouldAutoCreate = shouldAutoCreate(index, indexAndFlag.getValue() == null ? false : indexAndFlag.getValue(), state);
                 } catch (IndexNotFoundException e) {
                     shouldAutoCreate = false;
                     indicesThatCannotBeCreated.put(index, e);
@@ -442,8 +442,8 @@ boolean needToCheck() {
         return autoCreateIndex.needToCheck();
     }
 
-    boolean shouldAutoCreate(String index, ClusterState state) {
-        return autoCreateIndex.shouldAutoCreate(index, state);
+    boolean shouldAutoCreate(String index, boolean noAutoCreate, ClusterState state) {
+        return autoCreateIndex.shouldAutoCreate(index, noAutoCreate, state);
     }
 
     void createIndex(String index,

server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java

@@ -236,6 +236,11 @@ public OpType opType() {
         return OpType.DELETE;
     }
 
+    @Override
+    public boolean isNoAutoCreate() {
+        return false;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);

server/src/main/java/org/elasticsearch/action/index/IndexRequest.java

@@ -106,6 +106,8 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
 
     private boolean isPipelineResolved;
 
+    private boolean noAutoCreate;
+
     /**
      * Value for {@link #getAutoGeneratedTimestamp()} if the document has an external
      * provided ID.
@@ -150,6 +152,11 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio
         }
         ifSeqNo = in.readZLong();
         ifPrimaryTerm = in.readVLong();
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            noAutoCreate = in.readBoolean();
+        } else {
+            noAutoCreate = false;
+        }
     }
 
     public IndexRequest() {
@@ -656,6 +663,9 @@ private void writeBody(StreamOutput out) throws IOException {
         }
         out.writeZLong(ifSeqNo);
         out.writeVLong(ifPrimaryTerm);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeBoolean(noAutoCreate);
+        }
     }
 
     @Override
@@ -699,4 +709,14 @@ public long getAutoGeneratedTimestamp() {
     public long ramBytesUsed() {
         return SHALLOW_SIZE + RamUsageEstimator.sizeOf(id) + (source == null ? 0 : source.ramBytesUsed());
     }
+
+    @Override
+    public boolean isNoAutoCreate() {
+        return noAutoCreate;
+    }
+
+    public IndexRequest setNoAutoCreate(boolean noAutoCreate) {
+        this.noAutoCreate = noAutoCreate;
+        return this;
+    }
 }

server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java

@@ -222,4 +222,12 @@ public IndexRequestBuilder setPipeline(String pipeline) {
         request.setPipeline(pipeline);
         return this;
     }
+
+    /**
+     * Sets the no_auto_create flag to prevent this request from creating an index
+     */
+    public IndexRequestBuilder setNoAutoCreate(boolean noAutoCreate) {
+        request.setNoAutoCreate(noAutoCreate);
+        return this;
+    }
 }

server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.support;
 
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.Booleans;
@@ -63,10 +64,13 @@ public boolean needToCheck() {
      * Should the index be auto created?
      * @throws IndexNotFoundException if the index doesn't exist and shouldn't be auto created
      */
-    public boolean shouldAutoCreate(String index, ClusterState state) {
+    public boolean shouldAutoCreate(String index, boolean noAutoCreateFlag, ClusterState state) {
         if (resolver.hasIndexAbstraction(index, state)) {
             return false;
         }
+        if (noAutoCreateFlag)  {
+            throw new IndexNotFoundException("[" + DocWriteRequest.NO_AUTO_CREATE + "] request flag is [true]", index);
+        }
         // One volatile read, so that all checks are done against the same instance:
         final AutoCreate autoCreate = this.autoCreate;
         if (autoCreate.autoCreateIndex == false) {

server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java

@@ -116,7 +116,7 @@ public static void resolveAndValidateRouting(Metadata metadata, String concreteI
     @Override
     protected void doExecute(Task task, final UpdateRequest request, final ActionListener<UpdateResponse> listener) {
         // if we don't have a master, we don't have metadata, that's fine, let it find a master using create index API
-        if (autoCreateIndex.shouldAutoCreate(request.index(), clusterService.state())) {
+        if (autoCreateIndex.shouldAutoCreate(request.index(), request.isNoAutoCreate(), clusterService.state())) {
             client.admin().indices().create(new CreateIndexRequest()
                 .index(request.index())
                 .cause("auto(update api)")