Merge remote-tracking branch 'origin/main' into dot-prefix-validation

docs/changelog/111548.yaml

@@ -0,0 +1,6 @@
+pr: 111548
+summary: Json parsing exceptions should not cause 500 errors
+area: Infra/Core
+type: bug
+issues:
+ - 111542

docs/changelog/112282.yaml

@@ -0,0 +1,6 @@
+pr: 112282
+summary: Adds example plugin for custom ingest processor
+area: Ingest Node
+type: enhancement
+issues:
+ - 111539

docs/changelog/112409.yaml

@@ -0,0 +1,6 @@
+pr: 112409
+summary: Include reason when no nodes are found
+area: "Transform"
+type: bug
+issues:
+ - 112404

docs/changelog/112547.yaml

@@ -0,0 +1,5 @@
+pr: 112547
+summary: Remove reduce and `reduceContext` from `DelayedBucket`
+area: Aggregations
+type: enhancement
+issues: []

docs/plugins/development/creating-classic-plugins.asciidoc

@@ -32,12 +32,13 @@ for the plugin. If you need other resources, package them into a resources JAR.
 The {es} repository contains {es-repo}tree/main/plugins/examples[examples of plugins]. Some of these include:
 
 * a plugin with {es-repo}tree/main/plugins/examples/custom-settings[custom settings]
+* a plugin with a {es-repo}tree/main/plugins/examples/custom-processor[custom ingest processor]
 * adding {es-repo}tree/main/plugins/examples/rest-handler[custom rest endpoints]
 * adding a {es-repo}tree/main/plugins/examples/rescore[custom rescorer]
 * a script {es-repo}tree/main/plugins/examples/script-expert-scoring[implemented in Java]
 
 These examples provide the bare bones needed to get started. For more
-information about how to write a plugin, we recommend looking at the 
+information about how to write a plugin, we recommend looking at the
 {es-repo}tree/main/plugins/[source code of existing plugins] for inspiration.
 
 [discrete]
@@ -88,4 +89,4 @@ for more information.
 [[plugin-descriptor-file-classic]]
 ==== The plugin descriptor file for classic plugins
 
-include::plugin-descriptor-file.asciidoc[]
+include::plugin-descriptor-file.asciidoc[]

libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java

@@ -57,7 +57,8 @@ public Token nextToken() throws IOException {
         try {
             return convertToken(parser.nextToken());
         } catch (JsonEOFException e) {
-            throw new XContentEOFException(e);
+            JsonLocation location = e.getLocation();
+            throw new XContentEOFException(new XContentLocation(location.getLineNr(), location.getColumnNr()), "Unexpected end of file", e);
         } catch (JsonParseException e) {
             throw newXContentParseException(e);
         }

libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentEOFException.java

@@ -8,11 +8,9 @@
 
 package org.elasticsearch.xcontent;
 
-import java.io.IOException;
+public class XContentEOFException extends XContentParseException {
 
-public class XContentEOFException extends IOException {
-
-    public XContentEOFException(IOException cause) {
-        super(cause);
+    public XContentEOFException(XContentLocation location, String message, Exception cause) {
+        super(location, message, cause);
     }
 }

modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestUtils;
@@ -61,17 +62,19 @@ public Set<String> supportedCapabilities() {
 
     @Override
     public Set<String> supportedQueryParameters() {
-        return Set.of(
-            "name",
-            "include_defaults",
-            "timeout",
-            "master_timeout",
-            RestRequest.PATH_RESTRICTED,
-            IndicesOptions.WildcardOptions.EXPAND_WILDCARDS,
-            IndicesOptions.ConcreteTargetOptions.IGNORE_UNAVAILABLE,
-            IndicesOptions.WildcardOptions.ALLOW_NO_INDICES,
-            IndicesOptions.GatekeeperOptions.IGNORE_THROTTLED,
-            "verbose"
+        return Sets.union(
+            RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS,
+            Set.of(
+                "name",
+                "include_defaults",
+                "timeout",
+                "master_timeout",
+                IndicesOptions.WildcardOptions.EXPAND_WILDCARDS,
+                IndicesOptions.ConcreteTargetOptions.IGNORE_UNAVAILABLE,
+                IndicesOptions.WildcardOptions.ALLOW_NO_INDICES,
+                IndicesOptions.GatekeeperOptions.IGNORE_THROTTLED,
+                "verbose"
+            )
         );
     }
 }

modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml

@@ -114,14 +114,14 @@ setup:
 
   - match:  { responses.0.hits.total:     2  }
   - match:  { responses.1.error.root_cause.0.type: x_content_e_o_f_exception }
-  - match:  { responses.1.error.root_cause.0.reason: "/Unexpected.end.of.input/" }
+  - match:  { responses.1.error.root_cause.0.reason: "/\\[1:22\\].Unexpected.end.of.file/" }
   - match:  { responses.2.hits.total:     1  }
   - match:  { responses.3.error.root_cause.0.type: parsing_exception }
   - match:  { responses.3.error.root_cause.0.reason: "/unknown.query.\\[unknown\\]/" }
   - match:  { responses.4.error.root_cause.0.type: illegal_argument_exception }
   - match:  { responses.4.error.root_cause.0.reason: "[rest_total_hits_as_int] cannot be used if the tracking of total hits is not accurate, got 1" }
   - match:  { responses.0.status: 200 }
-  - match:  { responses.1.status: 500 }
+  - match:  { responses.1.status: 400 }
   - match:  { responses.2.status: 200 }
   - match:  { responses.3.status: 400 }
 

modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java

@@ -352,7 +352,7 @@ private static int getLevels(int treeLevels, double precisionInMeters, int defau
         public LegacyGeoShapeFieldMapper build(MapperBuilderContext context) {
             LegacyGeoShapeParser parser = new LegacyGeoShapeParser();
             GeoShapeFieldType ft = buildFieldType(parser, context);
-            return new LegacyGeoShapeFieldMapper(leafName(), ft, multiFieldsBuilder.build(this, context), copyTo, parser, this);
+            return new LegacyGeoShapeFieldMapper(leafName(), ft, builderParams(this, context), parser, this);
         }
     }
 
@@ -537,20 +537,18 @@ public PrefixTreeStrategy resolvePrefixTreeStrategy(String strategyName) {
     public LegacyGeoShapeFieldMapper(
         String simpleName,
         MappedFieldType mappedFieldType,
-        MultiFields multiFields,
-        CopyTo copyTo,
+        BuilderParams builderParams,
         LegacyGeoShapeParser parser,
         Builder builder
     ) {
         super(
             simpleName,
             mappedFieldType,
+            builderParams,
             builder.ignoreMalformed.get(),
             builder.coerce.get(),
             builder.ignoreZValue.get(),
             builder.orientation.get(),
-            multiFields,
-            copyTo,
             parser
         );
         this.indexCreatedVersion = builder.indexCreatedVersion;

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java

@@ -139,13 +139,11 @@ private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context) {
         @Override
         public MatchOnlyTextFieldMapper build(MapperBuilderContext context) {
             MatchOnlyTextFieldType tft = buildFieldType(context);
-            MultiFields multiFields = multiFieldsBuilder.build(this, context);
             return new MatchOnlyTextFieldMapper(
                 leafName(),
                 Defaults.FIELD_TYPE,
                 tft,
-                multiFields,
-                copyTo,
+                builderParams(this, context),
                 context.isSourceSynthetic(),
                 this
             );
@@ -382,12 +380,11 @@ private MatchOnlyTextFieldMapper(
         String simpleName,
         FieldType fieldType,
         MatchOnlyTextFieldType mappedFieldType,
-        MultiFields multiFields,
-        CopyTo copyTo,
+        BuilderParams builderParams,
         boolean storeSource,
         Builder builder
     ) {
-        super(simpleName, mappedFieldType, multiFields, copyTo, false, null);
+        super(simpleName, mappedFieldType, builderParams);
         assert mappedFieldType.getTextSearchInfo().isTokenized();
         assert mappedFieldType.hasDocValues() == false;
         this.fieldType = freezeAndDeduplicateFieldType(fieldType);
@@ -442,7 +439,7 @@ protected SyntheticSourceMode syntheticSourceMode() {
 
     @Override
     public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
-        if (copyTo.copyToFields().isEmpty() != true) {
+        if (copyTo().copyToFields().isEmpty() != true) {
             throw new IllegalArgumentException(
                 "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
             );

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java

@@ -98,8 +98,7 @@ public RankFeatureFieldMapper build(MapperBuilderContext context) {
                     positiveScoreImpact.getValue(),
                     nullValue.getValue()
                 ),
-                multiFieldsBuilder.build(this, context),
-                copyTo,
+                builderParams(this, context),
                 positiveScoreImpact.getValue(),
                 nullValue.getValue()
             );
@@ -172,12 +171,11 @@ public Query termQuery(Object value, SearchExecutionContext context) {
     private RankFeatureFieldMapper(
         String simpleName,
         MappedFieldType mappedFieldType,
-        MultiFields multiFields,
-        CopyTo copyTo,
+        BuilderParams builderParams,
         boolean positiveScoreImpact,
         Float nullValue
     ) {
-        super(simpleName, mappedFieldType, multiFields, copyTo, false, null);
+        super(simpleName, mappedFieldType, builderParams);
         this.positiveScoreImpact = positiveScoreImpact;
         this.nullValue = nullValue;
     }

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java

@@ -66,8 +66,7 @@ public RankFeaturesFieldMapper build(MapperBuilderContext context) {
             return new RankFeaturesFieldMapper(
                 leafName(),
                 new RankFeaturesFieldType(context.buildFullName(leafName()), meta.getValue(), positiveScoreImpact.getValue()),
-                multiFieldsBuilder.build(this, context),
-                copyTo,
+                builderParams(this, context),
                 positiveScoreImpact.getValue()
             );
         }
@@ -122,11 +121,10 @@ private static String indexedValueForSearch(Object value) {
     private RankFeaturesFieldMapper(
         String simpleName,
         MappedFieldType mappedFieldType,
-        MultiFields multiFields,
-        CopyTo copyTo,
+        BuilderParams builderParams,
         boolean positiveScoreImpact
     ) {
-        super(simpleName, mappedFieldType, multiFields, copyTo, false, null);
+        super(simpleName, mappedFieldType, builderParams);
         this.positiveScoreImpact = positiveScoreImpact;
     }
 

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java

@@ -197,14 +197,7 @@ public ScaledFloatFieldMapper build(MapperBuilderContext context) {
                 metric.getValue(),
                 indexMode
             );
-            return new ScaledFloatFieldMapper(
-                leafName(),
-                type,
-                multiFieldsBuilder.build(this, context),
-                copyTo,
-                context.isSourceSynthetic(),
-                this
-            );
+            return new ScaledFloatFieldMapper(leafName(), type, builderParams(this, context), context.isSourceSynthetic(), this);
         }
     }
 
@@ -470,12 +463,11 @@ public String toString() {
     private ScaledFloatFieldMapper(
         String simpleName,
         ScaledFloatFieldType mappedFieldType,
-        MultiFields multiFields,
-        CopyTo copyTo,
+        BuilderParams builderParams,
         boolean isSourceSynthetic,
         Builder builder
     ) {
-        super(simpleName, mappedFieldType, multiFields, copyTo);
+        super(simpleName, mappedFieldType, builderParams);
         this.isSourceSynthetic = isSourceSynthetic;
         this.indexed = builder.indexed.getValue();
         this.hasDocValues = builder.hasDocValues.getValue();
@@ -728,7 +720,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
                     + "] doesn't support synthetic source because it doesn't have doc values"
             );
         }
-        if (copyTo.copyToFields().isEmpty() != true) {
+        if (copyTo().copyToFields().isEmpty() != true) {
             throw new IllegalArgumentException(
                 "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
             );

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java

@@ -262,11 +262,10 @@ public SearchAsYouTypeFieldMapper build(MapperBuilderContext context) {
             return new SearchAsYouTypeFieldMapper(
                 leafName(),
                 ft,
-                copyTo,
+                builderParams(this, context),
                 indexAnalyzers,
                 prefixFieldMapper,
                 shingleFieldMappers,
-                multiFieldsBuilder.build(this, context),
                 this
             );
         }
@@ -498,7 +497,7 @@ static final class PrefixFieldMapper extends FieldMapper {
         final FieldType fieldType;
 
         PrefixFieldMapper(FieldType fieldType, PrefixFieldType mappedFieldType) {
-            super(mappedFieldType.name(), mappedFieldType, MultiFields.empty(), CopyTo.empty());
+            super(mappedFieldType.name(), mappedFieldType, BuilderParams.empty());
             this.fieldType = Mapper.freezeAndDeduplicateFieldType(fieldType);
         }
 
@@ -537,7 +536,7 @@ static final class ShingleFieldMapper extends FieldMapper {
         private final FieldType fieldType;
 
         ShingleFieldMapper(FieldType fieldType, ShingleFieldType mappedFieldtype) {
-            super(mappedFieldtype.name(), mappedFieldtype, MultiFields.empty(), CopyTo.empty());
+            super(mappedFieldtype.name(), mappedFieldtype, BuilderParams.empty());
             this.fieldType = freezeAndDeduplicateFieldType(fieldType);
         }
 
@@ -672,14 +671,13 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew
     public SearchAsYouTypeFieldMapper(
         String simpleName,
         SearchAsYouTypeFieldType mappedFieldType,
-        CopyTo copyTo,
+        BuilderParams builderParams,
         Map<String, NamedAnalyzer> indexAnalyzers,
         PrefixFieldMapper prefixField,
         ShingleFieldMapper[] shingleFields,
-        MultiFields multiFields,
         Builder builder
     ) {
-        super(simpleName, mappedFieldType, multiFields, copyTo, false, null);
+        super(simpleName, mappedFieldType, builderParams);
         this.prefixField = prefixField;
         this.shingleFields = shingleFields;
         this.maxShingleSize = builder.maxShingleSize.getValue();

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java

@@ -87,7 +87,7 @@ public TokenCountFieldMapper build(MapperBuilderContext context) {
                 nullValue.getValue(),
                 meta.getValue()
             );
-            return new TokenCountFieldMapper(leafName(), ft, multiFieldsBuilder.build(this, context), copyTo, this);
+            return new TokenCountFieldMapper(leafName(), ft, builderParams(this, context), this);
         }
     }
 
@@ -135,14 +135,8 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
     private final boolean enablePositionIncrements;
     private final Integer nullValue;
 
-    protected TokenCountFieldMapper(
-        String simpleName,
-        MappedFieldType defaultFieldType,
-        MultiFields multiFields,
-        CopyTo copyTo,
-        Builder builder
-    ) {
-        super(simpleName, defaultFieldType, multiFields, copyTo);
+    protected TokenCountFieldMapper(String simpleName, MappedFieldType defaultFieldType, BuilderParams builderParams, Builder builder) {
+        super(simpleName, defaultFieldType, builderParams);
         this.analyzer = builder.analyzer.getValue();
         this.enablePositionIncrements = builder.enablePositionIncrements.getValue();
         this.nullValue = builder.nullValue.getValue();

modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java

@@ -88,7 +88,7 @@ public Object valueForDisplay(Object value) {
     }
 
     protected ParentIdFieldMapper(String name, boolean eagerGlobalOrdinals) {
-        super(name, new ParentIdFieldType(name, eagerGlobalOrdinals), MultiFields.empty(), CopyTo.empty(), false, null);
+        super(name, new ParentIdFieldType(name, eagerGlobalOrdinals), BuilderParams.empty());
     }
 
     @Override

modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java

@@ -210,7 +210,7 @@ protected ParentJoinFieldMapper(
         boolean eagerGlobalOrdinals,
         List<Relations> relations
     ) {
-        super(simpleName, mappedFieldType, MultiFields.empty(), CopyTo.empty(), false, null);
+        super(simpleName, mappedFieldType, BuilderParams.empty());
         this.parentIdFields = parentIdFields;
         this.eagerGlobalOrdinals = eagerGlobalOrdinals;
         this.relations = relations;