Split RuntimeFieldType from corresponding MappedFieldType (#70695)

So far the runtime section supports only leaf field types, hence the internal representation is based on `RuntimeFieldType` that extends directly `MappedFieldType`. This is straightforward but it is limiting for e.g. an alias field that points to another field, or for object fields that are not queryable directly, hence should not be a MappedFieldType, yet their subfields do.

This commit makes `RuntimeFieldType` an interface, effectively splitting the definition of a runtime fields as defined and returned in the mappings, from its internal representation in terms of `MappedFieldType`.

The existing runtime script field types still extend `MappedFieldType` and now also implement the new interface, which makes the change rather simple.

modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java

@@ -273,7 +273,7 @@ public void parse(ParseContext context) throws IOException {
     }
 
     @Override
-    protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
+    protected void doXContentBody(XContentBuilder builder, Params params) throws IOException {
         builder.field("type", contentType());
         builder.field("eager_global_ordinals", eagerGlobalOrdinals);
         builder.startObject("relations");

server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java

@@ -128,6 +128,7 @@ private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesInd
 
                     // Check the ancestor of the field to find nested and object fields.
                     // Runtime fields are excluded since they can override any path.
+                    //TODO find a way to do this that does not require an instanceof check
                     if (ft instanceof RuntimeFieldType == false) {
                         int dotIndex = ft.name().lastIndexOf('.');
                         while (dotIndex > -1) {

server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

@@ -801,14 +801,14 @@ private static Mapper getLeafMapper(final ParseContext context,
         String fieldPath = context.path().pathAsText(fieldName);
         RuntimeFieldType runtimeFieldType = context.root().getRuntimeFieldType(fieldPath);
         if (runtimeFieldType != null) {
-            return new NoOpFieldMapper(subfields[subfields.length - 1], runtimeFieldType);
+            return new NoOpFieldMapper(subfields[subfields.length - 1], runtimeFieldType.asMappedFieldType().name());
         }
         return null;
     }
 
     private static class NoOpFieldMapper extends FieldMapper {
-        NoOpFieldMapper(String simpleName, RuntimeFieldType runtimeField) {
-            super(simpleName, new MappedFieldType(runtimeField.name(), false, false, false, TextSearchInfo.NONE, Collections.emptyMap()) {
+        NoOpFieldMapper(String simpleName, String fullName) {
+            super(simpleName, new MappedFieldType(fullName, false, false, false, TextSearchInfo.NONE, Collections.emptyMap()) {
                 @Override
                 public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
                     throw new UnsupportedOperationException();

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.support.AbstractXContentParser;
@@ -288,14 +289,13 @@ protected void checkIncomingMergeType(FieldMapper mergeWith) {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(simpleName());
-        boolean includeDefaults = params.paramAsBoolean("include_defaults", false);
-        doXContentBody(builder, includeDefaults, params);
+        doXContentBody(builder, params);
         return builder.endObject();
     }
 
-    protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
+    protected void doXContentBody(XContentBuilder builder, Params params) throws IOException {
         builder.field("type", contentType());
-        getMergeBuilder().toXContent(builder, includeDefaults);
+        getMergeBuilder().toXContent(builder, params);
         multiFields.toXContent(builder, params);
         copyTo.toXContent(builder, params);
     }
@@ -856,7 +856,7 @@ void check() {
     /**
      * A Builder for a ParametrizedFieldMapper
      */
-    public abstract static class Builder extends Mapper.Builder {
+    public abstract static class Builder extends Mapper.Builder implements ToXContentFragment {
 
         protected final MultiFields.Builder multiFieldsBuilder = new MultiFields.Builder();
         protected final CopyTo.Builder copyTo = new CopyTo.Builder();
@@ -916,10 +916,13 @@ protected String buildFullName(ContentPath contentPath) {
         /**
          * Writes the current builder parameter values as XContent
          */
-        public final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException {
+        @Override
+        public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            boolean includeDefaults = params.paramAsBoolean("include_defaults", false);
             for (Parameter<?> parameter : getParameters()) {
                 parameter.toXContent(builder, includeDefaults);
             }
+            return builder;
         }
 
         /**

server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java

@@ -69,7 +69,8 @@ final class FieldTypeLookup {
             fullNameToFieldType.put(aliasName, fullNameToFieldType.get(path));
         }
 
-        for (RuntimeFieldType runtimeFieldType : runtimeFieldTypes) {
+        for (RuntimeFieldType runtimeField : runtimeFieldTypes) {
+            MappedFieldType runtimeFieldType = runtimeField.asMappedFieldType();
             //this will override concrete fields with runtime fields that have the same name
             fullNameToFieldType.put(runtimeFieldType.name(), runtimeFieldType);
         }

server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java

@@ -145,8 +145,7 @@ public final XContentBuilder toXContent(XContentBuilder builder, Params params)
             return builder;
         }
         builder.startObject(simpleName());
-        boolean includeDefaults = params.paramAsBoolean("include_defaults", false);
-        getMergeBuilder().toXContent(builder, includeDefaults);
+        getMergeBuilder().toXContent(builder, params);
         return builder.endObject();
     }
 

server/src/main/java/org/elasticsearch/index/mapper/RuntimeFieldType.java

@@ -8,180 +8,53 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.search.MultiTermQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
-import org.apache.lucene.search.spans.SpanQuery;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.CheckedBiConsumer;
-import org.elasticsearch.common.geo.ShapeRelation;
-import org.elasticsearch.common.time.DateMathParser;
-import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.query.SearchExecutionContext;
 
 import java.io.IOException;
-import java.time.ZoneId;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.function.BiFunction;
 
-import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES;
-
 /**
- * Base implementation for a runtime field that can be defined as part of the runtime section of the index mappings
+ * Definition of a runtime field that can be defined as part of the runtime section of the index mappings
  */
-public abstract class RuntimeFieldType extends MappedFieldType implements ToXContentFragment {
-
-    private final CheckedBiConsumer<XContentBuilder, Boolean, IOException> toXContent;
-
-    protected RuntimeFieldType(String name, RuntimeFieldType.Builder builder) {
-        this(name, builder.meta(), builder::toXContent);
-    }
-
-    protected RuntimeFieldType(String name, Map<String, String> meta, CheckedBiConsumer<XContentBuilder, Boolean, IOException> toXContent) {
-        super(name, false, false, false, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta);
-        this.toXContent = toXContent;
-    }
+public interface RuntimeFieldType extends ToXContentFragment {
 
     @Override
-    public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+    default XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(name());
         builder.field("type", typeName());
-        boolean includeDefaults = params.paramAsBoolean("include_defaults", false);
-        doXContentBody(builder, includeDefaults);
+        doXContentBody(builder, params);
         builder.endObject();
         return builder;
     }
 
     /**
      * Prints out the parameters that subclasses expose
      */
-    final void doXContentBody(XContentBuilder builder, boolean includeDefaults) throws IOException {
-        toXContent.accept(builder, includeDefaults);
-    }
-
-    @Override
-    public final boolean isSearchable() {
-        return true;
-    }
-
-    @Override
-    public final boolean isAggregatable() {
-        return true;
-    }
-
-    @Override
-    public final Query rangeQuery(
-        Object lowerTerm,
-        Object upperTerm,
-        boolean includeLower,
-        boolean includeUpper,
-        ShapeRelation relation,
-        ZoneId timeZone,
-        DateMathParser parser,
-        SearchExecutionContext context
-    ) {
-        if (relation == ShapeRelation.DISJOINT) {
-            String message = "Runtime field [%s] of type [%s] does not support DISJOINT ranges";
-            throw new IllegalArgumentException(String.format(Locale.ROOT, message, name(), typeName()));
-        }
-        return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context);
-    }
-
-    protected abstract Query rangeQuery(
-        Object lowerTerm,
-        Object upperTerm,
-        boolean includeLower,
-        boolean includeUpper,
-        ZoneId timeZone,
-        DateMathParser parser,
-        SearchExecutionContext context
-    );
-
-    @Override
-    public Query fuzzyQuery(
-        Object value,
-        Fuzziness fuzziness,
-        int prefixLength,
-        int maxExpansions,
-        boolean transpositions,
-        SearchExecutionContext context
-    ) {
-        throw new IllegalArgumentException(unsupported("fuzzy", "keyword and text"));
-    }
-
-    @Override
-    public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, SearchExecutionContext context) {
-        throw new IllegalArgumentException(unsupported("prefix", "keyword, text and wildcard"));
-    }
-
-    @Override
-    public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, SearchExecutionContext context) {
-        throw new IllegalArgumentException(unsupported("wildcard", "keyword, text and wildcard"));
-    }
+    void doXContentBody(XContentBuilder builder, Params params) throws IOException;
 
-    @Override
-    public Query regexpQuery(
-        String value,
-        int syntaxFlags,
-        int matchFlags,
-        int maxDeterminizedStates,
-        MultiTermQuery.RewriteMethod method,
-        SearchExecutionContext context
-    ) {
-        throw new IllegalArgumentException(unsupported("regexp", "keyword and text"));
-    }
-
-    @Override
-    public Query phraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements) {
-        throw new IllegalArgumentException(unsupported("phrase", "text"));
-    }
-
-    @Override
-    public Query multiPhraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements) {
-        throw new IllegalArgumentException(unsupported("phrase", "text"));
-    }
-
-    @Override
-    public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions) {
-        throw new IllegalArgumentException(unsupported("phrase prefix", "text"));
-    }
-
-    @Override
-    public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRewriteMethod method, SearchExecutionContext context) {
-        throw new IllegalArgumentException(unsupported("span prefix", "text"));
-    }
-
-    private String unsupported(String query, String supported) {
-        return String.format(
-            Locale.ROOT,
-            "Can only use %s queries on %s fields - not on [%s] which is a runtime field of type [%s]",
-            query,
-            supported,
-            name(),
-            typeName()
-        );
-    }
+    /**
+     * Exposes the name of the runtime field
+     * @return name of the field
+     */
+    String name();
 
-    protected final void checkAllowExpensiveQueries(SearchExecutionContext context) {
-        if (context.allowExpensiveQueries() == false) {
-            throw new ElasticsearchException(
-                "queries cannot be executed against runtime fields while [" + ALLOW_EXPENSIVE_QUERIES.getKey() + "] is set to [false]."
-            );
-        }
-    }
+    /**
+     * Exposes the type of the runtime field
+     * @return type of the field
+     */
+    String typeName();
 
-    @Override
-    public final ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
-        return new DocValueFetcher(docValueFormat(format, null), context.getForField(this));
-    }
+    /**
+     * Exposes the {@link MappedFieldType} backing this runtime field, used to execute queries, run aggs etc.
+     * @return the {@link MappedFieldType} backing this runtime field
+     */
+    MappedFieldType asMappedFieldType();
 
     /**
      *  For runtime fields the {@link RuntimeFieldType.Parser} returns directly the {@link MappedFieldType}.
@@ -191,7 +64,7 @@ public final ValueFetcher valueFetcher(SearchExecutionContext context, String fo
      *  {@link RuntimeFieldType.Builder#parse(String, Mapper.TypeParser.ParserContext, Map)} and returns the corresponding
      *  {@link MappedFieldType}.
      */
-    public abstract static class Builder extends FieldMapper.Builder {
+    abstract class Builder extends FieldMapper.Builder {
         final FieldMapper.Parameter<Map<String, String>> meta = FieldMapper.Parameter.metaParam();
 
         protected Builder(String name) {
@@ -236,7 +109,7 @@ private void validate() {
      * Parser for a runtime field. Creates the appropriate {@link RuntimeFieldType} for a runtime field,
      * as defined in the runtime section of the index mappings.
      */
-    public static final class Parser {
+    final class Parser {
         private final BiFunction<String, Mapper.TypeParser.ParserContext, RuntimeFieldType.Builder> builderFunction;
 
         public Parser(BiFunction<String, Mapper.TypeParser.ParserContext, RuntimeFieldType.Builder> builderFunction) {
@@ -261,9 +134,9 @@ RuntimeFieldType parse(String name, Map<String, Object> node, Mapper.TypeParser.
      *                        translated to the removal of such runtime field
      * @return the parsed runtime fields
      */
-    public static Map<String, RuntimeFieldType> parseRuntimeFields(Map<String, Object> node,
-                                                                   Mapper.TypeParser.ParserContext parserContext,
-                                                                   boolean supportsRemoval) {
+    static Map<String, RuntimeFieldType> parseRuntimeFields(Map<String, Object> node,
+                                                            Mapper.TypeParser.ParserContext parserContext,
+                                                            boolean supportsRemoval) {
         Map<String, RuntimeFieldType> runtimeFields = new HashMap<>();
         Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
         while (iterator.hasNext()) {

server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

@@ -958,8 +958,9 @@ public static Query createPhrasePrefixQuery(TokenStream stream, String field, in
     }
 
     @Override
-    protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
+    protected void doXContentBody(XContentBuilder builder, Params params) throws IOException {
         // this is a pain, but we have to do this to maintain BWC
+        boolean includeDefaults = params.paramAsBoolean("include_defaults", false);
         builder.field("type", contentType());
         this.builder.index.toXContent(builder, includeDefaults);
         this.builder.store.toXContent(builder, includeDefaults);

server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java

@@ -106,7 +106,7 @@ public class SearchExecutionContext extends QueryRewriteContext {
     private boolean mapUnmappedFieldAsString;
     private NestedScope nestedScope;
     private final ValuesSourceRegistry valuesSourceRegistry;
-    private final Map<String, RuntimeFieldType> runtimeMappings;
+    private final Map<String, MappedFieldType> runtimeMappings;
 
     /**
      * Build a {@linkplain SearchExecutionContext}.
@@ -199,7 +199,7 @@ private SearchExecutionContext(int shardId,
                                    Index fullyQualifiedIndex,
                                    BooleanSupplier allowExpensiveQueries,
                                    ValuesSourceRegistry valuesSourceRegistry,
-                                   Map<String, RuntimeFieldType> runtimeMappings) {
+                                   Map<String, MappedFieldType> runtimeMappings) {
         super(xContentRegistry, namedWriteableRegistry, client, nowInMillis);
         this.shardId = shardId;
         this.shardRequestIndex = shardRequestIndex;
@@ -608,11 +608,18 @@ public Index getFullyQualifiedIndex() {
         return fullyQualifiedIndex;
     }
 
-    private static Map<String, RuntimeFieldType> parseRuntimeMappings(Map<String, Object> runtimeMappings, MapperService mapperService) {
+    private static Map<String, MappedFieldType> parseRuntimeMappings(Map<String, Object> runtimeMappings, MapperService mapperService) {
         if (runtimeMappings.isEmpty()) {
             return Collections.emptyMap();
         }
-        return RuntimeFieldType.parseRuntimeFields(new HashMap<>(runtimeMappings), mapperService.parserContext(), false);
+        Map<String, RuntimeFieldType> runtimeFields = RuntimeFieldType.parseRuntimeFields(new HashMap<>(runtimeMappings),
+            mapperService.parserContext(), false);
+        Map<String, MappedFieldType> runtimeFieldTypes = new HashMap<>();
+        for (RuntimeFieldType runtimeFieldType : runtimeFields.values()) {
+            MappedFieldType fieldType = runtimeFieldType.asMappedFieldType();
+            runtimeFieldTypes.put(fieldType.name(), fieldType);
+        }
+        return Collections.unmodifiableMap(runtimeFieldTypes);
     }
 
     /**