diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java index 9df9623926fc5..1c260d6f91ef7 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java @@ -1229,11 +1229,11 @@ public void testFieldCaps() throws IOException { assertEquals(2, ratingResponse.size()); FieldCapabilities expectedKeywordCapabilities = new FieldCapabilities( - "rating", "keyword", true, true, new String[]{"index2"}, null, null); + "rating", "keyword", true, true, new String[]{"index2"}, null, null, Collections.emptyMap()); assertEquals(expectedKeywordCapabilities, ratingResponse.get("keyword")); FieldCapabilities expectedLongCapabilities = new FieldCapabilities( - "rating", "long", true, true, new String[]{"index1"}, null, null); + "rating", "long", true, true, new String[]{"index1"}, null, null, Collections.emptyMap()); assertEquals(expectedLongCapabilities, ratingResponse.get("long")); // Check the capabilities for the 'field' field. @@ -1242,7 +1242,7 @@ public void testFieldCaps() throws IOException { assertEquals(1, fieldResponse.size()); FieldCapabilities expectedTextCapabilities = new FieldCapabilities( - "field", "text", true, false); + "field", "text", true, false, Collections.emptyMap()); assertEquals(expectedTextCapabilities, fieldResponse.get("text")); } diff --git a/docs/reference/mapping/params.asciidoc b/docs/reference/mapping/params.asciidoc index 51421afb4942e..f4a65aa111df1 100644 --- a/docs/reference/mapping/params.asciidoc +++ b/docs/reference/mapping/params.asciidoc @@ -8,15 +8,15 @@ parameters that are used by <>: The following mapping parameters are common to some or all field datatypes: * <> -* <> * <> * <> * <> * <> * <> +* <> * <> * <> -* <> +* <> * <> * <> * <> @@ -24,7 +24,8 @@ The following mapping parameters are common to some or all field datatypes: * <> * <> * <> -* <> +* <> +* <> * <> * <> * <> @@ -37,8 +38,6 @@ The following mapping parameters are common to some or all field datatypes: include::params/analyzer.asciidoc[] -include::params/normalizer.asciidoc[] - include::params/boost.asciidoc[] include::params/coerce.asciidoc[] @@ -49,10 +48,10 @@ include::params/doc-values.asciidoc[] include::params/dynamic.asciidoc[] -include::params/enabled.asciidoc[] - include::params/eager-global-ordinals.asciidoc[] +include::params/enabled.asciidoc[] + include::params/fielddata.asciidoc[] include::params/format.asciidoc[] @@ -69,8 +68,12 @@ include::params/index-phrases.asciidoc[] include::params/index-prefixes.asciidoc[] +include::params/meta.asciidoc[] + include::params/multi-fields.asciidoc[] +include::params/normalizer.asciidoc[] + include::params/norms.asciidoc[] include::params/null-value.asciidoc[] diff --git a/docs/reference/mapping/params/meta.asciidoc b/docs/reference/mapping/params/meta.asciidoc new file mode 100644 index 0000000000000..52d3ca7ff7cce --- /dev/null +++ b/docs/reference/mapping/params/meta.asciidoc @@ -0,0 +1,31 @@ +[[mapping-field-meta]] +=== `meta` + +Metadata attached to the field. This metadata is opaque to Elasticsearch, it is +only useful for multiple applications that work on the same indices to share +meta information about fields such as units + +[source,console] +------------ +PUT my_index +{ + "mappings": { + "properties": { + "latency": { + "type": "long", + "meta": { + "unit": "ms" + } + } + } + } +} +------------ +// TEST + +NOTE: Field metadata enforces at most 5 entries, that keys have a length that +is less than or equal to 20, and that values are strings whose length is less +than or equal to 50. + +NOTE: Field metadata is updatable by submitting a mapping update. The metadata +of the update will override the metadata of the existing field. diff --git a/docs/reference/mapping/types/boolean.asciidoc b/docs/reference/mapping/types/boolean.asciidoc index 116459d0660e4..ab8011a4c56e4 100644 --- a/docs/reference/mapping/types/boolean.asciidoc +++ b/docs/reference/mapping/types/boolean.asciidoc @@ -120,3 +120,6 @@ The following parameters are accepted by `boolean` fields: the <> field. Accepts `true` or `false` (default). +<>:: + + Metadata about the field. diff --git a/docs/reference/mapping/types/date.asciidoc b/docs/reference/mapping/types/date.asciidoc index 43ede27831b6a..4a9474dcfebf1 100644 --- a/docs/reference/mapping/types/date.asciidoc +++ b/docs/reference/mapping/types/date.asciidoc @@ -137,3 +137,7 @@ The following parameters are accepted by `date` fields: Whether the field value should be stored and retrievable separately from the <> field. Accepts `true` or `false` (default). + +<>:: + + Metadata about the field. diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc index 71419b378faae..e0ee14f99b0d0 100644 --- a/docs/reference/mapping/types/keyword.asciidoc +++ b/docs/reference/mapping/types/keyword.asciidoc @@ -115,6 +115,10 @@ The following parameters are accepted by `keyword` fields: when building a query for this field. Accepts `true` or `false` (default). +<>:: + + Metadata about the field. + NOTE: Indexes imported from 2.x do not support `keyword`. Instead they will attempt to downgrade `keyword` into `string`. This allows you to merge modern mappings with legacy mappings. Long lived indexes will have to be recreated diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc index 8280945b22777..e99869c7fe275 100644 --- a/docs/reference/mapping/types/numeric.asciidoc +++ b/docs/reference/mapping/types/numeric.asciidoc @@ -149,6 +149,10 @@ The following parameters are accepted by numeric types: the <> field. Accepts `true` or `false` (default). +<>:: + + Metadata about the field. + [[scaled-float-params]] ==== Parameters for `scaled_float` diff --git a/docs/reference/mapping/types/text.asciidoc b/docs/reference/mapping/types/text.asciidoc index 434d91fd4ec6a..f3bbb257fb85a 100644 --- a/docs/reference/mapping/types/text.asciidoc +++ b/docs/reference/mapping/types/text.asciidoc @@ -143,3 +143,7 @@ The following parameters are accepted by `text` fields: Whether term vectors should be stored for an <> field. Defaults to `no`. + +<>:: + + Metadata about the field. diff --git a/docs/reference/search/field-caps.asciidoc b/docs/reference/search/field-caps.asciidoc index f94ac492d7814..c6b945c055bf6 100644 --- a/docs/reference/search/field-caps.asciidoc +++ b/docs/reference/search/field-caps.asciidoc @@ -78,6 +78,12 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] The list of indices where this field is not aggregatable, or null if all indices have the same definition for the field. +`meta`:: + Merged metadata across all indices as a map of string keys to arrays of values. + A value length of 1 indicates that all indices had the same value for this key, + while a length of 2 or more indicates that not all indices had the same value + for this key. + [[search-field-caps-api-example]] ==== {api-examples-title} diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java index 54236c61b76dc..94ae0cee76bb7 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; @@ -35,6 +36,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; import static org.hamcrest.Matchers.containsString; @@ -353,4 +355,33 @@ public void testRejectIndexOptions() throws IOException { MapperParsingException e = expectThrows(MapperParsingException.class, () -> parser.parse("type", new CompressedXContent(mapping))); assertThat(e.getMessage(), containsString("index_options not allowed in field [foo] of type [scaled_float]")); } + + public void testMeta() throws Exception { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "scaled_float") + .field("meta", Collections.singletonMap("foo", "bar")) + .field("scaling_factor", 10.0) + .endObject().endObject().endObject().endObject()); + + DocumentMapper mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); + assertEquals(mapping, mapper.mappingSource().toString()); + + String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "scaled_float") + .field("scaling_factor", 10.0) + .endObject().endObject().endObject().endObject()); + mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE); + assertEquals(mapping2, mapper.mappingSource().toString()); + + String mapping3 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "scaled_float") + .field("meta", Collections.singletonMap("baz", "quux")) + .field("scaling_factor", 10.0) + .endObject().endObject().endObject().endObject()); + mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE); + assertEquals(mapping3, mapper.mappingSource().toString()); + } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml index 137512fe432bd..fcc1ba8104c9d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml @@ -317,4 +317,3 @@ setup: - match: {fields.misc.unmapped.searchable: false} - match: {fields.misc.unmapped.aggregatable: false} - match: {fields.misc.unmapped.indices: ["test2", "test3"]} - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/20_meta.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/20_meta.yml new file mode 100644 index 0000000000000..c2d565278717a --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/20_meta.yml @@ -0,0 +1,65 @@ +--- +"Merge metadata across multiple indices": + + - skip: + version: " - 7.99.99" + reason: Metadata support was added in 7.6 + + - do: + indices.create: + index: test1 + body: + mappings: + properties: + latency: + type: long + meta: + unit: ms + metric_type: gauge + + - do: + indices.create: + index: test2 + body: + mappings: + properties: + latency: + type: long + meta: + unit: ns + metric_type: gauge + + - do: + indices.create: + index: test3 + + - do: + field_caps: + index: test3 + fields: [latency] + + - is_false: fields.latency.long.meta.unit + + - do: + field_caps: + index: test1 + fields: [latency] + + - match: {fields.latency.long.meta.unit: ["ms"]} + - match: {fields.latency.long.meta.metric_type: ["gauge"]} + + - do: + field_caps: + index: test1,test3 + fields: [latency] + + - match: {fields.latency.long.meta.unit: ["ms"]} + - match: {fields.latency.long.meta.metric_type: ["gauge"]} + + - do: + field_caps: + index: test1,test2,test3 + fields: [latency] + + - match: {fields.latency.long.meta.unit: ["ms", "ns"]} + - match: {fields.latency.long.meta.metric_type: ["gauge"]} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml index 4b228ac0ecdb0..3a4044c8d71ad 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml @@ -108,3 +108,39 @@ - match: { error.type: "illegal_argument_exception" } - match: { error.reason: "Types cannot be provided in put mapping requests, unless the include_type_name parameter is set to true." } + +--- +"Update per-field metadata": + + - skip: + version: " - 7.99.99" + reason: "Per-field meta was introduced in 7.6" + + - do: + indices.create: + index: test_index + body: + mappings: + properties: + foo: + type: keyword + meta: + bar: baz + + - do: + indices.put_mapping: + index: test_index + body: + properties: + foo: + type: keyword + meta: + baz: quux + + - do: + indices.get_mapping: + index: test_index + + - is_false: test_index.mappings.properties.foo.meta.bar + - match: { test_index.mappings.properties.foo.meta.baz: "quux" } + diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java index 20f525716a218..15598a2a88a5c 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.fieldcaps; +import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -34,20 +35,33 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; /** * Describes the capabilities of a field optionally merged across multiple indices. */ public class FieldCapabilities implements Writeable, ToXContentObject { + private static final ParseField TYPE_FIELD = new ParseField("type"); private static final ParseField SEARCHABLE_FIELD = new ParseField("searchable"); private static final ParseField AGGREGATABLE_FIELD = new ParseField("aggregatable"); private static final ParseField INDICES_FIELD = new ParseField("indices"); private static final ParseField NON_SEARCHABLE_INDICES_FIELD = new ParseField("non_searchable_indices"); private static final ParseField NON_AGGREGATABLE_INDICES_FIELD = new ParseField("non_aggregatable_indices"); + private static final ParseField META_FIELD = new ParseField("meta"); + + private static Map> mapToMapOfSets(Map map) { + final Function, String> entryValueFunction = Map.Entry::getValue; + return map.entrySet().stream().collect( + Collectors.toUnmodifiableMap(Map.Entry::getKey, entryValueFunction.andThen(Set::of))); + } private final String name; private final String type; @@ -58,19 +72,23 @@ public class FieldCapabilities implements Writeable, ToXContentObject { private final String[] nonSearchableIndices; private final String[] nonAggregatableIndices; + private final Map> meta; + /** - * Constructor + * Constructor for a single index. * @param name The name of the field. * @param type The type associated with the field. * @param isSearchable Whether this field is indexed for search. * @param isAggregatable Whether this field can be aggregated on. + * @param meta Metadata about the field. */ - public FieldCapabilities(String name, String type, boolean isSearchable, boolean isAggregatable) { - this(name, type, isSearchable, isAggregatable, null, null, null); + public FieldCapabilities(String name, String type, boolean isSearchable, boolean isAggregatable, + Map meta) { + this(name, type, isSearchable, isAggregatable, null, null, null, mapToMapOfSets(Objects.requireNonNull(meta))); } /** - * Constructor + * Constructor for a set of indices. * @param name The name of the field * @param type The type associated with the field. * @param isSearchable Whether this field is indexed for search. @@ -81,12 +99,14 @@ public FieldCapabilities(String name, String type, boolean isSearchable, boolean * or null if the field is searchable in all indices. * @param nonAggregatableIndices The list of indices where this field is not aggregatable, * or null if the field is aggregatable in all indices. + * @param meta Merged metadata across indices. */ public FieldCapabilities(String name, String type, boolean isSearchable, boolean isAggregatable, String[] indices, String[] nonSearchableIndices, - String[] nonAggregatableIndices) { + String[] nonAggregatableIndices, + Map> meta) { this.name = name; this.type = type; this.isSearchable = isSearchable; @@ -94,6 +114,7 @@ public FieldCapabilities(String name, String type, this.indices = indices; this.nonSearchableIndices = nonSearchableIndices; this.nonAggregatableIndices = nonAggregatableIndices; + this.meta = Objects.requireNonNull(meta); } public FieldCapabilities(StreamInput in) throws IOException { @@ -104,6 +125,11 @@ public FieldCapabilities(StreamInput in) throws IOException { this.indices = in.readOptionalStringArray(); this.nonSearchableIndices = in.readOptionalStringArray(); this.nonAggregatableIndices = in.readOptionalStringArray(); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + meta = in.readMap(StreamInput::readString, i -> i.readSet(StreamInput::readString)); + } else { + meta = Collections.emptyMap(); + } } @Override @@ -115,6 +141,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalStringArray(indices); out.writeOptionalStringArray(nonSearchableIndices); out.writeOptionalStringArray(nonAggregatableIndices); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeMap(meta, StreamOutput::writeString, (o, set) -> o.writeCollection(set, StreamOutput::writeString)); + } } @Override @@ -132,6 +161,17 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (nonAggregatableIndices != null) { builder.field(NON_AGGREGATABLE_INDICES_FIELD.getPreferredName(), nonAggregatableIndices); } + if (meta.isEmpty() == false) { + builder.startObject("meta"); + List>> entries = new ArrayList<>(meta.entrySet()); + entries.sort(Comparator.comparing(Map.Entry::getKey)); // provide predictable order + for (Map.Entry> entry : entries) { + List values = new ArrayList<>(entry.getValue()); + values.sort(String::compareTo); // provide predictable order + builder.field(entry.getKey(), values); + } + builder.endObject(); + } builder.endObject(); return builder; } @@ -150,7 +190,8 @@ public static FieldCapabilities fromXContent(String name, XContentParser parser) (boolean) a[2], a[3] != null ? ((List) a[3]).toArray(new String[0]) : null, a[4] != null ? ((List) a[4]).toArray(new String[0]) : null, - a[5] != null ? ((List) a[5]).toArray(new String[0]) : null)); + a[5] != null ? ((List) a[5]).toArray(new String[0]) : null, + a[6] != null ? ((Map>) a[6]) : Collections.emptyMap())); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD); @@ -159,6 +200,8 @@ public static FieldCapabilities fromXContent(String name, XContentParser parser) PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), INDICES_FIELD); PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_SEARCHABLE_INDICES_FIELD); PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_AGGREGATABLE_INDICES_FIELD); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), + (parser, context) -> parser.map(HashMap::new, p -> Set.copyOf(p.list())), META_FIELD); } /** @@ -213,6 +256,13 @@ public String[] nonAggregatableIndices() { return nonAggregatableIndices; } + /** + * Return merged metadata across indices. + */ + public Map> meta() { + return meta; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -224,12 +274,13 @@ public boolean equals(Object o) { Objects.equals(type, that.type) && Arrays.equals(indices, that.indices) && Arrays.equals(nonSearchableIndices, that.nonSearchableIndices) && - Arrays.equals(nonAggregatableIndices, that.nonAggregatableIndices); + Arrays.equals(nonAggregatableIndices, that.nonAggregatableIndices) && + Objects.equals(meta, that.meta); } @Override public int hashCode() { - int result = Objects.hash(name, type, isSearchable, isAggregatable); + int result = Objects.hash(name, type, isSearchable, isAggregatable, meta); result = 31 * result + Arrays.hashCode(indices); result = 31 * result + Arrays.hashCode(nonSearchableIndices); result = 31 * result + Arrays.hashCode(nonAggregatableIndices); @@ -247,6 +298,7 @@ static class Builder { private boolean isSearchable; private boolean isAggregatable; private List indiceList; + private Map> meta; Builder(String name, String type) { this.name = name; @@ -254,15 +306,38 @@ static class Builder { this.isSearchable = true; this.isAggregatable = true; this.indiceList = new ArrayList<>(); + this.meta = new HashMap<>(); } - void add(String index, boolean search, boolean agg) { + private void add(String index, boolean search, boolean agg) { IndexCaps indexCaps = new IndexCaps(index, search, agg); indiceList.add(indexCaps); this.isSearchable &= search; this.isAggregatable &= agg; } + /** + * Collect capabilities of an index. + */ + void add(String index, boolean search, boolean agg, Map meta) { + add(index, search, agg); + for (Map.Entry entry : meta.entrySet()) { + this.meta.computeIfAbsent(entry.getKey(), key -> new HashSet<>()) + .add(entry.getValue()); + } + } + + /** + * Merge another capabilities instance. + */ + void merge(String index, boolean search, boolean agg, Map> meta) { + add(index, search, agg); + for (Map.Entry> entry : meta.entrySet()) { + this.meta.computeIfAbsent(entry.getKey(), key -> new HashSet<>()) + .addAll(entry.getValue()); + } + } + List getIndices() { return indiceList.stream().map(c -> c.name).collect(Collectors.toList()); } @@ -305,8 +380,12 @@ FieldCapabilities build(boolean withIndices) { } else { nonAggregatableIndices = null; } + final Function>, Set> entryValueFunction = Map.Entry::getValue; + Map> immutableMeta = meta.entrySet().stream() + .collect(Collectors.toUnmodifiableMap( + Map.Entry::getKey, entryValueFunction.andThen(Set::copyOf))); return new FieldCapabilities(name, type, isSearchable, isAggregatable, - indices, nonSearchableIndices, nonAggregatableIndices); + indices, nonSearchableIndices, nonAggregatableIndices, immutableMeta); } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index 3176d0d31390b..2ddec2c8378c6 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -176,7 +176,7 @@ private void addUnmappedFields(String[] indices, String field, Map> resp Map typeMap = responseMapBuilder.computeIfAbsent(field, f -> new HashMap<>()); FieldCapabilities.Builder builder = typeMap.computeIfAbsent(fieldCap.getType(), key -> new FieldCapabilities.Builder(field, key)); - builder.add(indexName, fieldCap.isSearchable(), fieldCap.isAggregatable()); + builder.merge(indexName, fieldCap.isSearchable(), fieldCap.isAggregatable(), fieldCap.meta()); } } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java index f391bf82eb944..9b482f60a150c 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java @@ -38,6 +38,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -89,7 +90,8 @@ protected FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesI if (ft != null) { if (indicesService.isMetaDataField(mapperService.getIndexSettings().getIndexVersionCreated(), field) || fieldPredicate.test(ft.name())) { - FieldCapabilities fieldCap = new FieldCapabilities(field, ft.typeName(), ft.isSearchable(), ft.isAggregatable()); + FieldCapabilities fieldCap = new FieldCapabilities(field, ft.typeName(), ft.isSearchable(), ft.isAggregatable(), + ft.meta()); responseMap.put(field, fieldCap); } else { continue; @@ -107,7 +109,7 @@ protected FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesI // no field type, it must be an object field ObjectMapper mapper = mapperService.getObjectMapper(parentField); String type = mapper.nested().isNested() ? "nested" : "object"; - FieldCapabilities fieldCap = new FieldCapabilities(parentField, type, false, false); + FieldCapabilities fieldCap = new FieldCapabilities(parentField, type, false, false, Collections.emptyMap()); responseMap.put(parentField, fieldCap); } dotIndex = parentField.lastIndexOf('.'); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 23753b881f20c..57e71ee25ae42 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -50,6 +50,7 @@ import java.util.Map; import java.util.HashMap; import java.util.Objects; +import java.util.TreeMap; import java.util.stream.StreamSupport; public abstract class FieldMapper extends Mapper implements Cloneable { @@ -223,6 +224,12 @@ protected void setupFieldType(BuilderContext context) { fieldType.setHasDocValues(defaultDocValues); } } + + /** Set metadata on this field. */ + public T meta(Map meta) { + fieldType.setMeta(meta); + return (T) this; + } } protected final Version indexCreatedVersion; @@ -427,6 +434,10 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, multiFields.toXContent(builder, params); copyTo.toXContent(builder, params); + + if (includeDefaults || fieldType().meta().isEmpty() == false) { + builder.field("meta", new TreeMap<>(fieldType().meta())); // ensure consistent order + } } protected final void doXContentAnalyzers(XContentBuilder builder, boolean includeDefaults) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 82a0239777e30..86dad273e71b0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -53,7 +53,9 @@ import java.io.IOException; import java.time.ZoneId; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; /** @@ -72,6 +74,7 @@ public abstract class MappedFieldType extends FieldType { private Object nullValue; private String nullValueAsString; // for sending null value to _all field private boolean eagerGlobalOrdinals; + private Map meta; protected MappedFieldType(MappedFieldType ref) { super(ref); @@ -85,6 +88,7 @@ protected MappedFieldType(MappedFieldType ref) { this.nullValue = ref.nullValue(); this.nullValueAsString = ref.nullValueAsString(); this.eagerGlobalOrdinals = ref.eagerGlobalOrdinals; + this.meta = ref.meta; } public MappedFieldType() { @@ -94,6 +98,7 @@ public MappedFieldType() { setOmitNorms(false); setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); setBoost(1.0f); + meta = Collections.emptyMap(); } @Override @@ -126,13 +131,14 @@ public boolean equals(Object o) { Objects.equals(eagerGlobalOrdinals, fieldType.eagerGlobalOrdinals) && Objects.equals(nullValue, fieldType.nullValue) && Objects.equals(nullValueAsString, fieldType.nullValueAsString) && - Objects.equals(similarity, fieldType.similarity); + Objects.equals(similarity, fieldType.similarity) && + Objects.equals(meta, fieldType.meta); } @Override public int hashCode() { return Objects.hash(super.hashCode(), name, boost, docValues, indexAnalyzer, searchAnalyzer, searchQuoteAnalyzer, - eagerGlobalOrdinals, similarity == null ? null : similarity.name(), nullValue, nullValueAsString); + eagerGlobalOrdinals, similarity == null ? null : similarity.name(), nullValue, nullValueAsString, meta); } // TODO: we need to override freeze() and add safety checks that all settings are actually set @@ -490,4 +496,18 @@ public static Term extractTerm(Query termQuery) { return ((TermQuery) termQuery).getTerm(); } + /** + * Get the metadata associated with this field. + */ + public Map meta() { + return meta; + } + + /** + * Associate metadata with this field. + */ + public void setMeta(Map meta) { + checkIfFrozen(); + this.meta = Map.copyOf(Objects.requireNonNull(meta)); + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java index cabadedcd7f20..4c2fdbdd73725 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java @@ -34,6 +34,8 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; import static org.elasticsearch.common.xcontent.support.XContentMapValues.isArray; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeFloatValue; @@ -144,7 +146,7 @@ private static void parseAnalyzersAndTermVectors(FieldMapper.Builder builder, St } } - public static void parseNorms(FieldMapper.Builder builder, String fieldName, Object propNode) { + public static void parseNorms(FieldMapper.Builder builder, String fieldName, Object propNode) { builder.omitNorms(XContentMapValues.nodeBooleanValue(propNode, fieldName + ".norms") == false); } @@ -152,8 +154,7 @@ public static void parseNorms(FieldMapper.Builder builder, String fieldName, Obj * Parse text field attributes. In addition to {@link #parseField common attributes} * this will parse analysis and term-vectors related settings. */ - @SuppressWarnings("unchecked") - public static void parseTextField(FieldMapper.Builder builder, String name, Map fieldNode, + public static void parseTextField(FieldMapper.Builder builder, String name, Map fieldNode, Mapper.TypeParser.ParserContext parserContext) { parseField(builder, name, fieldNode, parserContext); parseAnalyzersAndTermVectors(builder, name, fieldNode, parserContext); @@ -168,12 +169,58 @@ public static void parseTextField(FieldMapper.Builder builder, String name, Map< } } + /** + * Parse the {@code meta} key of the mapping. + */ + public static void parseMeta(FieldMapper.Builder builder, String name, Map fieldNode) { + Object metaObject = fieldNode.remove("meta"); + if (metaObject == null) { + // no meta + return; + } + if (metaObject instanceof Map == false) { + throw new MapperParsingException("[meta] must be an object, got " + metaObject.getClass().getSimpleName() + + "[" + metaObject + "] for field [" + name +"]"); + } + @SuppressWarnings("unchecked") + Map meta = (Map) metaObject; + if (meta.size() > 5) { + throw new MapperParsingException("[meta] can't have more than 5 entries, but got " + meta.size() + " on field [" + + name + "]"); + } + for (String key : meta.keySet()) { + if (key.codePointCount(0, key.length()) > 20) { + throw new MapperParsingException("[meta] keys can't be longer than 20 chars, but got [" + key + + "] for field [" + name + "]"); + } + } + for (Object value : meta.values()) { + if (value instanceof String) { + String sValue = (String) value; + if (sValue.codePointCount(0, sValue.length()) > 50) { + throw new MapperParsingException("[meta] values can't be longer than 50 chars, but got [" + value + + "] for field [" + name + "]"); + } + } else if (value == null) { + throw new MapperParsingException("[meta] values can't be null (field [" + name + "])"); + } else { + throw new MapperParsingException("[meta] values can only be strings, but got " + + value.getClass().getSimpleName() + "[" + value + "] for field [" + name + "]"); + } + } + final Function, Object> entryValueFunction = Map.Entry::getValue; + final Function stringCast = String.class::cast; + Map checkedMeta = meta.entrySet().stream() + .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, entryValueFunction.andThen(stringCast))); + builder.meta(checkedMeta); + } + /** * Parse common field attributes such as {@code doc_values} or {@code store}. */ - @SuppressWarnings("rawtypes") - public static void parseField(FieldMapper.Builder builder, String name, Map fieldNode, + public static void parseField(FieldMapper.Builder builder, String name, Map fieldNode, Mapper.TypeParser.ParserContext parserContext) { + parseMeta(builder, name, fieldNode); for (Iterator> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); final String propName = entry.getKey(); diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java index deeae3351ec96..6776d66c9df72 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java @@ -25,6 +25,9 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.Set; import static org.hamcrest.Matchers.equalTo; @@ -48,9 +51,9 @@ protected Writeable.Reader instanceReader() { public void testBuilder() { FieldCapabilities.Builder builder = new FieldCapabilities.Builder("field", "type"); - builder.add("index1", true, false); - builder.add("index2", true, false); - builder.add("index3", true, false); + builder.add("index1", true, false, Collections.emptyMap()); + builder.add("index2", true, false, Collections.emptyMap()); + builder.add("index3", true, false, Collections.emptyMap()); { FieldCapabilities cap1 = builder.build(false); @@ -59,6 +62,7 @@ public void testBuilder() { assertNull(cap1.indices()); assertNull(cap1.nonSearchableIndices()); assertNull(cap1.nonAggregatableIndices()); + assertEquals(Collections.emptyMap(), cap1.meta()); FieldCapabilities cap2 = builder.build(true); assertThat(cap2.isSearchable(), equalTo(true)); @@ -67,12 +71,13 @@ public void testBuilder() { assertThat(cap2.indices(), equalTo(new String[]{"index1", "index2", "index3"})); assertNull(cap2.nonSearchableIndices()); assertNull(cap2.nonAggregatableIndices()); + assertEquals(Collections.emptyMap(), cap2.meta()); } builder = new FieldCapabilities.Builder("field", "type"); - builder.add("index1", false, true); - builder.add("index2", true, false); - builder.add("index3", false, false); + builder.add("index1", false, true, Collections.emptyMap()); + builder.add("index2", true, false, Collections.emptyMap()); + builder.add("index3", false, false, Collections.emptyMap()); { FieldCapabilities cap1 = builder.build(false); assertThat(cap1.isSearchable(), equalTo(false)); @@ -80,6 +85,7 @@ public void testBuilder() { assertNull(cap1.indices()); assertThat(cap1.nonSearchableIndices(), equalTo(new String[]{"index1", "index3"})); assertThat(cap1.nonAggregatableIndices(), equalTo(new String[]{"index2", "index3"})); + assertEquals(Collections.emptyMap(), cap1.meta()); FieldCapabilities cap2 = builder.build(true); assertThat(cap2.isSearchable(), equalTo(false)); @@ -88,6 +94,30 @@ public void testBuilder() { assertThat(cap2.indices(), equalTo(new String[]{"index1", "index2", "index3"})); assertThat(cap2.nonSearchableIndices(), equalTo(new String[]{"index1", "index3"})); assertThat(cap2.nonAggregatableIndices(), equalTo(new String[]{"index2", "index3"})); + assertEquals(Collections.emptyMap(), cap2.meta()); + } + + builder = new FieldCapabilities.Builder("field", "type"); + builder.add("index1", true, true, Collections.emptyMap()); + builder.add("index2", true, true, Map.of("foo", "bar")); + builder.add("index3", true, true, Map.of("foo", "quux")); + { + FieldCapabilities cap1 = builder.build(false); + assertThat(cap1.isSearchable(), equalTo(true)); + assertThat(cap1.isAggregatable(), equalTo(true)); + assertNull(cap1.indices()); + assertNull(cap1.nonSearchableIndices()); + assertNull(cap1.nonAggregatableIndices()); + assertEquals(Map.of("foo", Set.of("bar", "quux")), cap1.meta()); + + FieldCapabilities cap2 = builder.build(true); + assertThat(cap2.isSearchable(), equalTo(true)); + assertThat(cap2.isAggregatable(), equalTo(true)); + assertThat(cap2.indices().length, equalTo(3)); + assertThat(cap2.indices(), equalTo(new String[]{"index1", "index2", "index3"})); + assertNull(cap2.nonSearchableIndices()); + assertNull(cap2.nonAggregatableIndices()); + assertEquals(Map.of("foo", Set.of("bar", "quux")), cap2.meta()); } } @@ -113,9 +143,23 @@ static FieldCapabilities randomFieldCaps(String fieldName) { nonAggregatableIndices[i] = randomAlphaOfLengthBetween(5, 20); } } + + Map> meta; + switch (randomInt(2)) { + case 0: + meta = Collections.emptyMap(); + break; + case 1: + meta = Map.of("foo", Set.of("bar")); + break; + default: + meta = Map.of("foo", Set.of("bar", "baz")); + break; + } + return new FieldCapabilities(fieldName, randomAlphaOfLengthBetween(5, 20), randomBoolean(), randomBoolean(), - indices, nonSearchableIndices, nonAggregatableIndices); + indices, nonSearchableIndices, nonAggregatableIndices, meta); } @Override @@ -127,7 +171,8 @@ protected FieldCapabilities mutateInstance(FieldCapabilities instance) { String[] indices = instance.indices(); String[] nonSearchableIndices = instance.nonSearchableIndices(); String[] nonAggregatableIndices = instance.nonAggregatableIndices(); - switch (between(0, 6)) { + Map> meta = instance.meta(); + switch (between(0, 7)) { case 0: name += randomAlphaOfLengthBetween(1, 10); break; @@ -169,7 +214,6 @@ protected FieldCapabilities mutateInstance(FieldCapabilities instance) { nonSearchableIndices = newNonSearchableIndices; break; case 6: - default: String[] newNonAggregatableIndices; int startNonAggregatablePos = 0; if (nonAggregatableIndices == null) { @@ -183,7 +227,18 @@ protected FieldCapabilities mutateInstance(FieldCapabilities instance) { } nonAggregatableIndices = newNonAggregatableIndices; break; + case 7: + Map> newMeta; + if (meta.isEmpty()) { + newMeta = Map.of("foo", Set.of("bar")); + } else { + newMeta = Collections.emptyMap(); + } + meta = newMeta; + break; + default: + throw new AssertionError(); } - return new FieldCapabilities(name, type, isSearchable, isAggregatable, indices, nonSearchableIndices, nonAggregatableIndices); + return new FieldCapabilities(name, type, isSearchable, isAggregatable, indices, nonSearchableIndices, nonAggregatableIndices, meta); } } diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java index 656dd5458e809..13a634177650c 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java @@ -152,19 +152,19 @@ public void testEmptyResponse() throws IOException { private static FieldCapabilitiesResponse createSimpleResponse() { Map titleCapabilities = new HashMap<>(); - titleCapabilities.put("text", new FieldCapabilities("title", "text", true, false)); + titleCapabilities.put("text", new FieldCapabilities("title", "text", true, false, Collections.emptyMap())); Map ratingCapabilities = new HashMap<>(); ratingCapabilities.put("long", new FieldCapabilities("rating", "long", true, false, new String[]{"index1", "index2"}, null, - new String[]{"index1"})); + new String[]{"index1"}, Collections.emptyMap())); ratingCapabilities.put("keyword", new FieldCapabilities("rating", "keyword", false, true, new String[]{"index3", "index4"}, new String[]{"index4"}, - null)); + null, Collections.emptyMap())); Map> responses = new HashMap<>(); responses.put("title", titleCapabilities); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index a6e6d5c79d1e8..1fd3f51ece916 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -46,6 +47,7 @@ import java.io.IOException; import java.util.Collection; +import java.util.Collections; import static org.hamcrest.Matchers.containsString; @@ -251,4 +253,30 @@ public void testEmptyName() throws IOException { ); assertThat(e.getMessage(), containsString("name cannot be empty string")); } + + public void testMeta() throws Exception { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "boolean") + .field("meta", Collections.singletonMap("foo", "bar")) + .endObject().endObject().endObject().endObject()); + + DocumentMapper mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); + assertEquals(mapping, mapper.mappingSource().toString()); + + String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "boolean") + .endObject().endObject().endObject().endObject()); + mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE); + assertEquals(mapping2, mapper.mappingSource().toString()); + + String mapping3 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "boolean") + .field("meta", Collections.singletonMap("baz", "quux")) + .endObject().endObject().endObject().endObject()); + mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE); + assertEquals(mapping3, mapper.mappingSource().toString()); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index daa70c865133f..477cf7de8285c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; @@ -39,6 +40,7 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.Collection; +import java.util.Collections; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.notNullValue; @@ -415,4 +417,30 @@ public void testIllegalFormatField() throws Exception { () -> parser.parse("type", new CompressedXContent(mapping))); assertEquals("Invalid format: [[test_format]]: Unknown pattern letter: t", e.getMessage()); } + + public void testMeta() throws Exception { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "date") + .field("meta", Collections.singletonMap("foo", "bar")) + .endObject().endObject().endObject().endObject()); + + DocumentMapper mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); + assertEquals(mapping, mapper.mappingSource().toString()); + + String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "date") + .endObject().endObject().endObject().endObject()); + mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE); + assertEquals(mapping2, mapper.mappingSource().toString()); + + String mapping3 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "date") + .field("meta", Collections.singletonMap("baz", "quux")) + .endObject().endObject().endObject().endObject()); + mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE); + assertEquals(mapping3, mapper.mappingSource().toString()); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index dad4b72cdf380..fd821f1170ce8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -46,6 +46,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -547,4 +548,30 @@ public void testSplitQueriesOnWhitespace() throws IOException { assertThat(ft.searchAnalyzer().name(), equalTo("my_lowercase")); assertTokenStreamContents(ft.searchAnalyzer().analyzer().tokenStream("", "Hello World"), new String[] {"hello world"}); } + + public void testMeta() throws Exception { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "keyword") + .field("meta", Collections.singletonMap("foo", "bar")) + .endObject().endObject().endObject().endObject()); + + DocumentMapper mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); + assertEquals(mapping, mapper.mappingSource().toString()); + + String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "keyword") + .endObject().endObject().endObject().endObject()); + mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE); + assertEquals(mapping2, mapper.mappingSource().toString()); + + String mapping3 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "keyword") + .field("meta", Collections.singletonMap("baz", "quux")) + .endObject().endObject().endObject().endObject()); + mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE); + assertEquals(mapping3, mapper.mappingSource().toString()); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 5d8d45687b1be..539d8bbdeaf59 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1247,4 +1247,30 @@ public void testSimpleMerge() throws IOException { assertThat(mapper.mappers().getMapper("b_field"), instanceOf(KeywordFieldMapper.class)); } } + + public void testMeta() throws Exception { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "text") + .field("meta", Collections.singletonMap("foo", "bar")) + .endObject().endObject().endObject().endObject()); + + DocumentMapper mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); + assertEquals(mapping, mapper.mappingSource().toString()); + + String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "text") + .endObject().endObject().endObject().endObject()); + mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE); + assertEquals(mapping2, mapper.mappingSource().toString()); + + String mapping3 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "text") + .field("meta", Collections.singletonMap("baz", "quux")) + .endObject().endObject().endObject().endObject()); + mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE); + assertEquals(mapping3, mapper.mappingSource().toString()); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java index 8fb30cf8d1c38..1a03cc7ca91b4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java @@ -19,6 +19,20 @@ package org.elasticsearch.index.mapper; +import static org.elasticsearch.index.analysis.AnalysisRegistry.DEFAULT_ANALYZER_NAME; +import static org.elasticsearch.index.analysis.AnalysisRegistry.DEFAULT_SEARCH_ANALYZER_NAME; +import static org.elasticsearch.index.analysis.AnalysisRegistry.DEFAULT_SEARCH_QUOTED_ANALYZER_NAME; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.standard.StandardAnalyzer; @@ -40,18 +54,7 @@ import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import static org.elasticsearch.index.analysis.AnalysisRegistry.DEFAULT_ANALYZER_NAME; -import static org.elasticsearch.index.analysis.AnalysisRegistry.DEFAULT_SEARCH_ANALYZER_NAME; -import static org.elasticsearch.index.analysis.AnalysisRegistry.DEFAULT_SEARCH_QUOTED_ANALYZER_NAME; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import org.hamcrest.Matchers; public class TypeParsersTests extends ESTestCase { @@ -227,4 +230,69 @@ public TokenStream create(TokenStream tokenStream) { return new CustomAnalyzer(null, new CharFilterFactory[0], new TokenFilterFactory[] { tokenFilter }); } + + public void testParseMeta() { + FieldMapper.Builder builder = new KeywordFieldMapper.Builder("foo"); + Mapper.TypeParser.ParserContext parserContext = new Mapper.TypeParser.ParserContext(null, null, null, null, null); + + { + Map mapping = new HashMap<>(Map.of("meta", 3)); + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> TypeParsers.parseField(builder, builder.name, mapping, parserContext)); + assertEquals("[meta] must be an object, got Integer[3] for field [foo]", e.getMessage()); + } + + { + Map mapping = new HashMap<>(Map.of("meta", Map.of("veryloooooooooooongkey", 3L))); + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> TypeParsers.parseField(builder, builder.name, mapping, parserContext)); + assertEquals("[meta] keys can't be longer than 20 chars, but got [veryloooooooooooongkey] for field [foo]", + e.getMessage()); + } + + { + Map mapping = new HashMap<>(Map.of("meta", Map.of( + "foo1", 3L, "foo2", 4L, "foo3", 5L, "foo4", 6L, "foo5", 7L, "foo6", 8L))); + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> TypeParsers.parseField(builder, builder.name, mapping, parserContext)); + assertEquals("[meta] can't have more than 5 entries, but got 6 on field [foo]", + e.getMessage()); + } + + { + Map mapping = new HashMap<>(Map.of("meta", Map.of("foo", Map.of("bar", "baz")))); + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> TypeParsers.parseField(builder, builder.name, mapping, parserContext)); + assertEquals("[meta] values can only be strings, but got Map1[{bar=baz}] for field [foo]", + e.getMessage()); + } + + { + Map mapping = new HashMap<>(Map.of("meta", Map.of("bar", "baz", "foo", 3))); + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> TypeParsers.parseField(builder, builder.name, mapping, parserContext)); + assertEquals("[meta] values can only be strings, but got Integer[3] for field [foo]", + e.getMessage()); + } + + { + Map meta = new HashMap<>(); + meta.put("foo", null); + Map mapping = new HashMap<>(Map.of("meta", meta)); + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> TypeParsers.parseField(builder, builder.name, mapping, parserContext)); + assertEquals("[meta] values can't be null (field [foo])", + e.getMessage()); + } + + { + String longString = IntStream.range(0, 51) + .mapToObj(Integer::toString) + .collect(Collectors.joining()); + Map mapping = new HashMap<>(Map.of("meta", Map.of("foo", longString))); + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> TypeParsers.parseField(builder, builder.name, mapping, parserContext)); + assertThat(e.getMessage(), Matchers.startsWith("[meta] values can't be longer than 50 chars")); + } + } } diff --git a/server/src/test/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/test/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index 7adc447a20736..254447abcf8eb 100644 --- a/server/src/test/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/test/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -119,12 +119,14 @@ public void testFieldAlias() { assertTrue(distance.containsKey("double")); assertEquals( - new FieldCapabilities("distance", "double", true, true, new String[] {"old_index"}, null, null), + new FieldCapabilities("distance", "double", true, true, new String[] {"old_index"}, null, null, + Collections.emptyMap()), distance.get("double")); assertTrue(distance.containsKey("text")); assertEquals( - new FieldCapabilities("distance", "text", true, false, new String[] {"new_index"}, null, null), + new FieldCapabilities("distance", "text", true, false, new String[] {"new_index"}, null, null, + Collections.emptyMap()), distance.get("text")); // Check the capabilities for the 'route_length_miles' alias. @@ -133,7 +135,7 @@ public void testFieldAlias() { assertTrue(routeLength.containsKey("double")); assertEquals( - new FieldCapabilities("route_length_miles", "double", true, true), + new FieldCapabilities("route_length_miles", "double", true, true, Collections.emptyMap()), routeLength.get("double")); } @@ -174,12 +176,14 @@ public void testWithUnmapped() { assertTrue(oldField.containsKey("long")); assertEquals( - new FieldCapabilities("old_field", "long", true, true, new String[] {"old_index"}, null, null), + new FieldCapabilities("old_field", "long", true, true, new String[] {"old_index"}, null, null, + Collections.emptyMap()), oldField.get("long")); assertTrue(oldField.containsKey("unmapped")); assertEquals( - new FieldCapabilities("old_field", "unmapped", false, false, new String[] {"new_index"}, null, null), + new FieldCapabilities("old_field", "unmapped", false, false, new String[] {"new_index"}, null, null, + Collections.emptyMap()), oldField.get("unmapped")); Map newField = response.getField("new_field"); @@ -187,7 +191,7 @@ public void testWithUnmapped() { assertTrue(newField.containsKey("long")); assertEquals( - new FieldCapabilities("new_field", "long", true, true), + new FieldCapabilities("new_field", "long", true, true, Collections.emptyMap()), newField.get("long")); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java index b732c6b5b42bf..18e3a64648759 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; @@ -29,6 +30,7 @@ import java.io.IOException; import java.util.Collection; +import java.util.Collections; import java.util.Set; import static org.hamcrest.Matchers.containsString; @@ -124,4 +126,33 @@ public void testEmptyName() throws IOException { } } + public void testMeta() throws Exception { + for (String type : TYPES) { + IndexService indexService = createIndex("test-" + type); + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", type) + .field("meta", Collections.singletonMap("foo", "bar")) + .endObject().endObject().endObject().endObject()); + + DocumentMapper mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); + assertEquals(mapping, mapper.mappingSource().toString()); + + String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", type) + .endObject().endObject().endObject().endObject()); + mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE); + assertEquals(mapping2, mapper.mappingSource().toString()); + + String mapping3 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", type) + .field("meta", Collections.singletonMap("baz", "quux")) + .endObject().endObject().endObject().endObject()); + mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE); + assertEquals(mapping3, mapper.mappingSource().toString()); + } + } + } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index b22f6eb0573df..0e5f9a9225178 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -43,6 +43,7 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.TypeParsers; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.indices.breaker.CircuitBreakerService; @@ -124,6 +125,7 @@ public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new HistogramFieldMapper.Builder(name); + TypeParsers.parseMeta(builder, name, node); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); String propName = entry.getKey(); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java index 055b01186cd61..76aa163d7e98e 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java @@ -11,10 +11,12 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.xpack.analytics.AnalyticsPlugin; @@ -22,6 +24,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import static org.hamcrest.Matchers.containsString; @@ -498,6 +501,33 @@ public void testNegativeCount() throws Exception { assertThat(e.getCause().getMessage(), containsString("[counts] elements must be >= 0 but got -3")); } + public void testMeta() throws Exception { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "histogram") + .field("meta", Collections.singletonMap("foo", "bar")) + .endObject().endObject().endObject().endObject()); + + IndexService indexService = createIndex("test"); + DocumentMapper mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); + assertEquals(mapping, mapper.mappingSource().toString()); + + String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "histogram") + .endObject().endObject().endObject().endObject()); + mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE); + assertEquals(mapping2, mapper.mappingSource().toString()); + + String mapping3 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "histogram") + .field("meta", Collections.singletonMap("baz", "quux")) + .endObject().endObject().endObject().endObject()); + mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE); + assertEquals(mapping3, mapper.mappingSource().toString()); + } + @Override protected Collection> getPlugins() { List> plugins = new ArrayList<>(super.getPlugins()); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java index 6b882d03f2919..6a273a72b254f 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java @@ -953,7 +953,7 @@ private MockFieldCapsResponseBuilder addNonAggregatableField(String field, Strin private MockFieldCapsResponseBuilder addField(String field, boolean isAggregatable, String... types) { Map caps = new HashMap<>(); for (String type : types) { - caps.put(type, new FieldCapabilities(field, type, true, isAggregatable)); + caps.put(type, new FieldCapabilities(field, type, true, isAggregatable, Collections.emptyMap())); } fieldCaps.put(field, caps); return this; diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index d57c090817d10..045f2b8928f33 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.sql.type.TypesTests; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -143,8 +144,10 @@ public void testMergeIncompatibleCapabilitiesOfObjectFields() throws Exception { addFieldCaps(fieldCaps, fieldName + ".keyword", "keyword", true, true); Map multi = new HashMap<>(); - multi.put("long", new FieldCapabilities(fieldName, "long", true, true, new String[] { "one-index" }, null, null)); - multi.put("text", new FieldCapabilities(fieldName, "text", true, false, new String[] { "another-index" }, null, null)); + multi.put("long", new FieldCapabilities(fieldName, "long", true, true, new String[] { "one-index" }, null, null, + Collections.emptyMap())); + multi.put("text", new FieldCapabilities(fieldName, "text", true, false, new String[] { "another-index" }, null, null, + Collections.emptyMap())); fieldCaps.put(fieldName, multi); @@ -214,7 +217,8 @@ public void testMultipleCompatibleIndicesWithDifferentFields() { public void testIndexWithNoMapping() { Map> versionFC = singletonMap("_version", - singletonMap("_index", new FieldCapabilities("_version", "_version", false, false))); + singletonMap("_index", new FieldCapabilities("_version", "_version", false, false, + Collections.emptyMap()))); assertTrue(IndexResolver.mergedMappings("*", new String[] { "empty" }, versionFC).isValid()); } @@ -289,7 +293,7 @@ private static class UpdateableFieldCapabilities extends FieldCapabilities { List nonAggregatableIndices = new ArrayList<>(); UpdateableFieldCapabilities(String name, String type, boolean isSearchable, boolean isAggregatable) { - super(name, type, isSearchable, isAggregatable); + super(name, type, isSearchable, isAggregatable, Collections.emptyMap()); } @Override @@ -323,7 +327,7 @@ private static void assertEqualsMaps(Map left, Map right) { private void addFieldCaps(Map> fieldCaps, String name, String type, boolean isSearchable, boolean isAggregatable) { Map cap = new HashMap<>(); - cap.put(name, new FieldCapabilities(name, type, isSearchable, isAggregatable)); + cap.put(name, new FieldCapabilities(name, type, isSearchable, isAggregatable, Collections.emptyMap())); fieldCaps.put(name, cap); } }