diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index 73900547ecf1c..adeb97fe351d5 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -185,11 +185,7 @@ protected void parseCreateField(ParseContext context) throws IOException { float value; if (context.externalValueSet()) { Object v = context.externalValue(); - if (v instanceof Number) { - value = ((Number) v).floatValue(); - } else { - value = Float.parseFloat(v.toString()); - } + value = objectToFloat(v); } else if (context.parser().currentToken() == Token.VALUE_NULL) { // skip return; @@ -209,6 +205,19 @@ protected void parseCreateField(ParseContext context) throws IOException { context.doc().addWithKey(name(), new FeatureField("_feature", name(), value)); } + private Float objectToFloat(Object value) { + if (value instanceof Number) { + return ((Number) value).floatValue(); + } else { + return Float.parseFloat(value.toString()); + } + } + + @Override + protected Float parseSourceValue(Object value) { + return objectToFloat(value); + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index c6d9303fcaf58..f8744280fc4a4 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -175,6 +175,11 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new AssertionError("parse is implemented directly"); } + @Override + protected Object parseSourceValue(Object value) { + return value; + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index e95b03d56d738..724d59f7ca2ca 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -499,6 +499,13 @@ private static double objectToDouble(Object value) { return doubleValue; } + @Override + protected Double parseSourceValue(Object value) { + double doubleValue = objectToDouble(value); + double scalingFactor = fieldType().getScalingFactor(); + return Math.round(doubleValue * scalingFactor) / scalingFactor; + } + private static class ScaledFloatIndexFieldData implements IndexNumericFieldData { private final IndexNumericFieldData scaledFieldData; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index d47793bd09929..bd3745d953f9d 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -473,6 +473,11 @@ protected void parseCreateField(ParseContext context) { throw new UnsupportedOperationException(); } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException(); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { @@ -510,6 +515,11 @@ protected void mergeOptions(FieldMapper other, List conflicts) { } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException(); + } + @Override protected String contentType() { return "shingle"; @@ -665,6 +675,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java index b66bcf23fada3..a42f1e217f4f7 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java @@ -150,6 +150,11 @@ protected void parseCreateField(ParseContext context) throws IOException { context.doc().addAll(NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, indexed, docValued, stored)); } + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + /** * Count position increments in a token stream. Package private for testing. * @param analyzer analyzer to create token stream diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java index 09c1768f28a90..2ffb8e870cd8c 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java @@ -23,9 +23,12 @@ import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttribute; import org.apache.lucene.document.FeatureField; import org.apache.lucene.index.IndexableField; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; @@ -180,4 +183,12 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep e.getCause().getMessage()); } + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + RankFeatureFieldMapper mapper = new RankFeatureFieldMapper.Builder("field").build(context); + + assertEquals(3.14f, mapper.parseSourceValue(3.14), 0.0001); + assertEquals(42.9f, mapper.parseSourceValue("42.9"), 0.0001); + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java index 79e15f49d2759..879df3b4304fb 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java @@ -21,9 +21,12 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; @@ -392,4 +395,15 @@ public void testMeta() throws Exception { new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE); assertEquals(mapping3, mapper.mappingSource().toString()); } + + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + ScaledFloatFieldMapper mapper = new ScaledFloatFieldMapper.Builder("field") + .scalingFactor(100) + .build(context); + + assertEquals(3.14, mapper.parseSourceValue(3.1415926), 0.00001); + assertEquals(3.14, mapper.parseSourceValue("3.1415"), 0.00001); + } } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java index 3ab36babfb33a..d0064d463f1e5 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java @@ -153,6 +153,11 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new IllegalStateException("Should never be called"); } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source."); + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index 12e03c15ddc91..831342b517e12 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -197,6 +197,11 @@ protected void parseCreateField(ParseContext context) throws IOException { context.doc().add(new SortedDocValuesField(fieldType().name(), binaryValue)); } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source."); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { ParentIdFieldMapper parentMergeWith = (ParentIdFieldMapper) other; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index c7ebabcdc803b..c07e5e8ac0dbc 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -367,6 +367,11 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new UnsupportedOperationException("parsing is implemented in parse(), this method should NEVER be called"); } + @Override + protected Object parseSourceValue(Object value) { + return value; + } + @Override public void parse(ParseContext context) throws IOException { context.path().add(simpleName()); diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index 0db99ca3ae87a..bc77a69cea6ac 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -389,6 +389,11 @@ public void parse(ParseContext context) throws IOException { processQuery(query, context); } + @Override + protected Object parseSourceValue(Object value) { + return value; + } + static void createQueryBuilderField(Version indexVersion, BinaryFieldMapper qbField, QueryBuilder queryBuilder, ParseContext context) throws IOException { try (ByteArrayOutputStream stream = new ByteArrayOutputStream()) { diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index 2e93ff99a2a3f..05ac6abfe575a 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -750,4 +750,9 @@ protected void parseCreateField(ParseContext context) throws IOException { createFieldNamesField(context); } } + + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java index 0384ed2f6780d..acdf1abafadd4 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java @@ -26,9 +26,12 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; @@ -477,4 +480,13 @@ public void testUpdateIgnoreAbove() throws IOException { indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); } + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + ICUCollationKeywordFieldMapper mapper = new ICUCollationKeywordFieldMapper.Builder("field").build(context); + + assertEquals("value", mapper.parseSourceValue("value")); + assertEquals("42", mapper.parseSourceValue(42L)); + assertEquals("true", mapper.parseSourceValue(true)); + } } diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 095d7bbd16b2f..5b593e34e1a35 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -599,6 +599,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index 5acc8c9a82280..1f4fdf59bc817 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -28,10 +28,12 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsResponse; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; @@ -44,8 +46,10 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.ParsedDocument; @@ -672,4 +676,17 @@ public void testEmptyName() throws IOException { assertThat(e.getMessage(), containsString("name cannot be empty string")); } + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + AnnotatedTextFieldMapper mapper = new AnnotatedTextFieldMapper.Builder("field") + .indexAnalyzer(indexService.getIndexAnalyzers().getDefaultIndexAnalyzer()) + .searchAnalyzer(indexService.getIndexAnalyzers().getDefaultSearchAnalyzer()) + .searchQuoteAnalyzer(indexService.getIndexAnalyzers().getDefaultSearchQuoteAnalyzer()) + .build(context); + + assertEquals("value", mapper.parseSourceValue("value")); + assertEquals("42", mapper.parseSourceValue(42L)); + assertEquals("true", mapper.parseSourceValue(true)); + } } diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index a8a51176142a9..f09fd6d150f03 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -170,9 +170,13 @@ protected void parseCreateField(ParseContext context) } } + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { } - } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml index f716e42f9e7eb..5653ae61a6c7d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml @@ -21,7 +21,7 @@ setup: index: test id: 1 body: - keyword: [ "first", "second" ] + keyword: [ "x", "y" ] integer_range: gte: 0 lte: 42 @@ -39,8 +39,8 @@ setup: - is_true: hits.hits.0._id - is_true: hits.hits.0._source - - match: { hits.hits.0.fields.keyword.0: first } - - match: { hits.hits.0.fields.keyword.1: second } + - match: { hits.hits.0.fields.keyword.0: x } + - match: { hits.hits.0.fields.keyword.1: y } - match: { hits.hits.0.fields.integer_range.0.gte: 0 } - match: { hits.hits.0.fields.integer_range.0.lte: 42 } @@ -65,7 +65,7 @@ setup: index: test id: 1 body: - keyword: [ "value" ] + keyword: [ "x" ] - do: catch: bad_request @@ -76,3 +76,49 @@ setup: - match: { error.root_cause.0.type: "illegal_argument_exception" } - match: { error.root_cause.0.reason: "Unable to retrieve the requested [fields] since _source is disabled in the mappings for index [test]" } + +--- +"Test ignore malformed": + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 1 + mappings: + properties: + keyword: + type: keyword + integer: + type: integer + ignore_malformed: true + + - do: + index: + index: test + id: 1 + body: + keyword: "x" + integer: 42 + + - do: + index: + index: test + id: 2 + body: + keyword: "y" + integer: "not an integer" + + - do: + indices.refresh: + index: [ test ] + + - do: + search: + index: test + body: + sort: [ keyword ] + fields: [ integer ] + + - match: { hits.hits.0.fields.integer.0: 42 } + - is_false: hits.hits.1.fields.integer diff --git a/server/src/main/java/org/elasticsearch/common/network/InetAddresses.java b/server/src/main/java/org/elasticsearch/common/network/InetAddresses.java index 2e68d8358f0b2..4afba58c9c630 100644 --- a/server/src/main/java/org/elasticsearch/common/network/InetAddresses.java +++ b/server/src/main/java/org/elasticsearch/common/network/InetAddresses.java @@ -384,4 +384,17 @@ public static Tuple parseCidr(String maskedAddress) { throw new IllegalArgumentException("Expected [ip/prefix] but was [" + maskedAddress + "]"); } } + + /** + * Given an address and prefix length, returns the string representation of the range in CIDR notation. + * + * See {@link #toAddrString} for details on how the address is represented. + */ + public static String toCidrString(InetAddress address, int prefixLength) { + return new StringBuilder() + .append(toAddrString(address)) + .append("/") + .append(prefixLength) + .toString(); + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index e76cf11c56603..4dbfe0c9a9b19 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -158,6 +158,11 @@ public FT fieldType() { protected abstract void setGeometryQueryBuilder(FT fieldType); } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException(); + } + public abstract static class TypeParser implements Mapper.TypeParser { protected abstract T newBuilder(String name, Map params); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index c63a8c46fd782..705160e88ff77 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -196,7 +196,11 @@ protected void parseCreateField(ParseContext context) throws IOException { // no doc values createFieldNamesField(context); } + } + @Override + protected Object parseSourceValue(Object value) { + return value; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 09c9ac3aa567e..a55a1c87c59eb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -28,6 +28,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; @@ -258,6 +259,16 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + public Boolean parseSourceValue(Object value) { + if (value instanceof Boolean) { + return (Boolean) value; + } else { + String textValue = value.toString(); + return Booleans.parseBoolean(textValue.toCharArray(), 0, textValue.length(), false); + } + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 2b5b981af4e42..c550f73195a8c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -602,6 +602,15 @@ private void parse(ParseContext parseContext, Token token, } } + @Override + protected List parseSourceValue(Object value) { + if (value instanceof List) { + return (List) value; + } else { + return List.of(value); + } + } + static class CompletionInputMetadata { public final String input; public final Map> contexts; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index e25de5ee7b843..e1c0285513ef2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -61,6 +61,7 @@ import java.time.Instant; import java.time.ZoneId; import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.Iterator; import java.util.List; import java.util.Locale; @@ -361,6 +362,7 @@ protected DateMathParser dateMathParser() { return dateMathParser; } + // Visible for testing. public long parse(String value) { return resolution.convert(DateFormatters.from(dateTimeFormatter().parse(value)).toInstant()); } @@ -632,6 +634,15 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + public String parseSourceValue(Object value) { + String date = value.toString(); + long timestamp = fieldType().parse(date); + + ZonedDateTime dateTime = fieldType().resolution().toInstant(timestamp).atZone(ZoneOffset.UTC); + return fieldType().dateTimeFormatter().format(dateTime); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { final DateFieldMapper d = (DateFieldMapper) other; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 77a17114518c3..52b85f5f0e38f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -37,6 +37,7 @@ import org.elasticsearch.index.mapper.FieldNamesFieldMapper.FieldNamesFieldType; import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.ArrayList; @@ -310,6 +311,47 @@ public void parse(ParseContext context) throws IOException { */ protected abstract void parseCreateField(ParseContext context) throws IOException; + /** + * Given access to a document's _source, return this field's values. + * + * In addition to pulling out the values, mappers can parse them into a standard form. This + * method delegates parsing to {@link #parseSourceValue} for parsing. Most mappers will choose + * to override {@link #parseSourceValue} -- for example numeric field mappers make sure to + * parse the source value into a number of the right type. + * + * Some mappers may need more flexibility and can override this entire method instead. + * + * @param lookup a lookup structure over the document's source. + * @return a list a standardized field values. + */ + public List lookupValues(SourceLookup lookup) { + Object sourceValue = lookup.extractValue(name()); + if (sourceValue == null) { + return List.of(); + } + + List values = new ArrayList<>(); + if (this instanceof ArrayValueMapperParser) { + return (List) parseSourceValue(sourceValue); + } else { + List sourceValues = sourceValue instanceof List ? (List) sourceValue : List.of(sourceValue); + for (Object value : sourceValues) { + Object parsedValue = parseSourceValue(value); + values.add(parsedValue); + } + } + return values; + } + + /** + * Given a value that has been extracted from a document's source, parse it into a standard + * format. This parsing logic should closely mirror the value parsing in + * {@link #parseCreateField} or {@link #parse}. + * + * Note that when overriding this method, {@link #lookupValues} should *not* be overridden. + */ + protected abstract Object parseSourceValue(Object value); + protected void createFieldNamesField(ParseContext context) { FieldNamesFieldType fieldNamesFieldType = context.docMapper().metadataMapper(FieldNamesFieldMapper.class).fieldType(); if (fieldNamesFieldType != null && fieldNamesFieldType.isEnabled()) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index e60638d073497..4a182c5aee6f2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -397,6 +397,12 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected String parseSourceValue(Object value) { + InetAddress address = InetAddresses.forString(value.toString()); + return InetAddresses.toAddrString(address); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { IpFieldMapper mergeWith = (IpFieldMapper) other; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 115bab34654e6..7a9dcd14942b5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -187,6 +187,11 @@ public Mapper.Builder parse(String name, Map node, ParserCont } } + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + public static final class KeywordFieldType extends StringFieldType { private NamedAnalyzer normalizer = null; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index 4aa98fe4a1123..c30c91d20fa4f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -70,6 +70,11 @@ public void postParse(ParseContext context) throws IOException { // do nothing } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source."); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 9c55caece6447..f1a77a932ad46 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1093,6 +1093,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected Number parseSourceValue(Object value) { + return fieldType().type.parse(value, coerce.value()); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { NumberFieldMapper m = (NumberFieldMapper) other; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 928b91feaf0d4..5286125d27908 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -52,6 +52,7 @@ import java.net.UnknownHostException; import java.time.ZoneId; import java.time.ZoneOffset; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -402,6 +403,26 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + @SuppressWarnings("unchecked") + protected Object parseSourceValue(Object value) { + RangeType rangeType = fieldType().rangeType(); + if (!(value instanceof Map)) { + assert rangeType == RangeType.IP; + Tuple ipRange = InetAddresses.parseCidr(value.toString()); + return InetAddresses.toCidrString(ipRange.v1(), ipRange.v2()); + } + + Map range = (Map) value; + Map parsedRange = new HashMap<>(); + for (Map.Entry entry : range.entrySet()) { + Object parsedValue = rangeType.parseValue(entry.getValue(), coerce.value(), fieldType().dateMathParser); + Object formattedValue = rangeType.formatValue(parsedValue, fieldType().dateTimeFormatter); + parsedRange.put(entry.getKey(), formattedValue); + } + return parsedRange; + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { RangeFieldMapper mergeWith = (RangeFieldMapper) other; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java index 86d79e3171573..f293d4ded6c61 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java @@ -36,14 +36,17 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; import java.net.InetAddress; +import java.time.Instant; import java.time.ZoneId; import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -69,8 +72,9 @@ public InetAddress parseTo(RangeFieldMapper.RangeFieldType fieldType, XContentPa InetAddress address = InetAddresses.forString(parser.text()); return included ? address : nextDown(address); } + @Override - public InetAddress parse(Object value, boolean coerce) { + public InetAddress parseValue(Object value, boolean coerce, @Nullable DateMathParser dateMathParser) { if (value instanceof InetAddress) { return (InetAddress) value; } else { @@ -80,6 +84,12 @@ public InetAddress parse(Object value, boolean coerce) { return InetAddresses.forString(value.toString()); } } + + @Override + public Object formatValue(Object value, DateFormatter dateFormatter) { + return InetAddresses.toAddrString((InetAddress) value); + } + @Override public InetAddress minValue() { return InetAddressPoint.MIN_VALUE; @@ -170,22 +180,34 @@ private Query createQuery(String field, Object lower, Object upper, boolean incl public Field getRangeField(String name, RangeFieldMapper.Range r) { return new LongRange(name, new long[] {((Number)r.from).longValue()}, new long[] {((Number)r.to).longValue()}); } - private Number parse(DateMathParser dateMathParser, String dateStr) { - return dateMathParser.parse(dateStr, () -> {throw new IllegalArgumentException("now is not used at indexing time");}) - .toEpochMilli(); - } @Override public Number parseFrom(RangeFieldMapper.RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) throws IOException { - Number value = parse(fieldType.dateMathParser, parser.text()); + Number value = parseValue(parser.text(), coerce, fieldType.dateMathParser); return included ? value : nextUp(value); } @Override public Number parseTo(RangeFieldMapper.RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) throws IOException{ - Number value = parse(fieldType.dateMathParser, parser.text()); + Number value = parseValue(parser.text(), coerce, fieldType.dateMathParser); return included ? value : nextDown(value); } + + @Override + public Long parseValue(Object dateStr, boolean coerce, @Nullable DateMathParser dateMathParser) { + assert dateMathParser != null; + return dateMathParser.parse(dateStr.toString(), () -> { + throw new IllegalArgumentException("now is not used at indexing time"); + }).toEpochMilli(); + } + + @Override + public Object formatValue(Object value, DateFormatter dateFormatter) { + long timestamp = (long) value; + ZonedDateTime dateTime = Instant.ofEpochMilli(timestamp).atZone(ZoneOffset.UTC); + return dateFormatter.format(dateTime); + } + @Override public Long minValue() { return Long.MIN_VALUE; @@ -243,6 +265,7 @@ public Query rangeQuery(String field, boolean hasDocValues, Object lowerTerm, Ob return createRangeQuery(field, hasDocValues, low, high, includeLower, includeUpper, relation); } + @Override public Query withinQuery(String field, Object from, Object to, boolean includeLower, boolean includeUpper) { return LONG.withinQuery(field, from, to, includeLower, includeUpper); @@ -598,6 +621,15 @@ public List createFields(ParseContext context, String name, Rang } return fields; } + + public Object parseValue(Object value, boolean coerce, @Nullable DateMathParser dateMathParser) { + return numberType.parse(value, coerce); + } + + public Object formatValue(Object value, DateFormatter formatter) { + return value; + } + /** parses from value. rounds according to included flag */ public Object parseFrom(RangeFieldMapper.RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) throws IOException { @@ -618,15 +650,12 @@ public Object parseTo(RangeFieldMapper.RangeFieldType fieldType, XContentParser public abstract Query withinQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo); public abstract Query containsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo); public abstract Query intersectsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo); - public Object parse(Object value, boolean coerce) { - return numberType.parse(value, coerce); - } public Query rangeQuery(String field, boolean hasDocValues, Object from, Object to, boolean includeFrom, boolean includeTo, ShapeRelation relation, @Nullable ZoneId timeZone, @Nullable DateMathParser dateMathParser, QueryShardContext context) { - Object lower = from == null ? minValue() : parse(from, false); - Object upper = to == null ? maxValue() : parse(to, false); + Object lower = from == null ? minValue() : parseValue(from, false, dateMathParser); + Object upper = to == null ? maxValue() : parseValue(to, false, dateMathParser); return createRangeQuery(field, hasDocValues, lower, upper, includeFrom, includeTo, relation); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index b8a6c63cd1492..3d8a852fd26b8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -479,6 +479,11 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new UnsupportedOperationException(); } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException(); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { @@ -505,6 +510,11 @@ protected void parseCreateField(ParseContext context) { throw new UnsupportedOperationException(); } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException(); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { @@ -828,6 +838,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + @Override public Iterator iterator() { List subIterators = new ArrayList<>(); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java index 1ede570a3bc41..64369aac9de9a 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java @@ -23,12 +23,15 @@ import org.apache.lucene.index.ReaderUtil; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.lookup.SourceLookup; +import java.util.HashSet; import java.util.Map; +import java.util.Set; /** * A fetch sub-phase for high-level field retrieval. Given a list of fields, it @@ -59,8 +62,22 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) { LeafReaderContext readerContext = context.searcher().getIndexReader().leaves().get(readerIndex); sourceLookup.setSegmentAndDocument(readerContext, hit.docId()); - Map fieldValues = fieldValueRetriever.retrieve(sourceLookup); + Set ignoredFields = getIgnoredFields(hit); + Map fieldValues = fieldValueRetriever.retrieve(sourceLookup, ignoredFields); hit.fields(fieldValues); } } + + private Set getIgnoredFields(SearchHit hit) { + DocumentField field = hit.field(IgnoredFieldMapper.NAME); + if (field == null) { + return Set.of(); + } + + Set ignoredFields = new HashSet<>(); + for (Object value : field.getValues()) { + ignoredFields.add((String) value); + } + return ignoredFields; + } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java index e9d1b88a9199f..bf879c85f043a 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java @@ -20,15 +20,14 @@ package org.elasticsearch.search.fetch.subphase; import org.elasticsearch.common.document.DocumentField; -import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.DocumentFieldMappers; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.search.lookup.SourceLookup; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -38,80 +37,60 @@ * Then given a specific document, it can retrieve the corresponding fields from the document's source. */ public class FieldValueRetriever { + private final DocumentFieldMappers fieldMappers; private final List fieldContexts; - private final Set sourcePaths; public static FieldValueRetriever create(MapperService mapperService, Collection fieldPatterns) { + DocumentFieldMappers fieldMappers = mapperService.documentMapper().mappers(); List fields = new ArrayList<>(); - Set sourcePaths = new HashSet<>(); for (String fieldPattern : fieldPatterns) { Collection concreteFields = mapperService.simpleMatchToFullName(fieldPattern); for (String field : concreteFields) { - MappedFieldType fieldType = mapperService.fieldType(field); - - if (fieldType != null) { + if (fieldMappers.getMapper(field) != null) { Set sourcePath = mapperService.sourcePath(field); fields.add(new FieldContext(field, sourcePath)); - sourcePaths.addAll(sourcePath); } } } - return new FieldValueRetriever(fields, sourcePaths); + return new FieldValueRetriever(fieldMappers, fields); } - private FieldValueRetriever(List fieldContexts, Set sourcePaths) { + private FieldValueRetriever(DocumentFieldMappers fieldMappers, + List fieldContexts) { + this.fieldMappers = fieldMappers; this.fieldContexts = fieldContexts; - this.sourcePaths = sourcePaths; } - @SuppressWarnings("unchecked") - public Map retrieve(SourceLookup sourceLookup) { - Map result = new HashMap<>(); - Map sourceValues = extractValues(sourceLookup, sourcePaths); - + public Map retrieve(SourceLookup sourceLookup, Set ignoredFields) { + Map documentFields = new HashMap<>(); for (FieldContext fieldContext : fieldContexts) { String field = fieldContext.fieldName; Set sourcePath = fieldContext.sourcePath; - List values = new ArrayList<>(); - for (String path : sourcePath) { - Object value = sourceValues.get(path); - if (value != null) { - if (value instanceof List) { - values.addAll((List) value); - } else { - values.add(value); - } - } + if (ignoredFields.contains(field)) { + continue; } - result.put(field, new DocumentField(field, values)); - } - return result; - } - /** - * For each of the provided paths, return its value in the source. Note that in contrast with - * {@link SourceLookup#extractRawValues}, array and object values can be returned. - */ - private static Map extractValues(SourceLookup sourceLookup, Set paths) { - Map result = new HashMap<>(paths.size()); - for (String path : paths) { - Object value = XContentMapValues.extractValue(path, sourceLookup); - if (value != null) { - result.put(path, value); + List parsedValues = new ArrayList<>(); + for (String path : sourcePath) { + FieldMapper fieldMapper = (FieldMapper) fieldMappers.getMapper(path); + List values = fieldMapper.lookupValues(sourceLookup); + parsedValues.addAll(values); } + documentFields.put(field, new DocumentField(field, parsedValues)); } - return result; + return documentFields; } private static class FieldContext { final String fieldName; final Set sourcePath; - FieldContext(String fieldName, Set sourcePath) { + FieldContext(String fieldName, + Set sourcePath) { this.fieldName = fieldName; this.sourcePath = sourcePath; } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java index e547f02ef0347..6393d4bf50ff0 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java @@ -132,6 +132,14 @@ public List extractRawValues(String path) { return XContentMapValues.extractRawValues(path, loadSourceIfNeeded()); } + /** + * For the provided path, return its value in the source. Note that in contrast with + * {@link SourceLookup#extractRawValues}, array and object values can be returned. + */ + public Object extractValue(String path) { + return XContentMapValues.extractValue(path, loadSourceIfNeeded()); + } + public Object filter(FetchSourceContext context) { return context.getFilter().apply(loadSourceIfNeeded()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index d779cb518d1bf..a99dc7324bfda 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -30,9 +30,12 @@ import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -280,9 +283,18 @@ public void testMeta() throws Exception { assertEquals(mapping3, mapper.mappingSource().toString()); } + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + BooleanFieldMapper mapper = new BooleanFieldMapper.Builder("field").build(context); + + assertTrue(mapper.parseSourceValue(true)); + assertFalse(mapper.parseSourceValue("false")); + assertFalse(mapper.parseSourceValue("")); + } + @Override protected BooleanFieldMapper.Builder newBuilder() { return new BooleanFieldMapper.Builder("boolean"); } - } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 425b2fba9bbe6..7b4a382ab09bc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -31,6 +31,8 @@ import org.apache.lucene.util.CharsRefBuilder; import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.RegExp; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; @@ -53,6 +55,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.function.Function; @@ -950,6 +953,20 @@ public void testLimitOfContextMappings() throws Throwable { CompletionFieldMapper.COMPLETION_CONTEXTS_LIMIT + "] has been exceeded")); } + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + CompletionFieldMapper mapper = new CompletionFieldMapper.Builder("field").build(context); + + assertEquals(List.of("value"), mapper.parseSourceValue("value")); + + List list = List.of("first", "second"); + assertEquals(list, mapper.parseSourceValue(list)); + + Map object = Map.of("input", List.of("first", "second"), "weight", "2.718"); + assertEquals(List.of(object), mapper.parseSourceValue(object)); + } + private Matcher suggestField(String value) { return Matchers.allOf(hasProperty(IndexableField::stringValue, equalTo(value)), Matchers.instanceOf(SuggestField.class)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index 2ca1b20a6f317..25a477b35c08c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -21,10 +21,13 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; +import org.elasticsearch.Version; import org.elasticsearch.bootstrap.JavaVersion; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; @@ -457,4 +460,40 @@ public void testMeta() throws Exception { assertEquals(mapping3, mapper.mappingSource().toString()); } + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + DateFieldMapper mapper = new DateFieldMapper.Builder("field").build(context); + String date = "2020-05-15T21:33:02.000Z"; + assertEquals(date, mapper.parseSourceValue(date)); + assertEquals(date, mapper.parseSourceValue(1589578382000L)); + + DateFieldMapper mapperWithFormat = new DateFieldMapper.Builder("field") + .format("yyyy/MM/dd||epoch_millis") + .build(context); + String dateInFormat = "1990/12/29"; + assertEquals(dateInFormat, mapperWithFormat.parseSourceValue(dateInFormat)); + assertEquals(dateInFormat, mapperWithFormat.parseSourceValue(662428800000L)); + + DateFieldMapper mapperWithMillis = new DateFieldMapper.Builder("field") + .format("epoch_millis") + .build(context); + String dateInMillis = "662428800000"; + assertEquals(dateInMillis, mapperWithMillis.parseSourceValue(dateInMillis)); + assertEquals(dateInMillis, mapperWithMillis.parseSourceValue(662428800000L)); + } + + public void testParseSourceValueNanos() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + DateFieldMapper mapper = new DateFieldMapper.Builder("field") + .format("strict_date_time||epoch_millis") + .withResolution(DateFieldMapper.Resolution.NANOSECONDS) + .build(context); + String date = "2020-05-15T21:33:02.123456789Z"; + assertEquals("2020-05-15T21:33:02.123456789Z", mapper.parseSourceValue(date)); + assertEquals("2020-05-15T21:33:02.123Z", mapper.parseSourceValue(1589578382123L)); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index d75967a069e8c..8eacfc3fe4278 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -115,6 +115,11 @@ static class FakeFieldMapper extends FieldMapper { protected void parseCreateField(ParseContext context) throws IOException { } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException(); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index fb730f675fbcf..281ef8bb9ad01 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -203,6 +203,11 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new UnsupportedOperationException(); } + @Override + protected Object parseSourceValue(Object value) { + return value; + } + @Override public Iterator iterator() { return Iterators.concat(super.iterator(), Arrays.asList(binMapper, boolMapper, pointMapper, shapeMapper, stringMapper).iterator()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java index 84a5f54b89082..5904ca15274bf 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java @@ -141,6 +141,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 37ea96e3df976..e1dbaa6667094 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -23,10 +23,13 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -278,4 +281,14 @@ public void testEmptyName() throws IOException { ); assertThat(e.getMessage(), containsString("name cannot be empty string")); } + + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + IpFieldMapper mapper = new IpFieldMapper.Builder("field").build(context); + + assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8::2:1")); + assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8:0:0:0:0:2:1")); + assertEquals("::1", mapper.parseSourceValue("0:0:0:0:0:0:0:1")); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java index 2f861ca6fc79c..8e06c938c8b7f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java @@ -20,10 +20,13 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; @@ -80,4 +83,14 @@ public void testStoreCidr() throws Exception { assertThat(storedField.stringValue(), containsString(strVal)); } } + + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + RangeFieldMapper mapper = new RangeFieldMapper.Builder("field", RangeType.IP).build(context); + Map range = Map.of("gte", "2001:db8:0:0:0:0:2:1"); + assertEquals(Map.of("gte", "2001:db8::2:1"), mapper.parseSourceValue(range)); + assertEquals("2001:db8::2:1/32", mapper.parseSourceValue("2001:db8:0:0:0:0:2:1/32")); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 8215a1f101fe2..ac478cdb7f199 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -26,6 +26,8 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; @@ -141,6 +143,9 @@ public void testDefaults() throws Exception { // used by TermVectorsService assertArrayEquals(new String[] { "1234" }, TermVectorsService.getValues(doc.rootDoc().getFields("field"))); + + FieldMapper fieldMapper = (FieldMapper) mapper.mappers().getMapper("field"); + assertEquals("1234", fieldMapper.parseSourceValue("1234")); } public void testIgnoreAbove() throws IOException { @@ -595,4 +600,14 @@ public void testMeta() throws Exception { new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE); assertEquals(mapping3, mapper.mappingSource().toString()); } + + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + KeywordFieldMapper mapper = new KeywordFieldMapper.Builder("field").build(context); + + assertEquals("value", mapper.parseSourceValue("value")); + assertEquals("42", mapper.parseSourceValue(42L)); + assertEquals("true", mapper.parseSourceValue(true)); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index e0984ded48835..d44288b45cee8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -22,9 +22,12 @@ import com.carrotsearch.randomizedtesting.annotations.Timeout; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -392,6 +395,15 @@ public void testEmptyName() throws IOException { } } + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + NumberFieldMapper mapper = new NumberFieldMapper.Builder("field", NumberType.INTEGER).build(context); + + assertEquals(3, mapper.parseSourceValue(3.14)); + assertEquals(42, mapper.parseSourceValue("42.9")); + } + @Timeout(millis = 30000) public void testOutOfRangeValues() throws IOException { final List> inputs = Arrays.asList( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index 67da23e6abe73..28f457e571d2e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -22,10 +22,13 @@ import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -40,6 +43,7 @@ import java.util.Collection; import java.util.HashSet; import java.util.Locale; +import java.util.Map; import static org.elasticsearch.index.query.RangeQueryBuilder.GTE_FIELD; import static org.elasticsearch.index.query.RangeQueryBuilder.GT_FIELD; @@ -480,4 +484,19 @@ public void testIllegalFormatField() throws Exception { assertEquals("Invalid format: [[test_format]]: Unknown pattern letter: t", e.getMessage()); } + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + RangeFieldMapper longMapper = new RangeFieldMapper.Builder("field", RangeType.LONG).build(context); + Map longRange = Map.of("gte", 3.14, "lt", "42.9"); + assertEquals(Map.of("gte", 3L, "lt", 42L), longMapper.parseSourceValue(longRange)); + + RangeFieldMapper dateMapper = new RangeFieldMapper.Builder("field", RangeType.DATE) + .format("yyyy/MM/dd||epoch_millis") + .build(context); + Map dateRange = Map.of("lt", "1990/12/29", "gte", 597429487111L); + assertEquals(Map.of("lt", "1990/12/29", "gte", "1988/12/06"), + dateMapper.parseSourceValue(dateRange)); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index d0657370f1c70..db42af63edcb2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -254,14 +254,14 @@ public void testDateRangeQueryUsingMappingFormat() { assertEquals(1466062190000L, formatter.parseMillis(to)); fieldType.setDateTimeFormatter(formatter); - final Query query = fieldType.rangeQuery(from, to, true, true, relation, null, null, context); + final Query query = fieldType.rangeQuery(from, to, true, true, relation, null, fieldType.dateMathParser(), context); assertEquals("field:", query.toString()); // compare lower and upper bounds with what we would get on a `date` field DateFieldType dateFieldType = new DateFieldType(); dateFieldType.setName(FIELDNAME); dateFieldType.setDateTimeFormatter(formatter); - final Query queryOnDateField = dateFieldType.rangeQuery(from, to, true, true, relation, null, null, context); + final Query queryOnDateField = dateFieldType.rangeQuery(from, to, true, true, relation, null, fieldType.dateMathParser(), context); assertEquals("field:[1465975790000 TO 1466062190999]", queryOnDateField.toString()); } @@ -486,9 +486,9 @@ private Object nextTo(Object from) throws Exception { } public void testParseIp() { - assertEquals(InetAddresses.forString("::1"), RangeType.IP.parse(InetAddresses.forString("::1"), randomBoolean())); - assertEquals(InetAddresses.forString("::1"), RangeType.IP.parse("::1", randomBoolean())); - assertEquals(InetAddresses.forString("::1"), RangeType.IP.parse(new BytesRef("::1"), randomBoolean())); + assertEquals(InetAddresses.forString("::1"), RangeType.IP.parseValue(InetAddresses.forString("::1"), randomBoolean(), null)); + assertEquals(InetAddresses.forString("::1"), RangeType.IP.parseValue("::1", randomBoolean(), null)); + assertEquals(InetAddresses.forString("::1"), RangeType.IP.parseValue(new BytesRef("::1"), randomBoolean(), null)); } public void testTermQuery() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 38137d5bd7bbc..9ad917f06f608 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -46,7 +46,9 @@ import org.apache.lucene.search.spans.SpanOrQuery; import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; @@ -1307,4 +1309,14 @@ public void testMeta() throws Exception { new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE); assertEquals(mapping3, mapper.mappingSource().toString()); } + + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + TextFieldMapper mapper = new TextFieldMapper.Builder("field").build(context); + + assertEquals("value", mapper.parseSourceValue("value")); + assertEquals("42", mapper.parseSourceValue(42L)); + assertEquals("true", mapper.parseSourceValue(true)); + } } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java index 92aa7eeacde4f..2b61fa2f8620e 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Set; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItems; @@ -65,8 +66,8 @@ public void testObjectValues() throws IOException { MapperService mapperService = createMapperService(); XContentBuilder source = XContentFactory.jsonBuilder().startObject() .startObject("float_range") - .field("gte", 0.0) - .field("lte", 2.718) + .field("gte", 0.0f) + .field("lte", 2.718f) .endObject() .endObject(); @@ -76,14 +77,59 @@ public void testObjectValues() throws IOException { DocumentField rangeField = fields.get("float_range"); assertNotNull(rangeField); assertThat(rangeField.getValues().size(), equalTo(1)); - assertThat(rangeField.getValue(), equalTo(Map.of("gte", 0.0, "lte", 2.718))); + assertThat(rangeField.getValue(), equalTo(Map.of("gte", 0.0f, "lte", 2.718f))); + } + + public void testNonExistentField() throws IOException { + MapperService mapperService = createMapperService(); + XContentBuilder source = XContentFactory.jsonBuilder().startObject() + .field("field", "value") + .endObject(); + + Map fields = retrieveFields(mapperService, source, "non-existent"); + assertThat(fields.size(), equalTo(0)); + } + + public void testArrayValueMappers() throws IOException { + MapperService mapperService = createMapperService(); + + XContentBuilder source = XContentFactory.jsonBuilder().startObject() + .array("completion", "first", "second") + .endObject(); + + Map fields = retrieveFields(mapperService, source, "completion"); + assertThat(fields.size(), equalTo(1)); + + DocumentField field = fields.get("completion"); + assertNotNull(field); + assertThat(field.getValues().size(), equalTo(2)); + assertThat(field.getValues(), hasItems("first", "second")); + + // Test a field with multiple geo-points. + source = XContentFactory.jsonBuilder().startObject() + .startObject("completion") + .array("input", "first", "second") + .field("weight", "2.718") + .endObject() + .endObject(); + + fields = retrieveFields(mapperService, source, "completion"); + assertThat(fields.size(), equalTo(1)); + + field = fields.get("completion"); + assertNotNull(field); + assertThat(field.getValues().size(), equalTo(1)); + + Map expected = Map.of("input", List.of("first", "second"), + "weight", "2.718"); + assertThat(field.getValues().get(0), equalTo(expected)); } public void testFieldNamesWithWildcard() throws IOException { MapperService mapperService = createMapperService();; XContentBuilder source = XContentFactory.jsonBuilder().startObject() .array("field", "first", "second") - .field("integer_field", "third") + .field("integer_field", 333) .startObject("object") .field("field", "fourth") .endObject() @@ -100,7 +146,7 @@ public void testFieldNamesWithWildcard() throws IOException { DocumentField otherField = fields.get("integer_field"); assertNotNull(otherField); assertThat(otherField.getValues().size(), equalTo(1)); - assertThat(otherField.getValues(), hasItems("third")); + assertThat(otherField.getValues(), hasItems(333)); DocumentField objectField = fields.get("object.field"); assertNotNull(objectField); @@ -108,7 +154,6 @@ public void testFieldNamesWithWildcard() throws IOException { assertThat(objectField.getValues(), hasItems("fourth")); } - public void testFieldAliases() throws IOException { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() .startObject("properties") @@ -226,15 +271,15 @@ private Map retrieveFields(MapperService mapperService, X sourceLookup.setSource(BytesReference.bytes(source)); FieldValueRetriever fetchFieldsLookup = FieldValueRetriever.create(mapperService, fieldPatterns); - return fetchFieldsLookup.retrieve(sourceLookup); + return fetchFieldsLookup.retrieve(sourceLookup, Set.of()); } - public MapperService createMapperService() throws IOException { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() .startObject("properties") .startObject("field").field("type", "keyword").endObject() .startObject("integer_field").field("type", "integer").endObject() + .startObject("completion").field("type", "completion").endObject() .startObject("float_range").field("type", "float_range").endObject() .startObject("object") .startObject("properties") diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index 08830205f464c..a797469431c0c 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -99,6 +99,11 @@ protected String contentType() { protected void parseCreateField(ParseContext context) throws IOException { } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException(); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index ae85f3a7d0780..ee435ee0403f4 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -170,6 +170,11 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new UnsupportedOperationException("Parsing is implemented in parse(), this method should NEVER be called"); } + @Override + protected Object parseSourceValue(Object value) { + return value; + } + public static class HistogramFieldType extends MappedFieldType { public HistogramFieldType() { diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 1efb8c10bf207..3c9b6395fa97f 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -38,6 +38,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.time.ZoneId; @@ -289,6 +290,18 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + public List lookupValues(SourceLookup lookup) { + return fieldType().value == null + ? List.of() + : List.of(fieldType().value); + } + + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException("This should never be called, since lookupValues is implemented directly."); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { ConstantKeywordFieldType newConstantKeywordFT = (ConstantKeywordFieldType) other.fieldType(); diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index 0e6b22b5a7a01..b97f79b594068 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -13,18 +13,21 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldMapperTestCase; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.junit.Before; import java.util.Collection; import java.util.Collections; +import java.util.List; public class ConstantKeywordFieldMapperTests extends FieldMapperTestCase { @@ -127,4 +130,27 @@ public void testMeta() throws Exception { new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE); assertEquals(mapping3, mapper.mappingSource().toString()); } + + public void testLookupValues() throws Exception { + IndexService indexService = createIndex("test"); + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "constant_keyword") + .endObject().endObject().endObject().endObject()); + DocumentMapper mapper = indexService.mapperService().merge("_doc", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); + assertEquals(mapping, mapper.mappingSource().toString()); + + FieldMapper fieldMapper = (FieldMapper) mapper.mappers().getMapper("field"); + List values = fieldMapper.lookupValues(new SourceLookup()); + assertTrue(values.isEmpty()); + + String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "constant_keyword") + .field("value", "foo").endObject().endObject().endObject().endObject()); + mapper = indexService.mapperService().merge("_doc", new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE); + + fieldMapper = (FieldMapper) mapper.mappers().getMapper("field"); + values = fieldMapper.lookupValues(new SourceLookup()); + assertEquals(1, values.size()); + assertEquals("foo", values.get(0)); + } } diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java index 6c01804c82d14..e95145bb0a269 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java @@ -617,6 +617,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected Object parseSourceValue(Object value) { + return value; + } + @Override protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { super.doXContentBody(builder, includeDefaults, params); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml index ad2e2677bf610..7a6c2a0d53b0d 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml @@ -184,3 +184,18 @@ setup: - match: {hits.hits.0._index: test1 } - match: {hits.hits.1._index: test1 } - match: {hits.hits.2._index: test2 } + +--- +"Field retrieval": + + - do: + search: + index: test* + body: + fields: [ foo ] + sort: [ { _index: asc } ] + + - match: { "hits.total.value": 3 } + - match: {hits.hits.0.fields.foo.0: bar } + - match: {hits.hits.1.fields.foo.0: bar } + - match: {hits.hits.2.fields.foo.0: baz } diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index 68d45fba6d149..7d19ec20ac300 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -224,6 +224,11 @@ public void parse(ParseContext context) throws IOException { context.doc().addWithKey(fieldType().name(), field); } + @Override + protected Object parseSourceValue(Object value) { + return value; + } + @Override protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { super.doXContentBody(builder, includeDefaults, params); diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java index 400d2a388aa56..b692443b70ef1 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java @@ -145,12 +145,16 @@ public void parse(ParseContext context) { throw new UnsupportedOperationException(ERROR_MESSAGE_7X); } - @Override protected void parseCreateField(ParseContext context) { throw new IllegalStateException("parse is implemented directly"); } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException(ERROR_MESSAGE_7X); + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 0a8edba374966..67882c0c4aa8e 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -893,6 +893,11 @@ protected void parseCreateField(ParseContext context) throws IOException { parseDoc.addAll(fields); } + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + // For internal use by Lucene only - used to define ngram index final MappedFieldType ngramFieldType;