From 51b6a4e71dc356714d85572812936ffd605e9359 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 9 Jun 2020 10:42:55 -0700 Subject: [PATCH] Add support for a 'format' option in fields retrieval. (#57855) The new `format` option allows for passing a custom date format: ``` POST logs-*/_search { "fields": [ "file.*", { "field": "event.timestamp", "format": "epoch_millis" }, ... ] } ``` Other API notes: * We use the same syntax as `docvalue_fields` for consistency. Under the hood, both `fields` and `docvalue_fields` use the same `FieldAndFormat` object to share serialization logic. * Only `date` and `date_range` fields support formatting currently. --- .../index/mapper/RankFeatureFieldMapper.java | 5 +- .../index/mapper/RankFeaturesFieldMapper.java | 5 +- .../index/mapper/ScaledFloatFieldMapper.java | 6 +- .../mapper/SearchAsYouTypeFieldMapper.java | 9 +- .../index/mapper/TokenCountFieldMapper.java | 6 +- .../mapper/RankFeatureFieldMapperTests.java | 4 +- .../mapper/ScaledFloatFieldMapperTests.java | 4 +- .../join/mapper/MetaJoinFieldMapper.java | 2 +- .../join/mapper/ParentIdFieldMapper.java | 2 +- .../join/mapper/ParentJoinFieldMapper.java | 5 +- .../percolator/PercolatorFieldMapper.java | 5 +- .../ICUCollationKeywordFieldMapper.java | 5 +- .../ICUCollationKeywordFieldMapperTests.java | 6 +- .../AnnotatedTextFieldMapper.java | 5 +- .../AnnotatedTextFieldMapperTests.java | 6 +- .../mapper/murmur3/Murmur3FieldMapper.java | 5 +- .../test/search/330_fetch_fields.yml | 47 +++++++ .../action/search/SearchRequestBuilder.java | 18 ++- .../mapper/AbstractGeometryFieldMapper.java | 2 +- .../index/mapper/BinaryFieldMapper.java | 5 +- .../index/mapper/BooleanFieldMapper.java | 6 +- .../index/mapper/CompletionFieldMapper.java | 6 +- .../index/mapper/DateFieldMapper.java | 10 +- .../index/mapper/FieldMapper.java | 10 +- .../index/mapper/IpFieldMapper.java | 6 +- .../index/mapper/KeywordFieldMapper.java | 5 +- .../index/mapper/MetadataFieldMapper.java | 2 +- .../index/mapper/NumberFieldMapper.java | 5 +- .../index/mapper/RangeFieldMapper.java | 9 +- .../index/mapper/TextFieldMapper.java | 9 +- .../index/query/InnerHitBuilder.java | 2 +- .../metrics/TopHitsAggregationBuilder.java | 2 +- .../metrics/TopHitsAggregatorFactory.java | 2 +- .../search/builder/SearchSourceBuilder.java | 43 ++++--- .../fetch/subphase/FetchDocValuesContext.java | 78 ------------ .../fetch/subphase/FetchDocValuesPhase.java | 1 - .../fetch/subphase/FetchFieldsContext.java | 6 +- .../search/fetch/subphase/FieldAndFormat.java | 115 ++++++++++++++++++ .../fetch/subphase/FieldValueRetriever.java | 30 +++-- .../index/mapper/BooleanFieldMapperTests.java | 6 +- .../mapper/CompletionFieldMapperTests.java | 6 +- .../index/mapper/DateFieldMapperTests.java | 23 ++-- .../mapper/DocumentFieldMapperTests.java | 2 +- .../index/mapper/ExternalMapper.java | 2 +- .../index/mapper/FakeStringFieldMapper.java | 2 +- .../index/mapper/IpFieldMapperTests.java | 6 +- .../index/mapper/IpRangeFieldMapperTests.java | 4 +- .../index/mapper/KeywordFieldMapperTests.java | 11 +- .../index/mapper/NumberFieldMapperTests.java | 4 +- .../index/mapper/ParametrizedMapperTests.java | 4 +- .../index/mapper/RangeFieldMapperTests.java | 20 ++- .../index/mapper/TextFieldMapperTests.java | 6 +- .../index/query/InnerHitBuilderTests.java | 2 +- .../subphase/FieldValueRetrieverTests.java | 34 +++++- .../index/mapper/MockFieldMapper.java | 2 +- .../mapper/HistogramFieldMapper.java | 5 +- .../mapper/ConstantKeywordFieldMapper.java | 8 +- .../ConstantKeywordFieldMapperTests.java | 4 +- .../mapper/FlatObjectFieldMapper.java | 5 +- .../ql/execution/search/QlSourceBuilder.java | 2 +- .../xpack/sql/action/SqlLicenseIT.java | 4 +- .../sql/action/SqlTranslateActionIT.java | 4 +- .../mapper/DenseVectorFieldMapper.java | 5 +- .../mapper/SparseVectorFieldMapper.java | 2 +- .../wildcard/mapper/WildcardFieldMapper.java | 5 +- 65 files changed, 466 insertions(+), 211 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index 75a97d45f5d2a..2b2677ec4a501 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -181,7 +181,10 @@ private Float objectToFloat(Object value) { } @Override - protected Float parseSourceValue(Object value) { + protected Float parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return objectToFloat(value); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index 760d6dbc6dfcf..b692b834448ee 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -160,7 +160,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value; } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index dce1ad292b9a1..ff896e35c092c 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -475,7 +475,11 @@ private static double objectToDouble(Object value) { } @Override - protected Double parseSourceValue(Object value) { + protected Double parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + double doubleValue = objectToDouble(value); double scalingFactor = fieldType().getScalingFactor(); return Math.round(doubleValue * scalingFactor) / scalingFactor; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index 7b26e229db2cc..48042c2d0ee2b 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -419,7 +419,7 @@ protected void parseCreateField(ParseContext context) { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException(); } @@ -465,7 +465,7 @@ protected void mergeOptions(FieldMapper other, List conflicts) { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException(); } @@ -588,7 +588,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value.toString(); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java index 51da52e37e22f..2b3041e0dc3a2 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java @@ -159,7 +159,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return value.toString(); } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java index e867370acc8ed..30df4394b7418 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java @@ -194,7 +194,7 @@ public void testParseSourceValue() { Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); RankFeatureFieldMapper mapper = new RankFeatureFieldMapper.Builder("field").build(context); - assertEquals(3.14f, mapper.parseSourceValue(3.14), 0.0001); - assertEquals(42.9f, mapper.parseSourceValue("42.9"), 0.0001); + assertEquals(3.14f, mapper.parseSourceValue(3.14, null), 0.0001); + assertEquals(42.9f, mapper.parseSourceValue("42.9", null), 0.0001); } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java index f573c25ba176d..bcbf6fb432f79 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java @@ -409,7 +409,7 @@ public void testParseSourceValue() { .scalingFactor(100) .build(context); - assertEquals(3.14, mapper.parseSourceValue(3.1415926), 0.00001); - assertEquals(3.14, mapper.parseSourceValue("3.1415"), 0.00001); + assertEquals(3.14, mapper.parseSourceValue(3.1415926, null), 0.00001); + assertEquals(3.14, mapper.parseSourceValue("3.1415", null), 0.00001); } } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java index 4a8ba4fc90a1a..404b7d50be00d 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java @@ -136,7 +136,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source."); } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index 75343d2a92d23..c81548c3d32de 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -186,7 +186,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source."); } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index 757651d028404..67217855ceef1 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -348,7 +348,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value; } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index e9ed2bbbebb58..895c792b53ded 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -368,7 +368,10 @@ public void parse(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value; } diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index 4684e28fead26..c43d715184cd0 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -733,7 +733,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value.toString(); } } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java index 28cd339e36bcd..1f69424a06829 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java @@ -491,8 +491,8 @@ public void testParseSourceValue() { Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); ICUCollationKeywordFieldMapper mapper = new ICUCollationKeywordFieldMapper.Builder("field").build(context); - assertEquals("value", mapper.parseSourceValue("value")); - assertEquals("42", mapper.parseSourceValue(42L)); - assertEquals("true", mapper.parseSourceValue(true)); + assertEquals("value", mapper.parseSourceValue("value", null)); + assertEquals("42", mapper.parseSourceValue(42L, null)); + assertEquals("true", mapper.parseSourceValue(true, null)); } } diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 5cde4e8ab264e..dcab066ca8dd5 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -584,7 +584,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value.toString(); } diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index a365aa24cf336..48d1ee50e224e 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -688,8 +688,8 @@ public void testParseSourceValue() { .build(context); AnnotatedTextFieldMapper mapper = (AnnotatedTextFieldMapper) fieldMapper; - assertEquals("value", mapper.parseSourceValue("value")); - assertEquals("42", mapper.parseSourceValue(42L)); - assertEquals("true", mapper.parseSourceValue(true)); + assertEquals("value", mapper.parseSourceValue("value", null)); + assertEquals("42", mapper.parseSourceValue(42L, null)); + assertEquals("true", mapper.parseSourceValue(true, null)); } } diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index ed00a4376fc42..ce33d3d25e11d 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -148,7 +148,10 @@ protected void parseCreateField(ParseContext context) } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value.toString(); } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml index 5653ae61a6c7d..02c3b2c592a88 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml @@ -46,6 +46,53 @@ setup: - match: { hits.hits.0.fields.integer_range.0.lte: 42 } --- +"Test date formatting": + - do: + indices.create: + index: test + body: + settings: + index.number_of_shards: 1 + mappings: + properties: + keyword: + type: keyword + date: + type: date + + - do: + index: + index: test + id: 1 + body: + keyword: "value" + date: "1990-12-29T22:30:00.000Z" + + - do: + indices.refresh: + index: [ test ] + + - do: + search: + index: test + body: + fields: + - field: date + format: "yyyy/MM/dd" + + - is_true: hits.hits.0._id + - is_true: hits.hits.0._source + - match: { hits.hits.0.fields.date.0: "1990/12/29" } + + - do: + catch: bad_request + search: + index: test + body: + fields: + - field: keyword + format: "yyyy/MM/dd" +--- "Test disable source": - do: indices.create: diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java index 5f175af80ef4c..42a56b308dafd 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java @@ -303,8 +303,24 @@ public SearchRequestBuilder addDocValueField(String name) { return addDocValueField(name, null); } + /** + * Adds a field to load and return. The field must be present in the document _source. + * + * @param name The field to load + */ public SearchRequestBuilder addFetchField(String name) { - sourceBuilder().fetchField(name); + sourceBuilder().fetchField(name, null); + return this; + } + + /** + * Adds a field to load and return. The field must be present in the document _source. + * + * @param name The field to load + * @param format TODO(jtibs): fill this in + */ + public SearchRequestBuilder addFetchField(String name, String format) { + sourceBuilder().fetchField(name, format); return this; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index e1a411ace33aa..7020ae0dcaa06 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -142,7 +142,7 @@ public Builder ignoreZValue(final boolean ignoreZValue) { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index bc61b4d891cea..fd055b495253a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -190,7 +190,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 97c4b12bbc6e7..ebc12c3c10a49 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -250,7 +250,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public Boolean parseSourceValue(Object value) { + public Boolean parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + if (value instanceof Boolean) { return (Boolean) value; } else { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 9da867d1e4422..2e57d68274e88 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -532,7 +532,11 @@ private void parse(ParseContext parseContext, Token token, } @Override - protected List parseSourceValue(Object value) { + protected List parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + if (value instanceof List) { return (List) value; } else { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index e321ab1d3189c..a423670addbb8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -544,12 +544,16 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public String parseSourceValue(Object value) { + public String parseSourceValue(Object value, String format) { String date = value.toString(); long timestamp = fieldType().parse(date); - ZonedDateTime dateTime = fieldType().resolution().toInstant(timestamp).atZone(ZoneOffset.UTC); - return fieldType().dateTimeFormatter().format(dateTime); + + DateFormatter dateTimeFormatter = fieldType().dateTimeFormatter(); + if (format != null) { + dateTimeFormatter = DateFormatter.forPattern(format).withLocale(dateTimeFormatter.locale()); + } + return dateTimeFormatter.format(dateTime); } public boolean getIgnoreMalformed() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 80d76e954e9be..a2f3fa6634be8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -24,6 +24,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; @@ -281,9 +282,10 @@ public void parse(ParseContext context) throws IOException { * Some mappers may need more flexibility and can override this entire method instead. * * @param lookup a lookup structure over the document's source. + * @param format an optional format string used when formatting values, for example a date format. * @return a list a standardized field values. */ - public List lookupValues(SourceLookup lookup) { + public List lookupValues(SourceLookup lookup, @Nullable String format) { Object sourceValue = lookup.extractValue(name()); if (sourceValue == null) { return List.of(); @@ -291,11 +293,11 @@ public List lookupValues(SourceLookup lookup) { List values = new ArrayList<>(); if (parsesArrayValue()) { - return (List) parseSourceValue(sourceValue); + return (List) parseSourceValue(sourceValue, format); } else { List sourceValues = sourceValue instanceof List ? (List) sourceValue : List.of(sourceValue); for (Object value : sourceValues) { - Object parsedValue = parseSourceValue(value); + Object parsedValue = parseSourceValue(value, format); values.add(parsedValue); } } @@ -309,7 +311,7 @@ public List lookupValues(SourceLookup lookup) { * * Note that when overriding this method, {@link #lookupValues} should *not* be overridden. */ - protected abstract Object parseSourceValue(Object value); + protected abstract Object parseSourceValue(Object value, @Nullable String format); protected void createFieldNamesField(ParseContext context) { FieldNamesFieldType fieldNamesFieldType = context.docMapper().metadataMapper(FieldNamesFieldMapper.class).fieldType(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 4be35adc759da..0bb1e6ed50a77 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -401,7 +401,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + InetAddress address = InetAddresses.forString(value.toString()); return InetAddresses.toAddrString(address); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index e905cc699f5bd..d9d41da1fc073 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -228,7 +228,10 @@ public Mapper.Builder parse(String name, Map node, ParserCont } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value.toString(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index 56c00ed8693f7..d77826a505d69 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -87,7 +87,7 @@ public void postParse(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index fd7f898ebc455..5abbf1e8871a7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1086,7 +1086,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Number parseSourceValue(Object value) { + protected Number parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return fieldType().type.parse(value, coerce.value()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index c2c141545fbba..0a3e1f648e1df 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -376,7 +376,7 @@ protected void parseCreateField(ParseContext context) throws IOException { @Override @SuppressWarnings("unchecked") - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { RangeType rangeType = fieldType().rangeType(); if (!(value instanceof Map)) { assert rangeType == RangeType.IP; @@ -384,11 +384,16 @@ protected Object parseSourceValue(Object value) { return InetAddresses.toCidrString(ipRange.v1(), ipRange.v2()); } + DateFormatter dateTimeFormatter = fieldType().dateTimeFormatter(); + if (format != null) { + dateTimeFormatter = DateFormatter.forPattern(format).withLocale(dateTimeFormatter.locale()); + } + Map range = (Map) value; Map parsedRange = new HashMap<>(); for (Map.Entry entry : range.entrySet()) { Object parsedValue = rangeType.parseValue(entry.getValue(), coerce.value(), fieldType().dateMathParser); - Object formattedValue = rangeType.formatValue(parsedValue, fieldType().dateTimeFormatter); + Object formattedValue = rangeType.formatValue(parsedValue, dateTimeFormatter); parsedRange.put(entry.getKey(), formattedValue); } return parsedRange; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index f92edddc9886e..bada725c33b40 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -497,7 +497,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException(); } @@ -528,7 +528,7 @@ protected void parseCreateField(ParseContext context) { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException(); } @@ -835,7 +835,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value.toString(); } diff --git a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java index 4c189c92f7c74..31c30aac86655 100644 --- a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java @@ -33,8 +33,8 @@ import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.search.fetch.StoredFieldsContext; -import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.sort.SortBuilder; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java index d10b226c6293b..9ca3cb639478c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java @@ -38,8 +38,8 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; import org.elasticsearch.search.fetch.StoredFieldsContext; -import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.sort.ScoreSortBuilder; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java index a0a480fbc55c3..ffee18e81cc24 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java @@ -26,8 +26,8 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.fetch.StoredFieldsContext; import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext; -import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.internal.SearchContext; diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index d43428cf64f0f..98066ad732379 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -46,8 +46,8 @@ import org.elasticsearch.search.aggregations.PipelineAggregationBuilder; import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.search.fetch.StoredFieldsContext; -import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.rescore.RescorerBuilder; @@ -167,7 +167,7 @@ public static HighlightBuilder highlight() { private List docValueFields; private List scriptFields; private FetchSourceContext fetchSourceContext; - private List fetchFields; + private List fetchFields; private AggregatorFactories.Builder aggregations; @@ -244,7 +244,9 @@ public SearchSourceBuilder(StreamInput in) throws IOException { trackTotalHitsUpTo = in.readOptionalInt(); if (in.getVersion().onOrAfter(Version.V_8_0_0)) { - fetchFields = in.readOptionalStringList(); + if (in.readBoolean()) { + fetchFields = in.readList(FieldAndFormat::new); + } } } @@ -302,7 +304,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalInt(trackTotalHitsUpTo); if (out.getVersion().onOrAfter(Version.V_8_0_0)) { - out.writeOptionalStringCollection(fetchFields); + out.writeBoolean(fetchFields != null); + if (fetchFields != null) { + out.writeList(fetchFields); + } } } @@ -835,18 +840,27 @@ public SearchSourceBuilder docValueField(String name) { /** * Gets the fields to load and return as part of the search request. */ - public List fetchFields() { + public List fetchFields() { return fetchFields; } /** * Adds a field to load and return as part of the search request. */ - public SearchSourceBuilder fetchField(String fieldName) { + public SearchSourceBuilder fetchField(String name) { + return fetchField(name, null); + } + + /** + * Adds a field to load and return as part of the search request. + * @param name the field name. + * @param format an optional format string used when formatting values, for example a date format. + */ + public SearchSourceBuilder fetchField(String name, @Nullable String format) { if (fetchFields == null) { fetchFields = new ArrayList<>(); } - fetchFields.add(fieldName); + fetchFields.add(new FieldAndFormat(name, format)); return this; } @@ -1148,7 +1162,7 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th } else if (FETCH_FIELDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { fetchFields = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - fetchFields.add(parser.text()); + fetchFields.add(FieldAndFormat.fromXContent(parser)); } } else if (INDICES_BOOST_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { @@ -1247,18 +1261,17 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t if (docValueFields != null) { builder.startArray(DOCVALUE_FIELDS_FIELD.getPreferredName()); for (FieldAndFormat docValueField : docValueFields) { - builder.startObject() - .field("field", docValueField.field); - if (docValueField.format != null) { - builder.field("format", docValueField.format); - } - builder.endObject(); + docValueField.toXContent(builder, params); } builder.endArray(); } if (fetchFields != null) { - builder.array(FETCH_FIELDS_FIELD.getPreferredName(), fetchFields); + builder.startArray(FETCH_FIELDS_FIELD.getPreferredName()); + for (FieldAndFormat docValueField : fetchFields) { + docValueField.toXContent(builder, params); + } + builder.endArray(); } if (scriptFields != null) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesContext.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesContext.java index c4449d0137779..3ab3003a27f6b 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesContext.java @@ -18,96 +18,18 @@ */ package org.elasticsearch.search.fetch.subphase; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.XContent; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MapperService; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.Objects; /** * All the required context to pull a field from the doc values. */ public class FetchDocValuesContext { - /** - * Wrapper around a field name and the format that should be used to - * display values of this field. - */ - public static final class FieldAndFormat implements Writeable { - - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("docvalues_field", - a -> new FieldAndFormat((String) a[0], (String) a[1])); - static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("field")); - PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), new ParseField("format")); - } - - /** - * Parse a {@link FieldAndFormat} from some {@link XContent}. - */ - public static FieldAndFormat fromXContent(XContentParser parser) throws IOException { - Token token = parser.currentToken(); - if (token.isValue()) { - return new FieldAndFormat(parser.text(), null); - } else { - return PARSER.apply(parser, null); - } - } - - /** The name of the field. */ - public final String field; - - /** The format of the field, or {@code null} if defaults should be used. */ - public final String format; - - /** Sole constructor. */ - public FieldAndFormat(String field, @Nullable String format) { - this.field = Objects.requireNonNull(field); - this.format = format; - } - - /** Serialization constructor. */ - public FieldAndFormat(StreamInput in) throws IOException { - this.field = in.readString(); - format = in.readOptionalString(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(field); - out.writeOptionalString(format); - } - - @Override - public int hashCode() { - int h = field.hashCode(); - h = 31 * h + Objects.hashCode(format); - return h; - } - - @Override - public boolean equals(Object obj) { - if (obj == null || getClass() != obj.getClass()) { - return false; - } - FieldAndFormat other = (FieldAndFormat) obj; - return field.equals(other.field) && Objects.equals(format, other.format); - } - - } - private final List fields; public static FetchDocValuesContext create(MapperService mapperService, diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java index 81838e447d280..576ae1d112582 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java @@ -33,7 +33,6 @@ import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.fetch.FetchSubPhase; -import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext.FieldAndFormat; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java index 7a42cb2c60e9c..84a390531204b 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java @@ -25,13 +25,13 @@ */ public class FetchFieldsContext { - private final List fields; + private final List fields; - public FetchFieldsContext(List fields) { + public FetchFieldsContext(List fields) { this.fields = fields; } - public List fields() { + public List fields() { return this.fields; } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java new file mode 100644 index 0000000000000..cf4edd13f5cd8 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java @@ -0,0 +1,115 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.fetch.subphase; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +/** + * Wrapper around a field name and the format that should be used to + * display values of this field. + */ +public final class FieldAndFormat implements Writeable, ToXContentObject { + private static final ParseField FIELD_FIELD = new ParseField("field"); + private static final ParseField FORMAT_FIELD = new ParseField("format"); + + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("fetch_field_and_format", + a -> new FieldAndFormat((String) a[0], (String) a[1])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), FIELD_FIELD); + PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), FORMAT_FIELD); + } + + /** + * Parse a {@link FieldAndFormat} from some {@link XContent}. + */ + public static FieldAndFormat fromXContent(XContentParser parser) throws IOException { + XContentParser.Token token = parser.currentToken(); + if (token.isValue()) { + return new FieldAndFormat(parser.text(), null); + } else { + return PARSER.apply(parser, null); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(FIELD_FIELD.getPreferredName(), field); + if (format != null) { + builder.field(FORMAT_FIELD.getPreferredName(), format); + } + builder.endObject(); + return builder; + } + + /** The name of the field. */ + public final String field; + + /** The format of the field, or {@code null} if defaults should be used. */ + public final String format; + + /** Sole constructor. */ + public FieldAndFormat(String field, @Nullable String format) { + this.field = Objects.requireNonNull(field); + this.format = format; + } + + /** Serialization constructor. */ + public FieldAndFormat(StreamInput in) throws IOException { + this.field = in.readString(); + format = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(field); + out.writeOptionalString(format); + } + + @Override + public int hashCode() { + int h = field.hashCode(); + h = 31 * h + Objects.hashCode(format); + return h; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || getClass() != obj.getClass()) { + return false; + } + FieldAndFormat other = (FieldAndFormat) obj; + return field.equals(other.field) && Objects.equals(format, other.format); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java index bf879c85f043a..49c2619fc0398 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.fetch.subphase; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.index.mapper.DocumentFieldMappers; import org.elasticsearch.index.mapper.FieldMapper; @@ -41,16 +42,19 @@ public class FieldValueRetriever { private final List fieldContexts; public static FieldValueRetriever create(MapperService mapperService, - Collection fieldPatterns) { + Collection fieldAndFormats) { DocumentFieldMappers fieldMappers = mapperService.documentMapper().mappers(); List fields = new ArrayList<>(); - for (String fieldPattern : fieldPatterns) { + for (FieldAndFormat fieldAndFormat : fieldAndFormats) { + String fieldPattern = fieldAndFormat.field; + String format = fieldAndFormat.format; + Collection concreteFields = mapperService.simpleMatchToFullName(fieldPattern); for (String field : concreteFields) { if (fieldMappers.getMapper(field) != null) { Set sourcePath = mapperService.sourcePath(field); - fields.add(new FieldContext(field, sourcePath)); + fields.add(new FieldContext(field, sourcePath, format)); } } } @@ -66,21 +70,22 @@ private FieldValueRetriever(DocumentFieldMappers fieldMappers, public Map retrieve(SourceLookup sourceLookup, Set ignoredFields) { Map documentFields = new HashMap<>(); - for (FieldContext fieldContext : fieldContexts) { - String field = fieldContext.fieldName; - Set sourcePath = fieldContext.sourcePath; - + for (FieldContext context : fieldContexts) { + String field = context.fieldName; if (ignoredFields.contains(field)) { continue; } List parsedValues = new ArrayList<>(); - for (String path : sourcePath) { + for (String path : context.sourcePath) { FieldMapper fieldMapper = (FieldMapper) fieldMappers.getMapper(path); - List values = fieldMapper.lookupValues(sourceLookup); + List values = fieldMapper.lookupValues(sourceLookup, context.format); parsedValues.addAll(values); } - documentFields.put(field, new DocumentField(field, parsedValues)); + + if (parsedValues.isEmpty() == false) { + documentFields.put(field, new DocumentField(field, parsedValues)); + } } return documentFields; } @@ -88,11 +93,14 @@ public Map retrieve(SourceLookup sourceLookup, Set sourcePath; + final @Nullable String format; FieldContext(String fieldName, - Set sourcePath) { + Set sourcePath, + @Nullable String format) { this.fieldName = fieldName; this.sourcePath = sourcePath; + this.format = format; } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index 8666ea0073442..5c34fb0017e79 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -301,8 +301,8 @@ public void testParseSourceValue() { Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); BooleanFieldMapper mapper = new BooleanFieldMapper.Builder("field").build(context); - assertTrue(mapper.parseSourceValue(true)); - assertFalse(mapper.parseSourceValue("false")); - assertFalse(mapper.parseSourceValue("")); + assertTrue(mapper.parseSourceValue(true, null)); + assertFalse(mapper.parseSourceValue("false", null)); + assertFalse(mapper.parseSourceValue("", null)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 90f4dfc26e22f..6564cdf6236fe 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -944,13 +944,13 @@ public void testParseSourceValue() { NamedAnalyzer defaultAnalyzer = new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer()); CompletionFieldMapper mapper = new CompletionFieldMapper.Builder("completion", defaultAnalyzer).build(context); - assertEquals(List.of("value"), mapper.parseSourceValue("value")); + assertEquals(List.of("value"), mapper.parseSourceValue("value", null)); List list = List.of("first", "second"); - assertEquals(list, mapper.parseSourceValue(list)); + assertEquals(list, mapper.parseSourceValue(list, null)); Map object = Map.of("input", List.of("first", "second"), "weight", "2.718"); - assertEquals(List.of(object), mapper.parseSourceValue(object)); + assertEquals(List.of(object), mapper.parseSourceValue(object, null)); } private Matcher suggestField(String value) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index 4ffd940716d0e..774fbb1758e4f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -455,25 +455,32 @@ public void testMeta() throws Exception { public void testParseSourceValue() { DateFieldMapper mapper = createMapper(Resolution.MILLISECONDS, null); String date = "2020-05-15T21:33:02.000Z"; - assertEquals(date, mapper.parseSourceValue(date)); - assertEquals(date, mapper.parseSourceValue(1589578382000L)); + assertEquals(date, mapper.parseSourceValue(date, null)); + assertEquals(date, mapper.parseSourceValue(1589578382000L, null)); DateFieldMapper mapperWithFormat = createMapper(Resolution.MILLISECONDS, "yyyy/MM/dd||epoch_millis"); String dateInFormat = "1990/12/29"; - assertEquals(dateInFormat, mapperWithFormat.parseSourceValue(dateInFormat)); - assertEquals(dateInFormat, mapperWithFormat.parseSourceValue(662428800000L)); + assertEquals(dateInFormat, mapperWithFormat.parseSourceValue(dateInFormat, null)); + assertEquals(dateInFormat, mapperWithFormat.parseSourceValue(662428800000L, null)); DateFieldMapper mapperWithMillis = createMapper(Resolution.MILLISECONDS, "epoch_millis"); String dateInMillis = "662428800000"; - assertEquals(dateInMillis, mapperWithMillis.parseSourceValue(dateInMillis)); - assertEquals(dateInMillis, mapperWithMillis.parseSourceValue(662428800000L)); + assertEquals(dateInMillis, mapperWithMillis.parseSourceValue(dateInMillis, null)); + assertEquals(dateInMillis, mapperWithMillis.parseSourceValue(662428800000L, null)); + } + + public void testParseSourceValueWithFormat() { + DateFieldMapper mapper = createMapper(Resolution.NANOSECONDS, "strict_date_time"); + String date = "1990-12-29T00:00:00.000Z"; + assertEquals("1990/12/29", mapper.parseSourceValue(date, "yyyy/MM/dd")); + assertEquals("662428800000", mapper.parseSourceValue(date, "epoch_millis")); } public void testParseSourceValueNanos() { DateFieldMapper mapper = createMapper(Resolution.NANOSECONDS, "strict_date_time||epoch_millis"); String date = "2020-05-15T21:33:02.123456789Z"; - assertEquals("2020-05-15T21:33:02.123456789Z", mapper.parseSourceValue(date)); - assertEquals("2020-05-15T21:33:02.123Z", mapper.parseSourceValue(1589578382123L)); + assertEquals("2020-05-15T21:33:02.123456789Z", mapper.parseSourceValue(date, null)); + assertEquals("2020-05-15T21:33:02.123Z", mapper.parseSourceValue(1589578382123L, null)); } private DateFieldMapper createMapper(Resolution resolution, String format) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index 5cfa4609d9931..fe621f6579aad 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -103,7 +103,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException(); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index 23ae5a9b6ec9c..44ee214e75594 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -201,7 +201,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { return value; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java index 09c2a899f13fc..8c68014522eaf 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java @@ -134,7 +134,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { return value.toString(); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 0706e382fe830..055162c1b0862 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -305,9 +305,9 @@ public void testParseSourceValue() { Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); IpFieldMapper mapper = new IpFieldMapper.Builder("field").build(context); - assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8::2:1")); - assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8:0:0:0:0:2:1")); - assertEquals("::1", mapper.parseSourceValue("0:0:0:0:0:0:0:1")); + assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8::2:1", null)); + assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8:0:0:0:0:2:1", null)); + assertEquals("::1", mapper.parseSourceValue("0:0:0:0:0:0:0:1", null)); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java index 8e06c938c8b7f..663fec56b5e6a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java @@ -90,7 +90,7 @@ public void testParseSourceValue() { RangeFieldMapper mapper = new RangeFieldMapper.Builder("field", RangeType.IP).build(context); Map range = Map.of("gte", "2001:db8:0:0:0:0:2:1"); - assertEquals(Map.of("gte", "2001:db8::2:1"), mapper.parseSourceValue(range)); - assertEquals("2001:db8::2:1/32", mapper.parseSourceValue("2001:db8:0:0:0:0:2:1/32")); + assertEquals(Map.of("gte", "2001:db8::2:1"), mapper.parseSourceValue(range, null)); + assertEquals("2001:db8::2:1/32", mapper.parseSourceValue("2001:db8:0:0:0:0:2:1/32", null)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 128a44953f7cd..8fd60b1aa41ea 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -170,7 +170,7 @@ public void testDefaults() throws Exception { assertArrayEquals(new String[] { "1234" }, TermVectorsService.getValues(doc.rootDoc().getFields("field"))); FieldMapper fieldMapper = (FieldMapper) mapper.mappers().getMapper("field"); - assertEquals("1234", fieldMapper.parseSourceValue("1234")); + assertEquals("1234", fieldMapper.parseSourceValue("1234", null)); } public void testIgnoreAbove() throws IOException { @@ -635,8 +635,11 @@ public void testParseSourceValue() { Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); KeywordFieldMapper mapper = new KeywordFieldMapper.Builder("field").build(context); - assertEquals("value", mapper.parseSourceValue("value")); - assertEquals("42", mapper.parseSourceValue(42L)); - assertEquals("true", mapper.parseSourceValue(true)); + assertEquals("value", mapper.parseSourceValue("value", null)); + assertEquals("42", mapper.parseSourceValue(42L, null)); + assertEquals("true", mapper.parseSourceValue(true, null)); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> mapper.parseSourceValue(true, "format")); + assertEquals("Field [field] of type [keyword] doesn't support formats.", e.getMessage()); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 7eefee2142419..9f89122c10aa1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -409,8 +409,8 @@ public void testParseSourceValue() { Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); NumberFieldMapper mapper = new NumberFieldMapper.Builder("field", NumberType.INTEGER).build(context); - assertEquals(3, mapper.parseSourceValue(3.14)); - assertEquals(42, mapper.parseSourceValue("42.9")); + assertEquals(3, mapper.parseSourceValue(3.14, null)); + assertEquals(42, mapper.parseSourceValue("42.9", null)); } @Timeout(millis = 30000) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index d034e1cd78f54..d4249ceacc714 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -185,8 +185,8 @@ protected void parseCreateField(ParseContext context) { } @Override - protected Object parseSourceValue(Object value) { - return value; + protected Object parseSourceValue(Object value, String format) { + return null; } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index 0659983e1ed62..b7f4f8cc7d962 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -497,13 +497,29 @@ public void testParseSourceValue() { RangeFieldMapper longMapper = new RangeFieldMapper.Builder("field", RangeType.LONG).build(context); Map longRange = Map.of("gte", 3.14, "lt", "42.9"); - assertEquals(Map.of("gte", 3L, "lt", 42L), longMapper.parseSourceValue(longRange)); + assertEquals(Map.of("gte", 3L, "lt", 42L), longMapper.parseSourceValue(longRange, null)); RangeFieldMapper dateMapper = new RangeFieldMapper.Builder("field", RangeType.DATE) .format("yyyy/MM/dd||epoch_millis") .build(context); Map dateRange = Map.of("lt", "1990/12/29", "gte", 597429487111L); assertEquals(Map.of("lt", "1990/12/29", "gte", "1988/12/06"), - dateMapper.parseSourceValue(dateRange)); + dateMapper.parseSourceValue(dateRange, null)); + } + + public void testParseSourceValueWithFormat() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + RangeFieldMapper longMapper = new RangeFieldMapper.Builder("field", RangeType.LONG).build(context); + Map longRange = Map.of("gte", 3.14, "lt", "42.9"); + assertEquals(Map.of("gte", 3L, "lt", 42L), longMapper.parseSourceValue(longRange, null)); + + RangeFieldMapper dateMapper = new RangeFieldMapper.Builder("field", RangeType.DATE) + .format("strict_date_time") + .build(context); + Map dateRange = Map.of("lt", "1990-12-29T00:00:00.000Z"); + assertEquals(Map.of("lt", "1990/12/29"), dateMapper.parseSourceValue(dateRange, "yyy/MM/dd")); + assertEquals(Map.of("lt", "662428800000"), dateMapper.parseSourceValue(dateRange, "epoch_millis")); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 3071849c99cdf..e3d632f409639 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1341,8 +1341,8 @@ public void testParseSourceValue() { FieldMapper fieldMapper = newBuilder().build(context); TextFieldMapper mapper = (TextFieldMapper) fieldMapper; - assertEquals("value", mapper.parseSourceValue("value")); - assertEquals("42", mapper.parseSourceValue(42L)); - assertEquals("true", mapper.parseSourceValue(true)); + assertEquals("value", mapper.parseSourceValue("value", null)); + assertEquals("42", mapper.parseSourceValue(42L, null)); + assertEquals("true", mapper.parseSourceValue(true, null)); } } diff --git a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java index bc2b303a8e817..1d0cc23cbb32b 100644 --- a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java @@ -32,8 +32,8 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilderTests; import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.search.sort.SortBuilder; diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java index 2b61fa2f8620e..a132be77fc593 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java @@ -48,7 +48,10 @@ public void testLeafValues() throws IOException { .endObject() .endObject(); - Map fields = retrieveFields(mapperService, source, List.of("field", "object.field")); + List fieldAndFormats = List.of( + new FieldAndFormat("field", null), + new FieldAndFormat("object.field", null)); + Map fields = retrieveFields(mapperService, source, fieldAndFormats); assertThat(fields.size(), equalTo(2)); DocumentField field = fields.get("field"); @@ -154,6 +157,27 @@ public void testFieldNamesWithWildcard() throws IOException { assertThat(objectField.getValues(), hasItems("fourth")); } + public void testDateFormat() throws IOException { + MapperService mapperService = createMapperService(); + XContentBuilder source = XContentFactory.jsonBuilder().startObject() + .field("field", "value") + .field("date_field", "1990-12-29T00:00:00.000Z") + .endObject(); + + Map fields = retrieveFields(mapperService, source, List.of( + new FieldAndFormat("field", null), + new FieldAndFormat("date_field", "yyyy/MM/dd"))); + assertThat(fields.size(), equalTo(2)); + + DocumentField field = fields.get("field"); + assertNotNull(field); + + DocumentField dateField = fields.get("date_field"); + assertNotNull(dateField); + assertThat(dateField.getValues().size(), equalTo(1)); + assertThat(dateField.getValue(), equalTo("1990/12/29")); + } + public void testFieldAliases() throws IOException { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() .startObject("properties") @@ -263,14 +287,15 @@ public void testObjectFields() throws IOException { } private Map retrieveFields(MapperService mapperService, XContentBuilder source, String fieldPattern) { - return retrieveFields(mapperService, source, List.of(fieldPattern)); + List fields = List.of(new FieldAndFormat(fieldPattern, null)); + return retrieveFields(mapperService, source, fields); } - private Map retrieveFields(MapperService mapperService, XContentBuilder source, List fieldPatterns) { + private Map retrieveFields(MapperService mapperService, XContentBuilder source, List fields) { SourceLookup sourceLookup = new SourceLookup(); sourceLookup.setSource(BytesReference.bytes(source)); - FieldValueRetriever fetchFieldsLookup = FieldValueRetriever.create(mapperService, fieldPatterns); + FieldValueRetriever fetchFieldsLookup = FieldValueRetriever.create(mapperService, fields); return fetchFieldsLookup.retrieve(sourceLookup, Set.of()); } @@ -279,6 +304,7 @@ public MapperService createMapperService() throws IOException { .startObject("properties") .startObject("field").field("type", "keyword").endObject() .startObject("integer_field").field("type", "integer").endObject() + .startObject("date_field").field("type", "date").endObject() .startObject("completion").field("type", "completion").endObject() .startObject("float_range").field("type", "float_range").endObject() .startObject("object") diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index 149ce7573c19a..05a9466f109ce 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -90,7 +90,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException(); } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 14c009a6beba7..eccda49a1a9e4 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -165,7 +165,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value; } diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 229b72be6d2c1..9184e28ac4086 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -264,14 +264,18 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public List lookupValues(SourceLookup lookup) { + public List lookupValues(SourceLookup lookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return fieldType().value == null ? List.of() : List.of(fieldType().value); } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException("This should never be called, since lookupValues is implemented directly."); } diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index acfd5689d6a2e..0552c63f26a68 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -146,7 +146,7 @@ public void testLookupValues() throws Exception { assertEquals(mapping, mapper.mappingSource().toString()); FieldMapper fieldMapper = (FieldMapper) mapper.mappers().getMapper("field"); - List values = fieldMapper.lookupValues(new SourceLookup()); + List values = fieldMapper.lookupValues(new SourceLookup(), null); assertTrue(values.isEmpty()); String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") @@ -155,7 +155,7 @@ public void testLookupValues() throws Exception { mapper = indexService.mapperService().merge("_doc", new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE); fieldMapper = (FieldMapper) mapper.mappers().getMapper("field"); - values = fieldMapper.lookupValues(new SourceLookup()); + values = fieldMapper.lookupValues(new SourceLookup(), null); assertEquals(1, values.size()); assertEquals("foo", values.get(0)); } diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java index 8c2f782b3d509..ee0e309fda7ed 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java @@ -565,7 +565,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value; } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java index c8e9d06d5f4d9..728711df05581 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java @@ -8,7 +8,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.script.Script; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext.FieldAndFormat; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import java.util.LinkedHashMap; import java.util.LinkedHashSet; diff --git a/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java b/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java index ce861d8f81a38..5441d4e94cd76 100644 --- a/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java +++ b/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java @@ -17,8 +17,8 @@ import org.elasticsearch.license.License.OperationMode; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import org.elasticsearch.transport.Netty4Plugin; import org.elasticsearch.transport.nio.NioTransportPlugin; @@ -154,7 +154,7 @@ public void testSqlTranslateActionLicense() throws Exception { .query("SELECT * FROM test").get(); SearchSourceBuilder source = response.source(); assertThat(source.docValueFields(), Matchers.contains( - new FetchDocValuesContext.FieldAndFormat("count", null))); + new FieldAndFormat("count", null))); FetchSourceContext fetchSource = source.fetchSource(); assertThat(fetchSource.includes(), Matchers.arrayContaining("data")); } diff --git a/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java b/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java index 251028be6b2f4..c6d6ce24b1435 100644 --- a/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java +++ b/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java @@ -8,8 +8,8 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.sort.SortBuilders; import static java.util.Collections.singletonList; @@ -35,7 +35,7 @@ public void testSqlTranslateAction() { assertTrue(fetch.fetchSource()); assertArrayEquals(new String[] { "data", "count" }, fetch.includes()); assertEquals( - singletonList(new FetchDocValuesContext.FieldAndFormat("date", "epoch_millis")), + singletonList(new FieldAndFormat("date", "epoch_millis")), source.docValueFields()); assertEquals(singletonList(SortBuilders.fieldSort("count").missing("_last").unmappedType("long")), source.sorts()); } diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index e9862f1a92591..51561a475b9a4 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -206,7 +206,10 @@ public void parse(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value; } diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java index 280bb98083988..91cdeb63d8577 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java @@ -143,7 +143,7 @@ protected void parseCreateField(ParseContext context) { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException(ERROR_MESSAGE_7X); } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 7ded93213f804..6a1c4f9ff43e6 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -951,7 +951,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value.toString(); }