From 2120a2930fc1e525191d708ad13464a8406a0c2b Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Thu, 28 May 2020 17:21:41 -0700 Subject: [PATCH 1/4] Pull FieldAndFormat into its own top-level class. --- .../index/query/InnerHitBuilder.java | 2 +- .../elasticsearch/search/SearchService.java | 8 +- .../metrics/TopHitsAggregationBuilder.java | 2 +- .../metrics/TopHitsAggregatorFactory.java | 2 +- .../search/builder/SearchSourceBuilder.java | 2 +- .../fetch/subphase/FetchDocValuesContext.java | 79 -------------- .../fetch/subphase/FetchDocValuesPhase.java | 1 - .../search/fetch/subphase/FieldAndFormat.java | 101 ++++++++++++++++++ .../index/query/InnerHitBuilderTests.java | 2 +- .../ql/execution/search/QlSourceBuilder.java | 2 +- .../xpack/sql/action/SqlLicenseIT.java | 4 +- .../sql/action/SqlTranslateActionIT.java | 4 +- 12 files changed, 116 insertions(+), 93 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java diff --git a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java index 4c189c92f7c74..31c30aac86655 100644 --- a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java @@ -33,8 +33,8 @@ import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.search.fetch.StoredFieldsContext; -import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.sort.SortBuilder; diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 5816b20146ec4..f5f86623f1124 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -87,6 +87,8 @@ import org.elasticsearch.search.fetch.ShardFetchRequest; import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext; import org.elasticsearch.search.fetch.subphase.FetchFieldsContext; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; + import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext.ScriptField; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.internal.AliasFilter; @@ -918,11 +920,11 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc context.fetchSourceContext(source.fetchSource()); } if (source.docValueFields() != null) { - List docValueFields = new ArrayList<>(); - for (FetchDocValuesContext.FieldAndFormat format : source.docValueFields()) { + List docValueFields = new ArrayList<>(); + for (FieldAndFormat format : source.docValueFields()) { Collection fieldNames = context.mapperService().simpleMatchToFullName(format.field); for (String fieldName: fieldNames) { - docValueFields.add(new FetchDocValuesContext.FieldAndFormat(fieldName, format.format)); + docValueFields.add(new FieldAndFormat(fieldName, format.format)); } } int maxAllowedDocvalueFields = context.mapperService().getIndexSettings().getMaxDocvalueFields(); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java index d10b226c6293b..9ca3cb639478c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java @@ -38,8 +38,8 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; import org.elasticsearch.search.fetch.StoredFieldsContext; -import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.sort.ScoreSortBuilder; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java index 96b54cf07e932..e870422c7e790 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java @@ -25,8 +25,8 @@ import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.fetch.StoredFieldsContext; import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext; -import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.internal.SearchContext; diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 3a360786d06c5..e06501818b797 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -48,8 +48,8 @@ import org.elasticsearch.search.aggregations.PipelineAggregationBuilder; import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.search.fetch.StoredFieldsContext; -import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.rescore.RescorerBuilder; diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesContext.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesContext.java index 5eaedac992fb1..4869d3e1fc107 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesContext.java @@ -18,92 +18,13 @@ */ package org.elasticsearch.search.fetch.subphase; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.XContent; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentParser.Token; - -import java.io.IOException; import java.util.List; -import java.util.Objects; /** * All the required context to pull a field from the doc values. */ public class FetchDocValuesContext { - /** - * Wrapper around a field name and the format that should be used to - * display values of this field. - */ - public static final class FieldAndFormat implements Writeable { - - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("docvalues_field", - a -> new FieldAndFormat((String) a[0], (String) a[1])); - static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("field")); - PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), new ParseField("format")); - } - - /** - * Parse a {@link FieldAndFormat} from some {@link XContent}. - */ - public static FieldAndFormat fromXContent(XContentParser parser) throws IOException { - Token token = parser.currentToken(); - if (token.isValue()) { - return new FieldAndFormat(parser.text(), null); - } else { - return PARSER.apply(parser, null); - } - } - - /** The name of the field. */ - public final String field; - - /** The format of the field, or {@code null} if defaults should be used. */ - public final String format; - - /** Sole constructor. */ - public FieldAndFormat(String field, @Nullable String format) { - this.field = Objects.requireNonNull(field); - this.format = format; - } - - /** Serialization constructor. */ - public FieldAndFormat(StreamInput in) throws IOException { - this.field = in.readString(); - format = in.readOptionalString(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(field); - out.writeOptionalString(format); - } - - @Override - public int hashCode() { - int h = field.hashCode(); - h = 31 * h + Objects.hashCode(format); - return h; - } - - @Override - public boolean equals(Object obj) { - if (obj == null || getClass() != obj.getClass()) { - return false; - } - FieldAndFormat other = (FieldAndFormat) obj; - return field.equals(other.field) && Objects.equals(format, other.format); - } - - } - private final List fields; public FetchDocValuesContext(List fields) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java index 762e076c5a89d..ab7eac82f166f 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java @@ -33,7 +33,6 @@ import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.fetch.FetchSubPhase; -import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext.FieldAndFormat; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java new file mode 100644 index 0000000000000..ad03482500a58 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java @@ -0,0 +1,101 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.fetch.subphase; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContent; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +/** + * Wrapper around a field name and the format that should be used to + * display values of this field. + */ +public final class FieldAndFormat implements Writeable { + + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("fetch_field_and_format", + a -> new FieldAndFormat((String) a[0], (String) a[1])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("field")); + PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), new ParseField("format")); + } + + /** + * Parse a {@link FieldAndFormat} from some {@link XContent}. + */ + public static FieldAndFormat fromXContent(XContentParser parser) throws IOException { + XContentParser.Token token = parser.currentToken(); + if (token.isValue()) { + return new FieldAndFormat(parser.text(), null); + } else { + return PARSER.apply(parser, null); + } + } + + /** The name of the field. */ + public final String field; + + /** The format of the field, or {@code null} if defaults should be used. */ + public final String format; + + /** Sole constructor. */ + public FieldAndFormat(String field, @Nullable String format) { + this.field = Objects.requireNonNull(field); + this.format = format; + } + + /** Serialization constructor. */ + public FieldAndFormat(StreamInput in) throws IOException { + this.field = in.readString(); + format = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(field); + out.writeOptionalString(format); + } + + @Override + public int hashCode() { + int h = field.hashCode(); + h = 31 * h + Objects.hashCode(format); + return h; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || getClass() != obj.getClass()) { + return false; + } + FieldAndFormat other = (FieldAndFormat) obj; + return field.equals(other.field) && Objects.equals(format, other.format); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java index bc2b303a8e817..1d0cc23cbb32b 100644 --- a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java @@ -32,8 +32,8 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilderTests; import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.search.sort.SortBuilder; diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java index c8e9d06d5f4d9..728711df05581 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java @@ -8,7 +8,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.script.Script; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext.FieldAndFormat; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import java.util.LinkedHashMap; import java.util.LinkedHashSet; diff --git a/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java b/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java index ce861d8f81a38..5441d4e94cd76 100644 --- a/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java +++ b/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java @@ -17,8 +17,8 @@ import org.elasticsearch.license.License.OperationMode; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import org.elasticsearch.transport.Netty4Plugin; import org.elasticsearch.transport.nio.NioTransportPlugin; @@ -154,7 +154,7 @@ public void testSqlTranslateActionLicense() throws Exception { .query("SELECT * FROM test").get(); SearchSourceBuilder source = response.source(); assertThat(source.docValueFields(), Matchers.contains( - new FetchDocValuesContext.FieldAndFormat("count", null))); + new FieldAndFormat("count", null))); FetchSourceContext fetchSource = source.fetchSource(); assertThat(fetchSource.includes(), Matchers.arrayContaining("data")); } diff --git a/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java b/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java index 251028be6b2f4..c6d6ce24b1435 100644 --- a/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java +++ b/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java @@ -8,8 +8,8 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.sort.SortBuilders; import static java.util.Collections.singletonList; @@ -35,7 +35,7 @@ public void testSqlTranslateAction() { assertTrue(fetch.fetchSource()); assertArrayEquals(new String[] { "data", "count" }, fetch.includes()); assertEquals( - singletonList(new FetchDocValuesContext.FieldAndFormat("date", "epoch_millis")), + singletonList(new FieldAndFormat("date", "epoch_millis")), source.docValueFields()); assertEquals(singletonList(SortBuilders.fieldSort("count").missing("_last").unmappedType("long")), source.sorts()); } From 1f5b92b7329b874c80e7b70597b58466942da0c5 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Sat, 30 May 2020 18:12:12 -0700 Subject: [PATCH 2/4] Only include non-empty field lists in the response. --- .../search/fetch/subphase/FieldValueRetriever.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java index bf879c85f043a..3a4ca1528f016 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java @@ -80,7 +80,10 @@ public Map retrieve(SourceLookup sourceLookup, Set values = fieldMapper.lookupValues(sourceLookup); parsedValues.addAll(values); } - documentFields.put(field, new DocumentField(field, parsedValues)); + + if (parsedValues.isEmpty() == false) { + documentFields.put(field, new DocumentField(field, parsedValues)); + } } return documentFields; } From a7d9c08a439ed7b668669d5c9d3778b65b21d0e1 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Mon, 8 Jun 2020 15:19:34 -0700 Subject: [PATCH 3/4] Pass formats to FieldValueRetriever. --- .../test/search/330_fetch_fields.yml | 35 +++++++++++++++++++ .../search/builder/SearchSourceBuilder.java | 24 +++++++++---- .../fetch/subphase/FetchFieldsContext.java | 6 ++-- .../fetch/subphase/FieldValueRetriever.java | 7 ++-- .../subphase/FieldValueRetrieverTests.java | 29 ++++++++++++--- 5 files changed, 86 insertions(+), 15 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml index 5653ae61a6c7d..b30d6e9c2993d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml @@ -45,6 +45,41 @@ setup: - match: { hits.hits.0.fields.integer_range.0.gte: 0 } - match: { hits.hits.0.fields.integer_range.0.lte: 42 } +--- +"Test date formatting": + - do: + indices.create: + index: test + body: + mappings: + properties: + date: + type: date + + - do: + index: + index: test + id: 1 + body: + date: "1990-12-29T22:30:00.000Z" + + - do: + indices.refresh: + index: [ test ] + + - do: + search: + index: test + body: + fields: + - field: date + format: "yyyy/MM/dd" + + - is_true: hits.hits.0._id + - is_true: hits.hits.0._source + + - match: { hits.hits.0.fields.date.0: "1990/12/29" } + --- "Test disable source": - do: diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index e06501818b797..f2f6750c4e509 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -172,7 +172,7 @@ public static HighlightBuilder highlight() { private List docValueFields; private List scriptFields; private FetchSourceContext fetchSourceContext; - private List fetchFields; + private List fetchFields; private AggregatorFactories.Builder aggregations; @@ -249,7 +249,9 @@ public SearchSourceBuilder(StreamInput in) throws IOException { trackTotalHitsUpTo = in.readOptionalInt(); if (in.getVersion().onOrAfter(Version.V_8_0_0)) { - fetchFields = in.readOptionalStringList(); + if (in.readBoolean()) { + fetchFields = in.readList(FieldAndFormat::new); + } } } @@ -307,7 +309,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalInt(trackTotalHitsUpTo); if (out.getVersion().onOrAfter(Version.V_8_0_0)) { - out.writeOptionalStringCollection(fetchFields); + out.writeBoolean(fetchFields != null); + if (fetchFields != null) { + out.writeList(fetchFields); + } } } @@ -840,7 +845,7 @@ public SearchSourceBuilder docValueField(String name) { /** * Gets the fields to load and return as part of the search request. */ - public List fetchFields() { + public List fetchFields() { return fetchFields; } @@ -848,10 +853,17 @@ public List fetchFields() { * Adds a field to load and return as part of the search request. */ public SearchSourceBuilder fetchField(String fieldName) { + return fetchField(fieldName, null); + } + + /** + * Adds a field to load and return as part of the search request. + */ + public SearchSourceBuilder fetchField(String fieldName, @Nullable String format) { if (fetchFields == null) { fetchFields = new ArrayList<>(); } - fetchFields.add(fieldName); + fetchFields.add(new FieldAndFormat(fieldName, format)); return this; } @@ -1153,7 +1165,7 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th } else if (FETCH_FIELDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { fetchFields = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - fetchFields.add(parser.text()); + fetchFields.add(FieldAndFormat.fromXContent(parser)); } } else if (INDICES_BOOST_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java index 7a42cb2c60e9c..84a390531204b 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java @@ -25,13 +25,13 @@ */ public class FetchFieldsContext { - private final List fields; + private final List fields; - public FetchFieldsContext(List fields) { + public FetchFieldsContext(List fields) { this.fields = fields; } - public List fields() { + public List fields() { return this.fields; } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java index 3a4ca1528f016..936e338b0e31d 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java @@ -41,11 +41,14 @@ public class FieldValueRetriever { private final List fieldContexts; public static FieldValueRetriever create(MapperService mapperService, - Collection fieldPatterns) { + Collection fieldAndFormats) { DocumentFieldMappers fieldMappers = mapperService.documentMapper().mappers(); List fields = new ArrayList<>(); - for (String fieldPattern : fieldPatterns) { + for (FieldAndFormat fieldAndFormat : fieldAndFormats) { + String fieldPattern = fieldAndFormat.field; + String format = fieldAndFormat.format; + Collection concreteFields = mapperService.simpleMatchToFullName(fieldPattern); for (String field : concreteFields) { if (fieldMappers.getMapper(field) != null) { diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java index 2b61fa2f8620e..a727b38185748 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java @@ -48,7 +48,10 @@ public void testLeafValues() throws IOException { .endObject() .endObject(); - Map fields = retrieveFields(mapperService, source, List.of("field", "object.field")); + List fieldAndFormats = List.of( + new FieldAndFormat("field", null), + new FieldAndFormat("object.field", null)); + Map fields = retrieveFields(mapperService, source, fieldAndFormats); assertThat(fields.size(), equalTo(2)); DocumentField field = fields.get("field"); @@ -154,6 +157,22 @@ public void testFieldNamesWithWildcard() throws IOException { assertThat(objectField.getValues(), hasItems("fourth")); } + public void testDateFormat() throws IOException { + MapperService mapperService = createMapperService(); + XContentBuilder source = XContentFactory.jsonBuilder().startObject() + .field("date_field", "1990-12-29T00:00:00.000Z") + .endObject(); + + FieldAndFormat fieldAndFormat = new FieldAndFormat("date_field", "yyyy/MM/dd"); + Map fields = retrieveFields(mapperService, source, List.of(fieldAndFormat)); + assertThat(fields.size(), equalTo(1)); + + DocumentField field = fields.get("date_field"); + assertNotNull(field); + assertThat(field.getValues().size(), equalTo(1)); + assertThat(field.getValue(), equalTo("1990/12/29")); + } + public void testFieldAliases() throws IOException { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() .startObject("properties") @@ -263,14 +282,15 @@ public void testObjectFields() throws IOException { } private Map retrieveFields(MapperService mapperService, XContentBuilder source, String fieldPattern) { - return retrieveFields(mapperService, source, List.of(fieldPattern)); + List fields = List.of(new FieldAndFormat(fieldPattern, null)); + return retrieveFields(mapperService, source, fields); } - private Map retrieveFields(MapperService mapperService, XContentBuilder source, List fieldPatterns) { + private Map retrieveFields(MapperService mapperService, XContentBuilder source, List fields) { SourceLookup sourceLookup = new SourceLookup(); sourceLookup.setSource(BytesReference.bytes(source)); - FieldValueRetriever fetchFieldsLookup = FieldValueRetriever.create(mapperService, fieldPatterns); + FieldValueRetriever fetchFieldsLookup = FieldValueRetriever.create(mapperService, fields); return fetchFieldsLookup.retrieve(sourceLookup, Set.of()); } @@ -279,6 +299,7 @@ public MapperService createMapperService() throws IOException { .startObject("properties") .startObject("field").field("type", "keyword").endObject() .startObject("integer_field").field("type", "integer").endObject() + .startObject("date_field").field("type", "date").endObject() .startObject("completion").field("type", "completion").endObject() .startObject("float_range").field("type", "float_range").endObject() .startObject("object") From 51b7de7954546c576001b683e2f724974143abc0 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Thu, 28 May 2020 17:09:25 -0700 Subject: [PATCH 4/4] Add support for passing a custom date format. --- .../index/mapper/RankFeatureFieldMapper.java | 5 +- .../index/mapper/RankFeaturesFieldMapper.java | 5 +- .../index/mapper/ScaledFloatFieldMapper.java | 6 +- .../mapper/SearchAsYouTypeFieldMapper.java | 9 +- .../index/mapper/TokenCountFieldMapper.java | 6 +- .../mapper/RankFeatureFieldMapperTests.java | 4 +- .../mapper/ScaledFloatFieldMapperTests.java | 4 +- .../join/mapper/MetaJoinFieldMapper.java | 2 +- .../join/mapper/ParentIdFieldMapper.java | 2 +- .../join/mapper/ParentJoinFieldMapper.java | 5 +- .../percolator/PercolatorFieldMapper.java | 5 +- .../ICUCollationKeywordFieldMapper.java | 5 +- .../ICUCollationKeywordFieldMapperTests.java | 6 +- .../AnnotatedTextFieldMapper.java | 5 +- .../AnnotatedTextFieldMapperTests.java | 6 +- .../mapper/murmur3/Murmur3FieldMapper.java | 5 +- .../test/search/330_fetch_fields.yml | 14 ++- .../action/search/SearchRequestBuilder.java | 18 +++- .../mapper/AbstractGeometryFieldMapper.java | 2 +- .../index/mapper/BinaryFieldMapper.java | 5 +- .../index/mapper/BooleanFieldMapper.java | 6 +- .../index/mapper/CompletionFieldMapper.java | 6 +- .../index/mapper/DateFieldMapper.java | 10 ++- .../index/mapper/FieldMapper.java | 10 ++- .../index/mapper/IpFieldMapper.java | 6 +- .../index/mapper/KeywordFieldMapper.java | 5 +- .../index/mapper/MetadataFieldMapper.java | 2 +- .../index/mapper/NumberFieldMapper.java | 6 +- .../index/mapper/RangeFieldMapper.java | 9 +- .../index/mapper/TextFieldMapper.java | 9 +- .../search/builder/SearchSourceBuilder.java | 23 ++--- .../search/fetch/subphase/FieldAndFormat.java | 22 ++++- .../fetch/subphase/FieldValueRetriever.java | 18 ++-- .../index/mapper/BooleanFieldMapperTests.java | 6 +- .../mapper/CompletionFieldMapperTests.java | 6 +- .../index/mapper/DateFieldMapperTests.java | 28 ++++-- .../mapper/DocumentFieldMapperTests.java | 2 +- .../index/mapper/ExternalMapper.java | 2 +- .../index/mapper/FakeStringFieldMapper.java | 2 +- .../index/mapper/IpFieldMapperTests.java | 6 +- .../index/mapper/IpRangeFieldMapperTests.java | 4 +- .../index/mapper/KeywordFieldMapperTests.java | 11 ++- .../index/mapper/NumberFieldMapperTests.java | 4 +- .../index/mapper/RangeFieldMapperTests.java | 20 ++++- .../index/mapper/TextFieldMapperTests.java | 6 +- .../subphase/FieldValueRetrieverTests.java | 17 ++-- .../index/mapper/MockFieldMapper.java | 2 +- .../mapper/HistogramFieldMapper.java | 5 +- .../mapper/ConstantKeywordFieldMapper.java | 8 +- .../ConstantKeywordFieldMapperTests.java | 4 +- .../mapper/FlatObjectFieldMapper.java | 5 +- .../mapper/DenseVectorFieldMapper.java | 5 +- .../mapper/SparseVectorFieldMapper.java | 2 +- .../wildcard/mapper/WildcardFieldMapper.java | 85 ++++++++++--------- 54 files changed, 322 insertions(+), 159 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index adeb97fe351d5..6b2c8fff8cd36 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -214,7 +214,10 @@ private Float objectToFloat(Object value) { } @Override - protected Float parseSourceValue(Object value) { + protected Float parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return objectToFloat(value); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index f8744280fc4a4..ce6af80f56a32 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -176,7 +176,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value; } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index 38ba8130711f3..8a2d771c53024 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -486,7 +486,11 @@ private static double objectToDouble(Object value) { } @Override - protected Double parseSourceValue(Object value) { + protected Double parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + double doubleValue = objectToDouble(value); double scalingFactor = fieldType().getScalingFactor(); return Math.round(doubleValue * scalingFactor) / scalingFactor; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index bd3745d953f9d..750bdaa55c672 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -474,7 +474,7 @@ protected void parseCreateField(ParseContext context) { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException(); } @@ -516,7 +516,7 @@ protected void mergeOptions(FieldMapper other, List conflicts) { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException(); } @@ -676,7 +676,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value.toString(); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java index a42f1e217f4f7..2b56e83be2926 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java @@ -151,7 +151,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return value.toString(); } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java index 2ffb8e870cd8c..90fa88042d4d1 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java @@ -188,7 +188,7 @@ public void testParseSourceValue() { Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); RankFeatureFieldMapper mapper = new RankFeatureFieldMapper.Builder("field").build(context); - assertEquals(3.14f, mapper.parseSourceValue(3.14), 0.0001); - assertEquals(42.9f, mapper.parseSourceValue("42.9"), 0.0001); + assertEquals(3.14f, mapper.parseSourceValue(3.14, null), 0.0001); + assertEquals(42.9f, mapper.parseSourceValue("42.9", null), 0.0001); } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java index 879df3b4304fb..e47cbe4228ca1 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java @@ -403,7 +403,7 @@ public void testParseSourceValue() { .scalingFactor(100) .build(context); - assertEquals(3.14, mapper.parseSourceValue(3.1415926), 0.00001); - assertEquals(3.14, mapper.parseSourceValue("3.1415"), 0.00001); + assertEquals(3.14, mapper.parseSourceValue(3.1415926, null), 0.00001); + assertEquals(3.14, mapper.parseSourceValue("3.1415", null), 0.00001); } } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java index 4a519947dcfa5..c8630a6a2a503 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java @@ -148,7 +148,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source."); } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index d2dc881ebde46..fef0357033ef8 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -192,7 +192,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source."); } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index 38ff03395e6fb..025cb41e1c2cd 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -362,7 +362,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value; } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index bc77a69cea6ac..889e5f3d8ca36 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -390,7 +390,10 @@ public void parse(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value; } diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index 68688e20b40cb..92b7317d2d47d 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -746,7 +746,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value.toString(); } } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java index acdf1abafadd4..1648259a03089 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java @@ -485,8 +485,8 @@ public void testParseSourceValue() { Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); ICUCollationKeywordFieldMapper mapper = new ICUCollationKeywordFieldMapper.Builder("field").build(context); - assertEquals("value", mapper.parseSourceValue("value")); - assertEquals("42", mapper.parseSourceValue(42L)); - assertEquals("true", mapper.parseSourceValue(true)); + assertEquals("value", mapper.parseSourceValue("value", null)); + assertEquals("42", mapper.parseSourceValue(42L, null)); + assertEquals("true", mapper.parseSourceValue(true, null)); } } diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 5b593e34e1a35..4a06f9fde9723 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -600,7 +600,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value.toString(); } diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index 1f4fdf59bc817..52d8ba61f3d87 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -685,8 +685,8 @@ public void testParseSourceValue() { .searchQuoteAnalyzer(indexService.getIndexAnalyzers().getDefaultSearchQuoteAnalyzer()) .build(context); - assertEquals("value", mapper.parseSourceValue("value")); - assertEquals("42", mapper.parseSourceValue(42L)); - assertEquals("true", mapper.parseSourceValue(true)); + assertEquals("value", mapper.parseSourceValue("value", null)); + assertEquals("42", mapper.parseSourceValue(42L, null)); + assertEquals("true", mapper.parseSourceValue(true, null)); } } diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index e61e72f11ce84..5b146c3bae099 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -164,7 +164,10 @@ protected void parseCreateField(ParseContext context) } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value.toString(); } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml index b30d6e9c2993d..02c3b2c592a88 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml @@ -51,8 +51,12 @@ setup: indices.create: index: test body: + settings: + index.number_of_shards: 1 mappings: properties: + keyword: + type: keyword date: type: date @@ -61,6 +65,7 @@ setup: index: test id: 1 body: + keyword: "value" date: "1990-12-29T22:30:00.000Z" - do: @@ -77,9 +82,16 @@ setup: - is_true: hits.hits.0._id - is_true: hits.hits.0._source - - match: { hits.hits.0.fields.date.0: "1990/12/29" } + - do: + catch: bad_request + search: + index: test + body: + fields: + - field: keyword + format: "yyyy/MM/dd" --- "Test disable source": - do: diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java index 5f175af80ef4c..42a56b308dafd 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java @@ -303,8 +303,24 @@ public SearchRequestBuilder addDocValueField(String name) { return addDocValueField(name, null); } + /** + * Adds a field to load and return. The field must be present in the document _source. + * + * @param name The field to load + */ public SearchRequestBuilder addFetchField(String name) { - sourceBuilder().fetchField(name); + sourceBuilder().fetchField(name, null); + return this; + } + + /** + * Adds a field to load and return. The field must be present in the document _source. + * + * @param name The field to load + * @param format TODO(jtibs): fill this in + */ + public SearchRequestBuilder addFetchField(String name, String format) { + sourceBuilder().fetchField(name, format); return this; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index fdd5ba0fc74d3..3132bdb0880e3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -159,7 +159,7 @@ public FT fieldType() { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index 56f451616d4c5..9e5a50d46ec3c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -193,7 +193,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 7508e10f51040..2c9ffb47e67a1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -253,7 +253,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public Boolean parseSourceValue(Object value) { + public Boolean parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + if (value instanceof Boolean) { return (Boolean) value; } else { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 26051c35cd745..ba7716d6db778 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -608,7 +608,11 @@ private void parse(ParseContext parseContext, Token token, } @Override - protected List parseSourceValue(Object value) { + protected List parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + if (value instanceof List) { return (List) value; } else { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 0e83f0375bb46..46fe28d68e4d4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -628,12 +628,16 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public String parseSourceValue(Object value) { + public String parseSourceValue(Object value, String format) { String date = value.toString(); long timestamp = fieldType().parse(date); - ZonedDateTime dateTime = fieldType().resolution().toInstant(timestamp).atZone(ZoneOffset.UTC); - return fieldType().dateTimeFormatter().format(dateTime); + + DateFormatter dateTimeFormatter = fieldType().dateTimeFormatter(); + if (format != null) { + dateTimeFormatter = DateFormatter.forPattern(format).withLocale(dateTimeFormatter.locale()); + } + return dateTimeFormatter.format(dateTime); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 820cf50d5cfb0..23440270fe3b0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -25,6 +25,7 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.elasticsearch.Version; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Setting; @@ -332,9 +333,10 @@ public void parse(ParseContext context) throws IOException { * Some mappers may need more flexibility and can override this entire method instead. * * @param lookup a lookup structure over the document's source. + * @param format an optional format string used when formatting values, for example a date format. * @return a list a standardized field values. */ - public List lookupValues(SourceLookup lookup) { + public List lookupValues(SourceLookup lookup, @Nullable String format) { Object sourceValue = lookup.extractValue(name()); if (sourceValue == null) { return List.of(); @@ -342,11 +344,11 @@ public List lookupValues(SourceLookup lookup) { List values = new ArrayList<>(); if (parsesArrayValue()) { - return (List) parseSourceValue(sourceValue); + return (List) parseSourceValue(sourceValue, format); } else { List sourceValues = sourceValue instanceof List ? (List) sourceValue : List.of(sourceValue); for (Object value : sourceValues) { - Object parsedValue = parseSourceValue(value); + Object parsedValue = parseSourceValue(value, format); values.add(parsedValue); } } @@ -360,7 +362,7 @@ public List lookupValues(SourceLookup lookup) { * * Note that when overriding this method, {@link #lookupValues} should *not* be overridden. */ - protected abstract Object parseSourceValue(Object value); + protected abstract Object parseSourceValue(Object value, @Nullable String format); protected void createFieldNamesField(ParseContext context) { FieldNamesFieldType fieldNamesFieldType = context.docMapper().metadataMapper(FieldNamesFieldMapper.class).fieldType(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index fb5ee2898a3e9..10eb827e62f42 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -392,7 +392,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + InetAddress address = InetAddresses.forString(value.toString()); return InetAddresses.toAddrString(address); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 6bc0bc6d3b6a6..ce752233047c7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -187,7 +187,10 @@ public Mapper.Builder parse(String name, Map node, ParserCont } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value.toString(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index c30c91d20fa4f..c5f2c58190643 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -71,7 +71,7 @@ public void postParse(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 3b9e5330725b7..408d5b2e0783d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.mapper; import com.fasterxml.jackson.core.JsonParseException; - import com.fasterxml.jackson.core.exc.InputCoercionException; import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; @@ -1088,7 +1087,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Number parseSourceValue(Object value) { + protected Number parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return fieldType().type.parse(value, coerce.value()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index cfc3b76c28604..0a0000d1025e7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -399,7 +399,7 @@ protected void parseCreateField(ParseContext context) throws IOException { @Override @SuppressWarnings("unchecked") - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { RangeType rangeType = fieldType().rangeType(); if (!(value instanceof Map)) { assert rangeType == RangeType.IP; @@ -407,11 +407,16 @@ protected Object parseSourceValue(Object value) { return InetAddresses.toCidrString(ipRange.v1(), ipRange.v2()); } + DateFormatter dateTimeFormatter = fieldType().dateTimeFormatter(); + if (format != null) { + dateTimeFormatter = DateFormatter.forPattern(format).withLocale(dateTimeFormatter.locale()); + } + Map range = (Map) value; Map parsedRange = new HashMap<>(); for (Map.Entry entry : range.entrySet()) { Object parsedValue = rangeType.parseValue(entry.getValue(), coerce.value(), fieldType().dateMathParser); - Object formattedValue = rangeType.formatValue(parsedValue, fieldType().dateTimeFormatter); + Object formattedValue = rangeType.formatValue(parsedValue, dateTimeFormatter); parsedRange.put(entry.getKey(), formattedValue); } return parsedRange; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 137825c8d6516..05f8a58b0c352 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -479,7 +479,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException(); } @@ -510,7 +510,7 @@ protected void parseCreateField(ParseContext context) { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException(); } @@ -839,7 +839,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value.toString(); } diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index f2f6750c4e509..c291dce8eac00 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -852,18 +852,20 @@ public List fetchFields() { /** * Adds a field to load and return as part of the search request. */ - public SearchSourceBuilder fetchField(String fieldName) { - return fetchField(fieldName, null); + public SearchSourceBuilder fetchField(String name) { + return fetchField(name, null); } /** * Adds a field to load and return as part of the search request. + * @param name the field name. + * @param format an optional format string used when formatting values, for example a date format. */ - public SearchSourceBuilder fetchField(String fieldName, @Nullable String format) { + public SearchSourceBuilder fetchField(String name, @Nullable String format) { if (fetchFields == null) { fetchFields = new ArrayList<>(); } - fetchFields.add(new FieldAndFormat(fieldName, format)); + fetchFields.add(new FieldAndFormat(name, format)); return this; } @@ -1264,18 +1266,17 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t if (docValueFields != null) { builder.startArray(DOCVALUE_FIELDS_FIELD.getPreferredName()); for (FieldAndFormat docValueField : docValueFields) { - builder.startObject() - .field("field", docValueField.field); - if (docValueField.format != null) { - builder.field("format", docValueField.format); - } - builder.endObject(); + docValueField.toXContent(builder, params); } builder.endArray(); } if (fetchFields != null) { - builder.array(FETCH_FIELDS_FIELD.getPreferredName(), fetchFields); + builder.startArray(FETCH_FIELDS_FIELD.getPreferredName()); + for (FieldAndFormat docValueField : fetchFields) { + docValueField.toXContent(builder, params); + } + builder.endArray(); } if (scriptFields != null) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java index ad03482500a58..cf4edd13f5cd8 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java @@ -25,7 +25,9 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContent; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; @@ -35,15 +37,17 @@ * Wrapper around a field name and the format that should be used to * display values of this field. */ -public final class FieldAndFormat implements Writeable { +public final class FieldAndFormat implements Writeable, ToXContentObject { + private static final ParseField FIELD_FIELD = new ParseField("field"); + private static final ParseField FORMAT_FIELD = new ParseField("format"); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("fetch_field_and_format", a -> new FieldAndFormat((String) a[0], (String) a[1])); static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("field")); - PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), new ParseField("format")); + PARSER.declareString(ConstructingObjectParser.constructorArg(), FIELD_FIELD); + PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), FORMAT_FIELD); } /** @@ -58,6 +62,17 @@ public static FieldAndFormat fromXContent(XContentParser parser) throws IOExcept } } + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(FIELD_FIELD.getPreferredName(), field); + if (format != null) { + builder.field(FORMAT_FIELD.getPreferredName(), format); + } + builder.endObject(); + return builder; + } + /** The name of the field. */ public final String field; @@ -97,5 +112,4 @@ public boolean equals(Object obj) { FieldAndFormat other = (FieldAndFormat) obj; return field.equals(other.field) && Objects.equals(format, other.format); } - } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java index 936e338b0e31d..49c2619fc0398 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.fetch.subphase; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.index.mapper.DocumentFieldMappers; import org.elasticsearch.index.mapper.FieldMapper; @@ -53,7 +54,7 @@ public static FieldValueRetriever create(MapperService mapperService, for (String field : concreteFields) { if (fieldMappers.getMapper(field) != null) { Set sourcePath = mapperService.sourcePath(field); - fields.add(new FieldContext(field, sourcePath)); + fields.add(new FieldContext(field, sourcePath, format)); } } } @@ -69,18 +70,16 @@ private FieldValueRetriever(DocumentFieldMappers fieldMappers, public Map retrieve(SourceLookup sourceLookup, Set ignoredFields) { Map documentFields = new HashMap<>(); - for (FieldContext fieldContext : fieldContexts) { - String field = fieldContext.fieldName; - Set sourcePath = fieldContext.sourcePath; - + for (FieldContext context : fieldContexts) { + String field = context.fieldName; if (ignoredFields.contains(field)) { continue; } List parsedValues = new ArrayList<>(); - for (String path : sourcePath) { + for (String path : context.sourcePath) { FieldMapper fieldMapper = (FieldMapper) fieldMappers.getMapper(path); - List values = fieldMapper.lookupValues(sourceLookup); + List values = fieldMapper.lookupValues(sourceLookup, context.format); parsedValues.addAll(values); } @@ -94,11 +93,14 @@ public Map retrieve(SourceLookup sourceLookup, Set sourcePath; + final @Nullable String format; FieldContext(String fieldName, - Set sourcePath) { + Set sourcePath, + @Nullable String format) { this.fieldName = fieldName; this.sourcePath = sourcePath; + this.format = format; } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index a99dc7324bfda..4b4d57982396d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -288,9 +288,9 @@ public void testParseSourceValue() { Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); BooleanFieldMapper mapper = new BooleanFieldMapper.Builder("field").build(context); - assertTrue(mapper.parseSourceValue(true)); - assertFalse(mapper.parseSourceValue("false")); - assertFalse(mapper.parseSourceValue("")); + assertTrue(mapper.parseSourceValue(true, null)); + assertFalse(mapper.parseSourceValue("false", null)); + assertFalse(mapper.parseSourceValue("", null)); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 7b4a382ab09bc..6ebb8a31c9c65 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -958,13 +958,13 @@ public void testParseSourceValue() { Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); CompletionFieldMapper mapper = new CompletionFieldMapper.Builder("field").build(context); - assertEquals(List.of("value"), mapper.parseSourceValue("value")); + assertEquals(List.of("value"), mapper.parseSourceValue("value", null)); List list = List.of("first", "second"); - assertEquals(list, mapper.parseSourceValue(list)); + assertEquals(list, mapper.parseSourceValue(list, null)); Map object = Map.of("input", List.of("first", "second"), "weight", "2.718"); - assertEquals(List.of(object), mapper.parseSourceValue(object)); + assertEquals(List.of(object), mapper.parseSourceValue(object, null)); } private Matcher suggestField(String value) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index 25a477b35c08c..24d9acf677ed0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -466,22 +466,34 @@ public void testParseSourceValue() { DateFieldMapper mapper = new DateFieldMapper.Builder("field").build(context); String date = "2020-05-15T21:33:02.000Z"; - assertEquals(date, mapper.parseSourceValue(date)); - assertEquals(date, mapper.parseSourceValue(1589578382000L)); + assertEquals(date, mapper.parseSourceValue(date, null)); + assertEquals(date, mapper.parseSourceValue(1589578382000L, null)); DateFieldMapper mapperWithFormat = new DateFieldMapper.Builder("field") .format("yyyy/MM/dd||epoch_millis") .build(context); String dateInFormat = "1990/12/29"; - assertEquals(dateInFormat, mapperWithFormat.parseSourceValue(dateInFormat)); - assertEquals(dateInFormat, mapperWithFormat.parseSourceValue(662428800000L)); + assertEquals(dateInFormat, mapperWithFormat.parseSourceValue(dateInFormat, null)); + assertEquals(dateInFormat, mapperWithFormat.parseSourceValue(662428800000L, null)); DateFieldMapper mapperWithMillis = new DateFieldMapper.Builder("field") .format("epoch_millis") .build(context); String dateInMillis = "662428800000"; - assertEquals(dateInMillis, mapperWithMillis.parseSourceValue(dateInMillis)); - assertEquals(dateInMillis, mapperWithMillis.parseSourceValue(662428800000L)); + assertEquals(dateInMillis, mapperWithMillis.parseSourceValue(dateInMillis, null)); + assertEquals(dateInMillis, mapperWithMillis.parseSourceValue(662428800000L, null)); + } + + public void testParseSourceValueWithFormat() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + DateFieldMapper mapper = new DateFieldMapper.Builder("field") + .format("strict_date_time") + .build(context); + String date = "1990-12-29T00:00:00.000Z"; + assertEquals("1990/12/29", mapper.parseSourceValue(date, "yyyy/MM/dd")); + assertEquals("662428800000", mapper.parseSourceValue(date, "epoch_millis")); } public void testParseSourceValueNanos() { @@ -493,7 +505,7 @@ public void testParseSourceValueNanos() { .withResolution(DateFieldMapper.Resolution.NANOSECONDS) .build(context); String date = "2020-05-15T21:33:02.123456789Z"; - assertEquals("2020-05-15T21:33:02.123456789Z", mapper.parseSourceValue(date)); - assertEquals("2020-05-15T21:33:02.123Z", mapper.parseSourceValue(1589578382123L)); + assertEquals("2020-05-15T21:33:02.123456789Z", mapper.parseSourceValue(date, null)); + assertEquals("2020-05-15T21:33:02.123Z", mapper.parseSourceValue(1589578382123L, null)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index 8eacfc3fe4278..8ac6a529f25b3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -116,7 +116,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException(); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index 281ef8bb9ad01..0f3433aa94e77 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -204,7 +204,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { return value; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java index 5904ca15274bf..c844646b392c2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java @@ -142,7 +142,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { return value.toString(); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index e1dbaa6667094..0af1e809dd849 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -287,8 +287,8 @@ public void testParseSourceValue() { Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); IpFieldMapper mapper = new IpFieldMapper.Builder("field").build(context); - assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8::2:1")); - assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8:0:0:0:0:2:1")); - assertEquals("::1", mapper.parseSourceValue("0:0:0:0:0:0:0:1")); + assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8::2:1", null)); + assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8:0:0:0:0:2:1", null)); + assertEquals("::1", mapper.parseSourceValue("0:0:0:0:0:0:0:1", null)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java index 8e06c938c8b7f..663fec56b5e6a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java @@ -90,7 +90,7 @@ public void testParseSourceValue() { RangeFieldMapper mapper = new RangeFieldMapper.Builder("field", RangeType.IP).build(context); Map range = Map.of("gte", "2001:db8:0:0:0:0:2:1"); - assertEquals(Map.of("gte", "2001:db8::2:1"), mapper.parseSourceValue(range)); - assertEquals("2001:db8::2:1/32", mapper.parseSourceValue("2001:db8:0:0:0:0:2:1/32")); + assertEquals(Map.of("gte", "2001:db8::2:1"), mapper.parseSourceValue(range, null)); + assertEquals("2001:db8::2:1/32", mapper.parseSourceValue("2001:db8:0:0:0:0:2:1/32", null)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index ac478cdb7f199..5bbdd649ccf5b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -145,7 +145,7 @@ public void testDefaults() throws Exception { assertArrayEquals(new String[] { "1234" }, TermVectorsService.getValues(doc.rootDoc().getFields("field"))); FieldMapper fieldMapper = (FieldMapper) mapper.mappers().getMapper("field"); - assertEquals("1234", fieldMapper.parseSourceValue("1234")); + assertEquals("1234", fieldMapper.parseSourceValue("1234", null)); } public void testIgnoreAbove() throws IOException { @@ -606,8 +606,11 @@ public void testParseSourceValue() { Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); KeywordFieldMapper mapper = new KeywordFieldMapper.Builder("field").build(context); - assertEquals("value", mapper.parseSourceValue("value")); - assertEquals("42", mapper.parseSourceValue(42L)); - assertEquals("true", mapper.parseSourceValue(true)); + assertEquals("value", mapper.parseSourceValue("value", null)); + assertEquals("42", mapper.parseSourceValue(42L, null)); + assertEquals("true", mapper.parseSourceValue(true, null)); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> mapper.parseSourceValue(true, "format")); + assertEquals("Field [field] of type [keyword] doesn't support formats.", e.getMessage()); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 6c9b29bd9f802..dc74a41c6d292 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -403,8 +403,8 @@ public void testParseSourceValue() { Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); NumberFieldMapper mapper = new NumberFieldMapper.Builder("field", NumberType.INTEGER).build(context); - assertEquals(3, mapper.parseSourceValue(3.14)); - assertEquals(42, mapper.parseSourceValue("42.9")); + assertEquals(3, mapper.parseSourceValue(3.14, null)); + assertEquals(42, mapper.parseSourceValue("42.9", null)); } @Timeout(millis = 30000) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index 28f457e571d2e..f69fd3b49304c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -490,13 +490,29 @@ public void testParseSourceValue() { RangeFieldMapper longMapper = new RangeFieldMapper.Builder("field", RangeType.LONG).build(context); Map longRange = Map.of("gte", 3.14, "lt", "42.9"); - assertEquals(Map.of("gte", 3L, "lt", 42L), longMapper.parseSourceValue(longRange)); + assertEquals(Map.of("gte", 3L, "lt", 42L), longMapper.parseSourceValue(longRange, null)); RangeFieldMapper dateMapper = new RangeFieldMapper.Builder("field", RangeType.DATE) .format("yyyy/MM/dd||epoch_millis") .build(context); Map dateRange = Map.of("lt", "1990/12/29", "gte", 597429487111L); assertEquals(Map.of("lt", "1990/12/29", "gte", "1988/12/06"), - dateMapper.parseSourceValue(dateRange)); + dateMapper.parseSourceValue(dateRange, null)); + } + + public void testParseSourceValueWithFormat() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + RangeFieldMapper longMapper = new RangeFieldMapper.Builder("field", RangeType.LONG).build(context); + Map longRange = Map.of("gte", 3.14, "lt", "42.9"); + assertEquals(Map.of("gte", 3L, "lt", 42L), longMapper.parseSourceValue(longRange, null)); + + RangeFieldMapper dateMapper = new RangeFieldMapper.Builder("field", RangeType.DATE) + .format("strict_date_time") + .build(context); + Map dateRange = Map.of("lt", "1990-12-29T00:00:00.000Z"); + assertEquals(Map.of("lt", "1990/12/29"), dateMapper.parseSourceValue(dateRange, "yyy/MM/dd")); + assertEquals(Map.of("lt", "662428800000"), dateMapper.parseSourceValue(dateRange, "epoch_millis")); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 9ad917f06f608..02abcda94164b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1315,8 +1315,8 @@ public void testParseSourceValue() { Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); TextFieldMapper mapper = new TextFieldMapper.Builder("field").build(context); - assertEquals("value", mapper.parseSourceValue("value")); - assertEquals("42", mapper.parseSourceValue(42L)); - assertEquals("true", mapper.parseSourceValue(true)); + assertEquals("value", mapper.parseSourceValue("value", null)); + assertEquals("42", mapper.parseSourceValue(42L, null)); + assertEquals("true", mapper.parseSourceValue(true, null)); } } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java index a727b38185748..a132be77fc593 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java @@ -160,17 +160,22 @@ public void testFieldNamesWithWildcard() throws IOException { public void testDateFormat() throws IOException { MapperService mapperService = createMapperService(); XContentBuilder source = XContentFactory.jsonBuilder().startObject() + .field("field", "value") .field("date_field", "1990-12-29T00:00:00.000Z") .endObject(); - FieldAndFormat fieldAndFormat = new FieldAndFormat("date_field", "yyyy/MM/dd"); - Map fields = retrieveFields(mapperService, source, List.of(fieldAndFormat)); - assertThat(fields.size(), equalTo(1)); + Map fields = retrieveFields(mapperService, source, List.of( + new FieldAndFormat("field", null), + new FieldAndFormat("date_field", "yyyy/MM/dd"))); + assertThat(fields.size(), equalTo(2)); - DocumentField field = fields.get("date_field"); + DocumentField field = fields.get("field"); assertNotNull(field); - assertThat(field.getValues().size(), equalTo(1)); - assertThat(field.getValue(), equalTo("1990/12/29")); + + DocumentField dateField = fields.get("date_field"); + assertNotNull(dateField); + assertThat(dateField.getValues().size(), equalTo(1)); + assertThat(dateField.getValue(), equalTo("1990/12/29")); } public void testFieldAliases() throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index a797469431c0c..bbf2a279aacd2 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -100,7 +100,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException(); } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index dddbb24300b22..db2123dbeb759 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -170,7 +170,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value; } diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index c17824fcc332d..bb062c5a31c5c 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -286,14 +286,18 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public List lookupValues(SourceLookup lookup) { + public List lookupValues(SourceLookup lookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return fieldType().value == null ? List.of() : List.of(fieldType().value); } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException("This should never be called, since lookupValues is implemented directly."); } diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index b97f79b594068..d53fd4f1a9bdf 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -140,7 +140,7 @@ public void testLookupValues() throws Exception { assertEquals(mapping, mapper.mappingSource().toString()); FieldMapper fieldMapper = (FieldMapper) mapper.mappers().getMapper("field"); - List values = fieldMapper.lookupValues(new SourceLookup()); + List values = fieldMapper.lookupValues(new SourceLookup(), null); assertTrue(values.isEmpty()); String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") @@ -149,7 +149,7 @@ public void testLookupValues() throws Exception { mapper = indexService.mapperService().merge("_doc", new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE); fieldMapper = (FieldMapper) mapper.mappers().getMapper("field"); - values = fieldMapper.lookupValues(new SourceLookup()); + values = fieldMapper.lookupValues(new SourceLookup(), null); assertEquals(1, values.size()); assertEquals("foo", values.get(0)); } diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java index 0ec128472023d..72f1aa3519eb9 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java @@ -617,7 +617,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value; } diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index 36364e9119224..6e21b9a2bd509 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -223,7 +223,10 @@ public void parse(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value; } diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java index d6023f3aa07b3..7bc0a55085e65 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java @@ -151,7 +151,7 @@ protected void parseCreateField(ParseContext context) { } @Override - protected Object parseSourceValue(Object value) { + protected Object parseSourceValue(Object value, String format) { throw new UnsupportedOperationException(ERROR_MESSAGE_7X); } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index c640ffbee21d5..bf370ec1a879c 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -223,7 +223,7 @@ public Mapper.Builder parse(String name, Map node, ParserCont public static final String TOKEN_END_STRING = TOKEN_START_STRING + TOKEN_START_STRING; public static final class WildcardFieldType extends MappedFieldType { - + static Analyzer lowercaseNormalizer = new LowercaseNormalizer(); public WildcardFieldType() { @@ -311,13 +311,13 @@ public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QuerySh verifyingBuilder.add(new BooleanClause(verifyingQuery, Occur.MUST)); return verifyingBuilder.build(); } else if (numWildcardChars == 0 || numWildcardStrings > 0) { - // We have no concrete characters and we're not a pure length query e.g. ??? + // We have no concrete characters and we're not a pure length query e.g. ??? return new DocValuesFieldExistsQuery(name()); } return verifyingQuery; } - + @Override public Query regexpQuery(String value, int flags, int maxDeterminizedStates, RewriteMethod method, QueryShardContext context) { if (value.length() == 0) { @@ -329,13 +329,13 @@ public Query regexpQuery(String value, int flags, int maxDeterminizedStates, Rew "[regexp] queries cannot be executed when '" + ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to false." ); } - + RegExp ngramRegex = new RegExp(addLineEndChars(toLowerCase(value)), flags); Query approxBooleanQuery = toApproximationQuery(ngramRegex); Query approxNgramQuery = rewriteBoolToNgramQuery(approxBooleanQuery); - - // MatchAll is a special case meaning the regex is known to match everything .* and + + // MatchAll is a special case meaning the regex is known to match everything .* and // there is no need for verification. if (approxNgramQuery instanceof MatchAllDocsQuery) { return existsQuery(context); @@ -346,28 +346,28 @@ public Query regexpQuery(String value, int flags, int maxDeterminizedStates, Rew }; AutomatonQueryOnBinaryDv verifyingQuery = new AutomatonQueryOnBinaryDv(name(), value, deferredAutomatonSupplier); - + // MatchAllButRequireVerificationQuery is a special case meaning the regex is reduced to a single - // clause which we can't accelerate at all and needs verification. Example would be ".." + // clause which we can't accelerate at all and needs verification. Example would be ".." if (approxNgramQuery instanceof MatchAllButRequireVerificationQuery) { return verifyingQuery; } - + // We can accelerate execution with the ngram query BooleanQuery.Builder verifyingBuilder = new BooleanQuery.Builder(); verifyingBuilder.add(new BooleanClause(approxNgramQuery, Occur.MUST)); verifyingBuilder.add(new BooleanClause(verifyingQuery, Occur.MUST)); return verifyingBuilder.build(); } - + // Convert a regular expression to a simplified query consisting of BooleanQuery and TermQuery objects // which captures as much of the logic as possible. Query can produce some false positives but shouldn't // produce any false negatives. // In addition to Term and BooleanQuery clauses there are MatchAllDocsQuery objects (e.g for .*) and - // a RegExpQuery if we can't resolve to any of the above. - // * If an expression resolves to a single MatchAllDocsQuery eg .* then a match all shortcut is possible with - // no verification needed. - // * If an expression resolves to a RegExpQuery eg ?? then only the verification + // a RegExpQuery if we can't resolve to any of the above. + // * If an expression resolves to a single MatchAllDocsQuery eg .* then a match all shortcut is possible with + // no verification needed. + // * If an expression resolves to a RegExpQuery eg ?? then only the verification // query is run. // * Anything else is a concrete query that should be run on the ngram index. public static Query toApproximationQuery(RegExp r) throws IllegalArgumentException { @@ -392,7 +392,7 @@ public static Query toApproximationQuery(RegExp r) throws IllegalArgumentExcepti // Repeat is zero or more times so zero matches = match all result = new MatchAllDocsQuery(); break; - + case REGEXP_REPEAT_MIN: case REGEXP_REPEAT_MINMAX: if (r.min > 0) { @@ -414,7 +414,7 @@ public static Query toApproximationQuery(RegExp r) throws IllegalArgumentExcepti // optimisation for .* queries - match all and no verification stage required. result = new MatchAllDocsQuery(); break; - // All other kinds of expression cannot be represented as a boolean or term query so return an object + // All other kinds of expression cannot be represented as a boolean or term query so return an object // that indicates verification is required case REGEXP_OPTIONAL: case REGEXP_INTERSECTION: @@ -422,7 +422,7 @@ public static Query toApproximationQuery(RegExp r) throws IllegalArgumentExcepti case REGEXP_CHAR_RANGE: case REGEXP_ANYCHAR: case REGEXP_INTERVAL: - case REGEXP_EMPTY: + case REGEXP_EMPTY: case REGEXP_AUTOMATON: result = new MatchAllButRequireVerificationQuery(); break; @@ -430,7 +430,7 @@ public static Query toApproximationQuery(RegExp r) throws IllegalArgumentExcepti assert result != null; // All regex types are understood and translated to a query. return result; } - + private static Query createConcatenationQuery(RegExp r) { // Create ANDs of expressions plus collapse consecutive TermQuerys into single longer ones ArrayList queries = new ArrayList<>(); @@ -447,7 +447,7 @@ private static Query createConcatenationQuery(RegExp r) { bAnd.add(new TermQuery(new Term("", sequence.toString())), Occur.MUST); sequence = new StringBuilder(); } - bAnd.add(query, Occur.MUST); + bAnd.add(query, Occur.MUST); } } if (sequence.length() > 0) { @@ -457,9 +457,9 @@ private static Query createConcatenationQuery(RegExp r) { if (combined.clauses().size() > 0) { return combined; } - // There's something in the regex we couldn't represent as a query - resort to a match all with verification + // There's something in the regex we couldn't represent as a query - resort to a match all with verification return new MatchAllButRequireVerificationQuery(); - + } private static Query createUnionQuery(RegExp r) { @@ -485,7 +485,7 @@ private static Query createUnionQuery(RegExp r) { return bOr.build(); } } - // There's something in the regex we couldn't represent as a query - resort to a match all with verification + // There's something in the regex we couldn't represent as a query - resort to a match all with verification return new MatchAllButRequireVerificationQuery(); } @@ -496,15 +496,15 @@ private static void findLeaves(RegExp exp, Kind kind, List queries) { } else { queries.add(toApproximationQuery(exp)); } - } - + } + private static String toLowerCase(String string) { return lowercaseNormalizer.normalize(null, string).utf8ToString(); } - + // Takes a BooleanQuery + TermQuery tree representing query logic and rewrites using ngrams of appropriate size. private Query rewriteBoolToNgramQuery(Query approxQuery) { - //TODO optimise more intelligently so we: + //TODO optimise more intelligently so we: // 1) favour full-length term queries eg abc over short eg a* when pruning too many clauses. // 2) make MAX_CLAUSES_IN_APPROXIMATION_QUERY a global cap rather than per-boolean clause. if (approxQuery == null) { @@ -533,13 +533,13 @@ private Query rewriteBoolToNgramQuery(Query approxQuery) { } if (approxQuery instanceof TermQuery) { TermQuery tq = (TermQuery) approxQuery; - + //Remove simple terms that are only string beginnings or ends. String s = tq.getTerm().text(); if (s.equals(WildcardFieldMapper.TOKEN_START_STRING) || s.equals(WildcardFieldMapper.TOKEN_END_STRING)) { return new MatchAllButRequireVerificationQuery(); } - + // Break term into tokens Set tokens = new LinkedHashSet<>(); getNgramTokens(tokens, s); @@ -553,8 +553,8 @@ private Query rewriteBoolToNgramQuery(Query approxQuery) { return approxQuery; } throw new IllegalStateException("Invalid query type found parsing regex query:" + approxQuery); - } - + } + static Query simplify(Query input) { if (input instanceof BooleanQuery == false) { return input; @@ -608,8 +608,8 @@ static Query simplify(Query input) { } return result; } - - + + static boolean isMatchAll(Query q) { return q instanceof MatchAllDocsQuery || q instanceof MatchAllButRequireVerificationQuery; } @@ -657,7 +657,7 @@ protected void getNgramTokens(Set tokens, String fragment) { throw new ElasticsearchParseException("Error parsing wildcard regex pattern fragment [" + fragment + "]"); } } - + private void addClause(String token, BooleanQuery.Builder bqBuilder, Occur occur) { assert token.codePointCount(0, token.length()) <= NGRAM_SIZE; @@ -693,9 +693,9 @@ public Query fuzzyQuery( BooleanQuery.Builder bqBuilder = new BooleanQuery.Builder(); //The approximation query can have a prefix and any number of ngrams. BooleanQuery.Builder approxBuilder = new BooleanQuery.Builder(); - + String postPrefixString = lowerSearchTerm; - + // Add all content prior to prefixLength as a MUST clause to the ngram index query if (prefixLength > 0) { Set prefixTokens = new LinkedHashSet<>(); @@ -730,7 +730,7 @@ public Query fuzzyQuery( } tokenizer.end(); tokenizer.close(); - + BooleanQuery.Builder ngramBuilder = new BooleanQuery.Builder(); int numClauses = 0; for (String token : postPrefixTokens) { @@ -744,7 +744,7 @@ public Query fuzzyQuery( ngramBuilder.setMinimumNumberShouldMatch(numClauses - fuzziness.asDistance(searchTerm)); approxBuilder.add(ngramBuilder.build(), Occur.MUST); } - + BooleanQuery ngramQ = approxBuilder.build(); if (ngramQ.clauses().size()>0) { bqBuilder.add(ngramQ, Occur.MUST); @@ -889,7 +889,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value) { + protected String parseSourceValue(Object value, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } return value.toString(); } @@ -900,8 +903,8 @@ void createFields(String value, Document parseDoc, Listfields) t if (value == null || value.length() > ignoreAbove) { return; } - // Always lower case the ngram index and value - helps with - // a) speed (less ngram variations to explore on disk and in RAM-based automaton) and + // Always lower case the ngram index and value - helps with + // a) speed (less ngram variations to explore on disk and in RAM-based automaton) and // b) uses less disk space String ngramValue = addLineEndChars(WildcardFieldType.toLowerCase(value)); Field ngramField = new Field(fieldType().name(), ngramValue, ngramFieldType); @@ -915,7 +918,7 @@ void createFields(String value, Document parseDoc, Listfields) t dvField.add(value.getBytes(StandardCharsets.UTF_8)); } } - + // Values held in the ngram index are encoded with special characters to denote start and end of values. static String addLineEndChars(String value) { return TOKEN_START_OR_END_CHAR + value + TOKEN_START_OR_END_CHAR + TOKEN_START_OR_END_CHAR;