Skip to content

Commit

Permalink
Add support for a 'format' option in fields retrieval. (#57855)
Browse files Browse the repository at this point in the history
The new `format` option allows for passing a custom date format:

```
POST logs-*/_search
{
  "fields": [
    "file.*",
    {
      "field": "event.timestamp",
      "format": "epoch_millis"
    },
    ...
  ]
}
```

Other API notes:
* We use the same syntax as `docvalue_fields` for consistency. Under the hood,
both `fields` and `docvalue_fields` use the same `FieldAndFormat` object to
share serialization logic.
* Only `date` and `date_range` fields support formatting currently.
  • Loading branch information
jtibshirani committed Jun 30, 2020
1 parent c3a3963 commit d218e6c
Show file tree
Hide file tree
Showing 65 changed files with 474 additions and 213 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,10 @@ private Float objectToFloat(Object value) {
}

@Override
protected Float parseSourceValue(Object value) {
protected Float parseSourceValue(Object value, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return objectToFloat(value);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,10 @@ protected void parseCreateField(ParseContext context) throws IOException {
}

@Override
protected Object parseSourceValue(Object value) {
protected Object parseSourceValue(Object value, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return value;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -497,7 +497,11 @@ private static double objectToDouble(Object value) {
}

@Override
protected Double parseSourceValue(Object value) {
protected Double parseSourceValue(Object value, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}

double doubleValue = objectToDouble(value);
double scalingFactor = fieldType().getScalingFactor();
return Math.round(doubleValue * scalingFactor) / scalingFactor;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -492,7 +492,7 @@ protected void parseCreateField(ParseContext context) {
}

@Override
protected Object parseSourceValue(Object value) {
protected Object parseSourceValue(Object value, String format) {
throw new UnsupportedOperationException();
}

Expand Down Expand Up @@ -538,7 +538,7 @@ protected void mergeOptions(FieldMapper other, List<String> conflicts) {
}

@Override
protected Object parseSourceValue(Object value) {
protected Object parseSourceValue(Object value, String format) {
throw new UnsupportedOperationException();
}

Expand Down Expand Up @@ -695,7 +695,10 @@ protected void parseCreateField(ParseContext context) throws IOException {
}

@Override
protected String parseSourceValue(Object value) {
protected String parseSourceValue(Object value, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return value.toString();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
}

@Override
protected String parseSourceValue(Object value) {
protected String parseSourceValue(Object value, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}

return value.toString();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ public void testParseSourceValue() {
Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
RankFeatureFieldMapper mapper = new RankFeatureFieldMapper.Builder("field").build(context);

assertEquals(3.14f, mapper.parseSourceValue(3.14), 0.0001);
assertEquals(42.9f, mapper.parseSourceValue("42.9"), 0.0001);
assertEquals(3.14f, mapper.parseSourceValue(3.14, null), 0.0001);
assertEquals(42.9f, mapper.parseSourceValue("42.9", null), 0.0001);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,7 @@ public void testParseSourceValue() {
.scalingFactor(100)
.build(context);

assertEquals(3.14, mapper.parseSourceValue(3.1415926), 0.00001);
assertEquals(3.14, mapper.parseSourceValue("3.1415"), 0.00001);
assertEquals(3.14, mapper.parseSourceValue(3.1415926, null), 0.00001);
assertEquals(3.14, mapper.parseSourceValue("3.1415", null), 0.00001);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
}

@Override
protected Object parseSourceValue(Object value) {
protected Object parseSourceValue(Object value, String format) {
throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source.");
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
}

@Override
protected Object parseSourceValue(Object value) {
protected Object parseSourceValue(Object value, String format) {
throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source.");
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,10 @@ protected void parseCreateField(ParseContext context) throws IOException {
}

@Override
protected Object parseSourceValue(Object value) {
protected Object parseSourceValue(Object value, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return value;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,10 @@ public void parse(ParseContext context) throws IOException {
}

@Override
protected Object parseSourceValue(Object value) {
protected Object parseSourceValue(Object value, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return value;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -754,7 +754,10 @@ protected void parseCreateField(ParseContext context) throws IOException {
}

@Override
protected String parseSourceValue(Object value) {
protected String parseSourceValue(Object value, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return value.toString();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -491,8 +491,8 @@ public void testParseSourceValue() {
Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
ICUCollationKeywordFieldMapper mapper = new ICUCollationKeywordFieldMapper.Builder("field").build(context);

assertEquals("value", mapper.parseSourceValue("value"));
assertEquals("42", mapper.parseSourceValue(42L));
assertEquals("true", mapper.parseSourceValue(true));
assertEquals("value", mapper.parseSourceValue("value", null));
assertEquals("42", mapper.parseSourceValue(42L, null));
assertEquals("true", mapper.parseSourceValue(true, null));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -596,7 +596,10 @@ protected void parseCreateField(ParseContext context) throws IOException {
}

@Override
protected String parseSourceValue(Object value) {
protected String parseSourceValue(Object value, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return value.toString();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -685,8 +685,8 @@ public void testParseSourceValue() {
.searchQuoteAnalyzer(indexService.getIndexAnalyzers().getDefaultSearchQuoteAnalyzer())
.build(context);

assertEquals("value", mapper.parseSourceValue("value"));
assertEquals("42", mapper.parseSourceValue(42L));
assertEquals("true", mapper.parseSourceValue(true));
assertEquals("value", mapper.parseSourceValue("value", null));
assertEquals("42", mapper.parseSourceValue(42L, null));
assertEquals("true", mapper.parseSourceValue(true, null));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,10 @@ protected void parseCreateField(ParseContext context)
}

@Override
protected String parseSourceValue(Object value) {
protected String parseSourceValue(Object value, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return value.toString();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,53 @@ setup:
- match: { hits.hits.0.fields.integer_range.0.lte: 42 }

---
"Test date formatting":
- do:
indices.create:
index: test
body:
settings:
index.number_of_shards: 1
mappings:
properties:
keyword:
type: keyword
date:
type: date

- do:
index:
index: test
id: 1
body:
keyword: "value"
date: "1990-12-29T22:30:00.000Z"

- do:
indices.refresh:
index: [ test ]

- do:
search:
index: test
body:
fields:
- field: date
format: "yyyy/MM/dd"

- is_true: hits.hits.0._id
- is_true: hits.hits.0._source
- match: { hits.hits.0.fields.date.0: "1990/12/29" }

- do:
catch: bad_request
search:
index: test
body:
fields:
- field: keyword
format: "yyyy/MM/dd"
---
"Test disable source":
- do:
indices.create:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -303,8 +303,24 @@ public SearchRequestBuilder addDocValueField(String name) {
return addDocValueField(name, null);
}

/**
* Adds a field to load and return. The field must be present in the document _source.
*
* @param name The field to load
*/
public SearchRequestBuilder addFetchField(String name) {
sourceBuilder().fetchField(name);
sourceBuilder().fetchField(name, null);
return this;
}

/**
* Adds a field to load and return. The field must be present in the document _source.
*
* @param name The field to load
* @param format TODO(jtibs): fill this in
*/
public SearchRequestBuilder addFetchField(String name, String format) {
sourceBuilder().fetchField(name, format);
return this;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ public Builder ignoreZValue(final boolean ignoreZValue) {
}

@Override
protected Object parseSourceValue(Object value) {
protected Object parseSourceValue(Object value, String format) {
throw new UnsupportedOperationException();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,10 @@ protected void parseCreateField(ParseContext context) throws IOException {
}

@Override
protected Object parseSourceValue(Object value) {
protected Object parseSourceValue(Object value, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return value;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
}

@Override
public Boolean parseSourceValue(Object value) {
public Boolean parseSourceValue(Object value, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}

if (value instanceof Boolean) {
return (Boolean) value;
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -624,7 +624,11 @@ private void parse(ParseContext parseContext, Token token,
}

@Override
protected List<?> parseSourceValue(Object value) {
protected List<?> parseSourceValue(Object value, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}

if (value instanceof List) {
return (List<?>) value;
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -628,12 +628,16 @@ protected void parseCreateField(ParseContext context) throws IOException {
}

@Override
public String parseSourceValue(Object value) {
public String parseSourceValue(Object value, String format) {
String date = value.toString();
long timestamp = fieldType().parse(date);

ZonedDateTime dateTime = fieldType().resolution().toInstant(timestamp).atZone(ZoneOffset.UTC);
return fieldType().dateTimeFormatter().format(dateTime);

DateFormatter dateTimeFormatter = fieldType().dateTimeFormatter();
if (format != null) {
dateTimeFormatter = DateFormatter.forPattern(format).withLocale(dateTimeFormatter.locale());
}
return dateTimeFormatter.format(dateTime);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexOptions;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
Expand Down Expand Up @@ -280,21 +281,22 @@ public void parse(ParseContext context) throws IOException {
* Some mappers may need more flexibility and can override this entire method instead.
*
* @param lookup a lookup structure over the document's source.
* @param format an optional format string used when formatting values, for example a date format.
* @return a list a standardized field values.
*/
public List<?> lookupValues(SourceLookup lookup) {
public List<?> lookupValues(SourceLookup lookup, @Nullable String format) {
Object sourceValue = lookup.extractValue(name());
if (sourceValue == null) {
return List.of();
}

List<Object> values = new ArrayList<>();
if (parsesArrayValue()) {
return (List<?>) parseSourceValue(sourceValue);
return (List<?>) parseSourceValue(sourceValue, format);
} else {
List<?> sourceValues = sourceValue instanceof List ? (List<?>) sourceValue : List.of(sourceValue);
for (Object value : sourceValues) {
Object parsedValue = parseSourceValue(value);
Object parsedValue = parseSourceValue(value, format);
values.add(parsedValue);
}
}
Expand All @@ -308,7 +310,7 @@ public List<?> lookupValues(SourceLookup lookup) {
*
* Note that when overriding this method, {@link #lookupValues} should *not* be overridden.
*/
protected abstract Object parseSourceValue(Object value);
protected abstract Object parseSourceValue(Object value, @Nullable String format);

protected void createFieldNamesField(ParseContext context) {
FieldNamesFieldType fieldNamesFieldType = context.docMapper().metadataMapper(FieldNamesFieldMapper.class).fieldType();
Expand Down
Loading

0 comments on commit d218e6c

Please sign in to comment.