diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc index 5c1671cb30dd3..1af10507793e6 100644 --- a/docs/reference/mapping/types.asciidoc +++ b/docs/reference/mapping/types.asciidoc @@ -9,7 +9,7 @@ document: === Core data types string:: <>, <> and <> -<>:: `long`, `integer`, `short`, `byte`, `double`, `float`, `half_float`, `scaled_float` +<>:: `long`, `integer`, `short`, `byte`, `double`, `float`, `half_float`, `scaled_float`, <> <>:: `date` <>:: `date_nanos` <>:: `boolean` @@ -136,3 +136,5 @@ include::types/shape.asciidoc[] include::types/constant-keyword.asciidoc[] include::types/wildcard.asciidoc[] + +include::types/unsigned_long.asciidoc[] diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc index afd0010135e03..1cc00932d2c02 100644 --- a/docs/reference/mapping/types/numeric.asciidoc +++ b/docs/reference/mapping/types/numeric.asciidoc @@ -15,6 +15,7 @@ The following numeric types are supported: `float`:: A single-precision 32-bit IEEE 754 floating point number, restricted to finite values. `half_float`:: A half-precision 16-bit IEEE 754 floating point number, restricted to finite values. `scaled_float`:: A floating point number that is backed by a `long`, scaled by a fixed `double` scaling factor. +`unsigned_long`:: An <> with a minimum value of 0 and a maximum value of +2^64^-1+. Below is an example of configuring a mapping with numeric fields: @@ -115,7 +116,7 @@ The following parameters are accepted by numeric types: <>:: Try to convert strings to numbers and truncate fractions for integers. - Accepts `true` (default) and `false`. + Accepts `true` (default) and `false`. Not applicable for unsigned_long. <>:: diff --git a/docs/reference/mapping/types/unsigned_long.asciidoc b/docs/reference/mapping/types/unsigned_long.asciidoc new file mode 100644 index 0000000000000..3a2b0ec3b2420 --- /dev/null +++ b/docs/reference/mapping/types/unsigned_long.asciidoc @@ -0,0 +1,147 @@ +[role="xpack"] +[testenv="basic"] + +[[unsigned-long]] +=== Unsigned long data type +++++ +Unsigned long +++++ + +Unsigned long is a numeric field type that represents an unsigned 64-bit +integer with a minimum value of 0 and a maximum value of +2^64^-1+ +(from 0 to 18446744073709551615). + +At index-time, an indexed value is converted to the singed long range: +[- 9223372036854775808, 9223372036854775807] by subtracting +2^63^+ from it +and stored as a singed long taking 8 bytes. +At query-time, the same conversion is done on query terms. + +[source,console] +-------------------------------------------------- +PUT my_index +{ + "mappings": { + "properties": { + "my_counter": { + "type": "unsigned_long" + } + } + } +} +-------------------------------------------------- + +Unsigned long can be indexed in a numeric or string form, +representing integer values in the range [0, 18446744073709551615]. +They can't have a decimal part. + +[source,console] +-------------------------------- +POST /my_index/_bulk?refresh +{"index":{"_id":1}} +{"my_counter": 0} +{"index":{"_id":2}} +{"my_counter": 9223372036854775808} +{"index":{"_id":3}} +{"my_counter": 18446744073709551614} +{"index":{"_id":4}} +{"my_counter": 18446744073709551615} +-------------------------------- +//TEST[continued] + +Term queries accept any numbers in a numeric or string form. + +[source,console] +-------------------------------- +GET /my_index/_search +{ + "query": { + "term" : { + "my_counter" : 18446744073709551615 + } + } +} +-------------------------------- +//TEST[continued] + +Range queries can contain ranges with decimal parts. +It is recommended to pass ranges as strings to ensure they are parsed +without any loss of precision. + +[source,console] +-------------------------------- +GET /my_index/_search +{ + "query": { + "range" : { + "my_counter" : { + "gte" : "9223372036854775808.5", + "lte" : "18446744073709551615" + } + } + } +} +-------------------------------- +//TEST[continued] + +WARNING: Unlike term and range queries, sorting and aggregations on +unsigned_long data may return imprecise results. For sorting and aggregations +double representation of unsigned longs is used, which means that long values +are first converted to double values. During this conversion, +for long values greater than +2^53^+ there could be some loss of +precision for the least significant digits. Long values less than +2^53^+ +are converted accurately. + +[source,console] +-------------------------------- +GET /my_index/_search +{ + "query": { + "match_all" : {} + }, + "sort" : {"my_counter" : "desc"} <1> +} +-------------------------------- +//TEST[continued] +<1> As both document values: "18446744073709551614" and "18446744073709551615" +are converted to the same double value: "1.8446744073709552E19", this +descending sort may return imprecise results, as the document with a lower +value of "18446744073709551614" may come before the document +with a higher value of "18446744073709551615". + +[[unsigned-long-params]] +==== Parameters for unsigned long fields + +The following parameters are accepted: + +[horizontal] + +<>:: + + Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + (default) or `false`. + +<>:: + + If `true`, malformed numbers are ignored. If `false` (default), malformed + numbers throw an exception and reject the whole document. + +<>:: + + Should the field be searchable? Accepts `true` (default) and `false`. + +<>:: + + Accepts a numeric value of the same `type` as the field which is + substituted for any explicit `null` values. Defaults to `null`, which + means the field is treated as missing. + +<>:: + + Whether the field value should be stored and retrievable separately from + the <> field. Accepts `true` or `false` + (default). + +<>:: + + Metadata about the field. diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java index 82a663bd9dc5d..cc4162a0a0439 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java @@ -206,6 +206,8 @@ Map map( long longValue() throws IOException; + long unsignedLongValue() throws IOException; + float floatValue() throws IOException; double doubleValue() throws IOException; diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java index 9a8686001e2dc..ad88c10325085 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java @@ -224,6 +224,11 @@ public long longValue() throws IOException { return parser.longValue(); } + @Override + public long unsignedLongValue() throws IOException { + return parser.unsignedLongValue(); + } + @Override public float floatValue() throws IOException { return parser.floatValue(); diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java index 7489222df2e76..ca17d923068f4 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java @@ -30,6 +30,7 @@ import org.elasticsearch.core.internal.io.IOUtils; import java.io.IOException; +import java.math.BigInteger; import java.nio.CharBuffer; public class JsonXContentParser extends AbstractXContentParser { @@ -166,6 +167,26 @@ public long doLongValue() throws IOException { return parser.getLongValue(); } + @Override + protected long doUnsignedLongValue() throws IOException { + JsonParser.NumberType numberType = parser.getNumberType(); + if ((numberType == JsonParser.NumberType.INT) || (numberType == JsonParser.NumberType.LONG)) { + long longValue = parser.getLongValue(); + if (longValue < 0) { + throw new IllegalArgumentException("Value [" + longValue + "] is out of range for unsigned long."); + } + return longValue; + } else if (numberType == JsonParser.NumberType.BIG_INTEGER) { + BigInteger bigIntegerValue = parser.getBigIntegerValue(); + if (bigIntegerValue.compareTo(BIGINTEGER_MAX_UNSIGNED_LONG_VALUE) > 0 || bigIntegerValue.compareTo(BigInteger.ZERO) < 0) { + throw new IllegalArgumentException("Value [" + bigIntegerValue + "] is out of range for unsigned long"); + } + return bigIntegerValue.longValue(); + } else { // for all other value types including numbers with decimal parts + throw new IllegalArgumentException("For input string: [" + parser.getValueAsString() + "]."); + } + } + @Override public float doFloatValue() throws IOException { return parser.getFloatValue(); diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java index 264af205e488b..f5244329f2e06 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java @@ -46,6 +46,8 @@ public abstract class AbstractXContentParser implements XContentParser { // references to this policy decision throughout the codebase and find // and change any code that needs to apply an alternative policy. public static final boolean DEFAULT_NUMBER_COERCE_POLICY = true; + public static BigInteger BIGINTEGER_MAX_UNSIGNED_LONG_VALUE = BigInteger.ONE.shiftLeft(64).subtract(BigInteger.ONE); // 2^64 -1 + private static void checkCoerceString(boolean coerce, Class clazz) { if (!coerce) { @@ -208,8 +210,26 @@ public long longValue(boolean coerce) throws IOException { return result; } + @Override + public long unsignedLongValue() throws IOException { + Token token = currentToken(); + if (token == Token.VALUE_STRING) { + return Long.parseUnsignedLong(text()); + } + long result = doUnsignedLongValue(); + return result; + } + protected abstract long doLongValue() throws IOException; + /** + * Returns an unsigned long value of the current numeric token. + * The method must check for proper boundaries: [0; 2^64-1], and also check that it doesn't have a decimal part. + * An exception is raised if any of the conditions is violated. + * Numeric tokens greater than Long.MAX_VALUE must be returned as negative values. + */ + protected abstract long doUnsignedLongValue() throws IOException; + @Override public float floatValue() throws IOException { return floatValue(DEFAULT_NUMBER_COERCE_POLICY); diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java index c54e71634d6ed..b2682615c06a5 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java @@ -73,6 +73,26 @@ protected long doLongValue() throws IOException { return numberValue().longValue(); } + @Override + protected long doUnsignedLongValue() throws IOException { + Number value = numberValue(); + if ((value instanceof Integer) || (value instanceof Long) || (value instanceof Short) || (value instanceof Byte)) { + long longValue = value.longValue(); + if (longValue < 0) { + throw new IllegalArgumentException("Value [" + longValue + "] is out of range for unsigned long."); + } + return longValue; + } else if (value instanceof BigInteger) { + BigInteger bigIntegerValue = (BigInteger) value; + if (bigIntegerValue.compareTo(BIGINTEGER_MAX_UNSIGNED_LONG_VALUE) > 0 || bigIntegerValue.compareTo(BigInteger.ZERO) < 0) { + throw new IllegalArgumentException("Value [" + bigIntegerValue + "] is out of range for unsigned long."); + } + return bigIntegerValue.longValue(); + } else { + throw new IllegalArgumentException("For input string: [" + value.toString() + "]."); + } + } + @Override protected float doFloatValue() throws IOException { return numberValue().floatValue(); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java index 6721716bb160e..b319ebf9fd5e7 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java @@ -87,7 +87,7 @@ public final ValuesSourceType getValuesSourceType() { * Values are casted to the provided targetNumericType type if it doesn't * match the field's numericType. */ - public final SortField sortField( + public SortField sortField( NumericType targetNumericType, Object missingValue, MultiValueMode sortMode, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java index 20b0086c1e4e2..d60d9dc0009cf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java @@ -236,6 +236,11 @@ public long longValue() throws IOException { return parser.longValue(); } + @Override + public long unsignedLongValue() throws IOException { + return parser.unsignedLongValue(); + } + @Override public float floatValue() throws IOException { return parser.floatValue(); diff --git a/x-pack/plugin/mapper-unsigned-long/build.gradle b/x-pack/plugin/mapper-unsigned-long/build.gradle new file mode 100644 index 0000000000000..11e535ab39cbc --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/build.gradle @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'unsigned-long' + description 'Module for the unsigned long field type' + classname 'org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin' + extendedPlugins = ['x-pack-core'] +} +archivesBaseName = 'x-pack-unsigned-long' + +dependencies { + compileOnly project(path: xpackModule('core'), configuration: 'default') + testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts') +} + +integTest.enabled = false diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java new file mode 100644 index 0000000000000..d8dfc096f74a7 --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -0,0 +1,519 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.unsignedlong; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.exc.InputCoercionException; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.IndexOrDocValuesQuery; +import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.SimpleMappedFieldType; +import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.TypeParsers; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.DocValueFormat; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +public class UnsignedLongFieldMapper extends FieldMapper { + protected static long MASK_2_63 = 0x8000000000000000L; + private static BigInteger BIGINTEGER_2_64_MINUS_ONE = BigInteger.ONE.shiftLeft(64).subtract(BigInteger.ONE); // 2^64 -1 + private static BigDecimal BIGDECIMAL_2_64_MINUS_ONE = new BigDecimal(BIGINTEGER_2_64_MINUS_ONE); + + public static final String CONTENT_TYPE = "unsigned_long"; + // use the same default as numbers + private static final FieldType FIELD_TYPE = new FieldType(); + static { + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + } + + public static class Builder extends FieldMapper.Builder { + + private Boolean ignoreMalformed; + private String nullValue; + + public Builder(String name) { + super(name, FIELD_TYPE); + builder = this; + } + + public Builder ignoreMalformed(boolean ignoreMalformed) { + this.ignoreMalformed = ignoreMalformed; + return builder; + } + + @Override + public Builder indexOptions(IndexOptions indexOptions) { + throw new MapperParsingException("index_options not allowed in field [" + name + "] of type [" + CONTENT_TYPE + "]"); + } + + protected Explicit ignoreMalformed(BuilderContext context) { + if (ignoreMalformed != null) { + return new Explicit<>(ignoreMalformed, true); + } + if (context.indexSettings() != null) { + return new Explicit<>(IGNORE_MALFORMED_SETTING.get(context.indexSettings()), false); + } + return NumberFieldMapper.Defaults.IGNORE_MALFORMED; + } + + public Builder nullValue(String nullValue) { + this.nullValue = nullValue; + return this; + } + + @Override + public UnsignedLongFieldMapper build(BuilderContext context) { + UnsignedLongFieldType type = new UnsignedLongFieldType(buildFullName(context), indexed, hasDocValues, meta); + return new UnsignedLongFieldMapper( + name, + fieldType, + type, + ignoreMalformed(context), + multiFieldsBuilder.build(this, context), + copyTo, + nullValue + ); + } + } + + public static class TypeParser implements Mapper.TypeParser { + @Override + public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + Builder builder = new Builder(name); + TypeParsers.parseField(builder, name, node, parserContext); + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String propName = entry.getKey(); + Object propNode = entry.getValue(); + if (propName.equals("null_value")) { + if (propNode == null) { + throw new MapperParsingException("Property [null_value] cannot be null."); + } + parseUnsignedLong(propNode); // confirm that null_value is a proper unsigned_long + String nullValue = (propNode instanceof BytesRef) ? ((BytesRef) propNode).utf8ToString() : propNode.toString(); + builder.nullValue(nullValue); + iterator.remove(); + } else if (propName.equals("ignore_malformed")) { + builder.ignoreMalformed(XContentMapValues.nodeBooleanValue(propNode, name + ".ignore_malformed")); + iterator.remove(); + } + } + return builder; + } + } + + public static final class UnsignedLongFieldType extends SimpleMappedFieldType { + + public UnsignedLongFieldType(String name, boolean indexed, boolean hasDocValues, Map meta) { + super(name, indexed, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + } + + public UnsignedLongFieldType(String name) { + this(name, true, true, Collections.emptyMap()); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public Query existsQuery(QueryShardContext context) { + if (hasDocValues()) { + return new DocValuesFieldExistsQuery(name()); + } else { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + failIfNotIndexed(); + Long longValue = parseTerm(value); + if (longValue == null) { + return new MatchNoDocsQuery(); + } + Query query = LongPoint.newExactQuery(name(), convertToSignedLong(longValue)); + if (boost() != 1f) { + query = new BoostQuery(query, boost()); + } + return query; + } + + @Override + public Query termsQuery(List values, QueryShardContext context) { + failIfNotIndexed(); + long[] lvalues = new long[values.size()]; + int upTo = 0; + for (int i = 0; i < values.size(); i++) { + Object value = values.get(i); + Long longValue = parseTerm(value); + if (longValue != null) { + lvalues[upTo++] = convertToSignedLong(longValue); + } + } + if (upTo == 0) { + return new MatchNoDocsQuery(); + } + if (upTo != lvalues.length) { + lvalues = Arrays.copyOf(lvalues, upTo); + } + Query query = LongPoint.newSetQuery(name(), lvalues); + if (boost() != 1f) { + query = new BoostQuery(query, boost()); + } + return query; + } + + @Override + public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { + failIfNotIndexed(); + long l = Long.MIN_VALUE; + long u = Long.MAX_VALUE; + if (lowerTerm != null) { + Long lt = parseLowerRangeTerm(lowerTerm, includeLower); + if (lt == null) return new MatchNoDocsQuery(); + l = convertToSignedLong(lt); + } + if (upperTerm != null) { + Long ut = parseUpperRangeTerm(upperTerm, includeUpper); + if (ut == null) return new MatchNoDocsQuery(); + u = convertToSignedLong(ut); + } + if (l > u) return new MatchNoDocsQuery(); + + Query query = LongPoint.newRangeQuery(name(), l, u); + if (hasDocValues()) { + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); + query = new IndexOrDocValuesQuery(query, dvQuery); + if (context.indexSortedOnField(name())) { + query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); + } + } + if (boost() != 1f) { + query = new BoostQuery(query, boost()); + } + return query; + } + + @Override + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + failIfNoDocValues(); + return (cache, breakerService, mapperService) -> { + final IndexNumericFieldData signedLongValues = new SortedNumericIndexFieldData.Builder( + name(), + IndexNumericFieldData.NumericType.LONG + ).build(cache, breakerService, mapperService); + return new UnsignedLongIndexFieldData(signedLongValues); + }; + } + + @Override + public Object valueForDisplay(Object value) { + if (value == null) { + return null; + } + return convertToOriginal(((Number) value).longValue()); + } + + @Override + public DocValueFormat docValueFormat(String format, ZoneId timeZone) { + if (timeZone != null) { + throw new IllegalArgumentException( + "Field [" + name() + "] of type [" + typeName() + "] does not support custom time zones" + ); + } + if (format == null) { + return DocValueFormat.RAW; + } else { + return new DocValueFormat.Decimal(format); + } + } + + @Override + public Function pointReaderIfPossible() { + if (isSearchable()) { + return (value) -> LongPoint.decodeDimension(value, 0); + } + return null; + } + + /** + * Parses value to unsigned long for Term Query + * @param value to to parse + * @return parsed value, if a value represents an unsigned long in the range [0, 18446744073709551615] + * null, if a value represents some other number + * throws an exception if a value is wrongly formatted number + */ + protected static Long parseTerm(Object value) { + if (value instanceof Number) { + if ((value instanceof Long) || (value instanceof Integer) || (value instanceof Short) || (value instanceof Byte)) { + long lv = ((Number) value).longValue(); + if (lv >= 0) { + return lv; + } + } else if (value instanceof BigInteger) { + BigInteger bigIntegerValue = (BigInteger) value; + if (bigIntegerValue.compareTo(BigInteger.ZERO) >= 0 && bigIntegerValue.compareTo(BIGINTEGER_2_64_MINUS_ONE) <= 0) { + return bigIntegerValue.longValue(); + } + } + } else { + String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString(); + try { + return Long.parseUnsignedLong(stringValue); + } catch (NumberFormatException e) { + // try again in case a number was negative or contained decimal + Double.parseDouble(stringValue); // throws an exception if it is an improper number + } + } + return null; // any other number: decimal or beyond the range of unsigned long + } + + /** + * Parses a lower term for a range query + * @param value to parse + * @param include whether a value should be included + * @return parsed value to long considering include parameter + * 0, if value is less than 0 + * a value truncated to long, if value is in range [0, 18446744073709551615] + * null, if value is higher than the maximum allowed value for unsigned long + * throws an exception is value represents wrongly formatted number + */ + protected static Long parseLowerRangeTerm(Object value, boolean include) { + if ((value instanceof Long) || (value instanceof Integer) || (value instanceof Short) || (value instanceof Byte)) { + long longValue = ((Number) value).longValue(); + if (longValue < 0) return 0L; // limit lowerTerm to min value for unsigned long: 0 + if (include == false) { // start from the next value + // for unsigned long, the next value for Long.MAX_VALUE is -9223372036854775808L + longValue = longValue == Long.MAX_VALUE ? Long.MIN_VALUE : ++longValue; + } + return longValue; + } + String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString(); + final BigDecimal bigDecimalValue = new BigDecimal(stringValue); // throws an exception if it is an improper number + if (bigDecimalValue.compareTo(BigDecimal.ZERO) <= 0) { + return 0L; // for values <=0, set lowerTerm to 0 + } + int c = bigDecimalValue.compareTo(BIGDECIMAL_2_64_MINUS_ONE); + if (c > 0 || (c == 0 && include == false)) { + return null; // lowerTerm is beyond maximum value + } + long longValue = bigDecimalValue.longValue(); + boolean hasDecimal = (bigDecimalValue.scale() > 0 && bigDecimalValue.stripTrailingZeros().scale() > 0); + if (include == false || hasDecimal) { + ++longValue; + } + return longValue; + } + + /** + * Parses an upper term for a range query + * @param value to parse + * @param include whether a value should be included + * @return parsed value to long considering include parameter + * null, if value is less that 0, as value is lower than the minimum allowed value for unsigned long + * a value truncated to long if value is in range [0, 18446744073709551615] + * -1 (unsigned long of 18446744073709551615) for values greater than 18446744073709551615 + * throws an exception is value represents wrongly formatted number + */ + protected static Long parseUpperRangeTerm(Object value, boolean include) { + if ((value instanceof Long) || (value instanceof Integer) || (value instanceof Short) || (value instanceof Byte)) { + long longValue = ((Number) value).longValue(); + if ((longValue < 0) || (longValue == 0 && include == false)) return null; // upperTerm is below minimum + longValue = include ? longValue : --longValue; + return longValue; + } + String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString(); + final BigDecimal bigDecimalValue = new BigDecimal(stringValue); // throws an exception if it is an improper number + int c = bigDecimalValue.compareTo(BigDecimal.ZERO); + if (c < 0 || (c == 0 && include == false)) { + return null; // upperTerm is below minimum + } + if (bigDecimalValue.compareTo(BIGDECIMAL_2_64_MINUS_ONE) > 0) { + return -1L; // limit upperTerm to max value for unsigned long: 18446744073709551615 + } + long longValue = bigDecimalValue.longValue(); + boolean hasDecimal = (bigDecimalValue.scale() > 0 && bigDecimalValue.stripTrailingZeros().scale() > 0); + if (include == false && hasDecimal == false) { + --longValue; + } + return longValue; + } + } + + private Explicit ignoreMalformed; + private final String nullValue; + private final Long nullValueNumeric; + + private UnsignedLongFieldMapper( + String simpleName, + FieldType fieldType, + UnsignedLongFieldType mappedFieldType, + Explicit ignoreMalformed, + MultiFields multiFields, + CopyTo copyTo, + String nullValue + ) { + super(simpleName, fieldType, mappedFieldType, multiFields, copyTo); + this.nullValue = nullValue; + this.nullValueNumeric = nullValue == null ? null : convertToSignedLong(parseUnsignedLong(nullValue)); + this.ignoreMalformed = ignoreMalformed; + } + + @Override + public UnsignedLongFieldType fieldType() { + return (UnsignedLongFieldType) super.fieldType(); + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + @Override + protected UnsignedLongFieldMapper clone() { + return (UnsignedLongFieldMapper) super.clone(); + } + + @Override + protected void parseCreateField(ParseContext context) throws IOException { + XContentParser parser = context.parser(); + Long numericValue; + if (context.externalValueSet()) { + numericValue = parseUnsignedLong(context.externalValue()); + } else if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { + numericValue = null; + } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING && parser.textLength() == 0) { + numericValue = null; + } else { + try { + numericValue = parser.unsignedLongValue(); + } catch (InputCoercionException | IllegalArgumentException | JsonParseException e) { + if (ignoreMalformed.value() && parser.currentToken().isValue()) { + context.addIgnoredField(mappedFieldType.name()); + return; + } else { + throw e; + } + } + } + if (numericValue == null) { + numericValue = nullValueNumeric; + if (numericValue == null) return; + } else { + numericValue = convertToSignedLong(numericValue); + } + + boolean docValued = fieldType().hasDocValues(); + boolean indexed = fieldType().isSearchable(); + boolean stored = fieldType.stored(); + + List fields = NumberFieldMapper.NumberType.LONG.createFields(fieldType().name(), numericValue, indexed, docValued, stored); + context.doc().addAll(fields); + if (docValued == false && (indexed || stored)) { + createFieldNamesField(context); + } + } + + @Override + protected void mergeOptions(FieldMapper other, List conflicts) { + UnsignedLongFieldMapper mergeWith = (UnsignedLongFieldMapper) other; + if (mergeWith.ignoreMalformed.explicit()) { + this.ignoreMalformed = mergeWith.ignoreMalformed; + } + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + super.doXContentBody(builder, includeDefaults, params); + + if (includeDefaults || ignoreMalformed.explicit()) { + builder.field("ignore_malformed", ignoreMalformed.value()); + } + if (nullValue != null) { + builder.field("null_value", nullValue); + } + } + + /** + * Parse object to unsigned long + * @param value must represent an unsigned long in rage [0;18446744073709551615] or an exception will be thrown + */ + private static long parseUnsignedLong(Object value) { + if ((value instanceof Long) || (value instanceof Integer) || (value instanceof Short) || (value instanceof Byte)) { + long lv = ((Number) value).longValue(); + if (lv < 0) { + throw new IllegalArgumentException("Value [" + lv + "] is out of range for unsigned long."); + } + return lv; + } + String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString(); + try { + return Long.parseUnsignedLong(stringValue); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("For input string: \"" + stringValue + "\""); + } + } + + /** + * Convert an unsigned long to the singed long by subtract 2^63 from it + * @param value – unsigned long value in the range [0; 2^64-1], values greater than 2^63-1 are negative + * @return signed long value in the range [-2^63; 2^63-1] + */ + private static long convertToSignedLong(long value) { + // subtracting 2^63 or 10000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 + // equivalent to flipping the first bit + return value ^ MASK_2_63; + } + + /** + * Convert a signed long to unsigned by adding 2^63 to it + * @param value – signed long value in the range [-2^63; 2^63-1] + * @return unsigned long value in the range [0; 2^64-1], values greater then 2^63-1 are negative + */ + protected static long convertToOriginal(long value) { + // adding 2^63 or 10000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 + // equivalent to flipping the first bit + return value ^ MASK_2_63; + } + +} diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongIndexFieldData.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongIndexFieldData.java new file mode 100644 index 0000000000000..0802fd4f192ed --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongIndexFieldData.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.unsignedlong; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.LeafNumericFieldData; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; + +public class UnsignedLongIndexFieldData extends IndexNumericFieldData { + private final IndexNumericFieldData signedLongFieldData; + + UnsignedLongIndexFieldData(IndexNumericFieldData signedLongFieldData) { + this.signedLongFieldData = signedLongFieldData; + } + + @Override + public String getFieldName() { + return signedLongFieldData.getFieldName(); + } + + @Override + public ValuesSourceType getValuesSourceType() { + return signedLongFieldData.getValuesSourceType(); + } + + @Override + public LeafNumericFieldData load(LeafReaderContext context) { + return new UnsignedLongLeafFieldData(signedLongFieldData.load(context)); + } + + @Override + public LeafNumericFieldData loadDirect(LeafReaderContext context) throws Exception { + return new UnsignedLongLeafFieldData(signedLongFieldData.loadDirect(context)); + } + + @Override + protected boolean sortRequiresCustomComparator() { + return true; + } + + @Override + public void clear() { + signedLongFieldData.clear(); + } + + @Override + public NumericType getNumericType() { + return NumericType.DOUBLE; + } + +} diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java new file mode 100644 index 0000000000000..5c3df93a1e430 --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.unsignedlong; + +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.elasticsearch.index.fielddata.FieldData; +import org.elasticsearch.index.fielddata.LeafNumericFieldData; +import org.elasticsearch.index.fielddata.NumericDoubleValues; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; + +import java.io.IOException; + +import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.convertToOriginal; + +public class UnsignedLongLeafFieldData implements LeafNumericFieldData { + private final LeafNumericFieldData signedLongFD; + + UnsignedLongLeafFieldData(LeafNumericFieldData signedLongFD) { + this.signedLongFD = signedLongFD; + } + + @Override + public SortedNumericDocValues getLongValues() { + return FieldData.castToLong(getDoubleValues()); + } + + @Override + public SortedNumericDoubleValues getDoubleValues() { + final SortedNumericDocValues values = signedLongFD.getLongValues(); + final NumericDocValues singleValues = DocValues.unwrapSingleton(values); + if (singleValues != null) { + return FieldData.singleton(new NumericDoubleValues() { + @Override + public boolean advanceExact(int doc) throws IOException { + return singleValues.advanceExact(doc); + } + + @Override + public double doubleValue() throws IOException { + return convertUnsignedLongToDouble(singleValues.longValue()); + } + }); + } else { + return new SortedNumericDoubleValues() { + + @Override + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); + } + + @Override + public double nextValue() throws IOException { + return convertUnsignedLongToDouble(values.nextValue()); + } + + @Override + public int docValueCount() { + return values.docValueCount(); + } + }; + } + } + + @Override + public ScriptDocValues getScriptValues() { + return new ScriptDocValues.Doubles(getDoubleValues()); + } + + @Override + public SortedBinaryDocValues getBytesValues() { + return FieldData.toString(getDoubleValues()); + } + + @Override + public long ramBytesUsed() { + return signedLongFD.ramBytesUsed(); + } + + @Override + public void close() { + signedLongFD.close(); + } + + private static double convertUnsignedLongToDouble(long value) { + if (value < 0L) { + return convertToOriginal(value); // add 2 ^ 63 + } else { + // add 2 ^ 63 as a double to make sure there is no overflow and final result is positive + return 0x1.0p63 + value; + } + } +} diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongMapperPlugin.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongMapperPlugin.java new file mode 100644 index 0000000000000..a3ea313403b53 --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongMapperPlugin.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.unsignedlong; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; + +import java.util.Map; + +import static java.util.Collections.singletonMap; + +public class UnsignedLongMapperPlugin extends Plugin implements MapperPlugin { + + public UnsignedLongMapperPlugin(Settings settings) {} + + @Override + public Map getMappers() { + return singletonMap(UnsignedLongFieldMapper.CONTENT_TYPE, new UnsignedLongFieldMapper.TypeParser()); + } + +} diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java new file mode 100644 index 0000000000000..9c4b2dd453c89 --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -0,0 +1,446 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.unsignedlong; + +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexableField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.FieldMapperTestCase; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.termvectors.TermVectorsService; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.junit.Before; + +import java.io.IOException; +import java.math.BigInteger; +import java.util.Collection; +import java.util.Set; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.containsString; + +public class UnsignedLongFieldMapperTests extends FieldMapperTestCase { + + IndexService indexService; + DocumentMapperParser parser; + + @Before + public void setup() { + indexService = createIndex("test"); + parser = indexService.mapperService().documentMapperParser(); + } + + @Override + protected Collection> getPlugins() { + return pluginList(UnsignedLongMapperPlugin.class, LocalStateCompositeXPackPlugin.class); + } + + @Override + protected Set unsupportedProperties() { + return Set.of("analyzer", "similarity"); + } + + @Override + protected UnsignedLongFieldMapper.Builder newBuilder() { + return new UnsignedLongFieldMapper.Builder("my_unsigned_long"); + } + + public void testDefaults() throws Exception { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .endObject() + .endObject() + .endObject() + .endObject() + ); + DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + // test that indexing values as string + { + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().field("my_unsigned_long", "18446744073709551615").endObject() + ), + XContentType.JSON + ) + ); + IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + assertEquals(2, fields.length); + IndexableField pointField = fields[0]; + assertEquals(1, pointField.fieldType().pointIndexDimensionCount()); + assertFalse(pointField.fieldType().stored()); + assertEquals(9223372036854775807L, pointField.numericValue().longValue()); + IndexableField dvField = fields[1]; + assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType()); + assertEquals(9223372036854775807L, dvField.numericValue().longValue()); + assertFalse(dvField.fieldType().stored()); + } + + // test indexing values as integer numbers + { + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "2", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().field("my_unsigned_long", 9223372036854775807L).endObject() + ), + XContentType.JSON + ) + ); + IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + assertEquals(2, fields.length); + IndexableField pointField = fields[0]; + assertEquals(-1L, pointField.numericValue().longValue()); + IndexableField dvField = fields[1]; + assertEquals(-1L, dvField.numericValue().longValue()); + } + + // test that indexing values as number with decimal is not allowed + { + ThrowingRunnable runnable = () -> mapper.parse( + new SourceToParse( + "test", + "3", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("my_unsigned_long", 10.5).endObject()), + XContentType.JSON + ) + ); + MapperParsingException e = expectThrows(MapperParsingException.class, runnable); + assertThat(e.getCause().getMessage(), containsString("For input string: [10.5]")); + } + } + + public void testNotIndexed() throws Exception { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .field("index", false) + .endObject() + .endObject() + .endObject() + .endObject() + ); + DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().field("my_unsigned_long", "18446744073709551615").endObject() + ), + XContentType.JSON + ) + ); + IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + assertEquals(1, fields.length); + IndexableField dvField = fields[0]; + assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType()); + assertEquals(9223372036854775807L, dvField.numericValue().longValue()); + } + + public void testNoDocValues() throws Exception { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .field("doc_values", false) + .endObject() + .endObject() + .endObject() + .endObject() + ); + DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().field("my_unsigned_long", "18446744073709551615").endObject() + ), + XContentType.JSON + ) + ); + IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + assertEquals(1, fields.length); + IndexableField pointField = fields[0]; + assertEquals(1, pointField.fieldType().pointIndexDimensionCount()); + assertEquals(9223372036854775807L, pointField.numericValue().longValue()); + } + + public void testStore() throws Exception { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .field("store", true) + .endObject() + .endObject() + .endObject() + .endObject() + ); + DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().field("my_unsigned_long", "18446744073709551615").endObject() + ), + XContentType.JSON + ) + ); + IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + assertEquals(3, fields.length); + IndexableField pointField = fields[0]; + assertEquals(1, pointField.fieldType().pointIndexDimensionCount()); + assertEquals(9223372036854775807L, pointField.numericValue().longValue()); + IndexableField dvField = fields[1]; + assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType()); + assertEquals(9223372036854775807L, dvField.numericValue().longValue()); + IndexableField storedField = fields[2]; + assertTrue(storedField.fieldType().stored()); + assertEquals(9223372036854775807L, storedField.numericValue().longValue()); + } + + public void testCoerceMappingParameterIsIllegal() throws Exception { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .field("coerce", false) + .endObject() + .endObject() + .endObject() + .endObject() + ); + ThrowingRunnable runnable = () -> parser.parse("_doc", new CompressedXContent(mapping)); + ; + MapperParsingException e = expectThrows(MapperParsingException.class, runnable); + assertEquals(e.getMessage(), "Mapping definition for [my_unsigned_long] has unsupported parameters: [coerce : false]"); + } + + public void testNullValue() throws IOException { + // test that if null value is not defined, field is not indexed + { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .endObject() + .endObject() + .endObject() + .endObject() + ); + DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().nullField("my_unsigned_long").endObject()), + XContentType.JSON + ) + ); + assertArrayEquals(new IndexableField[0], doc.rootDoc().getFields("my_unsigned_long")); + } + + // test that if null value is defined, it is used + { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .field("null_value", "18446744073709551615") + .endObject() + .endObject() + .endObject() + .endObject() + ); + DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().nullField("my_unsigned_long").endObject()), + XContentType.JSON + ) + ); + IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + assertEquals(2, fields.length); + IndexableField pointField = fields[0]; + assertEquals(9223372036854775807L, pointField.numericValue().longValue()); + IndexableField dvField = fields[1]; + assertEquals(9223372036854775807L, dvField.numericValue().longValue()); + } + } + + public void testIgnoreMalformed() throws Exception { + // test ignore_malformed is false by default + { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .endObject() + .endObject() + .endObject() + .endObject() + ); + DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + Object malformedValue1 = "a"; + ThrowingRunnable runnable = () -> mapper.parse( + new SourceToParse( + "test", + "_doc", + BytesReference.bytes(jsonBuilder().startObject().field("my_unsigned_long", malformedValue1).endObject()), + XContentType.JSON + ) + ); + MapperParsingException e = expectThrows(MapperParsingException.class, runnable); + assertThat(e.getCause().getMessage(), containsString("For input string: \"a\"")); + + Object malformedValue2 = Boolean.FALSE; + runnable = () -> mapper.parse( + new SourceToParse( + "test", + "_doc", + BytesReference.bytes(jsonBuilder().startObject().field("my_unsigned_long", malformedValue2).endObject()), + XContentType.JSON + ) + ); + e = expectThrows(MapperParsingException.class, runnable); + assertThat(e.getCause().getMessage(), containsString("Current token")); + assertThat(e.getCause().getMessage(), containsString("not numeric, can not use numeric value accessors")); + } + + // test ignore_malformed when set to true ignored malformed documents + { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .field("ignore_malformed", true) + .endObject() + .endObject() + .endObject() + .endObject() + ); + DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + Object malformedValue1 = "a"; + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(jsonBuilder().startObject().field("my_unsigned_long", malformedValue1).endObject()), + XContentType.JSON + ) + ); + IndexableField[] fields = doc.rootDoc().getFields("my_unsigned_long"); + assertEquals(0, fields.length); + assertArrayEquals(new String[] { "my_unsigned_long" }, TermVectorsService.getValues(doc.rootDoc().getFields("_ignored"))); + + Object malformedValue2 = Boolean.FALSE; + ParsedDocument doc2 = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(jsonBuilder().startObject().field("my_unsigned_long", malformedValue2).endObject()), + XContentType.JSON + ) + ); + IndexableField[] fields2 = doc2.rootDoc().getFields("my_unsigned_long"); + assertEquals(0, fields2.length); + assertArrayEquals(new String[] { "my_unsigned_long" }, TermVectorsService.getValues(doc2.rootDoc().getFields("_ignored"))); + } + } + + public void testIndexingOutOfRangeValues() throws Exception { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("my_unsigned_long") + .field("type", "unsigned_long") + .endObject() + .endObject() + .endObject() + .endObject() + ); + DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping)); + + for (Object outOfRangeValue : new Object[] { "-1", -1L, "18446744073709551616", new BigInteger("18446744073709551616") }) { + ThrowingRunnable runnable = () -> mapper.parse( + new SourceToParse( + "test", + "_doc", + BytesReference.bytes(jsonBuilder().startObject().field("my_unsigned_long", outOfRangeValue).endObject()), + XContentType.JSON + ) + ); + expectThrows(MapperParsingException.class, runnable); + } + } +} diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java new file mode 100644 index 0000000000000..e3e881c5389c6 --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.unsignedlong; + +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.UnsignedLongFieldType; +import java.util.List; + +import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.UnsignedLongFieldType.parseTerm; +import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.UnsignedLongFieldType.parseLowerRangeTerm; +import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.UnsignedLongFieldType.parseUpperRangeTerm; + +public class UnsignedLongFieldTypeTests extends FieldTypeTestCase { + + public void testTermQuery() { + UnsignedLongFieldType ft = new UnsignedLongFieldType("my_unsigned_long"); + + assertEquals(LongPoint.newExactQuery("my_unsigned_long", -9223372036854775808L), ft.termQuery(0, null)); + assertEquals(LongPoint.newExactQuery("my_unsigned_long", 0L), ft.termQuery("9223372036854775808", null)); + assertEquals(LongPoint.newExactQuery("my_unsigned_long", 9223372036854775807L), ft.termQuery("18446744073709551615", null)); + + assertEquals(new MatchNoDocsQuery(), ft.termQuery(-1L, null)); + assertEquals(new MatchNoDocsQuery(), ft.termQuery(10.5, null)); + assertEquals(new MatchNoDocsQuery(), ft.termQuery("18446744073709551616", null)); + + expectThrows(NumberFormatException.class, () -> ft.termQuery("18incorrectnumber", null)); + } + + public void testTermsQuery() { + UnsignedLongFieldType ft = new UnsignedLongFieldType("my_unsigned_long"); + + assertEquals( + LongPoint.newSetQuery("my_unsigned_long", -9223372036854775808L, 0L, 9223372036854775807L), + ft.termsQuery(List.of("0", "9223372036854775808", "18446744073709551615"), null) + ); + + assertEquals(new MatchNoDocsQuery(), ft.termsQuery(List.of(-9223372036854775808L, -1L), null)); + assertEquals(new MatchNoDocsQuery(), ft.termsQuery(List.of("-0.5", "3.14", "18446744073709551616"), null)); + + expectThrows(NumberFormatException.class, () -> ft.termsQuery(List.of("18incorrectnumber"), null)); + } + + public void testRangeQuery() { + UnsignedLongFieldType ft = new UnsignedLongFieldType("my_unsigned_long", true, false, null); + + assertEquals( + LongPoint.newRangeQuery("my_unsigned_long", -9223372036854775808L, -9223372036854775808L), + ft.rangeQuery(-1L, 0L, true, true, null) + ); + assertEquals( + LongPoint.newRangeQuery("my_unsigned_long", -9223372036854775808L, -9223372036854775808L), + ft.rangeQuery(0.0, 0.5, true, true, null) + ); + assertEquals( + LongPoint.newRangeQuery("my_unsigned_long", 0, 0), + ft.rangeQuery("9223372036854775807", "9223372036854775808", false, true, null) + ); + assertEquals( + LongPoint.newRangeQuery("my_unsigned_long", -9223372036854775808L, 9223372036854775806L), + ft.rangeQuery(null, "18446744073709551614.5", true, true, null) + ); + assertEquals( + LongPoint.newRangeQuery("my_unsigned_long", 9223372036854775807L, 9223372036854775807L), + ft.rangeQuery("18446744073709551615", "18446744073709551616", true, true, null) + ); + + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery(-1f, -0.5f, true, true, null)); + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery(-1L, 0L, true, false, null)); + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery(9223372036854775807L, 9223372036854775806L, true, true, null)); + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery("18446744073709551616", "18446744073709551616", true, true, null)); + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery("18446744073709551615", "18446744073709551616", false, true, null)); + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery(9223372036854775807L, 9223372036854775806L, true, true, null)); + + expectThrows(NumberFormatException.class, () -> ft.rangeQuery("18incorrectnumber", "18incorrectnumber", true, true, null)); + } + + public void testParseTermForTermQuery() { + // values that represent proper unsigned long number + assertEquals(0L, parseTerm("0").longValue()); + assertEquals(0L, parseTerm(0).longValue()); + assertEquals(9223372036854775807L, parseTerm(9223372036854775807L).longValue()); + assertEquals(-1L, parseTerm("18446744073709551615").longValue()); + + // values that represent numbers but not unsigned long and not in range of [0; 18446744073709551615] + assertEquals(null, parseTerm("-9223372036854775808.05")); + assertEquals(null, parseTerm(-9223372036854775808L)); + assertEquals(null, parseTerm(0.0)); + assertEquals(null, parseTerm(0.5)); + assertEquals(null, parseTerm("18446744073709551616")); + + // wrongly formatted numbers + expectThrows(NumberFormatException.class, () -> parseTerm("18incorrectnumber")); + } + + public void testParseLowerTermForRangeQuery() { + // values that are lower than min for lowerTerm are converted to 0 + assertEquals(0L, parseLowerRangeTerm(-9223372036854775808L, true).longValue()); + assertEquals(0L, parseLowerRangeTerm("-9223372036854775808", true).longValue()); + assertEquals(0L, parseLowerRangeTerm("-1", true).longValue()); + assertEquals(0L, parseLowerRangeTerm("-0.5", true).longValue()); + + assertEquals(0L, parseLowerRangeTerm(0L, true).longValue()); + assertEquals(0L, parseLowerRangeTerm("0", true).longValue()); + assertEquals(0L, parseLowerRangeTerm("0.0", true).longValue()); + assertEquals(1L, parseLowerRangeTerm("0.5", true).longValue()); + assertEquals(9223372036854775807L, parseLowerRangeTerm(9223372036854775806L, false).longValue()); + assertEquals(9223372036854775807L, parseLowerRangeTerm(9223372036854775807L, true).longValue()); + assertEquals(-9223372036854775808L, parseLowerRangeTerm(9223372036854775807L, false).longValue()); + assertEquals(-1L, parseLowerRangeTerm("18446744073709551614", false).longValue()); + assertEquals(-1L, parseLowerRangeTerm("18446744073709551614.1", true).longValue()); + assertEquals(-1L, parseLowerRangeTerm("18446744073709551615", true).longValue()); + + // values that are higher than max for lowerTerm don't return results + assertEquals(null, parseLowerRangeTerm("18446744073709551615", false)); + assertEquals(null, parseLowerRangeTerm("18446744073709551616", true)); + + // wrongly formatted numbers + expectThrows(NumberFormatException.class, () -> parseLowerRangeTerm("18incorrectnumber", true)); + } + + public void testParseUpperTermForRangeQuery() { + // values that are lower than min for upperTerm don't return results + assertEquals(null, parseUpperRangeTerm(-9223372036854775808L, true)); + assertEquals(null, parseUpperRangeTerm("-1", true)); + assertEquals(null, parseUpperRangeTerm("-0.5", true)); + assertEquals(null, parseUpperRangeTerm(0L, false)); + + assertEquals(0L, parseUpperRangeTerm(0L, true).longValue()); + assertEquals(0L, parseUpperRangeTerm("0", true).longValue()); + assertEquals(0L, parseUpperRangeTerm("0.0", true).longValue()); + assertEquals(0L, parseUpperRangeTerm("0.5", true).longValue()); + assertEquals(9223372036854775806L, parseUpperRangeTerm(9223372036854775807L, false).longValue()); + assertEquals(9223372036854775807L, parseUpperRangeTerm(9223372036854775807L, true).longValue()); + assertEquals(-2L, parseUpperRangeTerm("18446744073709551614.5", true).longValue()); + assertEquals(-2L, parseUpperRangeTerm("18446744073709551615", false).longValue()); + assertEquals(-1L, parseUpperRangeTerm("18446744073709551615", true).longValue()); + + // values that are higher than max for upperTerm are converted to "18446744073709551615" or -1 in singed representation + assertEquals(-1L, parseUpperRangeTerm("18446744073709551615.8", true).longValue()); + assertEquals(-1L, parseUpperRangeTerm("18446744073709551616", true).longValue()); + + // wrongly formatted numbers + expectThrows(NumberFormatException.class, () -> parseUpperRangeTerm("18incorrectnumber", true)); + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/10_basic.yml new file mode 100644 index 0000000000000..98bd40a0ca1ad --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/10_basic.yml @@ -0,0 +1,207 @@ +setup: + + - skip: + version: " - 7.99.99" + reason: "unsigned_long was added in 8.0" + + - do: + indices.create: + index: test1 + body: + mappings: + properties: + ul: + type: unsigned_long + + - do: + bulk: + index: test1 + refresh: true + body: | + { "index": {"_id" : "1"} } + { "ul": 0 } + { "index": {"_id" : "2"} } + { "ul": 9223372036854775807 } + { "index": {"_id" : "3"} } + { "ul": 9223372036854775808 } + { "index": {"_id" : "4"} } + { "ul": 18446744073709551614 } + { "index": {"_id" : "5"} } + { "ul": 18446744073709551615 } + +--- +"Exist query": + + - do: + search: + index: test1 + body: + size: 0 + query: + exists: + field: ul + + - match: { "hits.total.value": 5 } + + +--- +"Term query": + + - do: + search: + index: test1 + body: + query: + term: + ul: 0 + - match: { "hits.total.value": 1 } + - match: {hits.hits.0._id: "1" } + + - do: + search: + index: test1 + body: + query: + term: + ul: 18446744073709551615 + - match: { "hits.total.value": 1 } + - match: {hits.hits.0._id: "5" } + + - do: + search: + index: test1 + body: + query: + term: + ul: 18446744073709551616 + - match: { "hits.total.value": 0 } + +--- +"Terms query": + + - do: + search: + index: test1 + body: + size: 0 + query: + terms: + ul: [0, 9223372036854775808, 18446744073709551615] + + - match: { "hits.total.value": 3 } + +--- +"Range query": + + - do: + search: + index: test1 + body: + size: 0 + query: + range: + ul: + gte: 0 + - match: { "hits.total.value": 5 } + + - do: + search: + index: test1 + body: + size: 0 + query: + range: + ul: + gte: 0.5 + - match: { "hits.total.value": 4 } + + - do: + search: + index: test1 + body: + size: 0 + query: + range: + ul: + lte: 18446744073709551615 + - match: { "hits.total.value": 5 } + + - do: + search: + index: test1 + body: + query: + range: + ul: + lte: "18446744073709551614.5" # this must be string, as number gets converted to double with loss of precision + - match: { "hits.total.value": 4 } + +--- +"Sort": + + - do: + search: + index: test1 + body: + sort: [ { ul: asc } ] + + - match: { "hits.total.value": 5 } + - match: {hits.hits.0._id: "1" } + - match: {hits.hits.0.sort: [0.0] } + # as sort is based on double representation, there is some loss of precision during converting longs to doubles + # thus both 9223372036854775807 and 9223372036854775808 are converted to 9.223372036854776E18 + # both 18446744073709551614 and 18446744073709551615 are converted to 1.8446744073709552E19 + # hence, we can't assert ids for the following sort results: + - match: {hits.hits.1.sort: [9.223372036854776E18] } # could be docs with _id 2 or 3 + - match: {hits.hits.2.sort: [9.223372036854776E18] } # could be docs with _id 2 or 3 + - match: {hits.hits.3.sort: [1.8446744073709552E19] } # could be docs with _id 4 or 5 + - match: {hits.hits.4.sort: [1.8446744073709552E19] } # could be docs with _id 4 or 5 + +--- +"Aggs": + + - do: + search: + index: test1 + body: + size: 0 + aggs: + ul_terms: + terms: + field: ul + - length: { aggregations.ul_terms.buckets: 3 } + - match: { aggregations.ul_terms.buckets.0.key: 9.223372036854776E18 } + - match: { aggregations.ul_terms.buckets.1.key: 1.8446744073709552E19 } + - match: { aggregations.ul_terms.buckets.2.key: 0.0 } + + - do: + search: + index: test1 + body: + size: 0 + aggs: + ul_histogram: + histogram: + field: ul + interval: 9223372036854775808 + - length: { aggregations.ul_histogram.buckets: 3 } + - match: { aggregations.ul_histogram.buckets.0.key: 0.0 } + - match: { aggregations.ul_histogram.buckets.1.key: 9.223372036854776E18 } + - match: { aggregations.ul_histogram.buckets.2.key: 1.8446744073709552E19 } + + - do: + search: + index: test1 + body: + size: 0 + aggs: + ul_range: + range: + field: ul + ranges: [ + { "to": 9223372036854775807 }, + { "from": 9223372036854775807} + ] + - length: { aggregations.ul_range.buckets: 2 } + - match: { aggregations.ul_range.buckets.0.doc_count: 1 } + - match: { aggregations.ul_range.buckets.1.doc_count: 4 } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/20_null_value.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/20_null_value.yml new file mode 100644 index 0000000000000..cd30440d24ab7 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/20_null_value.yml @@ -0,0 +1,79 @@ +--- +"Null value": + - skip: + version: " - 7.99.99" + reason: "unsigned_long was added in 8.0" + + - do: + indices.create: + index: test1 + body: + mappings: + properties: + ul: + type: unsigned_long + null_value: 17446744073709551615 + + - do: + bulk: + index: test1 + refresh: true + body: | + { "index": {"_id" : "1"} } + { "ul": 0 } + { "index": {"_id" : "2"} } + { "ul": null } + { "index": {"_id" : "3"} } + { "ul": ""} + { "index": {"_id" : "4"} } + { "ul": 18446744073709551615 } + { "index": {"_id" : "5"} } + {} + + # term query + - do: + search: + index: test1 + body: + query: + term: + ul: 17446744073709551615 + - match: { "hits.total.value": 2 } + - match: {hits.hits.0._id: "2" } + - match: {hits.hits.1._id: "3" } + + + # asc sort + - do: + search: + index: test1 + body: + sort: { ul : { order: asc, missing : "_last" } } + - match: { "hits.total.value": 5 } + - match: {hits.hits.0._id: "1" } + - match: {hits.hits.0.sort: [0.0] } + - match: {hits.hits.1._id: "2" } + - match: {hits.hits.1.sort: [1.7446744073709552E19] } + - match: {hits.hits.2._id: "3" } + - match: {hits.hits.2.sort: [1.7446744073709552E19] } + - match: {hits.hits.3._id: "4" } + - match: {hits.hits.3.sort: [1.8446744073709552E19] } + - match: {hits.hits.4._id: "5" } + + # desc sort + - do: + search: + index: test1 + body: + sort: { ul: { order: desc, missing: "_first" } } + - match: { "hits.total.value": 5 } + - match: {hits.hits.0._id: "5" } + - match: {hits.hits.1._id: "4" } + - match: {hits.hits.1.sort: [1.8446744073709552E19] } + - match: {hits.hits.2._id: "2" } + - match: {hits.hits.2.sort: [1.7446744073709552E19] } + - match: {hits.hits.3._id: "3" } + - match: {hits.hits.3.sort: [1.7446744073709552E19] } + - match: {hits.hits.4._id: "1" } + - match: {hits.hits.4.sort: [0.0] } + diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/30_multi_fields.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/30_multi_fields.yml new file mode 100644 index 0000000000000..c7ee1ac4ac76e --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/30_multi_fields.yml @@ -0,0 +1,72 @@ +--- +"Multi keyword and unsigned_long fields": + - skip: + version: " - 7.99.99" + reason: "unsigned_long was added in 8.0" + + - do: + indices.create: + index: test1 + body: + mappings: + properties: + counter: + type: keyword + fields: + ul: + type: unsigned_long + + - do: + bulk: + index: test1 + refresh: true + body: | + { "index": {"_id" : "1"} } + { "counter": 0 } + { "index": {"_id" : "2"} } + { "counter": 9223372036854775808 } + { "index": {"_id" : "3"} } + { "counter": "9223372036854775808" } + { "index": {"_id" : "4"} } + { "counter": 18446744073709551614 } + { "index": {"_id" : "5"} } + { "counter": 18446744073709551615 } + + # term query + - do: + search: + index: test1 + body: + query: + term: + counter.ul: 9223372036854775808 + - match: { "hits.total.value": 2 } + - match: {hits.hits.0._id: "2" } + - match: {hits.hits.1._id: "3" } + + + # asc sort by keyword + - do: + search: + index: test1 + body: + sort: { counter : { order: asc} } + - match: { "hits.total.value": 5 } + - match: {hits.hits.0._id: "1" } + - match: {hits.hits.1._id: "4" } + - match: {hits.hits.2._id: "5" } + - match: {hits.hits.3._id: "2" } + - match: {hits.hits.4._id: "3" } + + # asc sort by unsigned long + - do: + search: + index: test1 + body: + sort: { counter.ul: { order: asc} } + - match: { "hits.total.value": 5 } + - match: {hits.hits.0._id: "1" } + - match: {hits.hits.1._id: "2" } + - match: {hits.hits.2._id: "3" } + - match: {hits.hits.3._id: "4" } + - match: {hits.hits.4._id: "5" } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/40_different_numeric.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/40_different_numeric.yml new file mode 100644 index 0000000000000..ba37f57ee886c --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/unsigned_long/40_different_numeric.yml @@ -0,0 +1,102 @@ +--- +"Different numeric types": + - skip: + version: " - 7.99.99" + reason: "unsigned_long was added in 8.0" + + - do: + indices.create: + index: test1 + body: + mappings: + properties: + my_counter: + type: double + + - do: + indices.create: + index: test2 + body: + mappings: + properties: + my_counter: + type: long + + - do: + indices.create: + index: test3 + body: + mappings: + properties: + my_counter: + type: unsigned_long + + - do: + bulk: + index: test1 + refresh: true + body: | + { "index": {"_id" : "1"} } + { "my_counter": 0 } + { "index": {"_id" : "2"} } + { "my_counter": 1000000 } + { "index": {"_id" : "3"} } + { "my_counter": 9223372036854775807 } + - do: + bulk: + index: test2 + refresh: true + body: | + { "index": {"_id" : "1"} } + { "my_counter": 0 } + { "index": {"_id" : "2"} } + { "my_counter": 1000000 } + { "index": {"_id" : "3"} } + { "my_counter": 9223372036854775807 } + + - do: + bulk: + index: test3 + refresh: true + body: | + { "index": {"_id" : "1"} } + { "my_counter": 0 } + { "index": {"_id" : "2"} } + { "my_counter": 1000000 } + { "index": {"_id" : "3"} } + { "my_counter": 18446744073709551615 } + + + - do: + search: + index: test* + body: + size: 0 + query: + range: + my_counter: + gte: 0 + - match: { "hits.total.value": 9 } + + - do: + search: + index: test* + body: + size: 0 + query: + range: + my_counter: + gt: 0 + lt: 9223372036854775807 + - match: { "hits.total.value": 3 } + + - do: + search: + index: test* + body: + size: 0 + query: + range: + my_counter: + gte: 9223372036854775807 + - match: { "hits.total.value": 3 }