From b7a1c8182aae742babfca8794e8dfe3deaf1f164 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 20 Jul 2020 17:03:44 -0400 Subject: [PATCH 1/5] Add double script fields This adds support for `double` typed `script` fields. --- .../fielddata/plain/LeafDoubleFieldData.java | 4 +- .../index/mapper/NumberFieldMapper.java | 56 +++-- .../DoubleScriptFieldScript.java | 47 +++- .../fielddata/ScriptDoubleDocValues.java | 43 ++++ .../fielddata/ScriptDoubleFieldData.java | 123 ++++++++++ .../mapper/ScriptDoubleMappedFieldType.java | 103 ++++++++ .../mapper/ScriptFieldMapper.java | 14 ++ .../query/AbstractDoubleScriptFieldQuery.java | 77 ++++++ .../query/AbstractLongScriptFieldQuery.java | 3 +- .../query/DoubleScriptFieldExistsQuery.java | 31 +++ .../query/DoubleScriptFieldRangeQuery.java | 72 ++++++ .../query/DoubleScriptFieldTermQuery.java | 57 +++++ .../query/DoubleScriptFieldTermsQuery.java | 73 ++++++ .../DoubleScriptFieldScriptTests.java | 13 +- .../ScriptDoubleMappedFieldTypeTests.java | 225 ++++++++++++++++++ .../mapper/ScriptFieldMapperTests.java | 20 ++ ...bstractDoubleScriptFieldQueryTestCase.java | 49 ++++ .../DoubleScriptFieldExistsQueryTests.java | 41 ++++ .../DoubleScriptFieldTermQueryTests.java | 59 +++++ .../DoubleScriptFieldTermsQueryTests.java | 107 +++++++++ .../test/runtime_fields/30_double.yml | 203 ++++++++++++++++ 21 files changed, 1381 insertions(+), 39 deletions(-) create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleDocValues.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleFieldData.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldType.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractDoubleScriptFieldQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldExistsQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldRangeQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermsQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldTypeTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractDoubleScriptFieldQueryTestCase.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldExistsQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermsQueryTests.java create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/30_double.yml diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java index 46cd3a5ccaef2..3ab7847f641d7 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java @@ -34,11 +34,11 @@ /** * Specialization of {@link LeafNumericFieldData} for floating-point numerics. */ -abstract class LeafDoubleFieldData implements LeafNumericFieldData { +public abstract class LeafDoubleFieldData implements LeafNumericFieldData { private final long ramBytesUsed; - LeafDoubleFieldData(long ramBytesUsed) { + protected LeafDoubleFieldData(long ramBytesUsed) { this.ramBytesUsed = ramBytesUsed; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 9edf0f0a8a7d1..5b8c95233fcf4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -416,28 +416,16 @@ public Query termsQuery(String field, List values) { public Query rangeQuery(String field, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, boolean hasDocValues, QueryShardContext context) { - double l = Double.NEGATIVE_INFINITY; - double u = Double.POSITIVE_INFINITY; - if (lowerTerm != null) { - l = parse(lowerTerm, false); - if (includeLower == false) { - l = DoublePoint.nextUp(l); - } - } - if (upperTerm != null) { - u = parse(upperTerm, false); - if (includeUpper == false) { - u = DoublePoint.nextDown(u); + return doubleRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (l, u) -> { + Query query = DoublePoint.newRangeQuery(field, l, u); + if (hasDocValues) { + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(field, + NumericUtils.doubleToSortableLong(l), + NumericUtils.doubleToSortableLong(u)); + query = new IndexOrDocValuesQuery(query, dvQuery); } - } - Query query = DoublePoint.newRangeQuery(field, l, u); - if (hasDocValues) { - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(field, - NumericUtils.doubleToSortableLong(l), - NumericUtils.doubleToSortableLong(u)); - query = new IndexOrDocValuesQuery(query, dvQuery); - } - return query; + return query; + }); } @Override @@ -844,7 +832,7 @@ static double signum(Object value) { /** * Converts an Object to a double by checking it against known types first */ - private static double objectToDouble(Object value) { + public static double objectToDouble(Object value) { double doubleValue; if (value instanceof Number) { @@ -882,6 +870,30 @@ public static long objectToLong(Object value, boolean coerce) { return Numbers.toLong(stringValue, coerce); } + public static Query doubleRangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + BiFunction builder + ) { + double l = Double.NEGATIVE_INFINITY; + double u = Double.POSITIVE_INFINITY; + if (lowerTerm != null) { + l = objectToDouble(lowerTerm); + if (includeLower == false) { + l = DoublePoint.nextUp(l); + } + } + if (upperTerm != null) { + u = objectToDouble(upperTerm); + if (includeUpper == false) { + u = DoublePoint.nextDown(u); + } + } + return builder.apply(l, u); + } + /** * Processes query bounds into {@code long}s and delegates the * provided {@code builder} to build a range query. diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java index 8b66738b8bc74..6824c21f4586f 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.runtimefields; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.painless.spi.Whitelist; import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; @@ -16,10 +17,9 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.function.DoubleConsumer; public abstract class DoubleScriptFieldScript extends AbstractScriptFieldScript { - static final ScriptContext CONTEXT = new ScriptContext<>("double_script_field", Factory.class); + public static final ScriptContext CONTEXT = new ScriptContext<>("double_script_field", Factory.class); static List whitelist() { return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "double_whitelist.txt")); @@ -32,14 +32,47 @@ public interface Factory extends ScriptFactory { } public interface LeafFactory { - DoubleScriptFieldScript newInstance(LeafReaderContext ctx, DoubleConsumer sync) throws IOException; + DoubleScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; } - private final DoubleConsumer sync; + private double[] values = new double[1]; + private int count; - public DoubleScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx, DoubleConsumer sync) { + public DoubleScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { super(params, searchLookup, ctx); - this.sync = sync; + } + + /** + * Execute the script for the provided {@code docId}. + */ + public final void runForDoc(int docId) { + count = 0; + setDocument(docId); + execute(); + } + + /** + * Values from the last time {@link #runForDoc(int)} was called. This array + * is mutable and will change with the next call of {@link #runForDoc(int)}. + * It is also oversized and will contain garbage at all indices at and + * above {@link #count()}. + */ + public final double[] values() { + return values; + } + + /** + * The number of results produced the last time {@link #runForDoc(int)} was called. + */ + public final int count() { + return count; + } + + private void collectValue(double v) { + if (values.length < count + 1) { + values = ArrayUtil.grow(values, count + 1); + } + values[count++] = v; } public static class Value { @@ -50,7 +83,7 @@ public Value(DoubleScriptFieldScript script) { } public void value(double v) { - script.sync.accept(v); + script.collectValue(v); } } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleDocValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleDocValues.java new file mode 100644 index 0000000000000..135229e260eba --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleDocValues.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; + +import java.io.IOException; +import java.util.Arrays; + +public final class ScriptDoubleDocValues extends SortedNumericDoubleValues { + private final DoubleScriptFieldScript script; + private int cursor; + + ScriptDoubleDocValues(DoubleScriptFieldScript script) { + this.script = script; + } + + @Override + public boolean advanceExact(int docId) { + script.runForDoc(docId); + if (script.count() == 0) { + return false; + } + Arrays.sort(script.values(), 0, script.count()); + cursor = 0; + return true; + } + + @Override + public double nextValue() throws IOException { + return script.values()[cursor++]; + } + + @Override + public int docValueCount() { + return script.count(); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleFieldData.java new file mode 100644 index 0000000000000..5e71f3f51bfec --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleFieldData.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.SearchLookupAware; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.fielddata.plain.LeafDoubleFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.script.Script; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; + +import java.io.IOException; + +public final class ScriptDoubleFieldData extends IndexNumericFieldData implements SearchLookupAware { + + public static class Builder implements IndexFieldData.Builder { + + private final Script script; + private final DoubleScriptFieldScript.Factory scriptFactory; + + public Builder(Script script, DoubleScriptFieldScript.Factory scriptFactory) { + this.script = script; + this.scriptFactory = scriptFactory; + } + + @Override + public ScriptDoubleFieldData build( + IndexSettings indexSettings, + MappedFieldType fieldType, + IndexFieldDataCache cache, + CircuitBreakerService breakerService, + MapperService mapperService + ) { + return new ScriptDoubleFieldData(fieldType.name(), script, scriptFactory); + } + } + + private final String fieldName; + private final Script script; + private final DoubleScriptFieldScript.Factory scriptFactory; + private final SetOnce leafFactory = new SetOnce<>(); + + private ScriptDoubleFieldData(String fieldName, Script script, DoubleScriptFieldScript.Factory scriptFactory) { + this.fieldName = fieldName; + this.script = script; + this.scriptFactory = scriptFactory; + } + + @Override + public void setSearchLookup(SearchLookup searchLookup) { + this.leafFactory.set(scriptFactory.newFactory(script.getParams(), searchLookup)); + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.NUMERIC; + } + + @Override + public ScriptDoubleLeafFieldData load(LeafReaderContext context) { + try { + return loadDirect(context); + } catch (Exception e) { + throw ExceptionsHelper.convertToElastic(e); + } + } + + @Override + public ScriptDoubleLeafFieldData loadDirect(LeafReaderContext context) throws IOException { + return new ScriptDoubleLeafFieldData(new ScriptDoubleDocValues(leafFactory.get().newInstance(context))); + } + + @Override + public NumericType getNumericType() { + return NumericType.DOUBLE; + } + + @Override + protected boolean sortRequiresCustomComparator() { + return true; + } + + @Override + public void clear() {} + + public static class ScriptDoubleLeafFieldData extends LeafDoubleFieldData { + private final ScriptDoubleDocValues scriptDoubleDocValues; + + ScriptDoubleLeafFieldData(ScriptDoubleDocValues scriptDoubleDocValues) { + super(0); + this.scriptDoubleDocValues = scriptDoubleDocValues; + } + + @Override + public SortedNumericDoubleValues getDoubleValues() { + return scriptDoubleDocValues; + } + + @Override + public void close() {} + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldType.java new file mode 100644 index 0000000000000..53ae6c0739e67 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldType.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.mapper; + +import com.carrotsearch.hppc.LongHashSet; +import com.carrotsearch.hppc.LongSet; + +import org.apache.lucene.search.Query; +import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.common.time.DateMathParser; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.script.Script; +import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.fielddata.ScriptDoubleFieldData; +import org.elasticsearch.xpack.runtimefields.query.DoubleScriptFieldExistsQuery; +import org.elasticsearch.xpack.runtimefields.query.DoubleScriptFieldRangeQuery; +import org.elasticsearch.xpack.runtimefields.query.DoubleScriptFieldTermQuery; +import org.elasticsearch.xpack.runtimefields.query.DoubleScriptFieldTermsQuery; + +import java.time.ZoneId; +import java.util.List; +import java.util.Map; + +public class ScriptDoubleMappedFieldType extends AbstractScriptMappedFieldType { + private final DoubleScriptFieldScript.Factory scriptFactory; + + ScriptDoubleMappedFieldType(String name, Script script, DoubleScriptFieldScript.Factory scriptFactory, Map meta) { + super(name, script, meta); + this.scriptFactory = scriptFactory; + } + + @Override + protected String runtimeType() { + return NumberType.DOUBLE.typeName(); + } + + @Override + public Object valueForDisplay(Object value) { + return value; // These should come back as a Double + } + + @Override + public ScriptDoubleFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + // TODO once we get SearchLookup as an argument, we can already call scriptFactory.newFactory here and pass through the result + return new ScriptDoubleFieldData.Builder(script, scriptFactory); + } + + private DoubleScriptFieldScript.LeafFactory leafFactory(QueryShardContext context) { + return scriptFactory.newFactory(script.getParams(), context.lookup()); + } + + @Override + public Query existsQuery(QueryShardContext context) { + checkAllowExpensiveQueries(context); + return new DoubleScriptFieldExistsQuery(script, leafFactory(context), name()); + } + + @Override + public Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + ShapeRelation relation, + ZoneId timeZone, + DateMathParser parser, + QueryShardContext context + ) { + checkAllowExpensiveQueries(context); + return NumberType.doubleRangeQuery( + lowerTerm, + upperTerm, + includeLower, + includeUpper, + (l, u) -> new DoubleScriptFieldRangeQuery(script, leafFactory(context), name(), l, u) + ); + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + checkAllowExpensiveQueries(context); + return new DoubleScriptFieldTermQuery(script, leafFactory(context), name(), NumberType.objectToDouble(value)); + } + + @Override + public Query termsQuery(List values, QueryShardContext context) { + if (values.isEmpty()) { + return Queries.newMatchAllQuery(); + } + LongSet terms = new LongHashSet(values.size()); + for (Object value : values) { + terms.add(Double.doubleToLongBits(NumberType.objectToDouble(value))); + } + checkAllowExpensiveQueries(context); + return new DoubleScriptFieldTermsQuery(script, leafFactory(context), name(), terms); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java index cc81089d6e14d..b5ad804292c6c 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java @@ -18,6 +18,7 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptType; +import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; @@ -74,6 +75,19 @@ protected String contentType() { public static class Builder extends ParametrizedFieldMapper.Builder { static final Map> FIELD_TYPE_RESOLVER = Map.of( + NumberType.DOUBLE.typeName(), + (builder, context) -> { + DoubleScriptFieldScript.Factory factory = builder.scriptCompiler.compile( + builder.script.getValue(), + DoubleScriptFieldScript.CONTEXT + ); + return new ScriptDoubleMappedFieldType( + builder.buildFullName(context), + builder.script.getValue(), + factory, + builder.meta.getValue() + ); + }, KeywordFieldMapper.CONTENT_TYPE, (builder, context) -> { StringScriptFieldScript.Factory factory = builder.scriptCompiler.compile( diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractDoubleScriptFieldQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractDoubleScriptFieldQuery.java new file mode 100644 index 0000000000000..5c4841085e26f --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractDoubleScriptFieldQuery.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.elasticsearch.script.Script; +import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; + +import java.io.IOException; +import java.util.Objects; + +/** + * Abstract base class for building queries based on {@link DoubleScriptFieldScript}. + */ +abstract class AbstractDoubleScriptFieldQuery extends AbstractScriptFieldQuery { + private final DoubleScriptFieldScript.LeafFactory leafFactory; + + AbstractDoubleScriptFieldQuery(Script script, DoubleScriptFieldScript.LeafFactory leafFactory, String fieldName) { + super(script, fieldName); + this.leafFactory = Objects.requireNonNull(leafFactory); + } + + /** + * Does the value match this query? + */ + protected abstract boolean matches(double[] values, int count); + + @Override + public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return false; // scripts aren't really cacheable at this point + } + + @Override + public Scorer scorer(LeafReaderContext ctx) throws IOException { + DoubleScriptFieldScript script = leafFactory.newInstance(ctx); + DocIdSetIterator approximation = DocIdSetIterator.all(ctx.reader().maxDoc()); + TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { + @Override + public boolean matches() throws IOException { + script.runForDoc(approximation().docID()); + return AbstractDoubleScriptFieldQuery.this.matches(script.values(), script.count()); + } + + @Override + public float matchCost() { + return MATCH_COST; + } + }; + return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); + } + }; + } + + @Override + public final void visit(QueryVisitor visitor) { + // No subclasses contain any Terms because those have to be strings. + if (visitor.acceptField(fieldName())) { + visitor.visitLeaf(this); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractLongScriptFieldQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractLongScriptFieldQuery.java index 7ebdb1047374d..89ffd5452fd48 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractLongScriptFieldQuery.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractLongScriptFieldQuery.java @@ -18,13 +18,12 @@ import org.apache.lucene.search.Weight; import org.elasticsearch.script.Script; import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; -import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; import java.io.IOException; import java.util.Objects; /** - * Abstract base class for building queries based on {@link StringScriptFieldScript}. + * Abstract base class for building queries based on {@link LongScriptFieldScript}. */ abstract class AbstractLongScriptFieldQuery extends AbstractScriptFieldQuery { private final LongScriptFieldScript.LeafFactory leafFactory; diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldExistsQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldExistsQuery.java new file mode 100644 index 0000000000000..620b6569457d1 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldExistsQuery.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.elasticsearch.script.Script; +import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; + +public class DoubleScriptFieldExistsQuery extends AbstractDoubleScriptFieldQuery { + public DoubleScriptFieldExistsQuery(Script script, DoubleScriptFieldScript.LeafFactory leafFactory, String fieldName) { + super(script, leafFactory, fieldName); + } + + @Override + protected boolean matches(double[] values, int count) { + return count > 0; + } + + @Override + public final String toString(String field) { + if (fieldName().contentEquals(field)) { + return getClass().getSimpleName(); + } + return fieldName() + ":" + getClass().getSimpleName(); + } + + // Superclass's equals and hashCode are great for this class +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldRangeQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldRangeQuery.java new file mode 100644 index 0000000000000..8bb407d3902a6 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldRangeQuery.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.elasticsearch.script.Script; +import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; + +import java.util.Objects; + +public class DoubleScriptFieldRangeQuery extends AbstractDoubleScriptFieldQuery { + private final double lowerValue; + private final double upperValue; + + public DoubleScriptFieldRangeQuery( + Script script, + DoubleScriptFieldScript.LeafFactory leafFactory, + String fieldName, + double lowerValue, + double upperValue + ) { + super(script, leafFactory, fieldName); + this.lowerValue = lowerValue; + this.upperValue = upperValue; + assert lowerValue <= upperValue; + } + + @Override + protected boolean matches(double[] values, int count) { + for (int i = 0; i < count; i++) { + if (lowerValue <= values[i] && values[i] <= upperValue) { + return true; + } + } + return false; + } + + @Override + public final String toString(String field) { + StringBuilder b = new StringBuilder(); + if (false == fieldName().contentEquals(field)) { + b.append(fieldName()).append(':'); + } + b.append('[').append(lowerValue).append(" TO ").append(upperValue).append(']'); + return b.toString(); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), lowerValue, upperValue); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + DoubleScriptFieldRangeQuery other = (DoubleScriptFieldRangeQuery) obj; + return lowerValue == other.lowerValue && upperValue == other.upperValue; + } + + double lowerValue() { + return lowerValue; + } + + double upperValue() { + return upperValue; + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermQuery.java new file mode 100644 index 0000000000000..2949aee788885 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermQuery.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.elasticsearch.script.Script; +import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; + +import java.util.Objects; + +public class DoubleScriptFieldTermQuery extends AbstractDoubleScriptFieldQuery { + private final double term; + + public DoubleScriptFieldTermQuery(Script script, DoubleScriptFieldScript.LeafFactory leafFactory, String fieldName, double term) { + super(script, leafFactory, fieldName); + this.term = term; + } + + @Override + protected boolean matches(double[] values, int count) { + for (int i = 0; i < count; i++) { + if (term == values[i]) { + return true; + } + } + return false; + } + + @Override + public final String toString(String field) { + if (fieldName().contentEquals(field)) { + return Double.toString(term); + } + return fieldName() + ":" + term; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), term); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + DoubleScriptFieldTermQuery other = (DoubleScriptFieldTermQuery) obj; + return term == other.term; + } + + double term() { + return term; + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermsQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermsQuery.java new file mode 100644 index 0000000000000..990430807e783 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermsQuery.java @@ -0,0 +1,73 @@ +/* +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import com.carrotsearch.hppc.LongSet; +import com.carrotsearch.hppc.cursors.LongCursor; + +import org.elasticsearch.script.Script; +import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; + +import java.util.Arrays; +import java.util.Objects; + +public class DoubleScriptFieldTermsQuery extends AbstractDoubleScriptFieldQuery { + private final LongSet terms; + + /** + * Build the query. + * @param terms The terms converted to a long with {@link Double#doubleToLongBits(double)}. + */ + public DoubleScriptFieldTermsQuery(Script script, DoubleScriptFieldScript.LeafFactory leafFactory, String fieldName, LongSet terms) { + super(script, leafFactory, fieldName); + this.terms = terms; + } + + @Override + protected boolean matches(double[] values, int count) { + for (int i = 0; i < count; i++) { + if (terms.contains(Double.doubleToLongBits(values[i]))) { + return true; + } + } + return false; + } + + @Override + public final String toString(String field) { + double[] termsArray = terms(); + Arrays.sort(termsArray); + if (fieldName().equals(field)) { + return Arrays.toString(termsArray); + } + return fieldName() + ":" + Arrays.toString(termsArray); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), terms); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + DoubleScriptFieldTermsQuery other = (DoubleScriptFieldTermsQuery) obj; + return terms.equals(other.terms); + } + + double[] terms() { + double[] result = new double[terms.size()]; + int i = 0; + for (LongCursor lc : terms) { + result[i++] = Double.longBitsToDouble(lc.value); + } + return result; + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java index 88d1232531e56..0f2e2d81e7f0c 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java @@ -115,13 +115,14 @@ protected DoubleScriptFieldScript.LeafFactory newLeafFactory( @Override protected IntFunction> newInstance(DoubleScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context) throws IOException { - List results = new ArrayList<>(); - DoubleScriptFieldScript script = leafFactory.newInstance(context, results::add); + DoubleScriptFieldScript script = leafFactory.newInstance(context); return docId -> { - results.clear(); - script.setDocument(docId); - script.execute(); - return results; + script.runForDoc(docId); + List list = new ArrayList<>(script.count()); + for (int i = 0; i < script.count(); i++) { + list.add(script.values()[i]); + } + return list; }; } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldTypeTests.java new file mode 100644 index 0000000000000..39e034a721aa8 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldTypeTests.java @@ -0,0 +1,225 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.mapper; + +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.painless.PainlessPlugin; +import org.elasticsearch.plugins.ExtensiblePlugin.ExtensionLoader; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.RuntimeFields; +import org.elasticsearch.xpack.runtimefields.RuntimeFieldsPainlessExtension; +import org.elasticsearch.xpack.runtimefields.fielddata.ScriptDoubleFieldData; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; + +import static java.util.Collections.emptyMap; +import static org.hamcrest.Matchers.equalTo; + +public class ScriptDoubleMappedFieldTypeTests extends AbstractScriptMappedFieldTypeTestCase { + public void testDocValues() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [1.0]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [3.14, 1.4]}")))); + List results = new ArrayList<>(); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + ScriptDoubleMappedFieldType ft = build("for (def v : source.foo) {value(v + params.param)}", Map.of("param", 1)); + IndexMetadata imd = IndexMetadata.builder("test") + .settings(Settings.builder().put("index.version.created", Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + ScriptDoubleFieldData ifd = ft.fielddataBuilder("test").build(new IndexSettings(imd, Settings.EMPTY), ft, null, null, null); + ifd.setSearchLookup(mockContext().lookup()); + searcher.search(new MatchAllDocsQuery(), new Collector() { + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + SortedNumericDoubleValues dv = ifd.load(context).getDoubleValues(); + return new LeafCollector() { + @Override + public void setScorer(Scorable scorer) throws IOException {} + + @Override + public void collect(int doc) throws IOException { + if (dv.advanceExact(doc)) { + for (int i = 0; i < dv.docValueCount(); i++) { + results.add(dv.nextValue()); + } + } + } + }; + } + }); + assertThat(results, equalTo(List.of(2.0, 2.4, 4.140000000000001))); + } + } + } + + public void testExistsQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [1]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": []}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + assertThat(searcher.count(build("for (def v : source.foo) {value(v)}").existsQuery(mockContext())), equalTo(1)); + } + } + } + + public void testExistsQueryIsExpensive() throws IOException { + checkExpensiveQuery(ScriptDoubleMappedFieldType::existsQuery); + } + + public void testRangeQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 2}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 2.5}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + assertThat( + searcher.count(build("value(source.foo)").rangeQuery("2", "3", true, true, null, null, null, mockContext())), + equalTo(2) + ); + assertThat( + searcher.count(build("value(source.foo)").rangeQuery(2, 3, true, true, null, null, null, mockContext())), + equalTo(2) + ); + assertThat( + searcher.count(build("value(source.foo)").rangeQuery(1.1, 3, true, true, null, null, null, mockContext())), + equalTo(2) + ); + assertThat( + searcher.count(build("value(source.foo)").rangeQuery(1.1, 3, false, true, null, null, null, mockContext())), + equalTo(2) + ); + assertThat( + searcher.count(build("value(source.foo)").rangeQuery(2, 3, false, true, null, null, null, mockContext())), + equalTo(1) + ); + assertThat( + searcher.count(build("value(source.foo)").rangeQuery(2.5, 3, true, true, null, null, null, mockContext())), + equalTo(1) + ); + assertThat( + searcher.count(build("value(source.foo)").rangeQuery(2.5, 3, false, true, null, null, null, mockContext())), + equalTo(0) + ); + } + } + } + + public void testRangeQueryIsExpensive() throws IOException { + checkExpensiveQuery( + (ft, ctx) -> ft.rangeQuery(randomLong(), randomLong(), randomBoolean(), randomBoolean(), null, null, null, ctx) + ); + } + + public void testTermQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 2}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + assertThat(searcher.count(build("value(source.foo)").termQuery("1", mockContext())), equalTo(1)); + assertThat(searcher.count(build("value(source.foo)").termQuery(1, mockContext())), equalTo(1)); + assertThat(searcher.count(build("value(source.foo)").termQuery(1.1, mockContext())), equalTo(0)); + assertThat( + searcher.count(build("value(source.foo + params.param)", Map.of("param", 1)).termQuery(2, mockContext())), + equalTo(1) + ); + } + } + } + + public void testTermQueryIsExpensive() throws IOException { + checkExpensiveQuery((ft, ctx) -> ft.termQuery(randomLong(), ctx)); + } + + public void testTermsQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 2.1}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + assertThat(searcher.count(build("value(source.foo)").termsQuery(List.of("1"), mockContext())), equalTo(1)); + assertThat(searcher.count(build("value(source.foo)").termsQuery(List.of(1), mockContext())), equalTo(1)); + assertThat(searcher.count(build("value(source.foo)").termsQuery(List.of(1.1), mockContext())), equalTo(0)); + assertThat(searcher.count(build("value(source.foo)").termsQuery(List.of(1.1, 2.1), mockContext())), equalTo(1)); + assertThat(searcher.count(build("value(source.foo)").termsQuery(List.of(2.1, 1), mockContext())), equalTo(2)); + } + } + } + + public void testTermsQueryIsExpensive() throws IOException { + checkExpensiveQuery((ft, ctx) -> ft.termsQuery(List.of(randomLong()), ctx)); + } + + private ScriptDoubleMappedFieldType build(String code) throws IOException { + return build(new Script(code)); + } + + private ScriptDoubleMappedFieldType build(String code, Map params) throws IOException { + return build(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, code, params)); + } + + private ScriptDoubleMappedFieldType build(Script script) throws IOException { + PainlessPlugin painlessPlugin = new PainlessPlugin(); + painlessPlugin.loadExtensions(new ExtensionLoader() { + @Override + @SuppressWarnings("unchecked") // We only ever load painless extensions here so it is fairly safe. + public List loadExtensions(Class extensionPointType) { + return (List) List.of(new RuntimeFieldsPainlessExtension()); + } + }); + ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, List.of(painlessPlugin, new RuntimeFields())); + try (ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts)) { + DoubleScriptFieldScript.Factory factory = scriptService.compile(script, DoubleScriptFieldScript.CONTEXT); + return new ScriptDoubleMappedFieldType("test", script, factory, emptyMap()); + } + } + + private void checkExpensiveQuery(BiConsumer queryBuilder) throws IOException { + ScriptDoubleMappedFieldType ft = build("value(1)"); + Exception e = expectThrows(ElasticsearchException.class, () -> queryBuilder.accept(ft, mockContext(false))); + assertThat( + e.getMessage(), + equalTo("queries cannot be executed against [script] fields while [search.allow_expensive_queries] is set to [false].") + ); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapperTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapperTests.java index e9c0faaca80b8..c53536d890009 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapperTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapperTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; +import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; import org.elasticsearch.xpack.runtimefields.RuntimeFields; import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; @@ -114,6 +115,13 @@ public void testUnsupportedRuntimeType() throws Exception { assertEquals("Failed to parse mapping: runtime_type [unsupported] not supported", exc.getMessage()); } + public void testDouble() throws IOException { + MapperService mapperService = createIndex("test", Settings.EMPTY, mapping("double")).mapperService(); + FieldMapper mapper = (FieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + assertThat(mapper, instanceOf(ScriptFieldMapper.class)); + assertEquals(Strings.toString(mapping("double")), Strings.toString(mapperService.documentMapper())); + } + public void testKeyword() throws IOException { MapperService mapperService = createIndex("test", Settings.EMPTY, mapping("keyword")).mapperService(); FieldMapper mapper = (FieldMapper) mapperService.documentMapper().mappers().getMapper("field"); @@ -210,6 +218,18 @@ public FactoryType compile( } private Object dummyScriptFactory(ScriptContext context) { + if (context == DoubleScriptFieldScript.CONTEXT) { + return (DoubleScriptFieldScript.Factory) (params, lookup) -> ctx -> new DoubleScriptFieldScript( + params, + lookup, + ctx + ) { + @Override + public void execute() { + new DoubleScriptFieldScript.Value(this).value(1.0); + } + }; + } if (context == StringScriptFieldScript.CONTEXT) { return (StringScriptFieldScript.Factory) (params, lookup) -> ctx -> new StringScriptFieldScript( params, diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractDoubleScriptFieldQueryTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractDoubleScriptFieldQueryTestCase.java new file mode 100644 index 0000000000000..59bd856c3cfd6 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractDoubleScriptFieldQueryTestCase.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.util.automaton.ByteRunAutomaton; +import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; + +public abstract class AbstractDoubleScriptFieldQueryTestCase extends + AbstractScriptFieldQueryTestCase { + + protected final DoubleScriptFieldScript.LeafFactory leafFactory = mock(DoubleScriptFieldScript.LeafFactory.class); + + @Override + public final void testVisit() { + T query = createTestInstance(); + List leavesVisited = new ArrayList<>(); + query.visit(new QueryVisitor() { + @Override + public void consumeTerms(Query query, Term... terms) { + fail(); + } + + @Override + public void consumeTermsMatching(Query query, String field, Supplier automaton) { + fail(); + } + + @Override + public void visitLeaf(Query query) { + leavesVisited.add(query); + } + }); + assertThat(leavesVisited, equalTo(List.of(query))); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldExistsQueryTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldExistsQueryTests.java new file mode 100644 index 0000000000000..f8889aa15ab5e --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldExistsQueryTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import static org.hamcrest.Matchers.equalTo; + +public class DoubleScriptFieldExistsQueryTests extends AbstractDoubleScriptFieldQueryTestCase { + @Override + protected DoubleScriptFieldExistsQuery createTestInstance() { + return new DoubleScriptFieldExistsQuery(randomScript(), leafFactory, randomAlphaOfLength(5)); + } + + @Override + protected DoubleScriptFieldExistsQuery copy(DoubleScriptFieldExistsQuery orig) { + return new DoubleScriptFieldExistsQuery(orig.script(), leafFactory, orig.fieldName()); + } + + @Override + protected DoubleScriptFieldExistsQuery mutate(DoubleScriptFieldExistsQuery orig) { + if (randomBoolean()) { + new DoubleScriptFieldExistsQuery(randomValueOtherThan(orig.script(), this::randomScript), leafFactory, orig.fieldName()); + } + return new DoubleScriptFieldExistsQuery(orig.script(), leafFactory, orig.fieldName() + "modified"); + } + + @Override + public void testMatches() { + assertTrue(createTestInstance().matches(new double[0], randomIntBetween(1, Integer.MAX_VALUE))); + assertFalse(createTestInstance().matches(new double[0], 0)); + assertFalse(createTestInstance().matches(new double[] { 1, 2, 3 }, 0)); + } + + @Override + protected void assertToString(DoubleScriptFieldExistsQuery query) { + assertThat(query.toString(query.fieldName()), equalTo("DoubleScriptFieldExistsQuery")); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermQueryTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermQueryTests.java new file mode 100644 index 0000000000000..2fe5038bce413 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermQueryTests.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.elasticsearch.script.Script; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.equalTo; + +public class DoubleScriptFieldTermQueryTests extends AbstractDoubleScriptFieldQueryTestCase { + @Override + protected DoubleScriptFieldTermQuery createTestInstance() { + return new DoubleScriptFieldTermQuery(randomScript(), leafFactory, randomAlphaOfLength(5), randomDouble()); + } + + @Override + protected DoubleScriptFieldTermQuery copy(DoubleScriptFieldTermQuery orig) { + return new DoubleScriptFieldTermQuery(orig.script(), leafFactory, orig.fieldName(), orig.term()); + } + + @Override + protected DoubleScriptFieldTermQuery mutate(DoubleScriptFieldTermQuery orig) { + Script script = orig.script(); + String fieldName = orig.fieldName(); + double term = orig.term(); + switch (randomInt(2)) { + case 0: + script = randomValueOtherThan(script, this::randomScript); + break; + case 1: + fieldName += "modified"; + break; + case 2: + term = randomValueOtherThan(term, ESTestCase::randomDouble); + break; + default: + fail(); + } + return new DoubleScriptFieldTermQuery(script, leafFactory, fieldName, term); + } + + @Override + public void testMatches() { + DoubleScriptFieldTermQuery query = new DoubleScriptFieldTermQuery(randomScript(), leafFactory, "test", 3.14); + assertTrue(query.matches(new double[] { 3.14 }, 1)); // Match because value matches + assertFalse(query.matches(new double[] { 2 }, 1)); // No match because wrong value + assertFalse(query.matches(new double[] { 2, 3.14 }, 1)); // No match because value after count of values + assertTrue(query.matches(new double[] { 2, 3.14 }, 2)); // Match because one value matches + } + + @Override + protected void assertToString(DoubleScriptFieldTermQuery query) { + assertThat(query.toString(query.fieldName()), equalTo(Double.toString(query.term()))); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermsQueryTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermsQueryTests.java new file mode 100644 index 0000000000000..d2d3a2f427e37 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermsQueryTests.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import com.carrotsearch.hppc.LongHashSet; +import com.carrotsearch.hppc.LongSet; + +import org.elasticsearch.script.Script; + +import java.util.Arrays; +import java.util.List; + +import static java.util.stream.Collectors.toList; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.startsWith; + +public class DoubleScriptFieldTermsQueryTests extends AbstractDoubleScriptFieldQueryTestCase { + @Override + protected DoubleScriptFieldTermsQuery createTestInstance() { + LongSet terms = new LongHashSet(); + int count = between(1, 100); + while (terms.size() < count) { + terms.add(Double.doubleToLongBits(randomDouble())); + } + return new DoubleScriptFieldTermsQuery(randomScript(), leafFactory, randomAlphaOfLength(5), terms); + } + + @Override + protected DoubleScriptFieldTermsQuery copy(DoubleScriptFieldTermsQuery orig) { + LongSet terms = new LongHashSet(); + for (double term : orig.terms()) { + terms.add(Double.doubleToLongBits(term)); + } + return new DoubleScriptFieldTermsQuery(orig.script(), leafFactory, orig.fieldName(), terms); + } + + @Override + protected DoubleScriptFieldTermsQuery mutate(DoubleScriptFieldTermsQuery orig) { + Script script = orig.script(); + String fieldName = orig.fieldName(); + LongSet terms = new LongHashSet(); + for (double term : orig.terms()) { + terms.add(Double.doubleToLongBits(term)); + } + switch (randomInt(2)) { + case 0: + script = randomValueOtherThan(script, this::randomScript); + break; + case 1: + fieldName += "modified"; + break; + case 2: + terms = new LongHashSet(terms); + while (false == terms.add(Double.doubleToLongBits(randomDouble()))) { + // Random double was already in the set + } + break; + default: + fail(); + } + return new DoubleScriptFieldTermsQuery(script, leafFactory, fieldName, terms); + } + + @Override + public void testMatches() { + DoubleScriptFieldTermsQuery query = new DoubleScriptFieldTermsQuery( + randomScript(), + leafFactory, + "test", + LongHashSet.from(Double.doubleToLongBits(0.1), Double.doubleToLongBits(0.2), Double.doubleToLongBits(7.5)) + ); + assertTrue(query.matches(new double[] { 0.1 }, 1)); + assertTrue(query.matches(new double[] { 0.2 }, 1)); + assertTrue(query.matches(new double[] { 7.5 }, 1)); + assertTrue(query.matches(new double[] { 0.1, 0 }, 2)); + assertTrue(query.matches(new double[] { 0, 0.1 }, 2)); + assertFalse(query.matches(new double[] { 0 }, 1)); + assertFalse(query.matches(new double[] { 0, 0.1 }, 1)); + } + + @Override + protected void assertToString(DoubleScriptFieldTermsQuery query) { + String toString = query.toString(query.fieldName()); + assertThat(toString, startsWith("[")); + assertThat(toString, endsWith("]")); + List list = Arrays.asList(toString.substring(1, toString.length() - 1).split(",")) + .stream() + .map(Double::parseDouble) + .collect(toList()); + + // Assert that the list is sorted + for (int i = 1; i < list.size(); i++) { + assertThat(list.get(i), greaterThan(list.get(i - 1))); + } + + // Assert that all terms are in the list + for (double term : query.terms()) { + assertThat(list, hasItem(term)); + } + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/30_double.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/30_double.yml new file mode 100644 index 0000000000000..7d864f2b35bf4 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/30_double.yml @@ -0,0 +1,203 @@ +--- +setup: + - do: + indices.create: + index: sensor + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + timestamp: + type: date + temperature: + type: long + voltage: + type: double + node: + type: keyword + voltage_percent: + type: script + runtime_type: double + script: + source: | + for (double v : doc['voltage']) { + value(v / params.max); + } + params: + max: 5.8 + top_voltage: + type: script + runtime_type: double + script: + source: | + for (double vp: doc['voltage_percent']) { + if (vp >= params.cutoff) { + value(vp); + } + } + params: + cutoff: 0.9 + + - do: + bulk: + index: sensor + refresh: true + body: | + {"index":{}} + {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} + {"index":{}} + {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} + {"index":{}} + {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} + {"index":{}} + {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} + {"index":{}} + {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} + {"index":{}} + {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} + +--- +"get mapping": + - do: + indices.get_mapping: + index: sensor + - match: {sensor.mappings.properties.voltage_percent.type: script } + - match: {sensor.mappings.properties.voltage_percent.runtime_type: double } + - match: + sensor.mappings.properties.voltage_percent.script.source: | + for (double v : doc['voltage']) { + value(v / params.max); + } + - match: {sensor.mappings.properties.voltage_percent.script.params: {max: 5.8} } + - match: {sensor.mappings.properties.voltage_percent.script.lang: painless } + - match: {sensor.mappings.properties.top_voltage.type: script } + - match: {sensor.mappings.properties.top_voltage.runtime_type: double } + - match: + sensor.mappings.properties.top_voltage.script.source: | + for (double vp: doc['voltage_percent']) { + if (vp >= params.cutoff) { + value(vp); + } + } + - match: {sensor.mappings.properties.top_voltage.script.params: {cutoff: 0.9} } + - match: {sensor.mappings.properties.top_voltage.script.lang: painless } + +--- +"docvalue_fields": + - do: + search: + index: sensor + body: + sort: timestamp + docvalue_fields: [voltage_percent] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.voltage_percent: [0.6896551724137931] } + - match: {hits.hits.1.fields.voltage_percent: [0.7241379310344828] } + - match: {hits.hits.2.fields.voltage_percent: [0.9655172413793103] } + - match: {hits.hits.3.fields.voltage_percent: [0.8793103448275862] } + - match: {hits.hits.4.fields.voltage_percent: [1.0] } + - match: {hits.hits.5.fields.voltage_percent: [0.896551724137931] } + +--- +"terms agg": + - do: + search: + index: sensor + body: + aggs: + v10: + terms: + field: voltage_percent + - match: {hits.total.value: 6} + - match: {aggregations.v10.buckets.0.key: 0.6896551724137931} + - match: {aggregations.v10.buckets.0.doc_count: 1} + - match: {aggregations.v10.buckets.1.key: 0.7241379310344828} + - match: {aggregations.v10.buckets.1.doc_count: 1} + +--- +"exists query": + - do: + search: + index: sensor + body: + query: + exists: + field: top_voltage + sort: timestamp + - match: {hits.total.value: 2} + - match: {hits.hits.0._source.voltage: 5.6} + - match: {hits.hits.1._source.voltage: 5.8} + +--- +"range query": + - do: + search: + index: sensor + body: + query: + range: + voltage_percent: + lt: .7 + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 4.0} + + - do: + search: + index: sensor + body: + query: + range: + voltage_percent: + gt: 1 + - match: {hits.total.value: 0} + + - do: + search: + index: sensor + body: + query: + range: + voltage_percent: + gte: 1 + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} + + - do: + search: + index: sensor + body: + query: + range: + voltage_percent: + gte: .7 + lte: .8 + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 4.2} + +--- +"term query": + - do: + search: + index: sensor + body: + query: + term: + voltage_percent: 1.0 + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} + +--- +"terms query": + - do: + search: + index: sensor + body: + query: + terms: + voltage_percent: [0.6896551724137931, 1] + sort: timestamp + - match: {hits.total.value: 2} + - match: {hits.hits.0._source.voltage: 4.0} + - match: {hits.hits.1._source.voltage: 5.8} From 491728b66d8751a641b96b601a85ebdcd63c5922 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 21 Jul 2020 13:05:16 -0400 Subject: [PATCH 2/5] Add tests for error from unimplemented queries --- ...tNonTextScriptMappedFieldTypeTestCase.java | 52 +++++++++++++++++++ ...AbstractScriptMappedFieldTypeTestCase.java | 25 +++++++-- .../ScriptDoubleMappedFieldTypeTests.java | 22 ++++++-- .../ScriptKeywordMappedFieldTypeTests.java | 17 ++++-- .../ScriptLongMappedFieldTypeTests.java | 24 +++++++-- 5 files changed, 123 insertions(+), 17 deletions(-) create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractNonTextScriptMappedFieldTypeTestCase.java diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractNonTextScriptMappedFieldTypeTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractNonTextScriptMappedFieldTypeTestCase.java new file mode 100644 index 0000000000000..66f7a5d811c31 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractNonTextScriptMappedFieldTypeTestCase.java @@ -0,0 +1,52 @@ +package org.elasticsearch.xpack.runtimefields.mapper; + +import org.apache.lucene.util.automaton.Operations; +import org.elasticsearch.common.unit.Fuzziness; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +abstract class AbstractNonTextScriptMappedFieldTypeTestCase extends AbstractScriptMappedFieldTypeTestCase { + protected abstract AbstractScriptMappedFieldType simpleMappedFieldType() throws IOException; + + public void testFuzzyQueryIsError() throws IOException { + assertQueryOnlyOnTextAndKeyword( + "fuzzy", + () -> simpleMappedFieldType().fuzzyQuery("cat", Fuzziness.AUTO, 0, 1, true, mockContext()) + ); + } + + public void testPrefixQueryIsError() throws IOException { + assertQueryOnlyOnTextKeywordAndWildcard("prefix", () -> simpleMappedFieldType().prefixQuery("cat", null, mockContext())); + } + + public void testRegexpQueryIsError() throws IOException { + assertQueryOnlyOnTextAndKeyword( + "regexp", + () -> simpleMappedFieldType().regexpQuery("cat", 0, Operations.DEFAULT_MAX_DETERMINIZED_STATES, null, mockContext()) + ); + } + + public void testWildcardQueryIsError() throws IOException { + assertQueryOnlyOnTextKeywordAndWildcard("wildcard", () -> simpleMappedFieldType().wildcardQuery("cat", null, mockContext())); + } + + private void assertQueryOnlyOnTextAndKeyword(String queryName, ThrowingRunnable buildQuery) { + Exception e = expectThrows(Exception.class, buildQuery); + assertThat( + e.getMessage(), + equalTo("Can only use " + queryName + " queries on keyword and text fields - not on [test] which is of type [script]") + ); + } + + private void assertQueryOnlyOnTextKeywordAndWildcard(String queryName, ThrowingRunnable buildQuery) { + Exception e = expectThrows(Exception.class, buildQuery); + assertThat( + e.getMessage(), + equalTo( + "Can only use " + queryName + " queries on keyword, text and wildcard fields - not on [test] which is of type [script]" + ) + ); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldTypeTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldTypeTestCase.java index 1b7845baa6c1f..518b30af46c93 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldTypeTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldTypeTestCase.java @@ -11,20 +11,39 @@ import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; + import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; abstract class AbstractScriptMappedFieldTypeTestCase extends ESTestCase { - protected QueryShardContext mockContext() { + public abstract void testDocValues() throws IOException; + + public abstract void testExistsQuery() throws IOException; + + public abstract void testExistsQueryIsExpensive() throws IOException; + + public abstract void testRangeQuery() throws IOException; + + public abstract void testRangeQueryIsExpensive() throws IOException; + + public abstract void testTermQuery() throws IOException; + + public abstract void testTermQueryIsExpensive() throws IOException; + + public abstract void testTermsQuery() throws IOException; + + public abstract void testTermsQueryIsExpensive() throws IOException; + + protected static QueryShardContext mockContext() { return mockContext(true); } - protected QueryShardContext mockContext(boolean allowExpensiveQueries) { + protected static QueryShardContext mockContext(boolean allowExpensiveQueries) { MapperService mapperService = mock(MapperService.class); QueryShardContext context = mock(QueryShardContext.class); when(context.allowExpensiveQueries()).thenReturn(allowExpensiveQueries); when(context.lookup()).thenReturn(new SearchLookup(mapperService, mft -> null)); return context; } - } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldTypeTests.java index 39e034a721aa8..63485d13325b9 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldTypeTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldTypeTests.java @@ -45,7 +45,8 @@ import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.equalTo; -public class ScriptDoubleMappedFieldTypeTests extends AbstractScriptMappedFieldTypeTestCase { +public class ScriptDoubleMappedFieldTypeTests extends AbstractNonTextScriptMappedFieldTypeTestCase { + @Override public void testDocValues() throws IOException { try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [1.0]}")))); @@ -90,6 +91,7 @@ public void collect(int doc) throws IOException { } } + @Override public void testExistsQuery() throws IOException { try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [1]}")))); @@ -101,6 +103,7 @@ public void testExistsQuery() throws IOException { } } + @Override public void testExistsQueryIsExpensive() throws IOException { checkExpensiveQuery(ScriptDoubleMappedFieldType::existsQuery); } @@ -150,6 +153,7 @@ public void testRangeQueryIsExpensive() throws IOException { ); } + @Override public void testTermQuery() throws IOException { try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); @@ -167,10 +171,12 @@ public void testTermQuery() throws IOException { } } + @Override public void testTermQueryIsExpensive() throws IOException { checkExpensiveQuery((ft, ctx) -> ft.termQuery(randomLong(), ctx)); } + @Override public void testTermsQuery() throws IOException { try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); @@ -186,19 +192,25 @@ public void testTermsQuery() throws IOException { } } + @Override public void testTermsQueryIsExpensive() throws IOException { checkExpensiveQuery((ft, ctx) -> ft.termsQuery(List.of(randomLong()), ctx)); } - private ScriptDoubleMappedFieldType build(String code) throws IOException { + @Override + protected ScriptDoubleMappedFieldType simpleMappedFieldType() throws IOException { + return build("value(source.foo)"); + } + + private static ScriptDoubleMappedFieldType build(String code) throws IOException { return build(new Script(code)); } - private ScriptDoubleMappedFieldType build(String code, Map params) throws IOException { + private static ScriptDoubleMappedFieldType build(String code, Map params) throws IOException { return build(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, code, params)); } - private ScriptDoubleMappedFieldType build(Script script) throws IOException { + private static ScriptDoubleMappedFieldType build(Script script) throws IOException { PainlessPlugin painlessPlugin = new PainlessPlugin(); painlessPlugin.loadExtensions(new ExtensionLoader() { @Override @@ -214,7 +226,7 @@ public List loadExtensions(Class extensionPointType) { } } - private void checkExpensiveQuery(BiConsumer queryBuilder) throws IOException { + private static void checkExpensiveQuery(BiConsumer queryBuilder) throws IOException { ScriptDoubleMappedFieldType ft = build("value(1)"); Exception e = expectThrows(ElasticsearchException.class, () -> queryBuilder.accept(ft, mockContext(false))); assertThat( diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldTypeTests.java index 4d1c990d94403..7605d79bb9f36 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldTypeTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldTypeTests.java @@ -49,6 +49,7 @@ import static org.hamcrest.Matchers.equalTo; public class ScriptKeywordMappedFieldTypeTests extends AbstractScriptMappedFieldTypeTestCase { + @Override public void testDocValues() throws IOException { try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [1]}")))); @@ -96,6 +97,7 @@ public void collect(int doc) throws IOException { } } + @Override public void testExistsQuery() throws IOException { try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [1]}")))); @@ -107,6 +109,7 @@ public void testExistsQuery() throws IOException { } } + @Override public void testExistsQueryIsExpensive() throws IOException { checkExpensiveQuery(ScriptKeywordMappedFieldType::existsQuery); } @@ -157,6 +160,7 @@ public void testPrefixQueryIsExpensive() throws IOException { checkExpensiveQuery((ft, ctx) -> ft.prefixQuery(randomAlphaOfLengthBetween(1, 1000), null, ctx)); } + @Override public void testRangeQuery() throws IOException { try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"cat\"}")))); @@ -172,6 +176,7 @@ public void testRangeQuery() throws IOException { } } + @Override public void testRangeQueryIsExpensive() throws IOException { checkExpensiveQuery( (ft, ctx) -> ft.rangeQuery( @@ -208,6 +213,7 @@ public void testRegexpQueryIsExpensive() throws IOException { checkExpensiveQuery((ft, ctx) -> ft.regexpQuery(randomAlphaOfLengthBetween(1, 1000), randomInt(0xFFFF), randomInt(), null, ctx)); } + @Override public void testTermQuery() throws IOException { try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); @@ -220,10 +226,12 @@ public void testTermQuery() throws IOException { } } + @Override public void testTermQueryIsExpensive() throws IOException { checkExpensiveQuery((ft, ctx) -> ft.termQuery(randomAlphaOfLengthBetween(1, 1000), ctx)); } + @Override public void testTermsQuery() throws IOException { try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); @@ -237,6 +245,7 @@ public void testTermsQuery() throws IOException { } } + @Override public void testTermsQueryIsExpensive() throws IOException { checkExpensiveQuery((ft, ctx) -> ft.termsQuery(randomList(100, () -> randomAlphaOfLengthBetween(1, 1000)), ctx)); } @@ -256,15 +265,15 @@ public void testWildcardQueryIsExpensive() throws IOException { checkExpensiveQuery((ft, ctx) -> ft.wildcardQuery(randomAlphaOfLengthBetween(1, 1000), null, ctx)); } - private ScriptKeywordMappedFieldType build(String code) throws IOException { + private static ScriptKeywordMappedFieldType build(String code) throws IOException { return build(new Script(code)); } - private ScriptKeywordMappedFieldType build(String code, Map params) throws IOException { + private static ScriptKeywordMappedFieldType build(String code, Map params) throws IOException { return build(new Script(ScriptType.INLINE, PainlessScriptEngine.NAME, code, params)); } - private ScriptKeywordMappedFieldType build(Script script) throws IOException { + private static ScriptKeywordMappedFieldType build(Script script) throws IOException { PainlessPlugin painlessPlugin = new PainlessPlugin(); painlessPlugin.loadExtensions(new ExtensionLoader() { @Override @@ -280,7 +289,7 @@ public List loadExtensions(Class extensionPointType) { } } - private void checkExpensiveQuery(BiConsumer queryBuilder) throws IOException { + private static void checkExpensiveQuery(BiConsumer queryBuilder) throws IOException { ScriptKeywordMappedFieldType ft = build("value('cat')"); Exception e = expectThrows(ElasticsearchException.class, () -> queryBuilder.accept(ft, mockContext(false))); assertThat( diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldTypeTests.java index 8474227a9c0d5..916985f93edef 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldTypeTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldTypeTests.java @@ -45,7 +45,8 @@ import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.equalTo; -public class ScriptLongMappedFieldTypeTests extends AbstractScriptMappedFieldTypeTestCase { +public class ScriptLongMappedFieldTypeTests extends AbstractNonTextScriptMappedFieldTypeTestCase { + @Override public void testDocValues() throws IOException { try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [1]}")))); @@ -90,6 +91,7 @@ public void collect(int doc) throws IOException { } } + @Override public void testExistsQuery() throws IOException { try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [1]}")))); @@ -101,10 +103,12 @@ public void testExistsQuery() throws IOException { } } + @Override public void testExistsQueryIsExpensive() throws IOException { checkExpensiveQuery(ScriptLongMappedFieldType::existsQuery); } + @Override public void testRangeQuery() throws IOException { try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); @@ -135,12 +139,14 @@ public void testRangeQuery() throws IOException { } } + @Override public void testRangeQueryIsExpensive() throws IOException { checkExpensiveQuery( (ft, ctx) -> ft.rangeQuery(randomLong(), randomLong(), randomBoolean(), randomBoolean(), null, null, null, ctx) ); } + @Override public void testTermQuery() throws IOException { try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); @@ -158,10 +164,12 @@ public void testTermQuery() throws IOException { } } + @Override public void testTermQueryIsExpensive() throws IOException { checkExpensiveQuery((ft, ctx) -> ft.termQuery(randomLong(), ctx)); } + @Override public void testTermsQuery() throws IOException { try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); @@ -177,19 +185,25 @@ public void testTermsQuery() throws IOException { } } + @Override public void testTermsQueryIsExpensive() throws IOException { checkExpensiveQuery((ft, ctx) -> ft.termsQuery(List.of(randomLong()), ctx)); } - private ScriptLongMappedFieldType build(String code) throws IOException { + @Override + protected ScriptLongMappedFieldType simpleMappedFieldType() throws IOException { + return build("value(source.foo)"); + } + + private static ScriptLongMappedFieldType build(String code) throws IOException { return build(new Script(code)); } - private ScriptLongMappedFieldType build(String code, Map params) throws IOException { + private static ScriptLongMappedFieldType build(String code, Map params) throws IOException { return build(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, code, params)); } - private ScriptLongMappedFieldType build(Script script) throws IOException { + private static ScriptLongMappedFieldType build(Script script) throws IOException { PainlessPlugin painlessPlugin = new PainlessPlugin(); painlessPlugin.loadExtensions(new ExtensionLoader() { @Override @@ -205,7 +219,7 @@ public List loadExtensions(Class extensionPointType) { } } - private void checkExpensiveQuery(BiConsumer queryBuilder) throws IOException { + private static void checkExpensiveQuery(BiConsumer queryBuilder) throws IOException { ScriptLongMappedFieldType ft = build("value(1)"); Exception e = expectThrows(ElasticsearchException.class, () -> queryBuilder.accept(ft, mockContext(false))); assertThat( From 0dd624a36d2c4372a97424b94aea3210b816e169 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 21 Jul 2020 13:53:35 -0400 Subject: [PATCH 3/5] WIP --- ...tNonTextScriptMappedFieldTypeTestCase.java | 2 + ...AbstractScriptMappedFieldTypeTestCase.java | 4 + .../ScriptDoubleMappedFieldTypeTests.java | 29 ++++++++ .../ScriptKeywordMappedFieldTypeTests.java | 73 +++++++++++++++++++ .../ScriptLongMappedFieldTypeTests.java | 29 ++++++++ 5 files changed, 137 insertions(+) diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractNonTextScriptMappedFieldTypeTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractNonTextScriptMappedFieldTypeTestCase.java index 66f7a5d811c31..a17bbbcceb350 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractNonTextScriptMappedFieldTypeTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractNonTextScriptMappedFieldTypeTestCase.java @@ -33,6 +33,7 @@ public void testWildcardQueryIsError() throws IOException { } private void assertQueryOnlyOnTextAndKeyword(String queryName, ThrowingRunnable buildQuery) { + // TODO use runtime type in the error message and a consistent exception type Exception e = expectThrows(Exception.class, buildQuery); assertThat( e.getMessage(), @@ -41,6 +42,7 @@ private void assertQueryOnlyOnTextAndKeyword(String queryName, ThrowingRunnable } private void assertQueryOnlyOnTextKeywordAndWildcard(String queryName, ThrowingRunnable buildQuery) { + // TODO use runtime type in the error message and a consistent exception type Exception e = expectThrows(Exception.class, buildQuery); assertThat( e.getMessage(), diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldTypeTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldTypeTestCase.java index 518b30af46c93..8ef949b7e6345 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldTypeTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldTypeTestCase.java @@ -19,6 +19,10 @@ abstract class AbstractScriptMappedFieldTypeTestCase extends ESTestCase { public abstract void testDocValues() throws IOException; + public abstract void testSort() throws IOException; + + public abstract void testUsedInScript() throws IOException; + public abstract void testExistsQuery() throws IOException; public abstract void testExistsQueryIsExpensive() throws IOException; diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldTypeTests.java index 63485d13325b9..a71910eb3cc39 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldTypeTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldTypeTests.java @@ -16,6 +16,9 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Scorable; import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; @@ -31,6 +34,7 @@ import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; import org.elasticsearch.xpack.runtimefields.RuntimeFields; import org.elasticsearch.xpack.runtimefields.RuntimeFieldsPainlessExtension; @@ -91,6 +95,31 @@ public void collect(int doc) throws IOException { } } + @Override + public void testSort() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [1.1]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [4.2]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [2.1]}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + IndexMetadata imd = IndexMetadata.builder("test") + .settings(Settings.builder().put("index.version.created", Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + ScriptDoubleMappedFieldType ft = build("for (def v : source.foo) { value(v)}"); + ScriptDoubleFieldData ifd = ft.fielddataBuilder("test").build(new IndexSettings(imd, Settings.EMPTY), ft, null, null, null); + ifd.setSearchLookup(mockContext().lookup()); + SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); + TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); + assertThat(reader.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [1.1]}")); + assertThat(reader.document(docs.scoreDocs[1].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [2.1]}")); + assertThat(reader.document(docs.scoreDocs[2].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [4.2]}")); + } + } + } + @Override public void testExistsQuery() throws IOException { try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldTypeTests.java index 7605d79bb9f36..c08c7fd8ab29a 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldTypeTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldTypeTests.java @@ -16,24 +16,34 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Scorable; import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.painless.PainlessPlugin; import org.elasticsearch.painless.PainlessScriptEngine; import org.elasticsearch.plugins.ExtensiblePlugin.ExtensionLoader; +import org.elasticsearch.script.ScoreScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptType; +import org.elasticsearch.script.ScoreScript.ExplanationHolder; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.xpack.runtimefields.RuntimeFields; import org.elasticsearch.xpack.runtimefields.RuntimeFieldsPainlessExtension; import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; @@ -97,6 +107,69 @@ public void collect(int doc) throws IOException { } } + @Override + public void testSort() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"a\"]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"d\"]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"b\"]}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + IndexMetadata imd = IndexMetadata.builder("test") + .settings(Settings.builder().put("index.version.created", Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + ScriptKeywordMappedFieldType ft = build("for (def v : source.foo) { value(v.toString())}"); + ScriptBinaryFieldData ifd = ft.fielddataBuilder("test").build(new IndexSettings(imd, Settings.EMPTY), ft, null, null, null); + ifd.setSearchLookup(mockContext().lookup()); + SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); + TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); + assertThat(reader.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [\"a\"]}")); + assertThat(reader.document(docs.scoreDocs[1].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [\"b\"]}")); + assertThat(reader.document(docs.scoreDocs[2].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [\"d\"]}")); + } + } + } + + @Override + public void testUsedInScript() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"a\"]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"aaa\"]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"aa\"]}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + IndexMetadata imd = IndexMetadata.builder("test") + .settings(Settings.builder().put("index.version.created", Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + ScriptKeywordMappedFieldType ft = build("for (def v : source.foo) { value(v.toString())}"); + ScriptBinaryFieldData ifd = ft.fielddataBuilder("test").build(new IndexSettings(imd, Settings.EMPTY), ft, null, null, null); + SearchLookup lookup = mockContext().lookup(); + ifd.setSearchLookup(lookup); + assertThat(searcher.count(new ScriptScoreQuery(new MatchAllDocsQuery(), new Script("test"), new ScoreScript.LeafFactory() { + @Override + public boolean needs_score() { + return false; + } + + @Override + public ScoreScript newInstance(LeafReaderContext ctx) throws IOException { + return new ScoreScript(Map.of(), lookup, ctx) { + @Override + public double execute(ExplanationHolder explanation) { + ScriptDocValues.Strings bytes = (ScriptDocValues.Strings) ifd.load(ctx).getScriptValues(); + return bytes.get(0).length(); + } + }; + } + }, 2f, "test", 0, Version.CURRENT)), equalTo(1)); + } + } + } + @Override public void testExistsQuery() throws IOException { try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldTypeTests.java index 916985f93edef..b5c21c8cc2144 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldTypeTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldTypeTests.java @@ -17,6 +17,9 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Scorable; import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; @@ -31,6 +34,7 @@ import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; import org.elasticsearch.xpack.runtimefields.RuntimeFields; import org.elasticsearch.xpack.runtimefields.RuntimeFieldsPainlessExtension; @@ -91,6 +95,31 @@ public void collect(int doc) throws IOException { } } + @Override + public void testSort() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [1]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [4]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [2]}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + IndexMetadata imd = IndexMetadata.builder("test") + .settings(Settings.builder().put("index.version.created", Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + ScriptLongMappedFieldType ft = build("for (def v : source.foo) { value(v)}"); + ScriptLongFieldData ifd = ft.fielddataBuilder("test").build(new IndexSettings(imd, Settings.EMPTY), ft, null, null, null); + ifd.setSearchLookup(mockContext().lookup()); + SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); + TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); + assertThat(reader.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [1]}")); + assertThat(reader.document(docs.scoreDocs[1].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [2]}")); + assertThat(reader.document(docs.scoreDocs[2].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [4]}")); + } + } + } + @Override public void testExistsQuery() throws IOException { try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { From f29bd64d89d5b358165773686fdd6440084a8478 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 21 Jul 2020 13:59:47 -0400 Subject: [PATCH 4/5] WIP --- .../AbstractScriptMappedFieldTypeTestCase.java | 13 ++++++++++++- .../ScriptKeywordMappedFieldTypeTests.java | 17 +++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldTypeTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldTypeTestCase.java index 8ef949b7e6345..76db6c559af43 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldTypeTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldTypeTestCase.java @@ -13,6 +13,8 @@ import java.io.IOException; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -44,10 +46,19 @@ protected static QueryShardContext mockContext() { } protected static QueryShardContext mockContext(boolean allowExpensiveQueries) { + return mockContext(allowExpensiveQueries, null); + } + + protected static QueryShardContext mockContext(boolean allowExpensiveQueries, AbstractScriptMappedFieldType mappedFieldType) { MapperService mapperService = mock(MapperService.class); + when(mapperService.fieldType(anyString())).thenReturn(mappedFieldType); QueryShardContext context = mock(QueryShardContext.class); + if (mappedFieldType != null) { + when(context.fieldMapper(anyString())).thenReturn(mappedFieldType); + when(context.getSearchAnalyzer(any())).thenReturn(mappedFieldType.getTextSearchInfo().getSearchAnalyzer()); + } when(context.allowExpensiveQueries()).thenReturn(allowExpensiveQueries); - when(context.lookup()).thenReturn(new SearchLookup(mapperService, mft -> null)); + when(context.lookup()).thenReturn(new SearchLookup(mapperService, mft -> mft.fielddataBuilder("test").build(indexSettings, fieldType, cache, breakerService, mapperService))); return context; } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldTypeTests.java index c08c7fd8ab29a..fb7c7644bc219 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldTypeTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldTypeTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.painless.PainlessPlugin; import org.elasticsearch.painless.PainlessScriptEngine; import org.elasticsearch.plugins.ExtensiblePlugin.ExtensionLoader; @@ -41,9 +40,7 @@ import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptType; -import org.elasticsearch.script.ScoreScript.ExplanationHolder; import org.elasticsearch.search.MultiValueMode; -import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.xpack.runtimefields.RuntimeFields; import org.elasticsearch.xpack.runtimefields.RuntimeFieldsPainlessExtension; import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; @@ -140,15 +137,7 @@ public void testUsedInScript() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"aa\"]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - IndexMetadata imd = IndexMetadata.builder("test") - .settings(Settings.builder().put("index.version.created", Version.CURRENT)) - .numberOfShards(1) - .numberOfReplicas(1) - .build(); - ScriptKeywordMappedFieldType ft = build("for (def v : source.foo) { value(v.toString())}"); - ScriptBinaryFieldData ifd = ft.fielddataBuilder("test").build(new IndexSettings(imd, Settings.EMPTY), ft, null, null, null); - SearchLookup lookup = mockContext().lookup(); - ifd.setSearchLookup(lookup); + QueryShardContext qsc = mockContext(true, build("for (def v : source.foo) { value(v.toString())}")); assertThat(searcher.count(new ScriptScoreQuery(new MatchAllDocsQuery(), new Script("test"), new ScoreScript.LeafFactory() { @Override public boolean needs_score() { @@ -157,10 +146,10 @@ public boolean needs_score() { @Override public ScoreScript newInstance(LeafReaderContext ctx) throws IOException { - return new ScoreScript(Map.of(), lookup, ctx) { + return new ScoreScript(Map.of(), qsc.lookup(), ctx) { @Override public double execute(ExplanationHolder explanation) { - ScriptDocValues.Strings bytes = (ScriptDocValues.Strings) ifd.load(ctx).getScriptValues(); + ScriptDocValues.Strings bytes = (ScriptDocValues.Strings) getDoc().get("test"); return bytes.get(0).length(); } }; From 76d9ee2a108e9668e79313b707028937f3d93e6f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 21 Jul 2020 16:02:57 -0400 Subject: [PATCH 5/5] precommit --- .../AbstractNonTextScriptMappedFieldTypeTestCase.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractNonTextScriptMappedFieldTypeTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractNonTextScriptMappedFieldTypeTestCase.java index a17bbbcceb350..4df5c6636fffc 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractNonTextScriptMappedFieldTypeTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractNonTextScriptMappedFieldTypeTestCase.java @@ -1,3 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + package org.elasticsearch.xpack.runtimefields.mapper; import org.apache.lucene.util.automaton.Operations; @@ -46,9 +52,7 @@ private void assertQueryOnlyOnTextKeywordAndWildcard(String queryName, ThrowingR Exception e = expectThrows(Exception.class, buildQuery); assertThat( e.getMessage(), - equalTo( - "Can only use " + queryName + " queries on keyword, text and wildcard fields - not on [test] which is of type [script]" - ) + equalTo("Can only use " + queryName + " queries on keyword, text and wildcard fields - not on [test] which is of type [script]") ); } }