diff --git a/docs/build.gradle b/docs/build.gradle index f8a8f465b7afe..048f4bc2c1d99 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -1355,10 +1355,26 @@ setups['reviews'] = ''' index: reviews refresh: true body: | + {"index": {"_id": "0"}} + {"product": "widget-foo", "rating": 4} {"index": {"_id": "1"}} {"product": "widget-foo", "rating": 1} {"index": {"_id": "2"}} {"product": "widget-foo", "rating": 5} + {"index": {"_id": "3"}} + {"product": "widget-foo", "rating": 2} + {"index": {"_id": "4"}} + {"product": "widget-foo", "rating": 1} + {"index": {"_id": "5"}} + {"product": "widget-foo", "rating": 4} + {"index": {"_id": "6"}} + {"product": "widget-foo", "rating": 1} + {"index": {"_id": "7"}} + {"product": "widget-foo", "rating": 2} + {"index": {"_id": "8"}} + {"product": "widget-foo", "rating": 5} + {"index": {"_id": "9"}} + {"product": "widget-foo", "rating": 5} ''' setups['remote_cluster'] = setups['host'] + ''' - do: diff --git a/docs/changelog/92767.yaml b/docs/changelog/92767.yaml new file mode 100644 index 0000000000000..1faf707f1ab3e --- /dev/null +++ b/docs/changelog/92767.yaml @@ -0,0 +1,5 @@ +pr: 92767 +summary: Upgrade t-digest and `HdrHistogram` library versions +area: Aggregations +type: upgrade +issues: [] diff --git a/docs/reference/aggregations/metrics/boxplot-aggregation.asciidoc b/docs/reference/aggregations/metrics/boxplot-aggregation.asciidoc index 4b15903108771..faa17fad8593f 100644 --- a/docs/reference/aggregations/metrics/boxplot-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/boxplot-aggregation.asciidoc @@ -55,9 +55,9 @@ The response will look like this: "load_time_boxplot": { "min": 0.0, "max": 990.0, - "q1": 165.0, - "q2": 445.0, - "q3": 725.0, + "q1": 170.0, + "q2": 450.0, + "q3": 730.0, "lower": 0.0, "upper": 990.0 } @@ -114,9 +114,9 @@ GET latency/_search "load_time_boxplot": { "min": 0.0, "max": 99.0, - "q1": 16.5, - "q2": 44.5, - "q3": 72.5, + "q1": 17.0, + "q2": 45.0, + "q3": 73.0, "lower": 0.0, "upper": 99.0 } diff --git a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc index 2eebe0efe9463..132e639a3f41a 100644 --- a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc @@ -54,13 +54,13 @@ percentiles: `[ 1, 5, 25, 50, 75, 95, 99 ]`. The response will look like this: "aggregations": { "load_time_outlier": { "values": { - "1.0": 5.0, - "5.0": 25.0, - "25.0": 165.0, - "50.0": 445.0, - "75.0": 725.0, - "95.0": 945.0, - "99.0": 985.0 + "1.0": 10.0, + "5.0": 30.0, + "25.0": 170.0, + "50.0": 450.0, + "75.0": 730.0, + "95.0": 950.0, + "99.0": 990.0 } } } @@ -128,31 +128,31 @@ Response: "values": [ { "key": 1.0, - "value": 5.0 + "value": 10.0 }, { "key": 5.0, - "value": 25.0 + "value": 30.0 }, { "key": 25.0, - "value": 165.0 + "value": 170.0 }, { "key": 50.0, - "value": 445.0 + "value": 450.0 }, { "key": 75.0, - "value": 725.0 + "value": 730.0 }, { "key": 95.0, - "value": 945.0 + "value": 950.0 }, { "key": 99.0, - "value": 985.0 + "value": 990.0 } ] } @@ -203,13 +203,13 @@ GET latency/_search "aggregations": { "load_time_outlier": { "values": { - "1.0": 0.5, - "5.0": 2.5, - "25.0": 16.5, - "50.0": 44.5, - "75.0": 72.5, - "95.0": 94.5, - "99.0": 98.5 + "1.0": 1.0, + "5.0": 3.0, + "25.0": 17.0, + "50.0": 45.0, + "75.0": 73.0, + "95.0": 95.0, + "99.0": 99.0 } } } diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 97e58a3175a29..ba7ec678f048a 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -1019,9 +1019,9 @@ - - - + + + @@ -3354,9 +3354,9 @@ - - - + + + diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/percentile_ranks_hdr_metric.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/percentile_ranks_hdr_metric.yml index 5c00bd1cf732a..a9c27ec8e29e7 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/percentile_ranks_hdr_metric.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/percentile_ranks_hdr_metric.yml @@ -50,10 +50,10 @@ basic: hdr: {} - match: { hits.total.value: 4 } - - close_to: { aggregations.percentile_ranks_int.values.50\\.0: { value: 25.0, error: 1} } - - close_to: { aggregations.percentile_ranks_int.values.99\\.0: { value: 50.0, error: 1} } - - close_to: { aggregations.percentile_ranks_double.values.50\\.0: { value: 25.0, error: 1} } - - close_to: { aggregations.percentile_ranks_double.values.99\\.0: { value: 50.0, error: 1} } + - close_to: { aggregations.percentile_ranks_int.values.50\\.0: { value: 25.0, error: 0.1} } + - close_to: { aggregations.percentile_ranks_int.values.99\\.0: { value: 50.0, error: 0.1} } + - close_to: { aggregations.percentile_ranks_double.values.50\\.0: { value: 25.0, error: 0.1} } + - close_to: { aggregations.percentile_ranks_double.values.99\\.0: { value: 50.0, error: 0.1} } --- set significant digits: @@ -79,10 +79,10 @@ set significant digits: number_of_significant_value_digits: 3 - match: { hits.total.value: 4 } - - close_to: { aggregations.percentile_ranks_int.values.50\\.0: { value: 25.0, error: 1} } - - close_to: { aggregations.percentile_ranks_int.values.99\\.0: { value: 50.0, error: 1} } - - close_to: { aggregations.percentile_ranks_double.values.50\\.0: { value: 25.0, error: 1} } - - close_to: { aggregations.percentile_ranks_double.values.99\\.0: { value: 50.0, error: 1} } + - close_to: { aggregations.percentile_ranks_int.values.50\\.0: { value: 25.0, error: 0.1} } + - close_to: { aggregations.percentile_ranks_int.values.99\\.0: { value: 50.0, error: 0.1} } + - close_to: { aggregations.percentile_ranks_double.values.50\\.0: { value: 25.0, error: 0.1} } + - close_to: { aggregations.percentile_ranks_double.values.99\\.0: { value: 50.0, error: 0.1} } --- filtered: @@ -112,8 +112,8 @@ filtered: number_of_significant_value_digits: 3 - match: { hits.total.value: 3 } - - close_to: { aggregations.percentile_ranks_int.values.50\\.0: { value: 0.0, error: 1} } - - close_to: { aggregations.percentile_ranks_double.values.50\\.0: { value: 0.0, error: 1} } + - close_to: { aggregations.percentile_ranks_int.values.50\\.0: { value: 0.0, error: 0.1} } + - close_to: { aggregations.percentile_ranks_double.values.50\\.0: { value: 0.0, error: 0.1} } --- missing field with missing param: @@ -134,8 +134,8 @@ missing field with missing param: number_of_significant_value_digits: 3 - match: { hits.total.value: 4 } - - close_to: { aggregations.percentile_ranks_missing.values.50\\.0: { value: 100.0, error: 1} } - - close_to: { aggregations.percentile_ranks_missing.values.99\\.0: { value: 100.0, error: 1} } + - close_to: { aggregations.percentile_ranks_missing.values.50\\.0: { value: 100.0, error: 0.1} } + - close_to: { aggregations.percentile_ranks_missing.values.99\\.0: { value: 100.0, error: 0.1} } --- missing field without missing param: @@ -224,6 +224,6 @@ non-keyed: - match: { hits.total.value: 4 } - match: { aggregations.percentile_ranks_int.values.0.key: 50} - - close_to: { aggregations.percentile_ranks_int.values.0.value: { value: 25, error: 1} } + - close_to: { aggregations.percentile_ranks_int.values.0.value: { value: 25, error: 0.1} } - match: { aggregations.percentile_ranks_int.values.1.key: 99} - - close_to: { aggregations.percentile_ranks_int.values.1.value: { value: 50, error: 1} } + - close_to: { aggregations.percentile_ranks_int.values.1.value: { value: 50, error: 0.1} } diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/percentile_ranks_tdigest_metric.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/percentile_ranks_tdigest_metric.yml index 966dde25da883..08376d7bae89d 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/percentile_ranks_tdigest_metric.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/percentile_ranks_tdigest_metric.yml @@ -41,17 +41,21 @@ basic: percentile_ranks_int: percentile_ranks: field: int - values: [50, 99] + values: [50, 100, 150, 200] percentile_ranks_double: percentile_ranks: field: double - values: [50, 99] + values: [50, 100, 150, 200] - match: { hits.total.value: 4 } - - close_to: { aggregations.percentile_ranks_int.values.50\\.0: { value: 37.0, error: 1} } - - close_to: { aggregations.percentile_ranks_int.values.99\\.0: { value: 61.5, error: 1} } - - close_to: { aggregations.percentile_ranks_double.values.50\\.0: { value: 37.0, error: 1} } - - close_to: { aggregations.percentile_ranks_double.values.99\\.0: { value: 61.5, error: 1} } + - close_to: { aggregations.percentile_ranks_int.values.50\\.0: { value: 25.0, error: 0.1 } } + - close_to: { aggregations.percentile_ranks_int.values.100\\.0: { value: 50.0, error: 0.1 } } + - close_to: { aggregations.percentile_ranks_int.values.150\\.0: { value: 75.0, error: 0.1 } } + - close_to: { aggregations.percentile_ranks_int.values.200\\.0: { value: 100.0, error: 0.1 } } + - close_to: { aggregations.percentile_ranks_double.values.50\\.0: { value: 25.0, error: 0.1 } } + - close_to: { aggregations.percentile_ranks_double.values.100\\.0: { value: 50.0, error: 0.1 } } + - close_to: { aggregations.percentile_ranks_double.values.150\\.0: { value: 75.0, error: 0.1 } } + - close_to: { aggregations.percentile_ranks_double.values.200\\.0: { value: 100.0, error: 0.1 } } --- filtered: @@ -70,15 +74,21 @@ filtered: percentile_ranks_int: percentile_ranks: field: int - values: [50] + values: [50, 100, 150, 200] percentile_ranks_double: percentile_ranks: field: double - values: [50] + values: [50, 100, 150, 200] - match: { hits.total.value: 3 } - - close_to: { aggregations.percentile_ranks_int.values.50\\.0: { value: 16.0, error: 1} } - - close_to: { aggregations.percentile_ranks_double.values.50\\.0: { value: 16.0, error: 1} } + - close_to: { aggregations.percentile_ranks_int.values.50\\.0: { value: 0.0, error: 0.1 } } + - close_to: { aggregations.percentile_ranks_int.values.100\\.0: { value: 33.3, error: 0.1 } } + - close_to: { aggregations.percentile_ranks_int.values.150\\.0: { value: 66.6, error: 0.1 } } + - close_to: { aggregations.percentile_ranks_int.values.200\\.0: { value: 100.0, error: 0.1 } } + - close_to: { aggregations.percentile_ranks_double.values.50\\.0: { value: 0.0, error: 0.1 } } + - close_to: { aggregations.percentile_ranks_double.values.100\\.0: { value: 33.3, error: 0.1 } } + - close_to: { aggregations.percentile_ranks_double.values.150\\.0: { value: 66.6, error: 0.1 } } + - close_to: { aggregations.percentile_ranks_double.values.200\\.0: { value: 100.0, error: 0.1 } } --- missing field with missing param: @@ -97,9 +107,9 @@ missing field with missing param: values: [50, 99] - match: { hits.total.value: 4 } - - close_to: { aggregations.percentile_ranks_missing.values.50\\.0: { value: 100.0, error: 1} } - - close_to: { aggregations.percentile_ranks_missing.values.99\\.0: { value: 100.0, error: 1} } - + - close_to: { aggregations.percentile_ranks_missing.values.50\\.0: { value: 100.0, error: 0.1} } + - close_to: { aggregations.percentile_ranks_missing.values.99\\.0: { value: 100.0, error: 0.1} } + --- missing field without missing param: - do: @@ -169,12 +179,15 @@ non-keyed test: percentile_ranks_int: percentile_ranks: field: int - values: [50, 99] + values: [50, 100, 150, 200] keyed: false - match: { hits.total.value: 4 } - - match: { aggregations.percentile_ranks_int.values.0.key: 50} - - close_to: { aggregations.percentile_ranks_int.values.0.value: { value: 37.0, error: 1} } - - match: { aggregations.percentile_ranks_int.values.1.key: 99} - - close_to: { aggregations.percentile_ranks_int.values.1.value: { value: 61.5, error: 1} } - + - match: { aggregations.percentile_ranks_int.values.0.key: 50 } + - close_to: { aggregations.percentile_ranks_int.values.0.value: { value: 25.0, error: 0.1 } } + - match: { aggregations.percentile_ranks_int.values.1.key: 100 } + - close_to: { aggregations.percentile_ranks_int.values.1.value: { value: 50.0, error: 0.1 } } + - match: { aggregations.percentile_ranks_int.values.2.key: 150 } + - close_to: { aggregations.percentile_ranks_int.values.2.value: { value: 75.0, error: 0.1 } } + - match: { aggregations.percentile_ranks_int.values.3.key: 200 } + - close_to: { aggregations.percentile_ranks_int.values.3.value: { value: 100.0, error: 0.1 } } diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/percentiles_hdr_metric.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/percentiles_hdr_metric.yml index 21f3ad31558af..b00065c356ef4 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/percentiles_hdr_metric.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/percentiles_hdr_metric.yml @@ -16,6 +16,21 @@ setup: string_field: type: keyword + - do: + indices.create: + index: test_2 + body: + settings: + number_of_replicas: 0 + number_of_shards: 5 + number_of_routing_shards: 5 + mappings: + properties: + int_field: + type: integer + double_field: + type: double + - do: bulk: refresh: true @@ -45,12 +60,38 @@ setup: double_field: 151.0 string_field: foo + - do: + bulk: + refresh: true + body: + - index: + _index: test_2 + _id: "1" + - int_field: [1, 2, 3, 1, 2, 3] + double_field: [1.2, 1.5, 1.2, 1.8] + - index: + _index: test_2 + _id: "2" + - int_field: 3 + double_field: 2.2 + - index: + _index: test_2 + _id: "3" + - int_field: [ 1, 2, 1, 2 ] + double_field: [ 1.2, 2.0 ] + - index: + _index: test_2 + _id: "4" + - int_field: 2 + double_field: 1.0 --- "Basic test": + - skip: + features: close_to - do: search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -63,28 +104,28 @@ setup: hdr: {} - - match: { hits.total: 4 } + - match: { hits.total.value: 4 } - length: { hits.hits: 4 } - - match: { aggregations.percentiles_int.values.1\.0: 1.0 } - - match: { aggregations.percentiles_int.values.5\.0: 1.0 } - - match: { aggregations.percentiles_int.values.25\.0: 1.0 } - - match: { aggregations.percentiles_int.values.50\.0: 51.0302734375 } - - match: { aggregations.percentiles_int.values.75\.0: 101.0615234375 } - - match: { aggregations.percentiles_int.values.95\.0: 151.1240234375 } - - match: { aggregations.percentiles_int.values.99\.0: 151.1240234375 } - - - match: { aggregations.percentiles_double.values.1\.0: 1.0 } - - match: { aggregations.percentiles_double.values.5\.0: 1.0 } - - match: { aggregations.percentiles_double.values.25\.0: 1.0 } - - match: { aggregations.percentiles_double.values.50\.0: 51.0302734375 } - - match: { aggregations.percentiles_double.values.75\.0: 101.0615234375 } - - match: { aggregations.percentiles_double.values.95\.0: 151.1240234375 } - - match: { aggregations.percentiles_double.values.99\.0: 151.1240234375 } + - close_to: { aggregations.percentiles_int.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.25\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.50\.0: { value: 51.03, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.75\.0: { value: 101.06, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.95\.0: { value: 151.12, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.99\.0: { value: 151.12, error: 0.1 } } + + - close_to: { aggregations.percentiles_double.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.25\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.50\.0: { value: 51.03, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.75\.0: { value: 101.06, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.95\.0: { value: 151.12, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.99\.0: { value: 151.12, error: 0.1 } } - do: search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -99,32 +140,34 @@ setup: number_of_significant_value_digits: 3 - - match: { hits.total: 4 } + - match: { hits.total.value: 4 } - length: { hits.hits: 4 } - - match: { aggregations.percentiles_int.values.1\.0: 1.0 } - - match: { aggregations.percentiles_int.values.5\.0: 1.0 } - - match: { aggregations.percentiles_int.values.25\.0: 1.0 } - - match: { aggregations.percentiles_int.values.50\.0: 51.0302734375 } - - match: { aggregations.percentiles_int.values.75\.0: 101.0615234375 } - - match: { aggregations.percentiles_int.values.95\.0: 151.1240234375 } - - match: { aggregations.percentiles_int.values.99\.0: 151.1240234375 } + - close_to: { aggregations.percentiles_int.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.25\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.50\.0: { value: 51.03, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.75\.0: { value: 101.06, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.95\.0: { value: 151.12, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.99\.0: { value: 151.12, error: 0.1 } } - - match: { aggregations.percentiles_double.values.1\.0: 1.0 } - - match: { aggregations.percentiles_double.values.5\.0: 1.0 } - - match: { aggregations.percentiles_double.values.25\.0: 1.0 } - - match: { aggregations.percentiles_double.values.50\.0: 51.0302734375 } - - match: { aggregations.percentiles_double.values.75\.0: 101.0615234375 } - - match: { aggregations.percentiles_double.values.95\.0: 151.1240234375 } - - match: { aggregations.percentiles_double.values.99\.0: 151.1240234375 } + - close_to: { aggregations.percentiles_double.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.25\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.50\.0: { value: 51.03, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.75\.0: { value: 101.06, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.95\.0: { value: 151.12, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.99\.0: { value: 151.12, error: 0.1 } } --- "Only aggs test": + - skip: + features: close_to - do: search: - rest_total_hits_as_int: true + index: test_1 body: size: 0 aggs: @@ -137,32 +180,34 @@ setup: field: double_field hdr: {} - - match: { hits.total: 4 } + - match: { hits.total.value: 4 } - length: { hits.hits: 0 } - - match: { aggregations.percentiles_int.values.1\.0: 1.0 } - - match: { aggregations.percentiles_int.values.5\.0: 1.0 } - - match: { aggregations.percentiles_int.values.25\.0: 1.0 } - - match: { aggregations.percentiles_int.values.50\.0: 51.0302734375 } - - match: { aggregations.percentiles_int.values.75\.0: 101.0615234375 } - - match: { aggregations.percentiles_int.values.95\.0: 151.1240234375 } - - match: { aggregations.percentiles_int.values.99\.0: 151.1240234375 } + - close_to: { aggregations.percentiles_int.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.25\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.50\.0: { value: 51.03, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.75\.0: { value: 101.06, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.95\.0: { value: 151.12, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.99\.0: { value: 151.12, error: 0.1 } } - - match: { aggregations.percentiles_double.values.1\.0: 1.0 } - - match: { aggregations.percentiles_double.values.5\.0: 1.0 } - - match: { aggregations.percentiles_double.values.25\.0: 1.0 } - - match: { aggregations.percentiles_double.values.50\.0: 51.0302734375 } - - match: { aggregations.percentiles_double.values.75\.0: 101.0615234375 } - - match: { aggregations.percentiles_double.values.95\.0: 151.1240234375 } - - match: { aggregations.percentiles_double.values.99\.0: 151.1240234375 } + - close_to: { aggregations.percentiles_double.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.25\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.50\.0: { value: 51.03, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.75\.0: { value: 101.06, error: 0.1 }} + - close_to: { aggregations.percentiles_double.values.95\.0: { value: 151.12, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.99\.0: { value: 151.12, error: 0.1 } } --- "Filtered test": + - skip: + features: close_to - do: search: - rest_total_hits_as_int: true + index: test_1 body: query: constant_score: @@ -180,32 +225,34 @@ setup: field: double_field hdr: {} - - match: { hits.total: 3 } + - match: { hits.total.value: 3 } - length: { hits.hits: 3 } - - match: { aggregations.percentiles_int.values.1\.0: 51.0 } - - match: { aggregations.percentiles_int.values.5\.0: 51.0 } - - match: { aggregations.percentiles_int.values.25\.0: 51.0 } - - match: { aggregations.percentiles_int.values.50\.0: 101.03125 } - - match: { aggregations.percentiles_int.values.75\.0: 101.03125 } - - match: { aggregations.percentiles_int.values.95\.0: 151.09375 } - - match: { aggregations.percentiles_int.values.99\.0: 151.09375 } + - close_to: { aggregations.percentiles_int.values.1\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.5\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.25\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.50\.0: { value: 101.03, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.75\.0: { value: 151.09, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.95\.0: { value: 151.09, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.99\.0: { value: 151.09, error: 0.1 } } - - match: { aggregations.percentiles_double.values.1\.0: 51.0 } - - match: { aggregations.percentiles_double.values.5\.0: 51.0 } - - match: { aggregations.percentiles_double.values.25\.0: 51.0 } - - match: { aggregations.percentiles_double.values.50\.0: 101.03125 } - - match: { aggregations.percentiles_double.values.75\.0: 101.03125 } - - match: { aggregations.percentiles_double.values.95\.0: 151.09375 } - - match: { aggregations.percentiles_double.values.99\.0: 151.09375 } + - close_to: { aggregations.percentiles_double.values.1\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.5\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.25\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.50\.0: { value: 101.03, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.75\.0: { value: 151.09, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.95\.0: { value: 151.09, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.99\.0: { value: 151.09, error: 0.1 } } --- "Missing field with missing param": + - skip: + features: close_to - do: search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_missing: @@ -214,23 +261,25 @@ setup: missing: 1.0 hdr: {} - - match: { hits.total: 4 } + - match: { hits.total.value: 4 } - length: { hits.hits: 4 } - - match: { aggregations.percentiles_missing.values.1\.0: 1.0 } - - match: { aggregations.percentiles_missing.values.5\.0: 1.0 } - - match: { aggregations.percentiles_missing.values.25\.0: 1.0 } - - match: { aggregations.percentiles_missing.values.50\.0: 1.0 } - - match: { aggregations.percentiles_missing.values.75\.0: 1.0 } - - match: { aggregations.percentiles_missing.values.95\.0: 1.0 } - - match: { aggregations.percentiles_missing.values.99\.0: 1.0 } + - close_to: { aggregations.percentiles_missing.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_missing.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_missing.values.25\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_missing.values.50\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_missing.values.75\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_missing.values.95\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_missing.values.99\.0: { value: 1.0, error: 0.1 } } --- "Missing field without missing param": + - skip: + features: close_to - do: search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_missing: @@ -238,16 +287,18 @@ setup: field: missing_field hdr: {} - - match: { hits.total: 4 } + - match: { hits.total.value: 4 } - length: { hits.hits: 4 } - is_false: aggregations.percentiles_missing.value --- "Metadata test": + - skip: + features: close_to - do: search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -257,17 +308,17 @@ setup: field: int_field hdr: {} - - match: { hits.total: 4 } + - match: { hits.total.value: 4 } - length: { hits.hits: 4 } - - match: { aggregations.percentiles_int.meta.foo: "bar" } - - match: { aggregations.percentiles_int.values.1\.0: 1.0 } - - match: { aggregations.percentiles_int.values.5\.0: 1.0 } - - match: { aggregations.percentiles_int.values.25\.0: 1.0 } - - match: { aggregations.percentiles_int.values.50\.0: 51.0302734375 } - - match: { aggregations.percentiles_int.values.75\.0: 101.0615234375 } - - match: { aggregations.percentiles_int.values.95\.0: 151.1240234375 } - - match: { aggregations.percentiles_int.values.99\.0: 151.1240234375 } + - match: { aggregations.percentiles_int.meta.foo: "bar" } + - close_to: { aggregations.percentiles_int.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.25\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.50\.0: { value: 51.03, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.75\.0: { value: 101.06, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.95\.0: { value: 151.12, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.99\.0: { value: 151.12, error: 0.1 } } --- "Invalid params test": @@ -275,7 +326,7 @@ setup: - do: catch: /\[numberOfSignificantValueDigits\] must be between 0 and 5/ search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -287,7 +338,7 @@ setup: - do: catch: /\[numberOfSignificantValueDigits\] must be between 0 and 5/ search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -299,7 +350,7 @@ setup: - do: catch: bad_request search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -311,7 +362,7 @@ setup: - do: catch: bad_request search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -323,7 +374,7 @@ setup: - do: catch: bad_request search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -335,7 +386,7 @@ setup: - do: catch: bad_request search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -347,7 +398,7 @@ setup: - do: catch: bad_request search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_string: @@ -357,10 +408,12 @@ setup: --- "Explicit Percents test": + - skip: + features: close_to - do: search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -375,25 +428,27 @@ setup: hdr: {} - - match: { hits.total: 4 } + - match: { hits.total.value: 4 } - length: { hits.hits: 4 } - - match: { aggregations.percentiles_int.values.5\.0: 1.0 } - - match: { aggregations.percentiles_int.values.25\.0: 1.0 } - - match: { aggregations.percentiles_int.values.50\.0: 51.0302734375 } + - close_to: { aggregations.percentiles_int.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.25\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.50\.0: { value: 51.03, error: 0.1 } } - - match: { aggregations.percentiles_double.values.5\.0: 1.0 } - - match: { aggregations.percentiles_double.values.25\.0: 1.0 } - - match: { aggregations.percentiles_double.values.50\.0: 51.0302734375 } + - close_to: { aggregations.percentiles_double.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.25\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.50\.0: { value: 51.03, error: 0.1 } } --- "Non-keyed test": + - skip: + features: close_to - do: search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -404,20 +459,21 @@ setup: hdr: {} - - match: { hits.total: 4 } + - match: { hits.total.value: 4 } - length: { hits.hits: 4 } - - match: { aggregations.percentiles_int.values.0.key: 5.0 } - - match: { aggregations.percentiles_int.values.0.value: 1.0 } + - close_to: { aggregations.percentiles_int.values.0.value: { value: 1.0, error: 0.1 } } - match: { aggregations.percentiles_int.values.1.key: 25.0 } - - match: { aggregations.percentiles_int.values.1.value: 1.0 } + - close_to: { aggregations.percentiles_int.values.1.value: { value: 1.0, error: 0.1 } } - match: { aggregations.percentiles_int.values.2.key: 50.0 } - - match: { aggregations.percentiles_int.values.2.value: 51.0302734375 } + - close_to: { aggregations.percentiles_int.values.2.value: { value: 51.03, error: 0.1 } } --- "Negative values test": + - skip: + features: close_to - do: index: @@ -428,7 +484,7 @@ setup: - do: search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -437,14 +493,53 @@ setup: hdr: {} - - match: { hits.total: 4 } + - match: { hits.total.value: 4 } - length: { hits.hits: 4 } - - match: { aggregations.percentiles_int.values.1\.0: 1.0 } - - match: { aggregations.percentiles_int.values.5\.0: 1.0 } - - match: { aggregations.percentiles_int.values.25\.0: 1.0 } - - match: { aggregations.percentiles_int.values.50\.0: 51.0302734375 } - - match: { aggregations.percentiles_int.values.75\.0: 101.0615234375 } - - match: { aggregations.percentiles_int.values.95\.0: 151.1240234375 } - - match: { aggregations.percentiles_int.values.99\.0: 151.1240234375 } + - close_to: { aggregations.percentiles_int.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.25\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.50\.0: { value: 51.03, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.75\.0: { value: 101.06, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.95\.0: { value: 151.12, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.99\.0: { value: 151.12, error: 0.1 } } - match: { _shards.failures.0.reason.type: array_index_out_of_bounds_exception } + +--- +"Multi-value fields test": + - skip: + features: close_to + + - do: + search: + index: test_2 + body: + aggs: + percentiles_int: + percentiles: + field: int_field + hdr: {} + percentiles_double: + percentiles: + field: double_field + hdr: {} + + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + + - close_to: { aggregations.percentiles_int.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.25\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.50\.0: { value: 2.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.75\.0: { value: 2.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.95\.0: { value: 3.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.99\.0: { value: 3.0, error: 0.1 } } + + - close_to: { aggregations.percentiles_double.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.25\.0: { value: 1.19, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.50\.0: { value: 1.19, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.75\.0: { value: 1.79, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.95\.0: { value: 2.20, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.99\.0: { value: 2.20, error: 0.1 } } diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/percentiles_tdigest_metric.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/percentiles_tdigest_metric.yml index 9a126730c63c6..8978d8f6ebc2a 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/percentiles_tdigest_metric.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/percentiles_tdigest_metric.yml @@ -14,6 +14,21 @@ setup: string_field: type: keyword + - do: + indices.create: + index: test_2 + body: + settings: + number_of_replicas: 0 + number_of_shards: 5 + number_of_routing_shards: 5 + mappings: + properties: + int_field: + type: integer + double_field: + type: double + - do: bulk: refresh: true @@ -43,12 +58,39 @@ setup: double_field: 151.0 string_field: foo + - do: + bulk: + refresh: true + body: + - index: + _index: test_2 + _id: "1" + - int_field: [ 1, 2, 3, 1, 2, 3 ] + double_field: [ 1.2, 1.5, 1.2, 1.8 ] + - index: + _index: test_2 + _id: "2" + - int_field: 3 + double_field: 2.2 + - index: + _index: test_2 + _id: "3" + - int_field: [ 1, 2, 1, 2 ] + double_field: [ 1.2, 2.0 ] + - index: + _index: test_2 + _id: "4" + - int_field: 2 + double_field: 1.0 + --- "Basic test": + - skip: + features: close_to - do: search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -58,28 +100,28 @@ setup: percentiles: field: double_field - - match: { hits.total: 4 } + - match: { hits.total.value: 4 } - length: { hits.hits: 4 } - - match: { aggregations.percentiles_int.values.1\.0: 1.0 } - - match: { aggregations.percentiles_int.values.5\.0: 1.0 } - - match: { aggregations.percentiles_int.values.25\.0: 26.0 } - - match: { aggregations.percentiles_int.values.50\.0: 76.0 } - - match: { aggregations.percentiles_int.values.75\.0: 126.0 } - - match: { aggregations.percentiles_int.values.95\.0: 151.0 } - - match: { aggregations.percentiles_int.values.99\.0: 151.0 } - - - match: { aggregations.percentiles_double.values.1\.0: 1.0 } - - match: { aggregations.percentiles_double.values.5\.0: 1.0 } - - match: { aggregations.percentiles_double.values.25\.0: 26.0 } - - match: { aggregations.percentiles_double.values.50\.0: 76.0 } - - match: { aggregations.percentiles_double.values.75\.0: 126.0 } - - match: { aggregations.percentiles_double.values.95\.0: 151.0 } - - match: { aggregations.percentiles_double.values.99\.0: 151.0 } + - close_to: { aggregations.percentiles_int.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.25\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.50\.0: { value: 101.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.75\.0: { value: 151.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.95\.0: { value: 151.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.99\.0: { value: 151.0, error: 0.1 } } + + - close_to: { aggregations.percentiles_double.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.25\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.50\.0: { value: 101.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.75\.0: { value: 151.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.95\.0: { value: 151.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.99\.0: { value: 151.0, error: 0.1 } } - do: search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -94,32 +136,34 @@ setup: compression: 200 - - match: { hits.total: 4 } + - match: { hits.total.value: 4 } - length: { hits.hits: 4 } - - match: { aggregations.percentiles_int.values.1\.0: 1.0 } - - match: { aggregations.percentiles_int.values.5\.0: 1.0 } - - match: { aggregations.percentiles_int.values.25\.0: 26.0 } - - match: { aggregations.percentiles_int.values.50\.0: 76.0 } - - match: { aggregations.percentiles_int.values.75\.0: 126.0 } - - match: { aggregations.percentiles_int.values.95\.0: 151.0 } - - match: { aggregations.percentiles_int.values.99\.0: 151.0 } + - close_to: { aggregations.percentiles_int.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.25\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.50\.0: { value: 101.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.75\.0: { value: 151.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.95\.0: { value: 151.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.99\.0: { value: 151.0, error: 0.1 } } - - match: { aggregations.percentiles_double.values.1\.0: 1.0 } - - match: { aggregations.percentiles_double.values.5\.0: 1.0 } - - match: { aggregations.percentiles_double.values.25\.0: 26.0 } - - match: { aggregations.percentiles_double.values.50\.0: 76.0 } - - match: { aggregations.percentiles_double.values.75\.0: 126.0 } - - match: { aggregations.percentiles_double.values.95\.0: 151.0 } - - match: { aggregations.percentiles_double.values.99\.0: 151.0 } + - close_to: { aggregations.percentiles_double.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.25\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.50\.0: { value: 101.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.75\.0: { value: 151.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.95\.0: { value: 151.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.99\.0: { value: 151.0, error: 0.1 } } --- "Only aggs test": + - skip: + features: close_to - do: search: - rest_total_hits_as_int: true + index: test_1 body: size: 0 aggs: @@ -130,33 +174,35 @@ setup: percentiles: field: double_field - - match: { hits.total: 4 } + - match: { hits.total.value: 4 } - length: { hits.hits: 0 } - - match: { aggregations.percentiles_int.values.1\.0: 1.0 } - - match: { aggregations.percentiles_int.values.5\.0: 1.0 } - - match: { aggregations.percentiles_int.values.25\.0: 26.0 } - - match: { aggregations.percentiles_int.values.50\.0: 76.0 } - - match: { aggregations.percentiles_int.values.75\.0: 126.0 } - - match: { aggregations.percentiles_int.values.95\.0: 151.0 } - - match: { aggregations.percentiles_int.values.99\.0: 151.0 } + - close_to: { aggregations.percentiles_int.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.25\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.50\.0: { value: 101.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.75\.0: { value: 151.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.95\.0: { value: 151.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.99\.0: { value: 151.0, error: 0.1 } } - - match: { aggregations.percentiles_double.values.1\.0: 1.0 } - - match: { aggregations.percentiles_double.values.5\.0: 1.0 } - - match: { aggregations.percentiles_double.values.25\.0: 26.0 } - - match: { aggregations.percentiles_double.values.50\.0: 76.0 } - - match: { aggregations.percentiles_double.values.75\.0: 126.0 } - - match: { aggregations.percentiles_double.values.95\.0: 151.0 } - - match: { aggregations.percentiles_double.values.99\.0: 151.0 } + - close_to: { aggregations.percentiles_double.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.25\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.50\.0: { value: 101.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.75\.0: { value: 151.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.95\.0: { value: 151.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.99\.0: { value: 151.0, error: 0.1 } } --- "Filtered test": + - skip: + features: close_to - do: search: - rest_total_hits_as_int: true + index: test_1 body: query: constant_score: @@ -172,31 +218,33 @@ setup: percentiles: field: double_field - - match: { hits.total: 3 } + - match: { hits.total.value: 3 } - length: { hits.hits: 3 } - - match: { aggregations.percentiles_int.values.1\.0: 51.0 } - - match: { aggregations.percentiles_int.values.5\.0: 51.0 } - - match: { aggregations.percentiles_int.values.25\.0: 63.5 } - - match: { aggregations.percentiles_int.values.50\.0: 101.0 } - - match: { aggregations.percentiles_int.values.75\.0: 138.5 } - - match: { aggregations.percentiles_int.values.95\.0: 151.0 } - - match: { aggregations.percentiles_int.values.99\.0: 151.0 } - - - match: { aggregations.percentiles_double.values.1\.0: 51.0 } - - match: { aggregations.percentiles_double.values.5\.0: 51.0 } - - match: { aggregations.percentiles_double.values.25\.0: 63.5 } - - match: { aggregations.percentiles_double.values.50\.0: 101.0 } - - match: { aggregations.percentiles_double.values.75\.0: 138.5 } - - match: { aggregations.percentiles_double.values.95\.0: 151.0 } - - match: { aggregations.percentiles_double.values.99\.0: 151.0 } + - close_to: { aggregations.percentiles_int.values.1\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.5\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.25\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.50\.0: { value: 101.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.75\.0: { value: 151.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.95\.0: { value: 151.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.99\.0: { value: 151.0, error: 0.1 } } + + - close_to: { aggregations.percentiles_double.values.1\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.5\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.25\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.50\.0: { value: 101.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.75\.0: { value: 151.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.95\.0: { value: 151.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.99\.0: { value: 151.0, error: 0.1 } } --- "Missing field with missing param": + - skip: + features: close_to - do: search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_missing: @@ -204,39 +252,41 @@ setup: field: missing_field missing: 1.0 - - match: { hits.total: 4 } + - match: { hits.total.value: 4 } - length: { hits.hits: 4 } - - match: { aggregations.percentiles_missing.values.1\.0: 1.0 } - - match: { aggregations.percentiles_missing.values.5\.0: 1.0 } - - match: { aggregations.percentiles_missing.values.25\.0: 1.0 } - - match: { aggregations.percentiles_missing.values.50\.0: 1.0 } - - match: { aggregations.percentiles_missing.values.75\.0: 1.0 } - - match: { aggregations.percentiles_missing.values.95\.0: 1.0 } - - match: { aggregations.percentiles_missing.values.99\.0: 1.0 } + - close_to: { aggregations.percentiles_missing.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_missing.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_missing.values.25\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_missing.values.50\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_missing.values.75\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_missing.values.95\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_missing.values.99\.0: { value: 1.0, error: 0.1 } } --- "Missing field without missing param": - do: search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_missing: percentiles: field: missing_field - - match: { hits.total: 4 } + - match: { hits.total.value: 4 } - length: { hits.hits: 4 } - is_false: aggregations.percentiles_missing.value --- "Metadata test": + - skip: + features: close_to - do: search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -245,18 +295,18 @@ setup: percentiles: field: int_field - - match: { hits.total: 4 } + - match: { hits.total.value: 4 } - length: { hits.hits: 4 } - match: { aggregations.percentiles_int.meta.foo: "bar" } - - match: { aggregations.percentiles_int.values.1\.0: 1.0 } - - match: { aggregations.percentiles_int.values.5\.0: 1.0 } - - match: { aggregations.percentiles_int.values.25\.0: 26.0 } - - match: { aggregations.percentiles_int.values.50\.0: 76.0 } - - match: { aggregations.percentiles_int.values.75\.0: 126.0 } - - match: { aggregations.percentiles_int.values.95\.0: 151.0 } - - match: { aggregations.percentiles_int.values.99\.0: 151.0 } + - close_to: { aggregations.percentiles_int.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.25\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.50\.0: { value: 101.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.75\.0: { value: 151.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.95\.0: { value: 151.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.99\.0: { value: 151.0, error: 0.1 } } --- "Invalid params test": @@ -264,7 +314,7 @@ setup: - do: catch: /\[compression\] must be greater than or equal to 0. Found \[-1.0\]/ search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -276,7 +326,7 @@ setup: - do: catch: bad_request search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -287,7 +337,7 @@ setup: - do: catch: bad_request search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -298,7 +348,7 @@ setup: - do: catch: bad_request search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -309,7 +359,7 @@ setup: - do: catch: bad_request search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_string: @@ -318,10 +368,12 @@ setup: --- "Explicit Percents test": + - skip: + features: close_to - do: search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: @@ -334,38 +386,81 @@ setup: percents: [5.0, 25.0, 50.0] - - match: { hits.total: 4 } + - match: { hits.total.value: 4 } - length: { hits.hits: 4 } - - match: { aggregations.percentiles_int.values.5\.0: 1.0 } - - match: { aggregations.percentiles_int.values.25\.0: 26.0 } - - match: { aggregations.percentiles_int.values.50\.0: 76.0 } + - close_to: { aggregations.percentiles_int.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.25\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.50\.0: { value: 101.0, error: 0.1 } } - - match: { aggregations.percentiles_double.values.5\.0: 1.0 } - - match: { aggregations.percentiles_double.values.25\.0: 26.0 } - - match: { aggregations.percentiles_double.values.50\.0: 76.0 } + - close_to: { aggregations.percentiles_double.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.25\.0: { value: 51.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.50\.0: { value: 101.0, error: 0.1 } } --- "Non-keyed test": + - skip: + features: close_to - do: search: - rest_total_hits_as_int: true + index: test_1 body: aggs: percentiles_int: percentiles: field: int_field - percents: [5.0, 25.0, 50.0] + percents: [5.0, 25.0, 50.0, 100.0] keyed: false - - match: { hits.total: 4 } + - match: { hits.total.value: 4 } - length: { hits.hits: 4 } - match: { aggregations.percentiles_int.values.0.key: 5.0 } - - match: { aggregations.percentiles_int.values.0.value: 1.0 } + - close_to: { aggregations.percentiles_int.values.0.value: { value: 1.0, error: 0.1 } } - match: { aggregations.percentiles_int.values.1.key: 25.0 } - - match: { aggregations.percentiles_int.values.1.value: 26.0 } + - close_to: { aggregations.percentiles_int.values.1.value: { value: 51.0, error: 0.1 } } - match: { aggregations.percentiles_int.values.2.key: 50.0 } - - match: { aggregations.percentiles_int.values.2.value: 76.0 } + - close_to: { aggregations.percentiles_int.values.2.value: { value: 101.0, error: 0.1 } } + - match: { aggregations.percentiles_int.values.3.key: 100.0 } + - close_to: { aggregations.percentiles_int.values.3.value: { value: 151.0, error: 0.1 } } + +--- +"Multi-value fields test": + - skip: + features: close_to + + - do: + search: + index: test_2 + body: + aggs: + percentiles_int: + percentiles: + field: int_field + hdr: {} + percentiles_double: + percentiles: + field: double_field + hdr: {} + + + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + + - close_to: { aggregations.percentiles_int.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.25\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.50\.0: { value: 2.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.75\.0: { value: 2.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.95\.0: { value: 3.0, error: 0.1 } } + - close_to: { aggregations.percentiles_int.values.99\.0: { value: 3.0, error: 0.1 } } + + - close_to: { aggregations.percentiles_double.values.1\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.5\.0: { value: 1.0, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.25\.0: { value: 1.19, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.50\.0: { value: 1.19, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.75\.0: { value: 1.79, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.95\.0: { value: 2.20, error: 0.1 } } + - close_to: { aggregations.percentiles_double.values.99\.0: { value: 2.20, error: 0.1 } } diff --git a/server/build.gradle b/server/build.gradle index 9e1985d5643c5..9fbac68914ffb 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -54,9 +54,9 @@ dependencies { implementation 'com.carrotsearch:hppc:0.8.1' // percentiles aggregation - api 'com.tdunning:t-digest:3.2' + api 'com.tdunning:t-digest:3.3' // precentil ranks aggregation - api 'org.hdrhistogram:HdrHistogram:2.1.9' + api 'org.hdrhistogram:HdrHistogram:2.1.12' // logging api "org.apache.logging.log4j:log4j-api:${versions.log4j}" @@ -228,7 +228,6 @@ tasks.named("thirdPartyAudit").configure { 'org.zeromq.ZMQ$Socket', 'org.zeromq.ZMQ', ) - ignoreMissingClasses 'javax.xml.bind.DatatypeConverter' } tasks.named("dependencyLicenses").configure { diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HDRPreAggregatedPercentilesAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HDRPreAggregatedPercentilesAggregatorTests.java index d15230c4ae65b..364d257bc93e1 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HDRPreAggregatedPercentilesAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HDRPreAggregatedPercentilesAggregatorTests.java @@ -31,15 +31,19 @@ import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Random; import java.util.function.Consumer; import static java.util.Collections.singleton; public class HDRPreAggregatedPercentilesAggregatorTests extends AggregatorTestCase { + protected Random random = new Random(17); + @Override protected List getSearchPlugins() { return List.of(new AnalyticsPlugin()); @@ -61,6 +65,12 @@ protected List getSupportedValuesSourceTypes() { ); } + private double[] makeDoubleArray(int size, int bound) { + double[] values = new double[size]; + Arrays.setAll(values, i -> random.nextDouble(bound)); + return values; + } + private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { DoubleHistogram histogram = new DoubleHistogram(3);// default for (double value : values) { @@ -101,14 +111,14 @@ public void testEmptyField() throws IOException { public void testSomeMatchesBinaryDocValues() throws IOException { testCase( new FieldExistsQuery("number"), - iw -> { iw.addDocument(singleton(getDocValue("number", new double[] { 60, 40, 20, 10 }))); }, + iw -> { iw.addDocument(singleton(getDocValue("number", makeDoubleArray(100_000, 100)))); }, hdr -> { // assertEquals(4L, hdr.state.getTotalCount()); - double approximation = 0.05d; - assertEquals(10.0d, hdr.percentile(25), approximation); - assertEquals(20.0d, hdr.percentile(50), approximation); - assertEquals(40.0d, hdr.percentile(75), approximation); - assertEquals(60.0d, hdr.percentile(99), approximation); + double approximation = 1.0d; + assertEquals(25.0d, hdr.percentile(25), approximation); + assertEquals(50.0d, hdr.percentile(50), approximation); + assertEquals(75.0d, hdr.percentile(75), approximation); + assertEquals(99.0d, hdr.percentile(99), approximation); assertTrue(AggregationInspectionHelper.hasValue(hdr)); } ); @@ -116,17 +126,17 @@ public void testSomeMatchesBinaryDocValues() throws IOException { public void testSomeMatchesMultiBinaryDocValues() throws IOException { testCase(new FieldExistsQuery("number"), iw -> { - iw.addDocument(singleton(getDocValue("number", new double[] { 60, 40, 20, 10 }))); - iw.addDocument(singleton(getDocValue("number", new double[] { 60, 40, 20, 10 }))); - iw.addDocument(singleton(getDocValue("number", new double[] { 60, 40, 20, 10 }))); - iw.addDocument(singleton(getDocValue("number", new double[] { 60, 40, 20, 10 }))); + iw.addDocument(singleton(getDocValue("number", makeDoubleArray(25_000, 100)))); + iw.addDocument(singleton(getDocValue("number", makeDoubleArray(25_000, 100)))); + iw.addDocument(singleton(getDocValue("number", makeDoubleArray(25_000, 100)))); + iw.addDocument(singleton(getDocValue("number", makeDoubleArray(25_000, 100)))); }, hdr -> { // assertEquals(16L, hdr.state.getTotalCount()); - double approximation = 0.05d; - assertEquals(10.0d, hdr.percentile(25), approximation); - assertEquals(20.0d, hdr.percentile(50), approximation); - assertEquals(40.0d, hdr.percentile(75), approximation); - assertEquals(60.0d, hdr.percentile(99), approximation); + double approximation = 1.0d; + assertEquals(25.0d, hdr.percentile(25), approximation); + assertEquals(50.0d, hdr.percentile(50), approximation); + assertEquals(75.0d, hdr.percentile(75), approximation); + assertEquals(99.0d, hdr.percentile(99), approximation); assertTrue(AggregationInspectionHelper.hasValue(hdr)); }); } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentilesAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentilesAggregatorTests.java index 5d810b16ac9a5..4b1e6b87f5511 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentilesAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentilesAggregatorTests.java @@ -27,8 +27,10 @@ import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Random; import java.util.function.Consumer; import static java.util.Collections.singleton; @@ -36,6 +38,8 @@ public class TDigestPreAggregatedPercentilesAggregatorTests extends AggregatorTestCase { + protected Random random = new Random(17); + @Override protected List getSearchPlugins() { return List.of(new AnalyticsPlugin()); @@ -46,6 +50,12 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy return new PercentilesAggregationBuilder("tdigest_percentiles").field(fieldName).percentilesConfig(new PercentilesConfig.TDigest()); } + private double[] makeDoubleArray(int size, int bound) { + double[] values = new double[size]; + Arrays.setAll(values, i -> random.nextDouble(bound)); + return values; + } + @Override protected List getSupportedValuesSourceTypes() { // Note: this is the same list as Core, plus Analytics @@ -79,14 +89,14 @@ public void testEmptyField() throws IOException { public void testSomeMatchesBinaryDocValues() throws IOException { testCase( new FieldExistsQuery("number"), - iw -> { iw.addDocument(singleton(histogramFieldDocValues("number", new double[] { 60, 40, 20, 10 }))); }, + iw -> { iw.addDocument(singleton(histogramFieldDocValues("number", makeDoubleArray(100_000, 100)))); }, hdr -> { // assertEquals(4L, hdr.state.getTotalCount()); - double approximation = 0.05d; - assertEquals(15.0d, hdr.percentile(25), approximation); - assertEquals(30.0d, hdr.percentile(50), approximation); - assertEquals(50.0d, hdr.percentile(75), approximation); - assertEquals(60.0d, hdr.percentile(99), approximation); + double approximation = 1.0d; + assertEquals(25.0d, hdr.percentile(25), approximation); + assertEquals(50.0d, hdr.percentile(50), approximation); + assertEquals(75.0d, hdr.percentile(75), approximation); + assertEquals(99.0d, hdr.percentile(100), approximation); assertTrue(AggregationInspectionHelper.hasValue(hdr)); } ); @@ -94,17 +104,17 @@ public void testSomeMatchesBinaryDocValues() throws IOException { public void testSomeMatchesMultiBinaryDocValues() throws IOException { testCase(new FieldExistsQuery("number"), iw -> { - iw.addDocument(singleton(histogramFieldDocValues("number", new double[] { 60, 40, 20, 10 }))); - iw.addDocument(singleton(histogramFieldDocValues("number", new double[] { 60, 40, 20, 10 }))); - iw.addDocument(singleton(histogramFieldDocValues("number", new double[] { 60, 40, 20, 10 }))); - iw.addDocument(singleton(histogramFieldDocValues("number", new double[] { 60, 40, 20, 10 }))); + iw.addDocument(singleton(histogramFieldDocValues("number", makeDoubleArray(25_000, 100)))); + iw.addDocument(singleton(histogramFieldDocValues("number", makeDoubleArray(25_000, 100)))); + iw.addDocument(singleton(histogramFieldDocValues("number", makeDoubleArray(25_000, 100)))); + iw.addDocument(singleton(histogramFieldDocValues("number", makeDoubleArray(25_000, 100)))); }, hdr -> { // assertEquals(16L, hdr.state.getTotalCount()); - double approximation = 0.05d; - assertEquals(15.0d, hdr.percentile(25), approximation); - assertEquals(30.0d, hdr.percentile(50), approximation); - assertEquals(50.0d, hdr.percentile(75), approximation); - assertEquals(60.0d, hdr.percentile(99), approximation); + double approximation = 1.0d; + assertEquals(25.0d, hdr.percentile(25), approximation); + assertEquals(50.0d, hdr.percentile(50), approximation); + assertEquals(75.0d, hdr.percentile(75), approximation); + assertEquals(99.0d, hdr.percentile(99), approximation); assertTrue(AggregationInspectionHelper.hasValue(hdr)); }); } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java index 2abdd3a6ace64..d3d931a0a632c 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregatorTests.java @@ -107,7 +107,7 @@ public void testMatchesSortedNumericDocValues() throws IOException { assertEquals(2, boxplot.getMin(), 0); assertEquals(10, boxplot.getMax(), 0); assertEquals(2, boxplot.getQ1(), 0); - assertEquals(3.5, boxplot.getQ2(), 0); + assertEquals(4, boxplot.getQ2(), 0); assertEquals(5, boxplot.getQ3(), 0); }); } @@ -124,7 +124,7 @@ public void testMatchesNumericDocValues() throws IOException { assertEquals(2, boxplot.getMin(), 0); assertEquals(10, boxplot.getMax(), 0); assertEquals(2, boxplot.getQ1(), 0); - assertEquals(3.5, boxplot.getQ2(), 0); + assertEquals(4, boxplot.getQ2(), 0); assertEquals(5, boxplot.getQ3(), 0); }); } @@ -142,7 +142,7 @@ public void testSomeMatchesSortedNumericDocValues() throws IOException { assertEquals(2, boxplot.getMin(), 0); assertEquals(10, boxplot.getMax(), 0); assertEquals(2, boxplot.getQ1(), 0); - assertEquals(3.5, boxplot.getQ2(), 0); + assertEquals(4, boxplot.getQ2(), 0); assertEquals(5, boxplot.getQ3(), 0); }); } @@ -160,7 +160,7 @@ public void testSomeMatchesNumericDocValues() throws IOException { assertEquals(2, boxplot.getMin(), 0); assertEquals(10, boxplot.getMax(), 0); assertEquals(2, boxplot.getQ1(), 0); - assertEquals(3.5, boxplot.getQ2(), 0); + assertEquals(4, boxplot.getQ2(), 0); assertEquals(5, boxplot.getQ3(), 0); }); } @@ -277,7 +277,7 @@ public void testEmptyBucket() throws IOException { assertEquals(1, boxplot.getMin(), 0); assertEquals(3, boxplot.getMax(), 0); assertEquals(1, boxplot.getQ1(), 0); - assertEquals(2, boxplot.getQ2(), 0); + assertEquals(3, boxplot.getQ2(), 0); assertEquals(3, boxplot.getQ3(), 0); assertNotNull(histo.getBuckets().get(1).getAggregations().asMap().get("boxplot")); @@ -293,7 +293,7 @@ public void testEmptyBucket() throws IOException { assertEquals(21, boxplot.getMin(), 0); assertEquals(23, boxplot.getMax(), 0); assertEquals(21, boxplot.getQ1(), 0); - assertEquals(22, boxplot.getQ2(), 0); + assertEquals(23, boxplot.getQ2(), 0); assertEquals(23, boxplot.getQ3(), 0); }, new AggTestConfig(histogram, fieldType)); } @@ -312,14 +312,14 @@ public void testFormatter() throws IOException { }, (Consumer) boxplot -> { assertEquals(1, boxplot.getMin(), 0); assertEquals(5, boxplot.getMax(), 0); - assertEquals(1.75, boxplot.getQ1(), 0); - assertEquals(3, boxplot.getQ2(), 0); - assertEquals(4.25, boxplot.getQ3(), 0); + assertEquals(2.0, boxplot.getQ1(), 0); + assertEquals(3.0, boxplot.getQ2(), 0); + assertEquals(4.0, boxplot.getQ3(), 0); assertEquals("0001.0", boxplot.getMinAsString()); assertEquals("0005.0", boxplot.getMaxAsString()); - assertEquals("0001.8", boxplot.getQ1AsString()); + assertEquals("0002.0", boxplot.getQ1AsString()); assertEquals("0003.0", boxplot.getQ2AsString()); - assertEquals("0004.2", boxplot.getQ3AsString()); + assertEquals("0004.0", boxplot.getQ3AsString()); }, new AggTestConfig(aggregationBuilder, fieldType)); } @@ -362,7 +362,7 @@ public void testValueScript() throws IOException { assertEquals(2, boxplot.getMin(), 0); assertEquals(8, boxplot.getMax(), 0); assertEquals(2, boxplot.getQ1(), 0); - assertEquals(5, boxplot.getQ2(), 0); + assertEquals(8, boxplot.getQ2(), 0); assertEquals(8, boxplot.getQ3(), 0); }, new AggTestConfig(aggregationBuilder, fieldType)); } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionEvaluationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionEvaluationIT.java index 3356604a16f84..3b695e128dff6 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionEvaluationIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionEvaluationIT.java @@ -100,13 +100,13 @@ private AucRoc.Result evaluateAucRoc(String actualField, String predictedField, public void testEvaluate_AucRoc_DoNotIncludeCurve() { AucRoc.Result aucrocResult = evaluateAucRoc(IS_PREDATOR_BOOLEAN_FIELD, IS_PREDATOR_PREDICTION_PROBABILITY_FIELD, false); - assertThat(aucrocResult.getValue(), is(closeTo(0.98, 0.001))); + assertThat(aucrocResult.getValue(), is(closeTo(0.978, 0.001))); assertThat(aucrocResult.getCurve(), hasSize(0)); } public void testEvaluate_AucRoc_IncludeCurve() { AucRoc.Result aucrocResult = evaluateAucRoc(IS_PREDATOR_BOOLEAN_FIELD, IS_PREDATOR_PREDICTION_PROBABILITY_FIELD, true); - assertThat(aucrocResult.getValue(), is(closeTo(0.98, 0.001))); + assertThat(aucrocResult.getValue(), is(closeTo(0.978, 0.001))); assertThat(aucrocResult.getCurve(), hasSize(greaterThan(0))); } diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java index b94371e35c61a..c03745bc09ab6 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java @@ -303,7 +303,7 @@ else if (type == Types.DATE) { } // and floats/doubles else if (type == Types.DOUBLE) { - assertEquals(msg, (double) expectedObject, (double) actualObject, lenientFloatingNumbers ? 1d : 0.0d); + assertEquals(msg, (double) expectedObject, (double) actualObject, 1d); } else if (type == Types.FLOAT) { assertEquals(msg, (float) expectedObject, (float) actualObject, lenientFloatingNumbers ? 1f : 0.0f); } else if (type == Types.OTHER) { diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/agg.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/agg.csv-spec index 6f7255755c3ff..1d9758ec40e77 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/agg.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/agg.csv-spec @@ -1062,7 +1062,7 @@ FROM test_emp; max | min | avg | sum | mad ------------------+---------------+-----------------+------------------+----------------- -155409.30000000002|23532.72 |964937.9295477575|-6307004.517507723|30811.76199261993 +155409.30000000002|23532.72 |964937.9295477575|-6307004.517507723|31892.952029520304 ; countWithScalars diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/constant-keyword.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/constant-keyword.csv-spec index 1c6dc0ac72dd4..adb6c556f366a 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/constant-keyword.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/constant-keyword.csv-spec @@ -89,7 +89,7 @@ SELECT extra_gender, FLOOR(AVG(salary)) AS avg, FLOOR(MAD(salary)) AS mad_s, MAD extra_gender | avg | mad_s | mad_l ---------------+-------------+-------------+--------------- -Female |48248 |10096 |1 +Female |48248 |10299 |1 ; caseGroupByAndHaving diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec index 9ce850f102066..3bd5d82b817e0 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec @@ -1553,10 +1553,10 @@ GROUP BY languages; ---------------+---------------+--------------- null |74999.0 |74992.0 1 |73717.0 |73712.0 -2 |73530.238 |69936.0 +2 |73578.0 |73584.0 3 |74970.0 |74992.0 4 |74572.0 |74608.0 -5 |66117.118 |56368.0 +5 |66817.0 |66864.0 // end::aggPercentileWithPercentileConfig ; diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/field-alias.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/field-alias.csv-spec index 6df5e02e63d17..cb59a73822853 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/field-alias.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/field-alias.csv-spec @@ -72,7 +72,7 @@ SELECT extra.info.gender AS gender, PERCENTILE(emp_no, 92.45) p1, PERCENTILE(emp gender:s | p1:d | p2:d null |10018.745 |10018.599999999999 F |10098.0085 |10096.119999999999 -M |10091.393 |10090.37 +M |10096.119999999999 |10090.37 ; multiplePercentilesWithoutComma @@ -80,7 +80,7 @@ SELECT extra.info.gender AS gender, PERCENTILE(emp_no, 91) p1, PERCENTILE(emp_no gender:s | p1:d | p2:d null |10018.599999999999 |10018.4 -F |10096.119999999999 |10093.74 +F |10098.0 |10093.74 M |10090.37 |10086.92 ; diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.csv-spec index 2bbde74ea06cc..10f4969df4723 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.csv-spec @@ -175,7 +175,7 @@ SELECT gender, MAD(salary_ul) AS mad FROM test_emp_copy GROUP BY gender ORDER BY gender | mad ---------------+--------------- -null |10789.0 +null |11805.0 F |12719.0 M |8905.0 ; @@ -232,7 +232,7 @@ FROM test_emp_copy; max | min | avg | sum | mad ------------------+---------------+-----------------+-----------------+----------------- -155409.30000000002|23532.72 |964937.9295477575|6306995.482492277|30811.76199261993 +155409.30000000002|23532.72 |964937.9295477575|6306995.482492277|31892.952029520304 ; aggregatesWithScalarsAndGroupByOrderByAggWithoutProjection @@ -255,9 +255,9 @@ GROUP BY gender ORDER BY gender; percentile | percentile_rank | gender -----------------+------------------+--------------- -86857.79999999999|32.69659025378865 |null -94042.92000000001|37.03569653103581 |F -87348.36 |44.337514210592246|M +88443.59999999999|30.0 |null +94222.8 |37.03569653103581 |F +87303.59999999999|45.614035087719294|M ; extendedStatsAggregateFunctionsWithScalars diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/boxplot.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/boxplot.yml index 3247920e29d83..326592c7dd5e4 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/boxplot.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/boxplot.yml @@ -55,9 +55,9 @@ setup: - match: { aggregations.plot.min: 2.0 } - match: { aggregations.plot.max: 15.0 } - - match: { aggregations.plot.q1: 2.75 } + - match: { aggregations.plot.q1: 3.0 } - match: { aggregations.plot.q2: 5.0 } - - match: { aggregations.plot.q3: 11.25 } + - match: { aggregations.plot.q3: 10.0 } --- "Search With Runtime Field": @@ -84,9 +84,9 @@ setup: - match: { aggregations.plot.min: 20.0 } - match: { aggregations.plot.max: 10000.0 } - - match: { aggregations.plot.q1: 27.5 } + - match: { aggregations.plot.q1: 30.0 } - match: { aggregations.plot.q2: 50.0 } - - match: { aggregations.plot.q3: 2612.5 } + - match: { aggregations.plot.q3: 150.0 } - match: { aggregations.plot.lower: 20.0 } - match: { aggregations.plot.upper: 150.0 } @@ -107,7 +107,7 @@ setup: - match: { aggregations.plot.min: 2.0 } - match: { aggregations.plot.max: 10000.0 } - match: { aggregations.plot.q1: 3 } - - match: { aggregations.plot.q2: 7.5 } + - match: { aggregations.plot.q2: 10.0 } - match: { aggregations.plot.q3: 15.0 } - match: { aggregations.plot.lower: 2.0 } - match: { aggregations.plot.upper: 15.0 } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/moving_percentiles.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/moving_percentiles.yml index 7ed941571642a..412da57e7dad4 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/moving_percentiles.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/moving_percentiles.yml @@ -62,6 +62,8 @@ setup: --- shift: + - skip: + features: close_to - do: search: index: "foo" @@ -87,19 +89,21 @@ shift: - length: { aggregations.histo.buckets: 3 } - match: { aggregations.histo.buckets.0.key_as_string: "2017-01-01T00:00:00.000Z" } - match: { aggregations.histo.buckets.0.doc_count: 3 } - - match: { aggregations.histo.buckets.0.percentiles.values.0.value: 4.0 } - - match: { aggregations.histo.buckets.0.moving_percentiles.values.0.value: 4.0 } + - close_to: { aggregations.histo.buckets.0.percentiles.values.0.value: { value: 4.0, error: 0.1 } } + - close_to: { aggregations.histo.buckets.0.moving_percentiles.values.0.value: { value: 4.0, error: 0.1 } } - match: { aggregations.histo.buckets.1.key_as_string: "2017-01-02T00:00:00.000Z" } - match: { aggregations.histo.buckets.1.doc_count: 2 } - - match: { aggregations.histo.buckets.1.percentiles.values.0.value: 5.0 } - - match: { aggregations.histo.buckets.1.moving_percentiles.values.0.value: 4.0 } + - close_to: { aggregations.histo.buckets.1.percentiles.values.0.value: { value: 5.2, error: 0.1 } } + - close_to: { aggregations.histo.buckets.1.moving_percentiles.values.0.value: { value: 4.0, error: 0.1 } } - match: { aggregations.histo.buckets.2.key_as_string: "2017-01-03T00:00:00.000Z" } - match: { aggregations.histo.buckets.2.doc_count: 1 } - - match: { aggregations.histo.buckets.2.percentiles.values.0.value: 7.0 } - - match: { aggregations.histo.buckets.2.moving_percentiles.values.0.value: 4.0 } + - close_to: { aggregations.histo.buckets.2.percentiles.values.0.value: { value: 7.6, error: 0.1 } } + - close_to: { aggregations.histo.buckets.2.moving_percentiles.values.0.value: { value: 4.0, error: 0.1 } } --- default choses tdigest: + - skip: + features: close_to - do: search: index: "foo" @@ -124,19 +128,21 @@ default choses tdigest: - length: { aggregations.histo.buckets: 3 } - match: { aggregations.histo.buckets.0.key_as_string: "2017-01-01T00:00:00.000Z" } - match: { aggregations.histo.buckets.0.doc_count: 3 } - - match: { aggregations.histo.buckets.0.percentiles.values.0.value: 4.0 } + - close_to: { aggregations.histo.buckets.0.percentiles.values.0.value: { value: 4.0, error: 0.1 } } - is_false: aggregations.histo.buckets.0.moving_percentiles - match: { aggregations.histo.buckets.1.key_as_string: "2017-01-02T00:00:00.000Z" } - match: { aggregations.histo.buckets.1.doc_count: 2 } - - match: { aggregations.histo.buckets.1.percentiles.values.0.value: 5.0 } - - match: { aggregations.histo.buckets.1.moving_percentiles.values.0.value: 4.0 } + - close_to: { aggregations.histo.buckets.1.percentiles.values.0.value: { value: 5.2, error: 0.1 } } + - close_to: { aggregations.histo.buckets.1.moving_percentiles.values.0.value: { value: 4.0, error: 0.1 } } - match: { aggregations.histo.buckets.2.key_as_string: "2017-01-03T00:00:00.000Z" } - match: { aggregations.histo.buckets.2.doc_count: 1 } - - match: { aggregations.histo.buckets.2.percentiles.values.0.value: 7.0 } - - match: { aggregations.histo.buckets.2.moving_percentiles.values.0.value: 4.0 } + - close_to: { aggregations.histo.buckets.2.percentiles.values.0.value: { value: 7.0, error: 0.1 } } + - close_to: { aggregations.histo.buckets.2.moving_percentiles.values.0.value: { value: 4.0, error: 0.1 } } --- tdest explicitly: + - skip: + features: close_to - do: search: index: "foo" @@ -162,19 +168,21 @@ tdest explicitly: - length: { aggregations.histo.buckets: 3 } - match: { aggregations.histo.buckets.0.key_as_string: "2017-01-01T00:00:00.000Z" } - match: { aggregations.histo.buckets.0.doc_count: 3 } - - match: { aggregations.histo.buckets.0.percentiles.values.0.value: 4.0 } + - close_to: { aggregations.histo.buckets.0.percentiles.values.0.value: { value: 4.0, error: 0.1 } } - is_false: aggregations.histo.buckets.0.moving_percentiles - match: { aggregations.histo.buckets.1.key_as_string: "2017-01-02T00:00:00.000Z" } - match: { aggregations.histo.buckets.1.doc_count: 2 } - - match: { aggregations.histo.buckets.1.percentiles.values.0.value: 5.0 } - - match: { aggregations.histo.buckets.1.moving_percentiles.values.0.value: 4.0 } + - close_to: { aggregations.histo.buckets.1.percentiles.values.0.value: { value: 5.2, error: 0.1 } } + - close_to: { aggregations.histo.buckets.1.moving_percentiles.values.0.value: { value: 4.0, error: 0.1 } } - match: { aggregations.histo.buckets.2.key_as_string: "2017-01-03T00:00:00.000Z" } - match: { aggregations.histo.buckets.2.doc_count: 1 } - - match: { aggregations.histo.buckets.2.percentiles.values.0.value: 7.0 } - - match: { aggregations.histo.buckets.2.moving_percentiles.values.0.value: 4.0 } + - close_to: { aggregations.histo.buckets.2.percentiles.values.0.value: { value: 7.6, error: 0.1 } } + - close_to: { aggregations.histo.buckets.2.moving_percentiles.values.0.value: { value: 4.0, error: 0.1 } } --- hdr: + - skip: + features: close_to - do: search: index: "foo" @@ -201,16 +209,16 @@ hdr: - length: { aggregations.histo.buckets: 3 } - match: { aggregations.histo.buckets.0.key_as_string: "2017-01-01T00:00:00.000Z" } - match: { aggregations.histo.buckets.0.doc_count: 3 } - - match: { aggregations.histo.buckets.0.percentiles.values.0.value: 4.24609375 } + - close_to: { aggregations.histo.buckets.0.percentiles.values.0.value: { value: 4.2, error: 0.1 } } - is_false: aggregations.histo.buckets.0.moving_percentiles - match: { aggregations.histo.buckets.1.key_as_string: "2017-01-02T00:00:00.000Z" } - match: { aggregations.histo.buckets.1.doc_count: 2 } - - match: { aggregations.histo.buckets.1.percentiles.values.0.value: 4.24609375 } - - match: { aggregations.histo.buckets.1.moving_percentiles.values.0.value: 4.24609375 } + - close_to: { aggregations.histo.buckets.1.percentiles.values.0.value: { value: 4.2, error: 0.1 } } + - close_to: { aggregations.histo.buckets.1.moving_percentiles.values.0.value: { value: 4.2, error: 0.1 } } - match: { aggregations.histo.buckets.2.key_as_string: "2017-01-03T00:00:00.000Z" } - match: { aggregations.histo.buckets.2.doc_count: 1 } - - match: { aggregations.histo.buckets.2.percentiles.values.0.value: 4.24609375 } - - match: { aggregations.histo.buckets.2.moving_percentiles.values.0.value: 4.24609375 } + - close_to: { aggregations.histo.buckets.2.percentiles.values.0.value: { value: 4.2, error: 0.1 } } + - close_to: { aggregations.histo.buckets.2.moving_percentiles.values.0.value: { value: 4.2, error: 0.1 } } --- bad path: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/percentiles_hdr.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/percentiles_hdr.yml new file mode 100644 index 0000000000000..fc836c148e6fd --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/percentiles_hdr.yml @@ -0,0 +1,162 @@ +--- +"percentiles hdr numeric field": + - skip: + version: " - 8.6.99" + reason: fixed in 8.7.0 + features: close_to + + - do: + indices.create: + index: test_1 + body: + mappings: + properties: + latency: + type: integer + + - do: + bulk: + index: test_1 + refresh: true + body: + - '{"index": {}}' + - '{"latency": 100}' + - '{"index": {}}' + - '{"latency": 120}' + - '{"index": {}}' + - '{"latency": 150}' + - '{"index": {}}' + - '{"latency": 140}' + - '{"index": {}}' + - '{"latency": 110}' + - '{"index": {}}' + - '{"latency": 110}' + - '{"index": {}}' + - '{"latency": 120}' + - '{"index": {}}' + - '{"latency": 160}' + - '{"index": {}}' + - '{"latency": 120}' + - '{"index": {}}' + - '{"latency": 100}' + - '{"index": {}}' + - '{"latency": 100}' + - '{"index": {}}' + - '{"latency": 130}' + - '{"index": {}}' + - '{"latency": 140}' + - '{"index": {}}' + - '{"latency": 130}' + - '{"index": {}}' + - '{"latency": 120}' + - '{"index": {}}' + - '{"latency": 160}' + + - do: + search: + index: test_1 + body: + aggs: + percentiles_histo: + percentiles: + field: latency + percents: [10.0, 50.0, 95.0, 96.0, 97.0, 98.0, 99.0, 100.0] + + - match: { hits.total.value: 16 } + - match: { hits.total.relation: "eq" } + - close_to: { aggregations.percentiles_histo.values.10\\.0: { value: 100.0, error: 0.000001 } } + - close_to: { aggregations.percentiles_histo.values.50\\.0: { value: 120.0, error: 0.000001 } } + - close_to: { aggregations.percentiles_histo.values.95\\.0: { value: 160.0, error: 0.000001 } } + - close_to: { aggregations.percentiles_histo.values.96\\.0: { value: 160.0, error: 0.000001 } } + - close_to: { aggregations.percentiles_histo.values.97\\.0: { value: 160.0, error: 0.000001 } } + - close_to: { aggregations.percentiles_histo.values.98\\.0: { value: 160.0, error: 0.000001 } } + - close_to: { aggregations.percentiles_histo.values.99\\.0: { value: 160.0, error: 0.000001 } } + - close_to: { aggregations.percentiles_histo.values.100\\.0: { value: 160.0, error: 0.000001 } } + +--- +"percentiles hdr histogram field": + - skip: + version: " - 8.6.99" + reason: fixed in 8.7.0 + features: close_to + + - do: + indices.create: + index: test_2 + body: + mappings: + properties: + latency: + type: histogram + + - do: + bulk: + index: test_2 + refresh: true + body: + - '{"index": {}}' + - '{"latency": {"counts": [10, 10, 10, 10, 10, 10, 10, 10, 10, 10], "values": [0.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0]}}' + + - do: + search: + index: test_2 + body: + aggs: + percentiles_histo: + percentiles: + hdr: {} + field: latency + percents: [10.0, 50.0, 95.0, 96.0, 97.0, 98.0, 99.0, 100.0] + + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - close_to: { aggregations.percentiles_histo.values.10\\.0: { value: 0.0, error: 0.1 } } + - close_to: { aggregations.percentiles_histo.values.50\\.0: { value: 40.02, error: 0.1 } } + - close_to: { aggregations.percentiles_histo.values.95\\.0: { value: 90.05, error: 0.1 } } + - close_to: { aggregations.percentiles_histo.values.96\\.0: { value: 90.05, error: 0.1 }} + - close_to: { aggregations.percentiles_histo.values.97\\.0: { value: 90.05, error: 0.1 } } + - close_to: { aggregations.percentiles_histo.values.98\\.0: { value: 90.05, error: 0.1 } } + - close_to: { aggregations.percentiles_histo.values.99\\.0: { value: 90.05, error: 0.1 } } + - close_to: { aggregations.percentiles_histo.values.100\\.0: { value: 90.05, error: 0.1 } } + +--- +"percentiles hdr histogram field large count sum": + - skip: + version: " - 8.6.99" + reason: fixed in 8.7.0 + features: close_to + + - do: + indices.create: + index: test_3 + body: + mappings: + properties: + latency: + type: histogram + + - do: + bulk: + index: test_3 + refresh: true + body: + - '{"index": {}}' + - '{"latency": {"counts": [2100000000, 300000000], "values": [1, 2]}}' + + - do: + search: + index: test_3 + body: + aggs: + percentiles_histo: + percentiles: + hdr: {} + field: latency + percents: [90.0, 95.0, 99.0] + + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - close_to: { aggregations.percentiles_histo.values.90\\.0: { value: 2.0, error: 0.1 } } + - close_to: { aggregations.percentiles_histo.values.95\\.0: { value: 2.0, error: 0.1 } } + - close_to: { aggregations.percentiles_histo.values.99\\.0: { value: 2.0, error: 0.1 } } + diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/percentiles_tdigest.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/percentiles_tdigest.yml new file mode 100644 index 0000000000000..a71b885a3cf52 --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/percentiles_tdigest.yml @@ -0,0 +1,160 @@ +--- +"percentiles t-digest numeric field": + - skip: + version: " - 8.6.99" + reason: fixed in 8.7.0 + features: close_to + + - do: + indices.create: + index: test_1 + body: + mappings: + properties: + latency: + type: integer + + - do: + bulk: + index: test_1 + refresh: true + body: + - '{"index": {}}' + - '{"latency": 100}' + - '{"index": {}}' + - '{"latency": 120}' + - '{"index": {}}' + - '{"latency": 150}' + - '{"index": {}}' + - '{"latency": 140}' + - '{"index": {}}' + - '{"latency": 110}' + - '{"index": {}}' + - '{"latency": 110}' + - '{"index": {}}' + - '{"latency": 120}' + - '{"index": {}}' + - '{"latency": 160}' + - '{"index": {}}' + - '{"latency": 120}' + - '{"index": {}}' + - '{"latency": 100}' + - '{"index": {}}' + - '{"latency": 100}' + - '{"index": {}}' + - '{"latency": 130}' + - '{"index": {}}' + - '{"latency": 140}' + - '{"index": {}}' + - '{"latency": 130}' + - '{"index": {}}' + - '{"latency": 120}' + - '{"index": {}}' + - '{"latency": 160}' + + - do: + search: + index: test_1 + body: + aggs: + percentiles_histo: + percentiles: + field: latency + percents: [10.0, 50.0, 95.0, 96.0, 97.0, 98.0, 99.0, 100.0] + + - match: { hits.total.value: 16 } + - match: { hits.total.relation: "eq" } + - close_to: { aggregations.percentiles_histo.values.10\\.0: { value: 100.0, error: 0.000001 } } + - close_to: { aggregations.percentiles_histo.values.50\\.0: { value: 120.0, error: 0.000001 } } + - close_to: { aggregations.percentiles_histo.values.95\\.0: { value: 160.0, error: 0.000001 } } + - close_to: { aggregations.percentiles_histo.values.96\\.0: { value: 160.0, error: 0.000001 } } + - close_to: { aggregations.percentiles_histo.values.97\\.0: { value: 160.0, error: 0.000001 } } + - close_to: { aggregations.percentiles_histo.values.98\\.0: { value: 160.0, error: 0.000001 } } + - close_to: { aggregations.percentiles_histo.values.99\\.0: { value: 160.0, error: 0.000001 } } + - close_to: { aggregations.percentiles_histo.values.100\\.0: { value: 160.0, error: 0.000001 } } + +--- +"percentiles t-digest histogram field": + - skip: + version: " - 8.6.99" + reason: fixed in 8.7.0 + features: close_to + + - do: + indices.create: + index: test_2 + body: + mappings: + properties: + latency: + type: histogram + + - do: + bulk: + index: test_2 + refresh: true + body: + - '{"index": {}}' + - '{"latency": {"counts": [10, 10, 10, 10, 10, 10, 10, 10, 10, 10], "values": [0.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0]}}' + + - do: + search: + index: test_2 + body: + aggs: + percentiles_histo: + percentiles: + field: latency + percents: [10.0, 50.0, 95.0, 96.0, 97.0, 98.0, 99.0, 100.0] + + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - close_to: { aggregations.percentiles_histo.values.10\\.0: { value: 5.0, error: 0.1 } } + - close_to: { aggregations.percentiles_histo.values.50\\.0: { value: 45.0, error: 0.1 } } + - close_to: { aggregations.percentiles_histo.values.95\\.0: { value: 90.0, error: 0.1 } } + - close_to: { aggregations.percentiles_histo.values.96\\.0: { value: 90.0, error: 0.1 }} + - close_to: { aggregations.percentiles_histo.values.97\\.0: { value: 90.0, error: 0.1 } } + - close_to: { aggregations.percentiles_histo.values.98\\.0: { value: 90.0, error: 0.1 } } + - close_to: { aggregations.percentiles_histo.values.99\\.0: { value: 90.0, error: 0.1 } } + - close_to: { aggregations.percentiles_histo.values.100\\.0: { value: 90.0, error: 0.1 } } + +--- +"percentiles t-digest histogram field large count sum": + - skip: + version: " - 8.6.99" + reason: fixed in 8.7.0 + features: close_to + + - do: + indices.create: + index: test_3 + body: + mappings: + properties: + latency: + type: histogram + + - do: + bulk: + index: test_3 + refresh: true + body: + - '{"index": {}}' + - '{"latency": {"counts": [2100000000, 300000000], "values": [1, 2]}}' + + - do: + search: + index: test_3 + body: + aggs: + percentiles_histo: + percentiles: + field: latency + percents: [90.0, 95.0, 99.0] + + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - close_to: { aggregations.percentiles_histo.values.90\\.0: { value: 1.125, error: 0.1 } } + - close_to: { aggregations.percentiles_histo.values.95\\.0: { value: 1.125, error: 0.1 } } + - close_to: { aggregations.percentiles_histo.values.99\\.0: { value: 1.125, error: 0.1 } } + diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/evaluate_data_frame.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/evaluate_data_frame.yml index 25adbf639548a..59a2a09b33480 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/evaluate_data_frame.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/evaluate_data_frame.yml @@ -192,6 +192,10 @@ setup: --- "Test outlier_detection auc_roc": + - skip: + features: close_to + reason: "t-digest and HdrHistogram library upgrade in version 8.7.0" + version: "- 8.6.99" - do: ml.evaluate_data_frame: body: > @@ -207,11 +211,15 @@ setup: } } } - - match: { outlier_detection.auc_roc.value: 0.9919403846153846 } + - close_to: { outlier_detection.auc_roc.value: { value: 0.95, error: 0.01 } } - is_false: outlier_detection.auc_roc.curve --- "Test outlier_detection auc_roc given actual_field is int": + - skip: + features: close_to + reason: "t-digest and HdrHistogram library upgrade in version 8.7.0" + version: "- 8.6.99" - do: ml.evaluate_data_frame: body: > @@ -227,11 +235,15 @@ setup: } } } - - match: { outlier_detection.auc_roc.value: 0.9919403846153846 } + - close_to: { outlier_detection.auc_roc.value: { value: 0.95, error: 0.01 } } - is_false: outlier_detection.auc_roc.curve --- "Test outlier_detection auc_roc include curve": + - skip: + features: close_to + reason: "t-digest and HdrHistogram library upgrade in version 8.7.0." + version: " - 8.6.99" - do: ml.evaluate_data_frame: body: > @@ -247,7 +259,7 @@ setup: } } } - - match: { outlier_detection.auc_roc.value: 0.9919403846153846 } + - close_to: { outlier_detection.auc_roc.value: { value: 0.95, error: 0.01 } } - is_true: outlier_detection.auc_roc.curve --- @@ -705,6 +717,10 @@ setup: } --- "Test classification auc_roc": + - skip: + features: close_to + reason: "t-digest and HdrHistogram library upgrade in version 8.7.0" + version: " - 8.6.99" - do: ml.evaluate_data_frame: body: > @@ -722,10 +738,14 @@ setup: } } } - - match: { classification.auc_roc.value: 0.7754152761810909 } + - close_to: { classification.auc_roc.value: { value: 0.73, error: 0.01 } } - is_false: classification.auc_roc.curve --- "Test classification auc_roc with default top_classes_field": + - skip: + features: close_to + reason: "t-digest and HdrHistogram library upgrade in version 8.7.0" + version: " - 8.6.99" - do: ml.evaluate_data_frame: body: > @@ -742,7 +762,7 @@ setup: } } } - - match: { classification.auc_roc.value: 0.7754152761810909 } + - close_to: { classification.auc_roc.value: { value: 0.73, error: 0.01 } } - is_false: classification.auc_roc.curve --- "Test classification accuracy with missing predicted_field":