From c21fb6d7d4f49daae09afe2101a59d609ff173f1 Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Tue, 29 Oct 2024 19:00:19 +0100 Subject: [PATCH] Cleanup --- quesma/model/README.md | 2 +- .../auto_date_histogram.go | 4 +++- quesma/model/where_visitor.go | 20 +++++++++++++------ .../pancake_sql_query_generation_test.go | 4 ---- quesma/queryparser/pancake_transformer.go | 15 ++++++++++++-- quesma/testdata/clients/clover.go | 6 +++--- quesma/testdata/unsupported_requests.go | 15 -------------- 7 files changed, 34 insertions(+), 32 deletions(-) diff --git a/quesma/model/README.md b/quesma/model/README.md index b6b8976dd..66d0e4d62 100644 --- a/quesma/model/README.md +++ b/quesma/model/README.md @@ -12,7 +12,7 @@ More info: https://www.elastic.co/guide/en/elasticsearch/reference/current/searc Metrics aggregation | Support | Bucket aggregation | Support | Pipeline aggregation | Support | ---------------------------|:----------------------:|------------------------------|:------------------:|------------------------|:------------------:| Avg | :white_check_mark: | Adjacency matrix | :x: | Average bucket | :white_check_mark: | - Cardinality | :white_check_mark: | Auto-interval date histogram | :x: | Bucket script | :x: | + Cardinality | :white_check_mark: | Auto-interval date histogram | :wavy_dash: | Bucket script | :wavy_dash: | Extended Stats | :white_check_mark:[^1] | Categorize text | :x: | Bucket count K-S test | :x: | Avg | :white_check_mark: | Children | :x: | Bucket correlation | :x: | Boxplot | :x: | Composite | :x: | Bucket selector | :x: | diff --git a/quesma/model/bucket_aggregations/auto_date_histogram.go b/quesma/model/bucket_aggregations/auto_date_histogram.go index 5269c9adb..0266a5ef0 100644 --- a/quesma/model/bucket_aggregations/auto_date_histogram.go +++ b/quesma/model/bucket_aggregations/auto_date_histogram.go @@ -26,11 +26,13 @@ func (query *AutoDateHistogram) AggregationType() model.AggregationType { } func (query *AutoDateHistogram) TranslateSqlResponseToJson(rows []model.QueryResultRow) model.JsonMap { - fmt.Println(rows) if len(rows) == 0 { logger.WarnWithCtx(query.ctx).Msgf("no rows returned for %s", query.String()) return make(model.JsonMap, 0) } + if len(rows) != 1 { + logger.WarnWithCtx(query.ctx).Msgf("unexpected (!= 1) number of rows returned for %s: %d.", query.String(), len(rows)) + } return model.JsonMap{ "buckets": []model.JsonMap{{ "key": query.key, diff --git a/quesma/model/where_visitor.go b/quesma/model/where_visitor.go index 40de24322..1de56fa76 100644 --- a/quesma/model/where_visitor.go +++ b/quesma/model/where_visitor.go @@ -1,11 +1,19 @@ +// Copyright Quesma, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 package model -// TODO: it's not 100% full/proper implementation, but works in the client case -func FindTimestampLowerBound(expr Expr) (InfixExpr, bool) { +// FindLowerBounds returns x if there is "x>=y" or "x>y" in the WHERE clause, but only as a single top-level expression. +// (I mean by that a>=0 is fine, a>=0 AND e2 [AND ...]] is also fine, but a>=0 OR e2 is not fine.) +// We achieve that by only descending for AND operators. +func FindLowerBounds(expr Expr) ([]InfixExpr, bool) { + if expr == nil { + return []InfixExpr{}, false + } + candidates := make([]InfixExpr, 0) visitor := NewBaseVisitor() visitor.OverrideVisitInfix = func(visitor *BaseExprVisitor, e InfixExpr) interface{} { - if e.Op == ">=" { + if e.Op == ">=" || e.Op == ">" { candidates = append(candidates, e) } else if e.Op == "AND" { e.Left.Accept(visitor) @@ -15,8 +23,8 @@ func FindTimestampLowerBound(expr Expr) (InfixExpr, bool) { } expr.Accept(visitor) - if len(candidates) == 1 { - return candidates[0], true + if len(candidates) >= 1 { + return candidates, true } - return InfixExpr{}, false + return []InfixExpr{}, false } diff --git a/quesma/queryparser/pancake_sql_query_generation_test.go b/quesma/queryparser/pancake_sql_query_generation_test.go index 80e1c360c..05c6d079b 100644 --- a/quesma/queryparser/pancake_sql_query_generation_test.go +++ b/quesma/queryparser/pancake_sql_query_generation_test.go @@ -64,10 +64,6 @@ func TestPancakeQueryGeneration(t *testing.T) { t.Skip("Unskip after merge of auto_date_histogram") } - if i != 126 { - t.Skip() - } - fmt.Println("i:", i, "test:", test.TestName) jsonp, err := types.ParseJSON(test.QueryRequestJson) diff --git a/quesma/queryparser/pancake_transformer.go b/quesma/queryparser/pancake_transformer.go index 100353572..f8db51ab3 100644 --- a/quesma/queryparser/pancake_transformer.go +++ b/quesma/queryparser/pancake_transformer.go @@ -378,12 +378,23 @@ func (a *pancakeTransformer) transformAutoDateHistogram(layers []*pancakeModelLa for _, layer := range layers { if layer.nextBucketAggregation != nil { if autoDateHistogram, ok := layer.nextBucketAggregation.queryType.(*bucket_aggregations.AutoDateHistogram); ok { - timestampLowerBound, ok := model.FindTimestampLowerBound(whereClause) + lowerBoundsInWhere, ok := model.FindLowerBounds(whereClause) if !ok { logger.WarnWithCtx(a.ctx).Msgf("could not find timestamp lower bound for auto_date_histogram %v", autoDateHistogram) continue } - if autoDateHistogram.GetField() != timestampLowerBound.Left { + var ( + timestampLowerBound model.InfixExpr + found bool + ) + for _, lowerBound := range lowerBoundsInWhere { + if lowerBound.Left == autoDateHistogram.GetField() { + timestampLowerBound = lowerBound + found = true + break + } + } + if !found { logger.WarnWithCtx(a.ctx).Msgf("auto_date_histogram field %s does not match timestamp lower bound %s", autoDateHistogram.GetField(), timestampLowerBound.Left) continue } diff --git a/quesma/testdata/clients/clover.go b/quesma/testdata/clients/clover.go index 045ef1c9f..cc2e31b55 100644 --- a/quesma/testdata/clients/clover.go +++ b/quesma/testdata/clients/clover.go @@ -432,11 +432,11 @@ var CloverTests = []testdata.AggregationTestCase{ "value": 202.0 }, "doc_count": 202, - "key": 1728518400000, - "key_as_string": "2024-10-10T00:00:00.000Z" + "key": 1728581627125, + "key_as_string": "2024-10-10T19:33:47.125+02:00" } ], - "interval": "7d", + "interval": "100y", "meta": { "dataViewId": "d3d7af60-4c81-11e8-b3d7-01146121b73d", "indexPatternString": "kibana_sample_data_flights", diff --git a/quesma/testdata/unsupported_requests.go b/quesma/testdata/unsupported_requests.go index 584e1e9df..22f95cc24 100644 --- a/quesma/testdata/unsupported_requests.go +++ b/quesma/testdata/unsupported_requests.go @@ -25,21 +25,6 @@ var UnsupportedQueriesTests = []UnsupportedQueryTestCase{ } }`, }, - { // [1] - TestName: "bucket aggregation: auto_date_histogram", - QueryType: "auto_date_histogram", - QueryRequestJson: ` - { - "aggs": { - "sales_over_time": { - "auto_date_histogram": { - "field": "date", - "buckets": 10 - } - } - } - }`, - }, { // [2] TestName: "bucket aggregation: categorize_text", QueryType: "categorize_text",