From f2feacb6b39f6b0a4c7e95747d8b4a3c96e5000a Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Thu, 31 Oct 2024 17:17:46 +0100 Subject: [PATCH 1/2] Minor fixes to query response renderer (#937) Only few very minor changes need to still be ported from https://github.com/QuesmaOrg/quesma/pull/867 It's first of them. --- quesma/queryparser/pancake_json_rendering.go | 36 +++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/quesma/queryparser/pancake_json_rendering.go b/quesma/queryparser/pancake_json_rendering.go index ca6815344..9aa5a3889 100644 --- a/quesma/queryparser/pancake_json_rendering.go +++ b/quesma/queryparser/pancake_json_rendering.go @@ -236,7 +236,9 @@ func (p *pancakeJSONRenderer) layerToJSON(remainingLayers []*pancakeModelLayer, default: metricRows = p.selectMetricRows(metric.InternalNamePrefix(), rows) } - result[metric.name] = metric.queryType.TranslateSqlResponseToJson(metricRows) + if metric.name != PancakeTotalCountMetricName { + result[metric.name] = metric.queryType.TranslateSqlResponseToJson(metricRows) + } // TODO: maybe add metadata also here? probably not needed } @@ -253,6 +255,9 @@ func (p *pancakeJSONRenderer) layerToJSON(remainingLayers []*pancakeModelLayer, if err != nil { return nil, err } + if layer.nextBucketAggregation.metadata != nil { + json["meta"] = layer.nextBucketAggregation.metadata + } result[layer.nextBucketAggregation.name] = json return result, nil } @@ -345,18 +350,30 @@ func (p *pancakeJSONRenderer) layerToJSON(remainingLayers []*pancakeModelLayer, return nil, fmt.Errorf("no key in bucket json, layer: %s", layer.nextBucketAggregation.name) } } + var ( + columnNameWithKey = layer.nextBucketAggregation.InternalNameForKey(0) // TODO: need all ids, multi_terms will probably not work now + found bool + subAggrKey any + currentBucketSubAggrRows []model.QueryResultRow + ) + if subAggrIdx < len(subAggrRows) { + subAggrKey, found = p.valueForColumn(subAggrRows[subAggrIdx], columnNameWithKey) + } - columnNameWithKey := layer.nextBucketAggregation.InternalNameForKey(0) // TODO: need all ids, multi_terms will probably not work now - subAggrKey, found := p.valueForColumn(subAggrRows[subAggrIdx], columnNameWithKey) if found && subAggrKey == key { - subAggr, err := p.layerToJSON(remainingLayers[1:], subAggrRows[subAggrIdx]) - if err != nil { - return nil, err - } - bucketArr[i] = util.MergeMaps(p.ctx, bucket, subAggr) + currentBucketSubAggrRows = subAggrRows[subAggrIdx] subAggrIdx++ } else { - bucketArr[i] = bucket + currentBucketSubAggrRows = []model.QueryResultRow{} + } + + subAggr, err := p.layerToJSON(remainingLayers[1:], currentBucketSubAggrRows) + if err != nil { + return nil, err + } + bucketArr[i] = util.MergeMaps(p.ctx, bucket, subAggr) + if _, exists = bucketArr[i][bucket_aggregations.OriginalKeyName]; exists { + delete(bucketArr[i], bucket_aggregations.OriginalKeyName) } } } @@ -367,6 +384,7 @@ func (p *pancakeJSONRenderer) layerToJSON(remainingLayers []*pancakeModelLayer, } result[layer.nextBucketAggregation.name] = buckets } + return result, nil } From 61339a9a1cd27126464ede9e7da25b819ccddef8 Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Thu, 31 Oct 2024 17:51:16 +0100 Subject: [PATCH 2/2] `auto_date_histogram` aggregation (#893) Only `bucket_nr=1` case is handled, `key` value for `auto_date_histogram`'s only bucket is extracted from `where` clause via a simple visitor. --- quesma/model/README.md | 2 +- .../auto_date_histogram.go | 59 ++++++++ quesma/model/expr.go | 5 +- .../pipeline_aggregations/bucket_script.go | 18 ++- quesma/model/where_visitor.go | 38 +++++ quesma/queryparser/filters_aggregation.go | 14 +- .../queryparser/pancake_aggregation_parser.go | 3 + .../pancake_aggregation_parser_buckets.go | 32 ++-- .../pancake_sql_query_generation_test.go | 2 +- quesma/queryparser/pancake_transformer.go | 18 +++ quesma/queryparser/pipeline_aggregations.go | 4 +- quesma/testdata/clients/clover.go | 140 +++++++++++++++++- quesma/testdata/unsupported_requests.go | 15 -- 13 files changed, 306 insertions(+), 44 deletions(-) create mode 100644 quesma/model/bucket_aggregations/auto_date_histogram.go create mode 100644 quesma/model/where_visitor.go diff --git a/quesma/model/README.md b/quesma/model/README.md index b6b8976dd..66d0e4d62 100644 --- a/quesma/model/README.md +++ b/quesma/model/README.md @@ -12,7 +12,7 @@ More info: https://www.elastic.co/guide/en/elasticsearch/reference/current/searc Metrics aggregation | Support | Bucket aggregation | Support | Pipeline aggregation | Support | ---------------------------|:----------------------:|------------------------------|:------------------:|------------------------|:------------------:| Avg | :white_check_mark: | Adjacency matrix | :x: | Average bucket | :white_check_mark: | - Cardinality | :white_check_mark: | Auto-interval date histogram | :x: | Bucket script | :x: | + Cardinality | :white_check_mark: | Auto-interval date histogram | :wavy_dash: | Bucket script | :wavy_dash: | Extended Stats | :white_check_mark:[^1] | Categorize text | :x: | Bucket count K-S test | :x: | Avg | :white_check_mark: | Children | :x: | Bucket correlation | :x: | Boxplot | :x: | Composite | :x: | Bucket selector | :x: | diff --git a/quesma/model/bucket_aggregations/auto_date_histogram.go b/quesma/model/bucket_aggregations/auto_date_histogram.go new file mode 100644 index 000000000..d46c8ec33 --- /dev/null +++ b/quesma/model/bucket_aggregations/auto_date_histogram.go @@ -0,0 +1,59 @@ +// Copyright Quesma, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 +package bucket_aggregations + +import ( + "context" + "fmt" + "quesma/logger" + "quesma/model" + "time" +) + +// TODO: only bucketsNr=1 is supported for now. Implement other cases. +type AutoDateHistogram struct { + ctx context.Context + field model.ColumnRef // name of the field, e.g. timestamp + bucketsNr int + key int64 +} + +// NewAutoDateHistogram creates a new AutoDateHistogram aggregation, during parsing. +// Key is set later, during pancake transformation. +func NewAutoDateHistogram(ctx context.Context, field model.ColumnRef, bucketsNr int) *AutoDateHistogram { + return &AutoDateHistogram{ctx: ctx, field: field, bucketsNr: bucketsNr} +} + +func (query *AutoDateHistogram) AggregationType() model.AggregationType { + return model.BucketAggregation +} + +func (query *AutoDateHistogram) TranslateSqlResponseToJson(rows []model.QueryResultRow) model.JsonMap { + if len(rows) == 0 { + logger.WarnWithCtx(query.ctx).Msgf("no rows returned for %s", query.String()) + return make(model.JsonMap, 0) + } + if len(rows) != 1 { + logger.WarnWithCtx(query.ctx).Msgf("unexpected (!= 1) number of rows returned for %s: %d.", query.String(), len(rows)) + } + return model.JsonMap{ + "buckets": []model.JsonMap{{ + "key": query.key, + "key_as_string": time.UnixMilli(query.key).Format("2006-01-02T15:04:05.000-07:00"), + "doc_count": rows[0].LastColValue(), + }}, + "interval": "100y", // seems working for bucketsNr=1 case. Will have to be changed for other cases. + } +} + +func (query *AutoDateHistogram) String() string { + return fmt.Sprintf("auto_date_histogram(field: %v, bucketsNr: %d)", model.AsString(query.field), query.bucketsNr) +} + +func (query *AutoDateHistogram) GetField() model.ColumnRef { + return query.field +} + +func (query *AutoDateHistogram) SetKey(key int64) { + query.key = key +} diff --git a/quesma/model/expr.go b/quesma/model/expr.go index 5fff78384..79433f907 100644 --- a/quesma/model/expr.go +++ b/quesma/model/expr.go @@ -9,7 +9,10 @@ type Expr interface { Accept(v ExprVisitor) interface{} } -var InvalidExpr = Expr(nil) +var ( + InvalidExpr = Expr(nil) + TrueExpr = NewLiteral(true) +) // ColumnRef is a reference to a column in a table, we can enrich it with more information (e.g. type used) as we go type ColumnRef struct { diff --git a/quesma/model/pipeline_aggregations/bucket_script.go b/quesma/model/pipeline_aggregations/bucket_script.go index e7c319291..d4b5b4f0f 100644 --- a/quesma/model/pipeline_aggregations/bucket_script.go +++ b/quesma/model/pipeline_aggregations/bucket_script.go @@ -16,8 +16,8 @@ type BucketScript struct { script string } -func NewBucketScript(ctx context.Context, script string) BucketScript { - return BucketScript{script: script, PipelineAggregation: newPipelineAggregation(ctx, "_count")} +func NewBucketScript(ctx context.Context, path, script string) BucketScript { + return BucketScript{script: script, PipelineAggregation: newPipelineAggregation(ctx, path)} } func (query BucketScript) AggregationType() model.AggregationType { @@ -28,8 +28,16 @@ func (query BucketScript) TranslateSqlResponseToJson(rows []model.QueryResultRow const defaultValue = 0. switch { case query.script == "params.numerator != null && params.denominator != null && params.denominator != 0 ? params.numerator / params.denominator : 0": - numerator := query.findFilterValue(rows, "numerator") - denominator := query.findFilterValue(rows, "denominator") + parent := query.GetPathToParent() + if len(parent) != 1 { + // TODO: research if this limitation can be removed, and do so if possible. + logger.WarnWithCtx(query.ctx).Msgf("unexpected parent path in bucket_script: %s. Returning default.", query.String()) + return model.JsonMap{"value": defaultValue} + } + + // replaceAll - hack but get the job done for the customer's case, and won't break anything in any other case. + numerator := query.findFilterValue(rows, strings.ReplaceAll(parent[0], "denominator", "numerator")) + denominator := query.findFilterValue(rows, strings.ReplaceAll(parent[0], "numerator", "denominator")) if denominator == 0 { return model.JsonMap{"value": defaultValue} } @@ -77,7 +85,7 @@ func (query BucketScript) findFilterValue(rows []model.QueryResultRow, filterNam } colName = strings.TrimSuffix(colName, "_col_0") if strings.HasSuffix(colName, "-"+filterName) { - return float64(util.ExtractInt64(col.Value)) + return util.ExtractNumeric64(col.Value) } } } diff --git a/quesma/model/where_visitor.go b/quesma/model/where_visitor.go new file mode 100644 index 000000000..97eee1bba --- /dev/null +++ b/quesma/model/where_visitor.go @@ -0,0 +1,38 @@ +// Copyright Quesma, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 +package model + +import ( + "math" +) + +// FindLowerBounds returns y if there is "x>=y" or "x>y" in the WHERE clause, but only as a single top-level expression. +// (I mean by that a>=0 is fine, a>=0 AND expr2 [AND ...]] is also fine (AND between all), but a>=0 OR e2 is not fine. +// a>=0 AND (expr2 OR expr3) is also fine, as on top level it's only an AND. +// We achieve that by only descending for AND operators. +// If there are multiple such expressions, we return the smallest one. +// +// TODO: add upper bound here too, when bucket_nr=1 in auto_date_histogram (only use case of this function), it's not needed. +func FindTimestampLowerBound(field ColumnRef, whereClause Expr) (timestamp int64, found bool) { + timestamp = math.MaxInt64 + visitor := NewBaseVisitor() + visitor.OverrideVisitInfix = func(visitor *BaseExprVisitor, e InfixExpr) interface{} { + if columnRef, ok := e.Left.(ColumnRef); ok && columnRef == field && e.Op == ">=" || e.Op == ">" { + if fun, ok := e.Right.(FunctionExpr); ok && fun.Name == "fromUnixTimestamp64Milli" && len(fun.Args) == 1 { + if rhs, ok := fun.Args[0].(LiteralExpr); ok { + if rhsInt64, ok := rhs.Value.(int64); ok { + timestamp = min(timestamp, rhsInt64) + found = true + } + } + } + } else if e.Op == "AND" { + e.Left.Accept(visitor) + e.Right.Accept(visitor) + } + return nil + } + + whereClause.Accept(visitor) + return +} diff --git a/quesma/queryparser/filters_aggregation.go b/quesma/queryparser/filters_aggregation.go index 446b0afec..c44cf4972 100644 --- a/quesma/queryparser/filters_aggregation.go +++ b/quesma/queryparser/filters_aggregation.go @@ -4,6 +4,7 @@ package queryparser import ( "quesma/logger" + "quesma/model" "quesma/model/bucket_aggregations" ) @@ -32,13 +33,18 @@ func (cw *ClickhouseQueryTranslator) parseFilters(queryMap QueryMap) (success bo } filters := make([]bucket_aggregations.Filter, 0, len(nestedMap)) - for name, filter := range nestedMap { - filterMap, ok := filter.(QueryMap) + for name, filterRaw := range nestedMap { + filterMap, ok := filterRaw.(QueryMap) if !ok { - logger.WarnWithCtx(cw.Ctx).Msgf("filter is not a map, but %T, value: %v. Skipping.", filter, filter) + logger.WarnWithCtx(cw.Ctx).Msgf("filter is not a map, but %T, value: %v. Skipping.", filterRaw, filterRaw) continue } - filters = append(filters, bucket_aggregations.NewFilter(name, cw.parseQueryMap(filterMap))) + filter := cw.parseQueryMap(filterMap) + if filter.WhereClause == nil { + filter.WhereClause = model.TrueExpr + filter.CanParse = true + } + filters = append(filters, bucket_aggregations.NewFilter(name, filter)) } return true, bucket_aggregations.NewFilters(cw.Ctx, filters) } diff --git a/quesma/queryparser/pancake_aggregation_parser.go b/quesma/queryparser/pancake_aggregation_parser.go index 898ae2fee..5a426e546 100644 --- a/quesma/queryparser/pancake_aggregation_parser.go +++ b/quesma/queryparser/pancake_aggregation_parser.go @@ -153,6 +153,9 @@ func (cw *ClickhouseQueryTranslator) pancakeParseAggregation(aggregationName str if filterRaw, ok := queryMap["filter"]; ok { if filter, ok := filterRaw.(QueryMap); ok { whereClause := cw.parseQueryMap(filter).WhereClause + if whereClause == nil { // empty filter <=> true + whereClause = model.TrueExpr + } aggregation.queryType = bucket_aggregations.NewFilterAgg(cw.Ctx, whereClause) } else { logger.WarnWithCtx(cw.Ctx).Msgf("filter is not a map, but %T, value: %v. Skipping", filterRaw, filterRaw) diff --git a/quesma/queryparser/pancake_aggregation_parser_buckets.go b/quesma/queryparser/pancake_aggregation_parser_buckets.go index 9830b499c..3d98d4a4d 100644 --- a/quesma/queryparser/pancake_aggregation_parser_buckets.go +++ b/quesma/queryparser/pancake_aggregation_parser_buckets.go @@ -120,6 +120,11 @@ func (cw *ClickhouseQueryTranslator) pancakeTryBucketAggregation(aggregation *pa delete(queryMap, "date_histogram") return success, nil } + if autoDateHistogram := cw.parseAutoDateHistogram(queryMap["auto_date_histogram"]); autoDateHistogram != nil { + aggregation.queryType = autoDateHistogram + delete(queryMap, "auto_date_histogram") + return + } for _, termsType := range []string{"terms", "significant_terms"} { termsRaw, ok := queryMap[termsType] if !ok { @@ -136,15 +141,8 @@ func (cw *ClickhouseQueryTranslator) pancakeTryBucketAggregation(aggregation *pa aggregation.filterOutEmptyKeyBucket = true } - size := 10 - if sizeRaw, ok := terms["size"]; ok { - if sizeParsed, ok := sizeRaw.(float64); ok { - size = int(sizeParsed) - } else { - logger.WarnWithCtx(cw.Ctx).Msgf("size is not an float64, but %T, value: %v. Using default", sizeRaw, sizeRaw) - } - } - + const defaultSize = 10 + size := cw.parseSize(terms, defaultSize) orderBy := cw.parseOrder(terms, queryMap, []model.Expr{fieldExpression}) aggregation.queryType = bucket_aggregations.NewTerms(cw.Ctx, termsType == "significant_terms", orderBy[0]) // TODO probably full, not [0] aggregation.selectedColumns = append(aggregation.selectedColumns, fieldExpression) @@ -346,6 +344,22 @@ func (cw *ClickhouseQueryTranslator) parseRandomSampler(randomSamplerRaw any) bu ) } +func (cw *ClickhouseQueryTranslator) parseAutoDateHistogram(paramsRaw any) *bucket_aggregations.AutoDateHistogram { + params, ok := paramsRaw.(QueryMap) + if !ok { + return nil + } + + fieldRaw := cw.parseFieldField(params, "auto_date_histogram") + var field model.ColumnRef + if field, ok = fieldRaw.(model.ColumnRef); !ok { + logger.WarnWithCtx(cw.Ctx).Msgf("field is not a string, but %T, value: %v. Skipping auto_date_histogram", fieldRaw, fieldRaw) + return nil + } + bucketsNr := cw.parseIntField(params, "buckets", 10) + return bucket_aggregations.NewAutoDateHistogram(cw.Ctx, field, bucketsNr) +} + func (cw *ClickhouseQueryTranslator) parseOrder(terms, queryMap QueryMap, fieldExpressions []model.Expr) []model.OrderByExpr { defaultDirection := model.DescOrder defaultOrderBy := model.NewOrderByExpr(model.NewCountFunc(), defaultDirection) diff --git a/quesma/queryparser/pancake_sql_query_generation_test.go b/quesma/queryparser/pancake_sql_query_generation_test.go index 05c6d079b..a181f68e3 100644 --- a/quesma/queryparser/pancake_sql_query_generation_test.go +++ b/quesma/queryparser/pancake_sql_query_generation_test.go @@ -61,7 +61,7 @@ func TestPancakeQueryGeneration(t *testing.T) { } if test.TestName == "multiple buckets_path(file:clients/clover,nr:1)" { - t.Skip("Unskip after merge of auto_date_histogram") + t.Skip("This needs fixing ASAP, easy to fix") } fmt.Println("i:", i, "test:", test.TestName) diff --git a/quesma/queryparser/pancake_transformer.go b/quesma/queryparser/pancake_transformer.go index 63b7a62d8..2f6c410a4 100644 --- a/quesma/queryparser/pancake_transformer.go +++ b/quesma/queryparser/pancake_transformer.go @@ -374,6 +374,23 @@ func (a *pancakeTransformer) createTopHitAndTopMetricsPancakes(pancake *pancakeM return } +// Auto date histogram is a date histogram, that automatically creates buckets based on time range. +// To do that we need parse WHERE clause which happens in this method. +func (a *pancakeTransformer) transformAutoDateHistogram(layers []*pancakeModelLayer, whereClause model.Expr) { + for _, layer := range layers { + if layer.nextBucketAggregation != nil { + if autoDateHistogram, ok := layer.nextBucketAggregation.queryType.(*bucket_aggregations.AutoDateHistogram); ok { + if tsLowerBound, found := model.FindTimestampLowerBound(autoDateHistogram.GetField(), whereClause); found { + autoDateHistogram.SetKey(tsLowerBound) + } else { + logger.WarnWithCtx(a.ctx).Msgf("could not find timestamp lower bound (field: %v, where clause: %v)", + autoDateHistogram.GetField(), whereClause) + } + } + } + } +} + func (a *pancakeTransformer) aggregationTreeToPancakes(topLevel pancakeAggregationTree) (pancakeResults []*pancakeModel, err error) { if len(topLevel.children) == 0 { return nil, fmt.Errorf("no top level aggregations found") @@ -398,6 +415,7 @@ func (a *pancakeTransformer) aggregationTreeToPancakes(topLevel pancakeAggregati } a.connectPipelineAggregations(layers) + a.transformAutoDateHistogram(layers, topLevel.whereClause) newPancake := pancakeModel{ layers: layers, diff --git a/quesma/queryparser/pipeline_aggregations.go b/quesma/queryparser/pipeline_aggregations.go index b5dc33d41..f3a007ad5 100644 --- a/quesma/queryparser/pipeline_aggregations.go +++ b/quesma/queryparser/pipeline_aggregations.go @@ -180,7 +180,7 @@ func (cw *ClickhouseQueryTranslator) parseBucketScriptBasic(queryMap QueryMap) ( return } if script, ok := scriptRaw.(string); ok { - return pipeline_aggregations.NewBucketScript(cw.Ctx, script), true + return pipeline_aggregations.NewBucketScript(cw.Ctx, bucketsPath, script), true } script, ok := scriptRaw.(QueryMap) @@ -204,7 +204,7 @@ func (cw *ClickhouseQueryTranslator) parseBucketScriptBasic(queryMap QueryMap) ( } // okay, we've checked everything, it's indeed a simple count - return pipeline_aggregations.NewBucketScript(cw.Ctx, ""), true + return pipeline_aggregations.NewBucketScript(cw.Ctx, bucketsPath, ""), true } func (cw *ClickhouseQueryTranslator) parseBucketsPath(shouldBeQueryMap any, aggregationName string) (bucketsPathStr string, success bool) { diff --git a/quesma/testdata/clients/clover.go b/quesma/testdata/clients/clover.go index 978edfc0d..3a182510e 100644 --- a/quesma/testdata/clients/clover.go +++ b/quesma/testdata/clients/clover.go @@ -333,16 +333,144 @@ var CloverTests = []testdata.AggregationTestCase{ ExpectedPancakeResults: []model.QueryResultRow{ {Cols: []model.QueryResultCol{ model.NewQueryResultCol("aggr__timeseries__count", int64(202)), + model.NewQueryResultCol("metric__timeseries__a2-numerator_col_0", int64(202)), + model.NewQueryResultCol("aggr__timeseries__a2-denominator__count", int64(202)), }}, }, ExpectedPancakeSQL: ` SELECT count(*) AS "aggr__timeseries__count", - countIf(True) AS - "name", - countIf(NOT ("field" = 'sth')) AS - "name" + countIf(NOT ("table.flower" = 'clover')) AS "metric__timeseries__a2-numerator_col_0", + countIf(true) AS "aggr__timeseries__a2-denominator__count" FROM __quesma_table_name - WHERE ("@timestamp">=parseDateTime64BestEffort('2024-10-11T09:58:03.723Z') AND - "@timestamp"<=parseDateTime64BestEffort('2024-10-11T10:13:03.723Z'))`, + WHERE ("@timestamp">=fromUnixTimestamp64Milli(1728640683723) AND "@timestamp"<= + fromUnixTimestamp64Milli(1728641583723))`, + }, + { // [2] + TestName: "simplest auto_date_histogram", + QueryRequestJson: ` + { + "aggs": { + "timeseries": { + "aggs": { + "469ef7fe-5927-42d1-918b-37c738c600f0": { + "bucket_script": { + "buckets_path": { + "count": "_count" + }, + "gap_policy": "skip", + "script": { + "lang": "expression", + "source": "count * 1" + } + } + } + }, + "auto_date_histogram": { + "buckets": 1, + "field": "timestamp" + }, + "meta": { + "dataViewId": "d3d7af60-4c81-11e8-b3d7-01146121b73d", + "indexPatternString": "kibana_sample_data_flights", + "intervalString": "54000000ms", + "normalized": true, + "panelId": "1a1d745d-0c21-4103-a2ae-df41d4fbd366", + "seriesId": "866fb08f-b9a4-43eb-a400-38ebb6c13aed", + "timeField": "timestamp" + } + } + }, + "query": { + "bool": { + "filter": [], + "must": [ + { + "range": { + "timestamp": { + "format": "strict_date_optional_time", + "gte": "2024-10-10T17:33:47.125Z", + "lte": "2024-10-11T08:33:47.125Z" + } + } + } + ], + "must_not": [], + "should": [] + } + }, + "runtime_mappings": { + "hour_of_day": { + "script": { + "source": "emit(doc['timestamp'].value.getHour());" + }, + "type": "long" + } + }, + "size": 0, + "timeout": "30000ms", + "track_total_hits": true + }`, + ExpectedResponse: ` + { + "completion_time_in_millis": 1728635627258, + "expiration_time_in_millis": 1728635687254, + "id": "FlhaTzBhMkpQU3lLMmlzNHhBeU9FMHcbaUp3ZGNYdDNSaGF3STVFZ2xWY3RuQTo2MzU4", + "is_partial": false, + "is_running": false, + "response": { + "_shards": { + "failed": 0, + "skipped": 0, + "successful": 1, + "total": 1 + }, + "aggregations": { + "timeseries": { + "buckets": [ + { + "469ef7fe-5927-42d1-918b-37c738c600f0": { + "value": 202.0 + }, + "doc_count": 202, + "key": 1728581627125, + "key_as_string": "2024-10-10T19:33:47.125+02:00" + } + ], + "interval": "100y", + "meta": { + "dataViewId": "d3d7af60-4c81-11e8-b3d7-01146121b73d", + "indexPatternString": "kibana_sample_data_flights", + "intervalString": "54000000ms", + "normalized": true, + "panelId": "1a1d745d-0c21-4103-a2ae-df41d4fbd366", + "seriesId": "866fb08f-b9a4-43eb-a400-38ebb6c13aed", + "timeField": "timestamp" + } + } + }, + "hits": { + "hits": [], + "max_score": null, + "total": { + "relation": "eq", + "value": 202 + } + }, + "timed_out": false, + "took": 4 + }, + "start_time_in_millis": 1728635627254 + }`, + ExpectedPancakeResults: []model.QueryResultRow{ + {Cols: []model.QueryResultCol{ + model.NewQueryResultCol("aggr__timeseries__count", int64(202)), + }}, + }, + ExpectedPancakeSQL: ` + SELECT count(*) AS "aggr__timeseries__count" + FROM __quesma_table_name + WHERE ("timestamp">=fromUnixTimestamp64Milli(1728581627125) AND "timestamp"<= + fromUnixTimestamp64Milli(1728635627125))`, + AdditionalAcceptableDifference: []string{"key_as_string"}, // timezone differences between local and github runs... There's always 2h difference between those, need to investigate. Maybe come back to .UTC() so there's no "+timezone" (e.g. +02:00)? }, } diff --git a/quesma/testdata/unsupported_requests.go b/quesma/testdata/unsupported_requests.go index 584e1e9df..22f95cc24 100644 --- a/quesma/testdata/unsupported_requests.go +++ b/quesma/testdata/unsupported_requests.go @@ -25,21 +25,6 @@ var UnsupportedQueriesTests = []UnsupportedQueryTestCase{ } }`, }, - { // [1] - TestName: "bucket aggregation: auto_date_histogram", - QueryType: "auto_date_histogram", - QueryRequestJson: ` - { - "aggs": { - "sales_over_time": { - "auto_date_histogram": { - "field": "date", - "buckets": 10 - } - } - } - }`, - }, { // [2] TestName: "bucket aggregation: categorize_text", QueryType: "categorize_text",