From 45a4cb27d546f41fbde9e0a17dc845976b0b1c7f Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Tue, 7 May 2024 16:00:05 -0400 Subject: [PATCH 01/21] tag name filtering added in the block Signed-off-by: Joe Elliott --- go.mod | 2 +- go.sum | 2 + modules/ingester/instance_search.go | 2 +- modules/querier/querier.go | 4 +- pkg/traceql/engine.go | 2 +- .../encoding/vparquet3/block_autocomplete.go | 248 ++++++++++++---- .../vparquet3/block_autocomplete_test.go | 279 ++++++++++++++++++ .../encoding/vparquet3/block_traceql_test.go | 2 + tempodb/encoding/vparquet3/wal_block.go | 2 +- .../segmentio/fasthash/fnv1a/hash.go | 80 ++++- .../segmentio/fasthash/fnv1a/hash32.go | 85 ++++-- vendor/modules.txt | 4 +- 12 files changed, 615 insertions(+), 97 deletions(-) diff --git a/go.mod b/go.mod index abfbf41f06b..8e5adb2807c 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,7 @@ require ( github.com/prometheus/common v0.49.0 github.com/prometheus/prometheus v0.48.1 github.com/prometheus/statsd_exporter v0.26.0 - github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e + github.com/segmentio/fasthash v1.0.3 github.com/sirupsen/logrus v1.9.3 // indirect github.com/sony/gobreaker v0.4.1 github.com/spf13/viper v1.18.2 diff --git a/go.sum b/go.sum index a60ae1ada9b..7a76c26d0a2 100644 --- a/go.sum +++ b/go.sum @@ -942,6 +942,8 @@ github.com/segmentio/encoding v0.3.6 h1:E6lVLyDPseWEulBmCmAKPanDd3jiyGDo5gMcugCR github.com/segmentio/encoding v0.3.6/go.mod h1:n0JeuIqEQrQoPDGsjo8UNd1iA0U8d8+oHAA4E3G3OxM= github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e h1:uO75wNGioszjmIzcY/tvdDYKRLVvzggtAmmJkn9j4GQ= github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e/go.mod h1:tm/wZFQ8e24NYaBGIlnO2WGCAi67re4HHuOm0sftE/M= +github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= +github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/sercand/kuberesolver/v5 v5.1.1 h1:CYH+d67G0sGBj7q5wLK61yzqJJ8gLLC8aeprPTHb6yY= github.com/sercand/kuberesolver/v5 v5.1.1/go.mod h1:Fs1KbKhVRnB2aDWN12NjKCB+RgYMWZJ294T3BtmVCpQ= github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y= diff --git a/modules/ingester/instance_search.go b/modules/ingester/instance_search.go index ce0ffa7832d..7c357d23a6e 100644 --- a/modules/ingester/instance_search.go +++ b/modules/ingester/instance_search.go @@ -260,7 +260,7 @@ func (i *instance) SearchTagsV2(ctx context.Context, scope string) (*tempopb.Sea atts := traceql.AllAttributeScopes() scopes = make([]string, 0, len(atts)+1) // +1 for intrinsic - scopes = append(scopes, api.ParamScopeIntrinsic) + scopes = append(scopes, api.ParamScopeIntrinsic) // jpe - search tags doesn't seem to have any "intrinsic" logic. should this be removed? for _, att := range atts { scopes = append(scopes, att.String()) } diff --git a/modules/querier/querier.go b/modules/querier/querier.go index 47bddf98f6d..77ee8ff0389 100644 --- a/modules/querier/querier.go +++ b/modules/querier/querier.go @@ -878,8 +878,8 @@ func (q *Querier) internalTagsSearchBlockV2(ctx context.Context, req *tempopb.Se if req.SearchReq.Scope == "" { // start with intrinsic scope and all traceql attribute scopes atts := traceql.AllAttributeScopes() - scopes = make([]string, 0, len(atts)+1) // +1 for intrinsic - scopes = append(scopes, api.ParamScopeIntrinsic) + scopes = make([]string, 0, len(atts)+1) // +1 for intrinsic + scopes = append(scopes, api.ParamScopeIntrinsic) // jpe - seachtagsv2 doesn't seem to have support for "intrinsics" remove? for _, att := range atts { scopes = append(scopes, att.String()) } diff --git a/pkg/traceql/engine.go b/pkg/traceql/engine.go index 28bbf815e5b..86c4a6a1fdf 100644 --- a/pkg/traceql/engine.go +++ b/pkg/traceql/engine.go @@ -219,7 +219,7 @@ func (e *Engine) createAutocompleteRequest(tag Attribute, pipeline Pipeline) Aut // return fmt.Errorf("cannot search for tag values for tag that is already used in query") // } // } - req.Conditions = append(req.Conditions, Condition{ + req.Conditions = append(req.Conditions, Condition{ // jpe should we move this into the fetch itself? it's weird we pass in both the tag through the request _and_ do the trickiness outside of adding it as OpNone Attribute: tag, Op: OpNone, }) diff --git a/tempodb/encoding/vparquet3/block_autocomplete.go b/tempodb/encoding/vparquet3/block_autocomplete.go index efa9a6f1d35..d78ac520d33 100644 --- a/tempodb/encoding/vparquet3/block_autocomplete.go +++ b/tempodb/encoding/vparquet3/block_autocomplete.go @@ -14,7 +14,76 @@ import ( "github.com/pkg/errors" ) -func (b *backendBlock) FetchTagValues(ctx context.Context, req traceql.AutocompleteRequest, cb traceql.AutocompleteCallback, opts common.SearchOptions) error { +// jpe - "AutoComplete" -> FetchTagValuesRequest and FetchTagNamesRequest +// jpe - what to share with below? +// jpe - require scope? +// jpe - add any ded col or well known col that has values. confirm this doesn't add reads +// jpe - explore using rep/def lvls to determine if a value exists at a row? confirm this doesn't add reads +// jpe - have to add an iterator for the generic attr name if it doesn't exist + +type tagRequest struct { + keys bool + tag traceql.Attribute +} + +func (r tagRequest) attributeRequested() bool { + return r.tag != (traceql.Attribute{}) +} + +func (r tagRequest) keysRequested() bool { + return r.keys +} + +// jpe - current searchTags takes one param at a time for scope. we should change it to take a list of scopes. otherwise we will +// be repeating this same search 2x +func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.AutocompleteRequest, cb common.TagCallback, opts common.SearchOptions) error { + err := checkConditions(req.Conditions) + if err != nil { + return errors.Wrap(err, "conditions invalid") + } + + mingledConditions, _, _, _, err := categorizeConditions(req.Conditions) + if err != nil { + return err + } + + // Last check. No conditions, use old path. It's much faster. + if len(req.Conditions) < 1 || mingledConditions { // jpe - fetch with {} puts in a condition for span start time? + return b.SearchTags(ctx, traceql.AttributeScopeResource, cb, opts) // jpe - remove hardcoded scope - should be SearchtagsV2? + } + + pf, _, err := b.openForSearch(ctx, opts) + if err != nil { + return err + } + + iter, err := autocompleteIter(ctx, req.Conditions, traceql.Attribute{}, pf, opts, b.meta.DedicatedColumns) // jpe call iter funcs directly to avoid nil cb + if err != nil { + return errors.Wrap(err, "creating fetch iter") + } + defer iter.Close() + + // fmt.Println("-------------") + // fmt.Println(iter) // jpe remove + + for { + // Exhaust the iterator + res, err := iter.Next() + if err != nil { + return err + } + if res == nil { + break + } + for _, oe := range res.OtherEntries { + cb(oe.Key) + } + } + + return nil +} + +func (b *backendBlock) FetchTagValues(ctx context.Context, req traceql.AutocompleteRequest, cb traceql.AutocompleteCallback, opts common.SearchOptions) error { // jpe swap to common.TagCallback err := checkConditions(req.Conditions) if err != nil { return errors.Wrap(err, "conditions invalid") @@ -35,7 +104,7 @@ func (b *backendBlock) FetchTagValues(ctx context.Context, req traceql.Autocompl return err } - iter, err := autocompleteIter(ctx, req, pf, opts, b.meta.DedicatedColumns) + iter, err := autocompleteIter(ctx, req.Conditions, req.TagName, pf, opts, b.meta.DedicatedColumns) // jpe call iter funcs directly to avoid nil cb if err != nil { return errors.Wrap(err, "creating fetch iter") } @@ -62,23 +131,12 @@ func (b *backendBlock) FetchTagValues(ctx context.Context, req traceql.Autocompl } // autocompleteIter creates an iterator that will collect values for a given attribute/tag. -func autocompleteIter(ctx context.Context, req traceql.AutocompleteRequest, pf *parquet.File, opts common.SearchOptions, dc backend.DedicatedColumns) (parquetquery.Iterator, error) { - iter, err := createDistinctIterator(ctx, req.Conditions, req.TagName, pf, opts, dc) - if err != nil { - return nil, fmt.Errorf("error creating iterator: %w", err) +func autocompleteIter(ctx context.Context, conds []traceql.Condition, tagName traceql.Attribute, pf *parquet.File, opts common.SearchOptions, dc backend.DedicatedColumns) (parquetquery.Iterator, error) { + tr := tagRequest{ // jpe - push this up? + tag: tagName, + keys: tagName == (traceql.Attribute{}), } - return iter, nil -} - -func createDistinctIterator( - ctx context.Context, - conds []traceql.Condition, - tag traceql.Attribute, - pf *parquet.File, - opts common.SearchOptions, - dc backend.DedicatedColumns, -) (parquetquery.Iterator, error) { // categorizeConditions conditions into span-level or resource-level _, spanConditions, resourceConditions, traceConditions, err := categorizeConditions(conds) if err != nil { @@ -90,15 +148,15 @@ func createDistinctIterator( var currentIter parquetquery.Iterator - if len(spanConditions) > 0 { - currentIter, err = createDistinctSpanIterator(makeIter, tag, currentIter, spanConditions, dc) + if len(spanConditions) > 0 || tr.keysRequested() { + currentIter, err = createDistinctSpanIterator(makeIter, tr, spanConditions, dc) if err != nil { return nil, errors.Wrap(err, "creating span iterator") } } - if len(resourceConditions) > 0 { - currentIter, err = createDistinctResourceIterator(makeIter, tag, currentIter, resourceConditions, dc) + if len(resourceConditions) > 0 || tr.keysRequested() { + currentIter, err = createDistinctResourceIterator(makeIter, tr, currentIter, resourceConditions, dc) if err != nil { return nil, errors.Wrap(err, "creating resource iterator") } @@ -118,8 +176,7 @@ func createDistinctIterator( // one span each. Spans are returned that match any of the given conditions. func createDistinctSpanIterator( makeIter makeIterFn, - tag traceql.Attribute, - primaryIter parquetquery.Iterator, + tr tagRequest, conditions []traceql.Condition, dedicatedColumns backend.DedicatedColumns, ) (parquetquery.Iterator, error) { @@ -134,7 +191,7 @@ func createDistinctSpanIterator( // TODO: Potentially problematic when wanted attribute is also part of a condition // e.g. { span.foo =~ ".*" && span.foo = } addSelectAs := func(attr traceql.Attribute, columnPath string, selectAs string) { - if attr == tag { + if attr == tr.tag { columnSelectAs[columnPath] = selectAs } else { columnSelectAs[columnPath] = "" // Don't select, just filter @@ -260,7 +317,7 @@ func createDistinctSpanIterator( iters = append(iters, makeIter(columnPath, orIfNeeded(predicates), columnSelectAs[columnPath])) } - attrIter, err := createDistinctAttributeIterator(makeIter, tag, genericConditions, DefinitionLevelResourceSpansILSSpanAttrs, + attrIter, err := createDistinctAttributeIterator(makeIter, tr, genericConditions, DefinitionLevelResourceSpansILSSpanAttrs, columnPathSpanAttrKey, columnPathSpanAttrString, columnPathSpanAttrInt, columnPathSpanAttrDouble, columnPathSpanAttrBool) if err != nil { return nil, errors.Wrap(err, "creating span attribute iterator") @@ -269,17 +326,13 @@ func createDistinctSpanIterator( iters = append(iters, attrIter) } - if primaryIter != nil { - iters = append([]parquetquery.Iterator{primaryIter}, iters...) - } - if len(columnPredicates) == 0 { // If no special+intrinsic+dedicated columns are being searched, // we can iterate over the generic attributes directly. return attrIter, nil } - spanCol := newDistinctValueCollector(mapSpanAttr) + spanCol := newDistinctValueCollector(mapSpanAttr, "span") // Left join here means the span id/start/end iterators + 1 are required, // and all other conditions are optional. Whatever matches is returned. @@ -288,7 +341,7 @@ func createDistinctSpanIterator( func createDistinctAttributeIterator( makeIter makeIterFn, - tag traceql.Attribute, + tr tagRequest, conditions []traceql.Condition, definitionLevel int, keyPath, strPath, intPath, floatPath, boolPath string, @@ -300,17 +353,16 @@ func createDistinctAttributeIterator( ) selectAs := func(key string, attr traceql.Attribute) string { - if tag == attr { + if tr.tag == attr { return key } return "" } for _, cond := range conditions { - if cond.Op == traceql.OpNone { // This means we have to scan all values, we don't know what type to expect - if tag == cond.Attribute { + if tr.tag == cond.Attribute { // If it's not the tag we're looking for, we can skip it attrKeys = append(attrKeys, cond.Attribute.Name) attrStringPreds = append(attrStringPreds, nil) @@ -380,19 +432,24 @@ func createDistinctAttributeIterator( valueIters = append(valueIters, makeIter(boolPath, orIfNeeded(boolPreds), "bool")) } - if len(valueIters) > 0 || len(iters) > 0 { + if len(valueIters) > 0 || len(iters) > 0 || tr.keysRequested() { if len(valueIters) > 0 { tagIter, err := parquetquery.NewLeftJoinIterator( definitionLevel, []parquetquery.Iterator{makeIter(keyPath, parquetquery.NewStringInPredicate(attrKeys), "key")}, valueIters, - newDistinctAttrCollector(scopeFromDefinitionLevel(definitionLevel)), + newDistinctAttrCollector(scopeFromDefinitionLevel(definitionLevel), false), // jpe - empty attribute signals we want tag names? ) if err != nil { return nil, fmt.Errorf("creating left join iterator: %w", err) } iters = append(iters, tagIter) } + + if tr.keysRequested() { + return keyNameIterator(makeIter, definitionLevel, keyPath, iters) + } + return parquetquery.NewJoinIterator( oneLevelUp(definitionLevel), iters, @@ -403,6 +460,23 @@ func createDistinctAttributeIterator( return nil, nil } +func keyNameIterator(makeIter makeIterFn, definitionLevel int, keyPath string, attrIters []parquetquery.Iterator) (parquetquery.Iterator, error) { + if len(attrIters) == 0 { + return parquetquery.NewJoinIterator( + oneLevelUp(definitionLevel), + []parquetquery.Iterator{makeIter(keyPath, newDistinctKeyPredicate(), "key")}, + newDistinctAttrCollector(scopeFromDefinitionLevel(definitionLevel), true), + ), nil + } + + return parquetquery.NewLeftJoinIterator( + oneLevelUp(definitionLevel), + attrIters, + []parquetquery.Iterator{makeIter(keyPath, newDistinctKeyPredicate(), "key")}, + newDistinctAttrCollector(scopeFromDefinitionLevel(definitionLevel), true), + ) +} + func oneLevelUp(definitionLevel int) int { switch definitionLevel { case DefinitionLevelResourceSpansILSSpanAttrs: @@ -415,7 +489,7 @@ func oneLevelUp(definitionLevel int) int { func createDistinctResourceIterator( makeIter makeIterFn, - tag traceql.Attribute, + tr tagRequest, spanIterator parquetquery.Iterator, conditions []traceql.Condition, dedicatedColumns backend.DedicatedColumns, @@ -433,7 +507,7 @@ func createDistinctResourceIterator( } addSelectAs := func(attr traceql.Attribute, columnPath string, selectAs string) { - if attr == tag { + if attr == tr.tag { columnSelectAs[columnPath] = selectAs } else { columnSelectAs[columnPath] = "" // Don't select, just filter @@ -456,7 +530,7 @@ func createDistinctResourceIterator( return nil, errors.Wrap(err, "creating predicate") } selectAs := cond.Attribute.Name - if tag != cond.Attribute { + if tr.tag != cond.Attribute { selectAs = "" } iters = append(iters, makeIter(entry.columnPath, pred, selectAs)) @@ -493,7 +567,7 @@ func createDistinctResourceIterator( iters = append(iters, makeIter(columnPath, orIfNeeded(predicates), columnSelectAs[columnPath])) } - attrIter, err := createDistinctAttributeIterator(makeIter, tag, genericConditions, DefinitionLevelResourceAttrs, + attrIter, err := createDistinctAttributeIterator(makeIter, tr, genericConditions, DefinitionLevelResourceAttrs, columnPathResourceAttrKey, columnPathResourceAttrString, columnPathResourceAttrInt, columnPathResourceAttrDouble, columnPathResourceAttrBool) if err != nil { return nil, errors.Wrap(err, "creating span attribute iterator") @@ -502,7 +576,7 @@ func createDistinctResourceIterator( iters = append(iters, attrIter) } - batchCol := newDistinctValueCollector(mapResourceAttr) + batchCol := newDistinctValueCollector(mapResourceAttr, "resource") // Put span iterator last, so it is only read when // the resource conditions are met. @@ -564,21 +638,25 @@ func createDistinctTraceIterator( // Final trace iterator // Join iterator means it requires matching resources to have been found // TraceCollor adds trace-level data to the spansets - return parquetquery.NewJoinIterator(DefinitionLevelTrace, traceIters, newDistinctValueCollector(mapTraceAttr)), nil + return parquetquery.NewJoinIterator(DefinitionLevelTrace, traceIters, newDistinctValueCollector(mapTraceAttr, "trace")), nil } var _ parquetquery.GroupPredicate = (*distinctAttrCollector)(nil) type distinctAttrCollector struct { - scope traceql.AttributeScope + scope traceql.AttributeScope + attrNames bool sentVals map[traceql.Static]struct{} + sentKeys map[string]struct{} } -func newDistinctAttrCollector(scope traceql.AttributeScope) *distinctAttrCollector { +func newDistinctAttrCollector(scope traceql.AttributeScope, attrNames bool) *distinctAttrCollector { return &distinctAttrCollector{ - scope: scope, - sentVals: make(map[traceql.Static]struct{}), + scope: scope, + sentVals: make(map[traceql.Static]struct{}), + sentKeys: make(map[string]struct{}), + attrNames: attrNames, } } @@ -596,15 +674,25 @@ func (d *distinctAttrCollector) KeepGroup(result *parquetquery.IteratorResult) b continue } - switch e.Key { - case "string": - val = traceql.NewStaticString(unsafeToString(e.Value.ByteArray())) - case "int": - val = traceql.NewStaticInt(int(e.Value.Int64())) - case "float": - val = traceql.NewStaticFloat(e.Value.Double()) - case "bool": - val = traceql.NewStaticBool(e.Value.Boolean()) + if d.attrNames { + if e.Key == "key" { + key := unsafeToString(e.Value.ByteArray()) + if _, ok := d.sentKeys[key]; !ok { + result.AppendOtherValue(key, traceql.Attribute{}) // jpe allocs? + d.sentKeys[key] = struct{}{} + } + } + } else { + switch e.Key { + case "string": + val = traceql.NewStaticString(unsafeToString(e.Value.ByteArray())) + case "int": + val = traceql.NewStaticInt(int(e.Value.Int64())) + case "float": + val = traceql.NewStaticFloat(e.Value.Double()) + case "bool": + val = traceql.NewStaticBool(e.Value.Boolean()) + } } } @@ -621,6 +709,50 @@ func (d *distinctAttrCollector) KeepGroup(result *parquetquery.IteratorResult) b return true } +// distinctKeyPredicate skips chunks that only contain keys that have been seen. +// jpe - this is approximate! document it as such +type distinctKeyPredicate struct { + sentKeys map[string]struct{} +} + +func newDistinctKeyPredicate() *distinctKeyPredicate { + return &distinctKeyPredicate{ + sentKeys: make(map[string]struct{}), + } +} + +func (d *distinctKeyPredicate) String() string { + return fmt.Sprintf("distinctKeyPredicate{}") +} + +func (d *distinctKeyPredicate) KeepColumnChunk(cc *parquetquery.ColumnChunkHelper) bool { // jpe name better and extend all the way to the tag call back, so we can test globally if something has been added + foundNew := false + + if dict := cc.Dictionary(); dict != nil { + l := dict.Len() + for i := 0; i < l; i++ { + v := dict.Index(int32(i)) + key := unsafeToString(v.ByteArray()) + + if _, ok := d.sentKeys[key]; !ok { + d.sentKeys[key] = struct{}{} + foundNew = true + break // jpe - noodle on this. it's a perf boost somehow? + } + } + } + + return foundNew +} + +func (d *distinctKeyPredicate) KeepPage(pq parquet.Page) bool { + return true +} + +func (d *distinctKeyPredicate) KeepValue(v parquet.Value) bool { + return true +} + type entry struct { Key string Value parquet.Value @@ -631,16 +763,18 @@ var _ parquetquery.GroupPredicate = (*distinctValueCollector)(nil) type distinctValueCollector struct { mapToStatic func(entry) traceql.Static sentVals map[traceql.Static]struct{} + name string } -func newDistinctValueCollector(mapToStatic func(entry) traceql.Static) *distinctValueCollector { +func newDistinctValueCollector(mapToStatic func(entry) traceql.Static, name string) *distinctValueCollector { return &distinctValueCollector{ mapToStatic: mapToStatic, sentVals: make(map[traceql.Static]struct{}), + name: name, } } -func (d distinctValueCollector) String() string { return "distinctValueCollector" } +func (d distinctValueCollector) String() string { return "distinctValueCollector(" + d.name + ")" } func (d distinctValueCollector) KeepGroup(result *parquetquery.IteratorResult) bool { for _, e := range result.Entries { diff --git a/tempodb/encoding/vparquet3/block_autocomplete_test.go b/tempodb/encoding/vparquet3/block_autocomplete_test.go index 7fd03f932f1..0bbe342b124 100644 --- a/tempodb/encoding/vparquet3/block_autocomplete_test.go +++ b/tempodb/encoding/vparquet3/block_autocomplete_test.go @@ -11,12 +11,196 @@ import ( "github.com/grafana/tempo/pkg/tempopb" "github.com/grafana/tempo/pkg/traceql" "github.com/grafana/tempo/pkg/util" + "github.com/grafana/tempo/pkg/util/test" "github.com/grafana/tempo/tempodb/backend" "github.com/grafana/tempo/tempodb/backend/local" "github.com/grafana/tempo/tempodb/encoding/common" "github.com/stretchr/testify/require" ) +func TestFetchTagNames(t *testing.T) { + testCases := []struct { + name string + query string + expectedValues []string + }{ + { + name: "no query - fall back to old search", // jpe - not working. not falling back to old search due to span start time cond? + query: "{}", + expectedValues: []string{ + "generic-01", + "generic-01-01", + "generic-01-02", + "generic-02", + "generic-02-01", + "span-same", + "resource-same", + }, + }, + { + name: "matches nothing", + query: "{span.generic-01-01=`bar`}", + expectedValues: []string{}, + }, + // span + { + name: "intrinsic span", + query: "{statusMessage=`msg-01-01`}", + expectedValues: []string{"generic-01", "generic-01-01", "span-same", "resource-same"}, + }, + { + name: "well known span", + query: "{span.http.method=`method-01-01`}", + expectedValues: []string{"generic-01", "generic-01-01", "span-same", "resource-same"}, + }, + { + name: "generic span", + query: "{span.generic-01-01=`foo`}", + expectedValues: []string{"generic-01", "generic-01-01", "span-same", "resource-same"}, + }, + { + name: "match two spans", + query: "{span.span-same=`foo`}", + expectedValues: []string{"generic-01", "generic-01-01", "span-same", "resource-same", "generic-02", "generic-02-01"}, + }, + // resource + { + name: "well known resource", + query: "{resource.cluster=`cluster-01`}", + expectedValues: []string{"generic-01", "generic-01-01", "generic-01-02", "span-same", "resource-same"}, + }, + { + name: "generic resource", + query: "{resource.generic-01=`bar`}", + expectedValues: []string{"generic-01", "generic-01-01", "generic-01-02", "span-same", "resource-same"}, + }, + { + name: "match two resources", + query: "{resource.resource-same=`foo`}", + expectedValues: []string{"generic-01", "generic-01-01", "generic-01-02", "span-same", "resource-same", "generic-02", "generic-02-01"}, + }, + // trace level match + { + name: "trace", + query: "{rootName=`root` }", + expectedValues: []string{"generic-01", "generic-01-01", "generic-01-02", "span-same", "resource-same", "generic-02", "generic-02-01"}, + }, + } + + strPtr := func(s string) *string { return &s } + tr := &Trace{ + TraceID: test.ValidTraceID(nil), + RootServiceName: "tr", + RootSpanName: "root", + ResourceSpans: []ResourceSpans{ + { + Resource: Resource{ + ServiceName: "svc-01", + Cluster: strPtr("cluster-01"), // well known + Attrs: []Attribute{ + {Key: "generic-01", Value: strPtr("bar")}, // generic + {Key: "resource-same", Value: strPtr("foo")}, + }, + DedicatedAttributes: DedicatedAttributes{ + String01: strPtr("dedicated-01"), + }, + }, + ScopeSpans: []ScopeSpans{ + { + Spans: []Span{ + { + SpanID: []byte("0101"), + Name: "span-01-01", + HttpMethod: strPtr("method-01-01"), // well known + StatusMessage: "msg-01-01", // intrinsic + Attrs: []Attribute{ + {Key: "generic-01-01", Value: strPtr("foo")}, // generic + {Key: "span-same", Value: strPtr("foo")}, // generic + }, + DedicatedAttributes: DedicatedAttributes{ + String01: strPtr("dedicated-01-01"), + }, + }, + { + SpanID: []byte("0102"), + Name: "span-01-02", + HttpMethod: strPtr("method-01-02"), // well known + StatusMessage: "msg-01-02", // intrinsic + Attrs: []Attribute{ + {Key: "generic-01-02", Value: strPtr("foo")}, // generic + }, + DedicatedAttributes: DedicatedAttributes{ + String01: strPtr("dedicated-01-02"), + }, + }, + }, + }, + }, + }, + { + Resource: Resource{ + ServiceName: "svc-02", + Cluster: strPtr("cluster-02"), // well known + Attrs: []Attribute{ + {Key: "generic-02", Value: strPtr("bar")}, // generic + {Key: "resource-same", Value: strPtr("foo")}, + }, + DedicatedAttributes: DedicatedAttributes{ + String01: strPtr("dedicated-02"), + }, + }, + ScopeSpans: []ScopeSpans{ + { + Spans: []Span{ + { + SpanID: []byte("0201"), + Name: "span-02-01", + HttpMethod: strPtr("method-02-01"), // well known + StatusMessage: "msg-02-01", // intrinsic + Attrs: []Attribute{ + {Key: "generic-02-01", Value: strPtr("foo")}, // generic + {Key: "span-same", Value: strPtr("foo")}, // generic + }, + DedicatedAttributes: DedicatedAttributes{ + String01: strPtr("dedicated-02-01"), + }, + }, + }, + }, + }, + }, + }, + } + + ctx := context.TODO() + block := makeBackendBlockWithTraces(t, []*Trace{tr}) + + opts := common.DefaultSearchOptions() + + for _, tc := range testCases { + t.Run(fmt.Sprintf("query: %s", tc.query), func(t *testing.T) { + distinctAttrNames := util.NewDistinctStringCollector(1_000_000) + req, err := traceql.ExtractFetchSpansRequest(tc.query) + require.NoError(t, err) + + // Build autocomplete request + autocompleteReq := traceql.AutocompleteRequest{ + Conditions: req.Conditions, + TagName: traceql.Attribute{}, + } + + err = block.FetchTagNames(ctx, autocompleteReq, distinctAttrNames.Collect, opts) + require.NoError(t, err) + + expectedValues := tc.expectedValues + actualValues := distinctAttrNames.Strings() + sort.Strings(expectedValues) + sort.Strings(actualValues) + require.Equal(t, expectedValues, actualValues) + }) + } +} + func TestFetchTagValues(t *testing.T) { testCases := []struct { name string @@ -281,6 +465,7 @@ func TestFetchTagValues(t *testing.T) { tagAtrr, err := traceql.ParseIdentifier(tc.tag) require.NoError(t, err) + // jpe - remove this if we move it into the FetchTagValues function autocompleteReq.Conditions = append(autocompleteReq.Conditions, traceql.Condition{ Attribute: tagAtrr, Op: traceql.OpNone, @@ -324,6 +509,29 @@ func BenchmarkFetchTagValues(b *testing.B) { tag: "resource.namespace", query: `{span.http.status_code=200}`, }, + { + tag: "resource.namespace", + query: `{span.http.status_code=200}`, + }, + // pathologic cases + /* + { + tag: "resource.k8s.node.name", + query: `{span.http.method="GET"}`, + }, + { + tag: "span.sampler.type", + query: `{span.http.method="GET"}`, + }, + { + tag: "span.sampler.type", + query: `{resource.k8s.node.name>"aaa"}`, + }, + { + tag: "resource.k8s.node.name", + query: `{span.sampler.type>"aaa"}`, + }, + */ } ctx := context.TODO() @@ -372,3 +580,74 @@ func BenchmarkFetchTagValues(b *testing.B) { }) } } + +func BenchmarkFetchTags(b *testing.B) { + testCases := []struct { + query string + }{ + { + query: `{resource.namespace="tempo-ops"}`, // well known/dedicated column + }, + { + query: `{resource.k8s.node.name>"h"}`, // generic attribute + }, + { + query: `{span.http.status_code=200}`, // well known/dedicated column + }, + { + query: `{span.sampler.type="probabilistic"}`, // generic attribute + }, + { + query: `{rootName="Memcache.Put"}`, // trace level + }, + // pathological cases + /* + { + query: `{resource.k8s.node.name>"aaa"}`, // generic attribute + }, + { + query: `{span.http.method="GET"}`, // well known/dedicated column + }, + { + query: `{span.sampler.type>"aaa"}`, // generic attribute + }, + */ + } + + ctx := context.TODO() + tenantID := "1" + // blockID := uuid.MustParse("3685ee3d-cbbf-4f36-bf28-93447a19dea6") + blockID := uuid.MustParse("00145f38-6058-4e57-b1ba-334db8edce23") + + r, _, _, err := local.New(&local.Config{ + // Path: path.Join("/Users/marty/src/tmp/"), + Path: path.Join("/Users/joe/testblock"), + }) + require.NoError(b, err) + + rr := backend.NewReader(r) + meta, err := rr.BlockMeta(ctx, blockID, tenantID) + require.NoError(b, err) + + block := newBackendBlock(meta, rr) + opts := common.DefaultSearchOptions() + + for _, tc := range testCases { + b.Run(fmt.Sprintf("query: %s", tc.query), func(b *testing.B) { + distinctStrings := util.NewDistinctStringCollector(1_000_000) + req, err := traceql.ExtractFetchSpansRequest(tc.query) + require.NoError(b, err) + + autocompleteReq := traceql.AutocompleteRequest{ + Conditions: req.Conditions, + TagName: traceql.Attribute{}, + } + b.ResetTimer() + + for i := 0; i < b.N; i++ { + err := block.FetchTagNames(ctx, autocompleteReq, distinctStrings.Collect, opts) + require.NoError(b, err) + } + }) + } +} diff --git a/tempodb/encoding/vparquet3/block_traceql_test.go b/tempodb/encoding/vparquet3/block_traceql_test.go index 2b03359cc8d..06e0c7466e1 100644 --- a/tempodb/encoding/vparquet3/block_traceql_test.go +++ b/tempodb/encoding/vparquet3/block_traceql_test.go @@ -444,6 +444,8 @@ func fullyPopulatedTestTrace(id common.ID) *Trace { Attrs: []Attribute{ {Key: "foo", Value: strPtr("abc")}, {Key: LabelServiceName, ValueInt: intPtr(123)}, // Different type than dedicated column + {Key: "asdf", ValueInt: intPtr(1234)}, // Different type than dedicated column + {Key: "other", ValueInt: intPtr(1234)}, // Different type than dedicated column }, DedicatedAttributes: DedicatedAttributes{ String01: strPtr("dedicated-resource-attr-value-1"), diff --git a/tempodb/encoding/vparquet3/wal_block.go b/tempodb/encoding/vparquet3/wal_block.go index 4831dfd1bba..33f908d44f5 100644 --- a/tempodb/encoding/vparquet3/wal_block.go +++ b/tempodb/encoding/vparquet3/wal_block.go @@ -706,7 +706,7 @@ func (b *walBlock) FetchTagValues(ctx context.Context, req traceql.AutocompleteR pf := file.parquetFile - iter, err := autocompleteIter(ctx, req, pf, opts, b.meta.DedicatedColumns) + iter, err := autocompleteIter(ctx, req.Conditions, req.TagName, pf, opts, b.meta.DedicatedColumns) if err != nil { return fmt.Errorf("creating fetch iter: %w", err) } diff --git a/vendor/github.com/segmentio/fasthash/fnv1a/hash.go b/vendor/github.com/segmentio/fasthash/fnv1a/hash.go index d2f3c966844..92849b1142b 100644 --- a/vendor/github.com/segmentio/fasthash/fnv1a/hash.go +++ b/vendor/github.com/segmentio/fasthash/fnv1a/hash.go @@ -14,6 +14,11 @@ func HashString64(s string) uint64 { return AddString64(Init64, s) } +// HashBytes64 returns the hash of u. +func HashBytes64(b []byte) uint64 { + return AddBytes64(Init64, b) +} + // HashUint64 returns the hash of u. func HashUint64(u uint64) uint64 { return AddUint64(Init64, u) @@ -34,24 +39,69 @@ func AddString64(h uint64, s string) uint64 { - BenchmarkHash64/hash_function-4 50000000 38.6 ns/op 932.35 MB/s 0 B/op 0 allocs/op */ + for len(s) >= 8 { + h = (h ^ uint64(s[0])) * prime64 + h = (h ^ uint64(s[1])) * prime64 + h = (h ^ uint64(s[2])) * prime64 + h = (h ^ uint64(s[3])) * prime64 + h = (h ^ uint64(s[4])) * prime64 + h = (h ^ uint64(s[5])) * prime64 + h = (h ^ uint64(s[6])) * prime64 + h = (h ^ uint64(s[7])) * prime64 + s = s[8:] + } + + if len(s) >= 4 { + h = (h ^ uint64(s[0])) * prime64 + h = (h ^ uint64(s[1])) * prime64 + h = (h ^ uint64(s[2])) * prime64 + h = (h ^ uint64(s[3])) * prime64 + s = s[4:] + } + + if len(s) >= 2 { + h = (h ^ uint64(s[0])) * prime64 + h = (h ^ uint64(s[1])) * prime64 + s = s[2:] + } + + if len(s) > 0 { + h = (h ^ uint64(s[0])) * prime64 + } + + return h +} + +// AddBytes64 adds the hash of b to the precomputed hash value h. +func AddBytes64(h uint64, b []byte) uint64 { + for len(b) >= 8 { + h = (h ^ uint64(b[0])) * prime64 + h = (h ^ uint64(b[1])) * prime64 + h = (h ^ uint64(b[2])) * prime64 + h = (h ^ uint64(b[3])) * prime64 + h = (h ^ uint64(b[4])) * prime64 + h = (h ^ uint64(b[5])) * prime64 + h = (h ^ uint64(b[6])) * prime64 + h = (h ^ uint64(b[7])) * prime64 + b = b[8:] + } + + if len(b) >= 4 { + h = (h ^ uint64(b[0])) * prime64 + h = (h ^ uint64(b[1])) * prime64 + h = (h ^ uint64(b[2])) * prime64 + h = (h ^ uint64(b[3])) * prime64 + b = b[4:] + } - i := 0 - n := (len(s) / 8) * 8 - - for i != n { - h = (h ^ uint64(s[i])) * prime64 - h = (h ^ uint64(s[i+1])) * prime64 - h = (h ^ uint64(s[i+2])) * prime64 - h = (h ^ uint64(s[i+3])) * prime64 - h = (h ^ uint64(s[i+4])) * prime64 - h = (h ^ uint64(s[i+5])) * prime64 - h = (h ^ uint64(s[i+6])) * prime64 - h = (h ^ uint64(s[i+7])) * prime64 - i += 8 + if len(b) >= 2 { + h = (h ^ uint64(b[0])) * prime64 + h = (h ^ uint64(b[1])) * prime64 + b = b[2:] } - for _, c := range s[i:] { - h = (h ^ uint64(c)) * prime64 + if len(b) > 0 { + h = (h ^ uint64(b[0])) * prime64 } return h diff --git a/vendor/github.com/segmentio/fasthash/fnv1a/hash32.go b/vendor/github.com/segmentio/fasthash/fnv1a/hash32.go index df1f3e5b07f..ac91e247034 100644 --- a/vendor/github.com/segmentio/fasthash/fnv1a/hash32.go +++ b/vendor/github.com/segmentio/fasthash/fnv1a/hash32.go @@ -14,6 +14,11 @@ func HashString32(s string) uint32 { return AddString32(Init32, s) } +// HashBytes32 returns the hash of u. +func HashBytes32(b []byte) uint32 { + return AddBytes32(Init32, b) +} + // HashUint32 returns the hash of u. func HashUint32(u uint32) uint32 { return AddUint32(Init32, u) @@ -21,23 +26,69 @@ func HashUint32(u uint32) uint32 { // AddString32 adds the hash of s to the precomputed hash value h. func AddString32(h uint32, s string) uint32 { - i := 0 - n := (len(s) / 8) * 8 - - for i != n { - h = (h ^ uint32(s[i])) * prime32 - h = (h ^ uint32(s[i+1])) * prime32 - h = (h ^ uint32(s[i+2])) * prime32 - h = (h ^ uint32(s[i+3])) * prime32 - h = (h ^ uint32(s[i+4])) * prime32 - h = (h ^ uint32(s[i+5])) * prime32 - h = (h ^ uint32(s[i+6])) * prime32 - h = (h ^ uint32(s[i+7])) * prime32 - i += 8 - } - - for _, c := range s[i:] { - h = (h ^ uint32(c)) * prime32 + for len(s) >= 8 { + h = (h ^ uint32(s[0])) * prime32 + h = (h ^ uint32(s[1])) * prime32 + h = (h ^ uint32(s[2])) * prime32 + h = (h ^ uint32(s[3])) * prime32 + h = (h ^ uint32(s[4])) * prime32 + h = (h ^ uint32(s[5])) * prime32 + h = (h ^ uint32(s[6])) * prime32 + h = (h ^ uint32(s[7])) * prime32 + s = s[8:] + } + + if len(s) >= 4 { + h = (h ^ uint32(s[0])) * prime32 + h = (h ^ uint32(s[1])) * prime32 + h = (h ^ uint32(s[2])) * prime32 + h = (h ^ uint32(s[3])) * prime32 + s = s[4:] + } + + if len(s) >= 2 { + h = (h ^ uint32(s[0])) * prime32 + h = (h ^ uint32(s[1])) * prime32 + s = s[2:] + } + + if len(s) > 0 { + h = (h ^ uint32(s[0])) * prime32 + } + + return h +} + +// AddBytes32 adds the hash of b to the precomputed hash value h. +func AddBytes32(h uint32, b []byte) uint32 { + for len(b) >= 8 { + h = (h ^ uint32(b[0])) * prime32 + h = (h ^ uint32(b[1])) * prime32 + h = (h ^ uint32(b[2])) * prime32 + h = (h ^ uint32(b[3])) * prime32 + h = (h ^ uint32(b[4])) * prime32 + h = (h ^ uint32(b[5])) * prime32 + h = (h ^ uint32(b[6])) * prime32 + h = (h ^ uint32(b[7])) * prime32 + b = b[8:] + } + + if len(b) >= 4 { + h = (h ^ uint32(b[0])) * prime32 + h = (h ^ uint32(b[1])) * prime32 + h = (h ^ uint32(b[2])) * prime32 + h = (h ^ uint32(b[3])) * prime32 + b = b[4:] + } + + if len(b) >= 2 { + h = (h ^ uint32(b[0])) * prime32 + h = (h ^ uint32(b[1])) * prime32 + b = b[2:] + } + + if len(b) > 0 { + h = (h ^ uint32(b[0])) * prime32 } return h diff --git a/vendor/modules.txt b/vendor/modules.txt index 8a2392e4f5e..5e3feb4d0b3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1174,8 +1174,8 @@ github.com/sean-/seed # github.com/segmentio/encoding v0.3.6 ## explicit; go 1.14 github.com/segmentio/encoding/thrift -# github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e -## explicit +# github.com/segmentio/fasthash v1.0.3 +## explicit; go 1.11 github.com/segmentio/fasthash/fnv1a # github.com/sercand/kuberesolver/v5 v5.1.1 ## explicit; go 1.18 From 4472ec36fc70376f716ad8cadc13624b872b9600 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Mon, 20 May 2024 14:26:43 -0400 Subject: [PATCH 02/21] add well known and dedicated cols Signed-off-by: Joe Elliott --- .../encoding/vparquet3/block_autocomplete.go | 32 +++++++++++++++++++ .../vparquet3/block_autocomplete_test.go | 6 ++++ 2 files changed, 38 insertions(+) diff --git a/tempodb/encoding/vparquet3/block_autocomplete.go b/tempodb/encoding/vparquet3/block_autocomplete.go index d78ac520d33..6a36bbae03e 100644 --- a/tempodb/encoding/vparquet3/block_autocomplete.go +++ b/tempodb/encoding/vparquet3/block_autocomplete.go @@ -80,6 +80,38 @@ func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.Autocomple } } + // currently just seeing if any row groups have values. future improvements: + // - only check those row groups that otherwise have a match in the iterators above + // - use rep/def levels to determine if a value exists at a row w/o actually testing values. + // atm i believe this requires reading the pages themselves b/c the rep/def lvls come w/ the page + hasValues := func(path string, pf *parquet.File) bool { + idx, _ := parquetquery.GetColumnIndexByPath(pf, path) + md := pf.Metadata() + for _, rg := range md.RowGroups { + col := rg.Columns[idx] + if col.MetaData.NumValues-col.MetaData.Statistics.NullCount > 0 { + return true + } + } + + return false + } + + // add all well known columns that have values + for name, entry := range wellKnownColumnLookups { + if hasValues(entry.columnPath, pf) { // jpe - resource vs span scope? + cb(name) + } + } + + // add all dedicated columns that have values + dedCols := dedicatedColumnsToColumnMapping(b.meta.DedicatedColumns) // jpe - scope again? + for name, col := range dedCols.mapping { + if hasValues(col.ColumnPath, pf) { + cb(name) + } + } + return nil } diff --git a/tempodb/encoding/vparquet3/block_autocomplete_test.go b/tempodb/encoding/vparquet3/block_autocomplete_test.go index 0bbe342b124..e3ac02bf9a0 100644 --- a/tempodb/encoding/vparquet3/block_autocomplete_test.go +++ b/tempodb/encoding/vparquet3/block_autocomplete_test.go @@ -194,6 +194,12 @@ func TestFetchTagNames(t *testing.T) { expectedValues := tc.expectedValues actualValues := distinctAttrNames.Strings() + + // add dedicated and well known columns to expected values. the code currently does not + // attempt to perfectly filter these, but instead adds them to the return if any values are present + expectedValues = append(expectedValues, "dedicated.span.1", "dedicated.resource.1") + expectedValues = append(expectedValues, "cluster", "http.method", "service.name") + sort.Strings(expectedValues) sort.Strings(actualValues) require.Equal(t, expectedValues, actualValues) From a34699f540c53441ec32f62e1aa4ed9e3d64d1fb Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Mon, 20 May 2024 15:33:30 -0400 Subject: [PATCH 03/21] Support for scopes at FetchTags Signed-off-by: Joe Elliott --- pkg/traceql/storage.go | 7 + .../encoding/vparquet3/block_autocomplete.go | 94 +++++++----- .../vparquet3/block_autocomplete_test.go | 139 +++++++++++------- tempodb/encoding/vparquet3/wal_block.go | 7 +- 4 files changed, 155 insertions(+), 92 deletions(-) diff --git a/pkg/traceql/storage.go b/pkg/traceql/storage.go index 3a504e23ddf..99e23461d2a 100644 --- a/pkg/traceql/storage.go +++ b/pkg/traceql/storage.go @@ -258,3 +258,10 @@ func NewSpansetFetcherWrapper(f func(ctx context.Context, req FetchSpansRequest) func (s SpansetFetcherWrapper) Fetch(ctx context.Context, request FetchSpansRequest) (FetchSpansResponse, error) { return s.f(ctx, request) } + +type FetchTagsRequest struct { + Conditions []Condition + Scope AttributeScope + // jpe - add effort param? + // TODO: Add start and end time? +} diff --git a/tempodb/encoding/vparquet3/block_autocomplete.go b/tempodb/encoding/vparquet3/block_autocomplete.go index 6a36bbae03e..9b424494aa6 100644 --- a/tempodb/encoding/vparquet3/block_autocomplete.go +++ b/tempodb/encoding/vparquet3/block_autocomplete.go @@ -14,29 +14,33 @@ import ( "github.com/pkg/errors" ) -// jpe - "AutoComplete" -> FetchTagValuesRequest and FetchTagNamesRequest -// jpe - what to share with below? // jpe - require scope? -// jpe - add any ded col or well known col that has values. confirm this doesn't add reads -// jpe - explore using rep/def lvls to determine if a value exists at a row? confirm this doesn't add reads -// jpe - have to add an iterator for the generic attr name if it doesn't exist +// jpe - add effort param? type tagRequest struct { - keys bool - tag traceql.Attribute + // applies to tag names and tag values. the conditions by which to return the filtered data + conditions []traceql.Condition + // scope requested. only used for tag names. A scope of None means all scopes. + scope traceql.AttributeScope + // tag requested. only used for tag values. if populated then return tag values for this tag, otherwise return tag names. + tag traceql.Attribute } -func (r tagRequest) attributeRequested() bool { - return r.tag != (traceql.Attribute{}) -} +func (r tagRequest) keysRequested(scope traceql.AttributeScope) bool { + if r.tag != (traceql.Attribute{}) { + return false + } + + // none scope means return all scopes + if r.scope == traceql.AttributeScopeNone { + return true + } -func (r tagRequest) keysRequested() bool { - return r.keys + return r.scope == scope } -// jpe - current searchTags takes one param at a time for scope. we should change it to take a list of scopes. otherwise we will -// be repeating this same search 2x -func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.AutocompleteRequest, cb common.TagCallback, opts common.SearchOptions) error { +// jpe - callback needs to take a scope somehow +func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsRequest, cb common.TagCallback, opts common.SearchOptions) error { err := checkConditions(req.Conditions) if err != nil { return errors.Wrap(err, "conditions invalid") @@ -57,7 +61,12 @@ func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.Autocomple return err } - iter, err := autocompleteIter(ctx, req.Conditions, traceql.Attribute{}, pf, opts, b.meta.DedicatedColumns) // jpe call iter funcs directly to avoid nil cb + tr := tagRequest{ + conditions: req.Conditions, + scope: req.Scope, + } + + iter, err := autocompleteIter(ctx, tr, pf, opts, b.meta.DedicatedColumns) // jpe call iter funcs directly to avoid nil cb if err != nil { return errors.Wrap(err, "creating fetch iter") } @@ -99,16 +108,32 @@ func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.Autocomple // add all well known columns that have values for name, entry := range wellKnownColumnLookups { + if entry.level != tr.scope && tr.scope != traceql.AttributeScopeNone { + continue + } + if hasValues(entry.columnPath, pf) { // jpe - resource vs span scope? cb(name) } } - // add all dedicated columns that have values - dedCols := dedicatedColumnsToColumnMapping(b.meta.DedicatedColumns) // jpe - scope again? - for name, col := range dedCols.mapping { - if hasValues(col.ColumnPath, pf) { - cb(name) + // add all span dedicated columns that have values + if tr.scope == traceql.AttributeScopeNone || tr.scope == traceql.AttributeScopeSpan { + dedCols := dedicatedColumnsToColumnMapping(b.meta.DedicatedColumns, backend.DedicatedColumnScopeSpan) + for name, col := range dedCols.mapping { + if hasValues(col.ColumnPath, pf) { + cb(name) + } + } + } + + // add all resource dedicated columns that have values + if tr.scope == traceql.AttributeScopeNone || tr.scope == traceql.AttributeScopeResource { + dedCols := dedicatedColumnsToColumnMapping(b.meta.DedicatedColumns, backend.DedicatedColumnScopeResource) + for name, col := range dedCols.mapping { + if hasValues(col.ColumnPath, pf) { + cb(name) + } } } @@ -136,7 +161,12 @@ func (b *backendBlock) FetchTagValues(ctx context.Context, req traceql.Autocompl return err } - iter, err := autocompleteIter(ctx, req.Conditions, req.TagName, pf, opts, b.meta.DedicatedColumns) // jpe call iter funcs directly to avoid nil cb + tr := tagRequest{ + conditions: req.Conditions, + tag: req.TagName, + } + + iter, err := autocompleteIter(ctx, tr, pf, opts, b.meta.DedicatedColumns) // jpe call iter funcs directly to avoid nil cb if err != nil { return errors.Wrap(err, "creating fetch iter") } @@ -163,14 +193,9 @@ func (b *backendBlock) FetchTagValues(ctx context.Context, req traceql.Autocompl } // autocompleteIter creates an iterator that will collect values for a given attribute/tag. -func autocompleteIter(ctx context.Context, conds []traceql.Condition, tagName traceql.Attribute, pf *parquet.File, opts common.SearchOptions, dc backend.DedicatedColumns) (parquetquery.Iterator, error) { - tr := tagRequest{ // jpe - push this up? - tag: tagName, - keys: tagName == (traceql.Attribute{}), - } - +func autocompleteIter(ctx context.Context, tr tagRequest, pf *parquet.File, opts common.SearchOptions, dc backend.DedicatedColumns) (parquetquery.Iterator, error) { // categorizeConditions conditions into span-level or resource-level - _, spanConditions, resourceConditions, traceConditions, err := categorizeConditions(conds) + _, spanConditions, resourceConditions, traceConditions, err := categorizeConditions(tr.conditions) if err != nil { return nil, err } @@ -180,14 +205,14 @@ func autocompleteIter(ctx context.Context, conds []traceql.Condition, tagName tr var currentIter parquetquery.Iterator - if len(spanConditions) > 0 || tr.keysRequested() { + if len(spanConditions) > 0 || tr.keysRequested(traceql.AttributeScopeSpan) { currentIter, err = createDistinctSpanIterator(makeIter, tr, spanConditions, dc) if err != nil { return nil, errors.Wrap(err, "creating span iterator") } } - if len(resourceConditions) > 0 || tr.keysRequested() { + if len(resourceConditions) > 0 || tr.keysRequested(traceql.AttributeScopeResource) { currentIter, err = createDistinctResourceIterator(makeIter, tr, currentIter, resourceConditions, dc) if err != nil { return nil, errors.Wrap(err, "creating resource iterator") @@ -464,13 +489,14 @@ func createDistinctAttributeIterator( valueIters = append(valueIters, makeIter(boolPath, orIfNeeded(boolPreds), "bool")) } - if len(valueIters) > 0 || len(iters) > 0 || tr.keysRequested() { + scope := scopeFromDefinitionLevel(definitionLevel) + if len(valueIters) > 0 || len(iters) > 0 || tr.keysRequested(scope) { if len(valueIters) > 0 { tagIter, err := parquetquery.NewLeftJoinIterator( definitionLevel, []parquetquery.Iterator{makeIter(keyPath, parquetquery.NewStringInPredicate(attrKeys), "key")}, valueIters, - newDistinctAttrCollector(scopeFromDefinitionLevel(definitionLevel), false), // jpe - empty attribute signals we want tag names? + newDistinctAttrCollector(scope, false), ) if err != nil { return nil, fmt.Errorf("creating left join iterator: %w", err) @@ -478,7 +504,7 @@ func createDistinctAttributeIterator( iters = append(iters, tagIter) } - if tr.keysRequested() { + if tr.keysRequested(scope) { return keyNameIterator(makeIter, definitionLevel, keyPath, iters) } diff --git a/tempodb/encoding/vparquet3/block_autocomplete_test.go b/tempodb/encoding/vparquet3/block_autocomplete_test.go index e3ac02bf9a0..6c115f8c5d5 100644 --- a/tempodb/encoding/vparquet3/block_autocomplete_test.go +++ b/tempodb/encoding/vparquet3/block_autocomplete_test.go @@ -20,70 +20,78 @@ import ( func TestFetchTagNames(t *testing.T) { testCases := []struct { - name string - query string - expectedValues []string + name string + query string + expectedSpanValues []string + expectedResourceValues []string }{ { name: "no query - fall back to old search", // jpe - not working. not falling back to old search due to span start time cond? query: "{}", - expectedValues: []string{ - "generic-01", + expectedSpanValues: []string{ "generic-01-01", "generic-01-02", - "generic-02", "generic-02-01", "span-same", - "resource-same", }, + expectedResourceValues: []string{"generic-01", "generic-02", "resource-same"}, }, { - name: "matches nothing", - query: "{span.generic-01-01=`bar`}", - expectedValues: []string{}, + name: "matches nothing", + query: "{span.generic-01-01=`bar`}", + expectedSpanValues: []string{}, + expectedResourceValues: []string{}, }, // span { - name: "intrinsic span", - query: "{statusMessage=`msg-01-01`}", - expectedValues: []string{"generic-01", "generic-01-01", "span-same", "resource-same"}, + name: "intrinsic span", + query: "{statusMessage=`msg-01-01`}", + expectedSpanValues: []string{"generic-01-01", "span-same"}, + expectedResourceValues: []string{"generic-01", "resource-same"}, }, { - name: "well known span", - query: "{span.http.method=`method-01-01`}", - expectedValues: []string{"generic-01", "generic-01-01", "span-same", "resource-same"}, + name: "well known span", + query: "{span.http.method=`method-01-01`}", + expectedSpanValues: []string{"generic-01-01", "span-same"}, + expectedResourceValues: []string{"generic-01", "resource-same"}, }, { - name: "generic span", - query: "{span.generic-01-01=`foo`}", - expectedValues: []string{"generic-01", "generic-01-01", "span-same", "resource-same"}, + name: "generic span", + query: "{span.generic-01-01=`foo`}", + expectedSpanValues: []string{"generic-01-01", "span-same"}, + expectedResourceValues: []string{"generic-01", "resource-same"}, }, { - name: "match two spans", - query: "{span.span-same=`foo`}", - expectedValues: []string{"generic-01", "generic-01-01", "span-same", "resource-same", "generic-02", "generic-02-01"}, + name: "match two spans", + query: "{span.span-same=`foo`}", + expectedSpanValues: []string{"generic-01-01", "span-same", "generic-02-01"}, + expectedResourceValues: []string{"generic-01", "resource-same", "generic-02"}, }, // resource { - name: "well known resource", - query: "{resource.cluster=`cluster-01`}", - expectedValues: []string{"generic-01", "generic-01-01", "generic-01-02", "span-same", "resource-same"}, + name: "well known resource", + query: "{resource.cluster=`cluster-01`}", + expectedSpanValues: []string{"generic-01-01", "generic-01-02", "span-same"}, + expectedResourceValues: []string{"generic-01", "resource-same"}, }, { - name: "generic resource", - query: "{resource.generic-01=`bar`}", - expectedValues: []string{"generic-01", "generic-01-01", "generic-01-02", "span-same", "resource-same"}, + name: "generic resource", + query: "{resource.generic-01=`bar`}", + expectedSpanValues: []string{"generic-01-01", "generic-01-02", "span-same"}, + expectedResourceValues: []string{"generic-01", "resource-same"}, }, { - name: "match two resources", - query: "{resource.resource-same=`foo`}", - expectedValues: []string{"generic-01", "generic-01-01", "generic-01-02", "span-same", "resource-same", "generic-02", "generic-02-01"}, + name: "match two resources", + query: "{resource.resource-same=`foo`}", + expectedSpanValues: []string{"generic-01-01", "generic-01-02", "span-same", "generic-02-01"}, + expectedResourceValues: []string{"generic-01", "resource-same", "generic-02"}, }, // trace level match { - name: "trace", - query: "{rootName=`root` }", - expectedValues: []string{"generic-01", "generic-01-01", "generic-01-02", "span-same", "resource-same", "generic-02", "generic-02-01"}, + name: "trace", + query: "{rootName=`root` }", + expectedSpanValues: []string{"generic-01-01", "generic-01-02", "span-same", "generic-02-01"}, + expectedResourceValues: []string{"generic-01", "resource-same", "generic-02"}, }, } @@ -178,32 +186,51 @@ func TestFetchTagNames(t *testing.T) { opts := common.DefaultSearchOptions() for _, tc := range testCases { - t.Run(fmt.Sprintf("query: %s", tc.query), func(t *testing.T) { - distinctAttrNames := util.NewDistinctStringCollector(1_000_000) - req, err := traceql.ExtractFetchSpansRequest(tc.query) - require.NoError(t, err) + for _, scope := range []traceql.AttributeScope{traceql.AttributeScopeSpan, traceql.AttributeScopeResource, traceql.AttributeScopeNone} { + expectedSpanValues := tc.expectedSpanValues + expectedResourceValues := tc.expectedResourceValues - // Build autocomplete request - autocompleteReq := traceql.AutocompleteRequest{ - Conditions: req.Conditions, - TagName: traceql.Attribute{}, + // add dedicated and well known columns to expected values. the code currently does not + // attempt to perfectly filter these, but instead adds them to the return if any values are present + dedicatedSpanValues := []string{"dedicated.span.1"} + dedicatedResourceValues := []string{"dedicated.resource.1"} + + wellKnownSpanValues := []string{"http.method"} + wellKnownResourceValues := []string{"cluster", "service.name"} + + var expectedValues []string + if scope == traceql.AttributeScopeSpan || scope == traceql.AttributeScopeNone { + expectedValues = append(expectedValues, expectedSpanValues...) + expectedValues = append(expectedValues, wellKnownSpanValues...) + expectedValues = append(expectedValues, dedicatedSpanValues...) + } + if scope == traceql.AttributeScopeResource || scope == traceql.AttributeScopeNone { + expectedValues = append(expectedValues, expectedResourceValues...) + expectedValues = append(expectedValues, wellKnownResourceValues...) + expectedValues = append(expectedValues, dedicatedResourceValues...) } - err = block.FetchTagNames(ctx, autocompleteReq, distinctAttrNames.Collect, opts) - require.NoError(t, err) + t.Run(fmt.Sprintf("query: %s-%s", tc.query, scope), func(t *testing.T) { + distinctAttrNames := util.NewDistinctStringCollector(1_000_000) + req, err := traceql.ExtractFetchSpansRequest(tc.query) + require.NoError(t, err) - expectedValues := tc.expectedValues - actualValues := distinctAttrNames.Strings() + // Build autocomplete request + autocompleteReq := traceql.FetchTagsRequest{ + Conditions: req.Conditions, + Scope: scope, + } - // add dedicated and well known columns to expected values. the code currently does not - // attempt to perfectly filter these, but instead adds them to the return if any values are present - expectedValues = append(expectedValues, "dedicated.span.1", "dedicated.resource.1") - expectedValues = append(expectedValues, "cluster", "http.method", "service.name") + err = block.FetchTagNames(ctx, autocompleteReq, distinctAttrNames.Collect, opts) + require.NoError(t, err) - sort.Strings(expectedValues) - sort.Strings(actualValues) - require.Equal(t, expectedValues, actualValues) - }) + actualValues := distinctAttrNames.Strings() + + sort.Strings(expectedValues) + sort.Strings(actualValues) + require.Equal(t, expectedValues, actualValues) + }) + } } } @@ -644,9 +671,9 @@ func BenchmarkFetchTags(b *testing.B) { req, err := traceql.ExtractFetchSpansRequest(tc.query) require.NoError(b, err) - autocompleteReq := traceql.AutocompleteRequest{ + autocompleteReq := traceql.FetchTagsRequest{ Conditions: req.Conditions, - TagName: traceql.Attribute{}, + Scope: traceql.AttributeScopeNone, } b.ResetTimer() diff --git a/tempodb/encoding/vparquet3/wal_block.go b/tempodb/encoding/vparquet3/wal_block.go index 33f908d44f5..a977e798c12 100644 --- a/tempodb/encoding/vparquet3/wal_block.go +++ b/tempodb/encoding/vparquet3/wal_block.go @@ -704,9 +704,12 @@ func (b *walBlock) FetchTagValues(ctx context.Context, req traceql.AutocompleteR } defer file.Close() - pf := file.parquetFile + tr := tagRequest{ + conditions: req.Conditions, + tag: req.TagName, + } - iter, err := autocompleteIter(ctx, req.Conditions, req.TagName, pf, opts, b.meta.DedicatedColumns) + iter, err := autocompleteIter(ctx, tr, file.parquetFile, opts, b.meta.DedicatedColumns) if err != nil { return fmt.Errorf("creating fetch iter: %w", err) } From 64bd7b419d6241bee9488a3b984fcc62d9ed7f71 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Tue, 21 May 2024 12:08:07 -0400 Subject: [PATCH 04/21] conditionally add selectAs in the trace level iterator Signed-off-by: Joe Elliott --- tempodb/encoding/vparquet3/block_autocomplete.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tempodb/encoding/vparquet3/block_autocomplete.go b/tempodb/encoding/vparquet3/block_autocomplete.go index 9b424494aa6..e92d0920a2f 100644 --- a/tempodb/encoding/vparquet3/block_autocomplete.go +++ b/tempodb/encoding/vparquet3/block_autocomplete.go @@ -220,7 +220,7 @@ func autocompleteIter(ctx context.Context, tr tagRequest, pf *parquet.File, opts } if len(traceConditions) > 0 { - currentIter, err = createDistinctTraceIterator(makeIter, currentIter, traceConditions) + currentIter, err = createDistinctTraceIterator(makeIter, tr, currentIter, traceConditions) if err != nil { return nil, errors.Wrap(err, "creating trace iterator") } @@ -647,12 +647,20 @@ func createDistinctResourceIterator( func createDistinctTraceIterator( makeIter makeIterFn, + tr tagRequest, resourceIter parquetquery.Iterator, conds []traceql.Condition, ) (parquetquery.Iterator, error) { var err error traceIters := make([]parquetquery.Iterator, 0, 3) + selectAs := func(attr traceql.Attribute, columnPath string) string { + if attr == tr.tag { + return columnPath + } + return "" + } + // add conditional iterators first. this way if someone searches for { traceDuration > 1s && span.foo = "bar"} the query will // be sped up by searching for traceDuration first. note that we can only set the predicates if all conditions is true. // otherwise we just pass the info up to the engine to make a choice @@ -667,7 +675,7 @@ func createDistinctTraceIterator( if err != nil { return nil, err } - traceIters = append(traceIters, makeIter(columnPathDurationNanos, pred, columnPathDurationNanos)) + traceIters = append(traceIters, makeIter(columnPathDurationNanos, pred, selectAs(cond.Attribute, columnPathDurationNanos))) case traceql.IntrinsicTraceRootSpan: var pred parquetquery.Predicate @@ -675,7 +683,7 @@ func createDistinctTraceIterator( if err != nil { return nil, err } - traceIters = append(traceIters, makeIter(columnPathRootSpanName, pred, columnPathRootSpanName)) + traceIters = append(traceIters, makeIter(columnPathRootSpanName, pred, selectAs(cond.Attribute, columnPathRootSpanName))) case traceql.IntrinsicTraceRootService: var pred parquetquery.Predicate @@ -683,7 +691,7 @@ func createDistinctTraceIterator( if err != nil { return nil, err } - traceIters = append(traceIters, makeIter(columnPathRootServiceName, pred, columnPathRootServiceName)) + traceIters = append(traceIters, makeIter(columnPathRootServiceName, pred, selectAs(cond.Attribute, columnPathRootServiceName))) } } From dcce20dc58b218e95874daf7c44f70c26a5ebad7 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Tue, 21 May 2024 13:29:35 -0400 Subject: [PATCH 05/21] add support for intrinsics Signed-off-by: Joe Elliott --- .../encoding/vparquet3/block_autocomplete.go | 72 ++++++++++++------- .../vparquet3/block_autocomplete_test.go | 32 +++++---- 2 files changed, 65 insertions(+), 39 deletions(-) diff --git a/tempodb/encoding/vparquet3/block_autocomplete.go b/tempodb/encoding/vparquet3/block_autocomplete.go index e92d0920a2f..af28b53790d 100644 --- a/tempodb/encoding/vparquet3/block_autocomplete.go +++ b/tempodb/encoding/vparquet3/block_autocomplete.go @@ -72,8 +72,8 @@ func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsR } defer iter.Close() - // fmt.Println("-------------") - // fmt.Println(iter) // jpe remove + fmt.Println("-------------") + fmt.Println(iter) // jpe remove for { // Exhaust the iterator @@ -314,14 +314,37 @@ func createDistinctSpanIterator( addSelectAs(cond.Attribute, columnPathSpanStatusMessage, columnPathSpanStatusMessage) continue + case traceql.IntrinsicNestedSetLeft: + pred, err := createIntPredicate(cond.Op, cond.Operands) + if err != nil { + return nil, err + } + addPredicate(columnPathSpanNestedSetLeft, pred) + addSelectAs(cond.Attribute, columnPathSpanNestedSetLeft, columnPathSpanNestedSetLeft) + continue + + case traceql.IntrinsicNestedSetRight: + pred, err := createIntPredicate(cond.Op, cond.Operands) + if err != nil { + return nil, err + } + addPredicate(columnPathSpanNestedSetRight, pred) + addSelectAs(cond.Attribute, columnPathSpanNestedSetRight, columnPathSpanNestedSetRight) + continue + + case traceql.IntrinsicNestedSetParent: + pred, err := createIntPredicate(cond.Op, cond.Operands) + if err != nil { + return nil, err + } + addPredicate(columnPathSpanParentID, pred) + addSelectAs(cond.Attribute, columnPathSpanParentID, columnPathSpanParentID) + continue + // TODO: Support structural operators case traceql.IntrinsicStructuralDescendant, traceql.IntrinsicStructuralChild, - traceql.IntrinsicStructuralSibling, - // nested set intrinsics should not be considered when autocompleting - traceql.IntrinsicNestedSetLeft, - traceql.IntrinsicNestedSetRight, - traceql.IntrinsicNestedSetParent: + traceql.IntrinsicStructuralSibling: continue } @@ -792,23 +815,24 @@ func (d *distinctKeyPredicate) String() string { } func (d *distinctKeyPredicate) KeepColumnChunk(cc *parquetquery.ColumnChunkHelper) bool { // jpe name better and extend all the way to the tag call back, so we can test globally if something has been added - foundNew := false - - if dict := cc.Dictionary(); dict != nil { - l := dict.Len() - for i := 0; i < l; i++ { - v := dict.Index(int32(i)) - key := unsafeToString(v.ByteArray()) - - if _, ok := d.sentKeys[key]; !ok { - d.sentKeys[key] = struct{}{} - foundNew = true - break // jpe - noodle on this. it's a perf boost somehow? - } - } - } - - return foundNew + // foundNew := false + + // if dict := cc.Dictionary(); dict != nil { + // l := dict.Len() + // for i := 0; i < l; i++ { + // v := dict.Index(int32(i)) + // key := unsafeToString(v.ByteArray()) + + // if _, ok := d.sentKeys[key]; !ok { + // d.sentKeys[key] = struct{}{} + // foundNew = true + // break // jpe - noodle on this. it's a perf boost somehow? + // } + // } + // } + + // return foundNew + return true } func (d *distinctKeyPredicate) KeepPage(pq parquet.Page) bool { diff --git a/tempodb/encoding/vparquet3/block_autocomplete_test.go b/tempodb/encoding/vparquet3/block_autocomplete_test.go index 6c115f8c5d5..a73ee5e3ea4 100644 --- a/tempodb/encoding/vparquet3/block_autocomplete_test.go +++ b/tempodb/encoding/vparquet3/block_autocomplete_test.go @@ -628,7 +628,7 @@ func BenchmarkFetchTags(b *testing.B) { query: `{span.http.status_code=200}`, // well known/dedicated column }, { - query: `{span.sampler.type="probabilistic"}`, // generic attribute + query: `{nestedSetParent=-1}`, // generic attribute }, { query: `{rootName="Memcache.Put"}`, // trace level @@ -666,21 +666,23 @@ func BenchmarkFetchTags(b *testing.B) { opts := common.DefaultSearchOptions() for _, tc := range testCases { - b.Run(fmt.Sprintf("query: %s", tc.query), func(b *testing.B) { - distinctStrings := util.NewDistinctStringCollector(1_000_000) - req, err := traceql.ExtractFetchSpansRequest(tc.query) - require.NoError(b, err) + for _, scope := range []traceql.AttributeScope{traceql.AttributeScopeSpan, traceql.AttributeScopeResource, traceql.AttributeScopeNone} { + b.Run(fmt.Sprintf("query: %s %s", tc.query, scope), func(b *testing.B) { + distinctStrings := util.NewDistinctStringCollector(1_000_000) + req, err := traceql.ExtractFetchSpansRequest(tc.query) + require.NoError(b, err) - autocompleteReq := traceql.FetchTagsRequest{ - Conditions: req.Conditions, - Scope: traceql.AttributeScopeNone, - } - b.ResetTimer() + autocompleteReq := traceql.FetchTagsRequest{ + Conditions: req.Conditions, + Scope: scope, + } + b.ResetTimer() - for i := 0; i < b.N; i++ { - err := block.FetchTagNames(ctx, autocompleteReq, distinctStrings.Collect, opts) - require.NoError(b, err) - } - }) + for i := 0; i < b.N; i++ { + err := block.FetchTagNames(ctx, autocompleteReq, distinctStrings.Collect, opts) + require.NoError(b, err) + } + }) + } } } From 9482f8d605453cab592d47559bfdfb04499f726e Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 28 Jun 2024 08:29:31 -0400 Subject: [PATCH 06/21] add scope Signed-off-by: Joe Elliott --- tempodb/encoding/vparquet3/block_autocomplete.go | 14 +++++--------- .../encoding/vparquet3/block_autocomplete_test.go | 1 - 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/tempodb/encoding/vparquet3/block_autocomplete.go b/tempodb/encoding/vparquet3/block_autocomplete.go index e0129ea4671..991d0b9cdcf 100644 --- a/tempodb/encoding/vparquet3/block_autocomplete.go +++ b/tempodb/encoding/vparquet3/block_autocomplete.go @@ -14,9 +14,6 @@ import ( "github.com/pkg/errors" ) -// jpe - require scope? -// jpe - add effort param? - type tagRequest struct { // applies to tag names and tag values. the conditions by which to return the filtered data conditions []traceql.Condition @@ -39,7 +36,6 @@ func (r tagRequest) keysRequested(scope traceql.AttributeScope) bool { return r.scope == scope } -// jpe - callback needs to take a scope somehow func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsRequest, cb common.TagsCallback, opts common.SearchOptions) error { err := checkConditions(req.Conditions) if err != nil { @@ -53,7 +49,7 @@ func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsR // Last check. No conditions, use old path. It's much faster. if len(req.Conditions) < 1 || mingledConditions { // jpe - fetch with {} puts in a condition for span start time? - return b.SearchTags(ctx, traceql.AttributeScopeResource, cb, opts) // jpe - remove hardcoded scope - should be SearchtagsV2? + return b.SearchTags(ctx, traceql.AttributeScopeResource, cb, opts) } pf, _, err := b.openForSearch(ctx, opts) @@ -66,7 +62,7 @@ func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsR scope: req.Scope, } - iter, err := autocompleteIter(ctx, tr, pf, opts, b.meta.DedicatedColumns) // jpe call iter funcs directly to avoid nil cb + iter, err := autocompleteIter(ctx, tr, pf, opts, b.meta.DedicatedColumns) if err != nil { return errors.Wrap(err, "creating fetch iter") } @@ -85,7 +81,7 @@ func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsR break } for _, oe := range res.OtherEntries { - cb(oe.Key, traceql.AttributeScopeResource) // jpe - set scope correctly + cb(oe.Key, oe.Value.(traceql.AttributeScope)) } } @@ -767,7 +763,7 @@ func (d *distinctAttrCollector) KeepGroup(result *parquetquery.IteratorResult) b if e.Key == "key" { key := unsafeToString(e.Value.ByteArray()) if _, ok := d.sentKeys[key]; !ok { - result.AppendOtherValue(key, traceql.Attribute{}) // jpe allocs? + result.AppendOtherValue(key, d.scope) d.sentKeys[key] = struct{}{} } } @@ -814,7 +810,7 @@ func (d *distinctKeyPredicate) String() string { return fmt.Sprintf("distinctKeyPredicate{}") } -func (d *distinctKeyPredicate) KeepColumnChunk(cc *parquetquery.ColumnChunkHelper) bool { // jpe name better and extend all the way to the tag call back, so we can test globally if something has been added +func (d *distinctKeyPredicate) KeepColumnChunk(cc *parquetquery.ColumnChunkHelper) bool { // foundNew := false // if dict := cc.Dictionary(); dict != nil { diff --git a/tempodb/encoding/vparquet3/block_autocomplete_test.go b/tempodb/encoding/vparquet3/block_autocomplete_test.go index b3e92bfdaf2..eb005687562 100644 --- a/tempodb/encoding/vparquet3/block_autocomplete_test.go +++ b/tempodb/encoding/vparquet3/block_autocomplete_test.go @@ -506,7 +506,6 @@ func TestFetchTagValues(t *testing.T) { tagAtrr, err := traceql.ParseIdentifier(tc.tag) require.NoError(t, err) - // jpe - remove this if we move it into the FetchTagValues function autocompleteReq.Conditions = append(autocompleteReq.Conditions, traceql.Condition{ Attribute: tagAtrr, Op: traceql.OpNone, From 59a65a022517a16a5bd92ac3331e4f3e563632d2 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 28 Jun 2024 09:20:03 -0400 Subject: [PATCH 07/21] all but vp4 Signed-off-by: Joe Elliott --- .../processor/localblocks/processor_test.go | 4 ++ pkg/traceql/storage.go | 2 + tempodb/encoding/common/interfaces.go | 5 +- tempodb/encoding/v2/backend_block.go | 4 ++ tempodb/encoding/v2/wal_block.go | 5 ++ tempodb/encoding/vparquet2/block.go | 5 +- tempodb/encoding/vparquet2/wal_block.go | 5 +- .../encoding/vparquet3/block_autocomplete.go | 40 ++++++++---- .../vparquet3/block_autocomplete_test.go | 6 +- tempodb/encoding/vparquet3/wal_block.go | 62 +++++++++++++++++++ 10 files changed, 119 insertions(+), 19 deletions(-) diff --git a/modules/generator/processor/localblocks/processor_test.go b/modules/generator/processor/localblocks/processor_test.go index 20ee2333886..18edca6dfd7 100644 --- a/modules/generator/processor/localblocks/processor_test.go +++ b/modules/generator/processor/localblocks/processor_test.go @@ -356,4 +356,8 @@ func (m *mockBlock) FetchTagValues(context.Context, traceql.FetchTagValuesReques return nil } +func (m *mockBlock) FetchTagNames(context.Context, traceql.FetchTagsRequest, traceql.FetchTagsCallback, common.SearchOptions) error { + return nil +} + func (m *mockBlock) BlockMeta() *backend.BlockMeta { return m.meta } diff --git a/pkg/traceql/storage.go b/pkg/traceql/storage.go index 53a1ad3f1f0..0a353327464 100644 --- a/pkg/traceql/storage.go +++ b/pkg/traceql/storage.go @@ -263,6 +263,8 @@ func (s SpansetFetcherWrapper) Fetch(ctx context.Context, request FetchSpansRequ return s.f(ctx, request) } +type FetchTagsCallback func(tag string, scope AttributeScope) bool + type FetchTagsRequest struct { Conditions []Condition Scope AttributeScope diff --git a/tempodb/encoding/common/interfaces.go b/tempodb/encoding/common/interfaces.go index 9af8ebfceb4..24bcc20c7c9 100644 --- a/tempodb/encoding/common/interfaces.go +++ b/tempodb/encoding/common/interfaces.go @@ -16,7 +16,7 @@ type Finder interface { } type ( - TagsCallback func(t string, scope traceql.AttributeScope) // flip args? + TagsCallback func(t string, scope traceql.AttributeScope) TagValuesCallback func(t string) bool TagValuesCallbackV2 func(traceql.Static) (stop bool) ) @@ -28,7 +28,8 @@ type Searcher interface { SearchTagValuesV2(ctx context.Context, tag traceql.Attribute, cb TagValuesCallbackV2, opts SearchOptions) error Fetch(context.Context, traceql.FetchSpansRequest, SearchOptions) (traceql.FetchSpansResponse, error) - FetchTagValues(context.Context, traceql.FetchTagValuesRequest, traceql.FetchTagValuesCallback, SearchOptions) error // jpe - remove this for TagValuesCallbackV2 + FetchTagValues(context.Context, traceql.FetchTagValuesRequest, traceql.FetchTagValuesCallback, SearchOptions) error + FetchTagNames(context.Context, traceql.FetchTagsRequest, traceql.FetchTagsCallback, SearchOptions) error } type SearchOptions struct { diff --git a/tempodb/encoding/v2/backend_block.go b/tempodb/encoding/v2/backend_block.go index 4183a09f9b6..55c6f851c82 100644 --- a/tempodb/encoding/v2/backend_block.go +++ b/tempodb/encoding/v2/backend_block.go @@ -168,3 +168,7 @@ func (b *BackendBlock) Fetch(context.Context, traceql.FetchSpansRequest, common. func (b *BackendBlock) FetchTagValues(context.Context, traceql.FetchTagValuesRequest, traceql.FetchTagValuesCallback, common.SearchOptions) error { return common.ErrUnsupported } + +func (b *BackendBlock) FetchTagNames(context.Context, traceql.FetchTagsRequest, traceql.FetchTagsCallback, common.SearchOptions) error { + return common.ErrUnsupported +} diff --git a/tempodb/encoding/v2/wal_block.go b/tempodb/encoding/v2/wal_block.go index 3109dc34331..d1a29da2c76 100644 --- a/tempodb/encoding/v2/wal_block.go +++ b/tempodb/encoding/v2/wal_block.go @@ -311,6 +311,11 @@ func (a *walBlock) FetchTagValues(context.Context, traceql.FetchTagValuesRequest return common.ErrUnsupported } +// FetchTagNames implements traceql.Searcher +func (b *walBlock) FetchTagNames(context.Context, traceql.FetchTagsRequest, traceql.FetchTagsCallback, common.SearchOptions) error { + return common.ErrUnsupported +} + func (a *walBlock) fullFilename() string { filename := a.fullFilenameSeparator("+") _, e1 := os.Stat(filename) diff --git a/tempodb/encoding/vparquet2/block.go b/tempodb/encoding/vparquet2/block.go index 85ccfa32a39..43087081d85 100644 --- a/tempodb/encoding/vparquet2/block.go +++ b/tempodb/encoding/vparquet2/block.go @@ -35,6 +35,9 @@ func (b *backendBlock) BlockMeta() *backend.BlockMeta { } func (b *backendBlock) FetchTagValues(context.Context, traceql.FetchTagValuesRequest, traceql.FetchTagValuesCallback, common.SearchOptions) error { - // TODO: Add support? + return common.ErrUnsupported +} + +func (b *backendBlock) FetchTagNames(context.Context, traceql.FetchTagsRequest, traceql.FetchTagsCallback, common.SearchOptions) error { return common.ErrUnsupported } diff --git a/tempodb/encoding/vparquet2/wal_block.go b/tempodb/encoding/vparquet2/wal_block.go index f4bf1ace929..bf1789b9c3f 100644 --- a/tempodb/encoding/vparquet2/wal_block.go +++ b/tempodb/encoding/vparquet2/wal_block.go @@ -672,7 +672,10 @@ func (b *walBlock) Fetch(ctx context.Context, req traceql.FetchSpansRequest, opt } func (b *walBlock) FetchTagValues(context.Context, traceql.FetchTagValuesRequest, traceql.FetchTagValuesCallback, common.SearchOptions) error { - // TODO: Add support? + return common.ErrUnsupported +} + +func (b *walBlock) FetchTagNames(context.Context, traceql.FetchTagsRequest, traceql.FetchTagsCallback, common.SearchOptions) error { return common.ErrUnsupported } diff --git a/tempodb/encoding/vparquet3/block_autocomplete.go b/tempodb/encoding/vparquet3/block_autocomplete.go index 991d0b9cdcf..f04b2780a48 100644 --- a/tempodb/encoding/vparquet3/block_autocomplete.go +++ b/tempodb/encoding/vparquet3/block_autocomplete.go @@ -36,7 +36,7 @@ func (r tagRequest) keysRequested(scope traceql.AttributeScope) bool { return r.scope == scope } -func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsRequest, cb common.TagsCallback, opts common.SearchOptions) error { +func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsRequest, cb traceql.FetchTagsCallback, opts common.SearchOptions) error { err := checkConditions(req.Conditions) if err != nil { return errors.Wrap(err, "conditions invalid") @@ -49,7 +49,9 @@ func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsR // Last check. No conditions, use old path. It's much faster. if len(req.Conditions) < 1 || mingledConditions { // jpe - fetch with {} puts in a condition for span start time? - return b.SearchTags(ctx, traceql.AttributeScopeResource, cb, opts) + return b.SearchTags(ctx, traceql.AttributeScopeResource, func(t string, scope traceql.AttributeScope) { + cb(t, scope) + }, opts) } pf, _, err := b.openForSearch(ctx, opts) @@ -81,10 +83,18 @@ func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsR break } for _, oe := range res.OtherEntries { - cb(oe.Key, oe.Value.(traceql.AttributeScope)) + if cb(oe.Key, oe.Value.(traceql.AttributeScope)) { + return nil // We have enough values + } } } + tagNamesForSpecialColumns(req.Scope, pf, b.meta.DedicatedColumns, cb) + + return nil +} + +func tagNamesForSpecialColumns(scope traceql.AttributeScope, pf *parquet.File, dcs backend.DedicatedColumns, cb traceql.FetchTagsCallback) { // currently just seeing if any row groups have values. future improvements: // - only check those row groups that otherwise have a match in the iterators above // - use rep/def levels to determine if a value exists at a row w/o actually testing values. @@ -104,36 +114,40 @@ func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsR // add all well known columns that have values for name, entry := range wellKnownColumnLookups { - if entry.level != tr.scope && tr.scope != traceql.AttributeScopeNone { + if entry.level != scope && scope != traceql.AttributeScopeNone { continue } if hasValues(entry.columnPath, pf) { - cb(name, entry.level) + if cb(name, entry.level) { + return + } } } // add all span dedicated columns that have values - if tr.scope == traceql.AttributeScopeNone || tr.scope == traceql.AttributeScopeSpan { - dedCols := dedicatedColumnsToColumnMapping(b.meta.DedicatedColumns, backend.DedicatedColumnScopeSpan) + if scope == traceql.AttributeScopeNone || scope == traceql.AttributeScopeSpan { + dedCols := dedicatedColumnsToColumnMapping(dcs, backend.DedicatedColumnScopeSpan) for name, col := range dedCols.mapping { if hasValues(col.ColumnPath, pf) { - cb(name, traceql.AttributeScopeSpan) + if cb(name, traceql.AttributeScopeSpan) { + return + } } } } // add all resource dedicated columns that have values - if tr.scope == traceql.AttributeScopeNone || tr.scope == traceql.AttributeScopeResource { - dedCols := dedicatedColumnsToColumnMapping(b.meta.DedicatedColumns, backend.DedicatedColumnScopeResource) + if scope == traceql.AttributeScopeNone || scope == traceql.AttributeScopeResource { + dedCols := dedicatedColumnsToColumnMapping(dcs, backend.DedicatedColumnScopeResource) for name, col := range dedCols.mapping { if hasValues(col.ColumnPath, pf) { - cb(name, traceql.AttributeScopeResource) + if cb(name, traceql.AttributeScopeResource) { + return + } } } } - - return nil } func (b *backendBlock) FetchTagValues(ctx context.Context, req traceql.FetchTagValuesRequest, cb traceql.FetchTagValuesCallback, opts common.SearchOptions) error { diff --git a/tempodb/encoding/vparquet3/block_autocomplete_test.go b/tempodb/encoding/vparquet3/block_autocomplete_test.go index eb005687562..15d5164c291 100644 --- a/tempodb/encoding/vparquet3/block_autocomplete_test.go +++ b/tempodb/encoding/vparquet3/block_autocomplete_test.go @@ -221,8 +221,9 @@ func TestFetchTagNames(t *testing.T) { Scope: scope, } - err = block.FetchTagNames(ctx, autocompleteReq, func(t string, scope traceql.AttributeScope) { + err = block.FetchTagNames(ctx, autocompleteReq, func(t string, scope traceql.AttributeScope) bool { distinctAttrNames.Collect(scope.String(), t) + return false }, opts) require.NoError(t, err) @@ -686,8 +687,9 @@ func BenchmarkFetchTags(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - err := block.FetchTagNames(ctx, autocompleteReq, func(t string, scope traceql.AttributeScope) { + err := block.FetchTagNames(ctx, autocompleteReq, func(t string, scope traceql.AttributeScope) bool { distinctStrings.Collect(scope.String(), t) + return false }, opts) require.NoError(b, err) } diff --git a/tempodb/encoding/vparquet3/wal_block.go b/tempodb/encoding/vparquet3/wal_block.go index af3b55a96b9..b735d45fc44 100644 --- a/tempodb/encoding/vparquet3/wal_block.go +++ b/tempodb/encoding/vparquet3/wal_block.go @@ -744,6 +744,68 @@ func (b *walBlock) FetchTagValues(ctx context.Context, req traceql.FetchTagValue return nil } +func (b *walBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsRequest, cb traceql.FetchTagsCallback, opts common.SearchOptions) error { + err := checkConditions(req.Conditions) + if err != nil { + return fmt.Errorf("conditions invalid: %w", err) + } + + mingledConditions, _, _, _, err := categorizeConditions(req.Conditions) + if err != nil { + return err + } + + if len(req.Conditions) < 1 || mingledConditions { // jpe - fetch with {} puts in a condition for span start time? + return b.SearchTags(ctx, traceql.AttributeScopeResource, func(t string, scope traceql.AttributeScope) { + cb(t, scope) + }, opts) + } + + blockFlushes := b.readFlushes() + for _, page := range blockFlushes { + file, err := page.file(ctx) + if err != nil { + return fmt.Errorf("error opening file %s: %w", page.path, err) + } + defer file.Close() + + tr := tagRequest{ + conditions: req.Conditions, + scope: req.Scope, + } + + iter, err := autocompleteIter(ctx, tr, file.parquetFile, opts, b.meta.DedicatedColumns) + if err != nil { + return fmt.Errorf("creating fetch iter: %w", err) + } + + for { + // Exhaust the iterator + res, err := iter.Next() + if err != nil { + iter.Close() + return err + } + if res == nil { + break + } + for _, oe := range res.OtherEntries { + if cb(oe.Key, oe.Value.(traceql.AttributeScope)) { + iter.Close() + return nil // We have enough values + } + } + } + iter.Close() + + // add well known + tagNamesForSpecialColumns(req.Scope, file.parquetFile, b.meta.DedicatedColumns, cb) + } + + // combine iters? + return nil +} + func (b *walBlock) walPath() string { filename := fmt.Sprintf("%s+%s+%s", b.meta.BlockID, b.meta.TenantID, VersionString) return filepath.Join(b.path, filename) From b26f8bad319f0e7aedfb222dd41d9cf2d6ef311d Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 28 Jun 2024 09:25:47 -0400 Subject: [PATCH 08/21] all todos Signed-off-by: Joe Elliott --- pkg/traceql/engine.go | 2 +- pkg/traceql/storage.go | 1 - .../encoding/vparquet3/block_autocomplete.go | 56 ++----------------- .../vparquet3/block_autocomplete_test.go | 2 +- tempodb/encoding/vparquet3/wal_block.go | 2 +- 5 files changed, 7 insertions(+), 56 deletions(-) diff --git a/pkg/traceql/engine.go b/pkg/traceql/engine.go index 35a5b0623fb..166b5356fef 100644 --- a/pkg/traceql/engine.go +++ b/pkg/traceql/engine.go @@ -219,7 +219,7 @@ func (e *Engine) createAutocompleteRequest(tag Attribute, pipeline Pipeline) Fet // return fmt.Errorf("cannot search for tag values for tag that is already used in query") // } // } - req.Conditions = append(req.Conditions, Condition{ // jpe should we move this into the fetch itself? it's weird we pass in both the tag through the request _and_ do the trickiness outside of adding it as OpNone + req.Conditions = append(req.Conditions, Condition{ Attribute: tag, Op: OpNone, }) diff --git a/pkg/traceql/storage.go b/pkg/traceql/storage.go index 0a353327464..6efe3cdf876 100644 --- a/pkg/traceql/storage.go +++ b/pkg/traceql/storage.go @@ -268,6 +268,5 @@ type FetchTagsCallback func(tag string, scope AttributeScope) bool type FetchTagsRequest struct { Conditions []Condition Scope AttributeScope - // jpe - add effort param? // TODO: Add start and end time? } diff --git a/tempodb/encoding/vparquet3/block_autocomplete.go b/tempodb/encoding/vparquet3/block_autocomplete.go index f04b2780a48..0b0f527e648 100644 --- a/tempodb/encoding/vparquet3/block_autocomplete.go +++ b/tempodb/encoding/vparquet3/block_autocomplete.go @@ -48,7 +48,7 @@ func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsR } // Last check. No conditions, use old path. It's much faster. - if len(req.Conditions) < 1 || mingledConditions { // jpe - fetch with {} puts in a condition for span start time? + if len(req.Conditions) < 1 || mingledConditions { return b.SearchTags(ctx, traceql.AttributeScopeResource, func(t string, scope traceql.AttributeScope) { cb(t, scope) }, opts) @@ -70,9 +70,6 @@ func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsR } defer iter.Close() - fmt.Println("-------------") - fmt.Println(iter) // jpe remove - for { // Exhaust the iterator res, err := iter.Next() @@ -176,7 +173,7 @@ func (b *backendBlock) FetchTagValues(ctx context.Context, req traceql.FetchTagV tag: req.TagName, } - iter, err := autocompleteIter(ctx, tr, pf, opts, b.meta.DedicatedColumns) // jpe call iter funcs directly to avoid nil cb + iter, err := autocompleteIter(ctx, tr, pf, opts, b.meta.DedicatedColumns) if err != nil { return errors.Wrap(err, "creating fetch iter") } @@ -555,7 +552,7 @@ func keyNameIterator(makeIter makeIterFn, definitionLevel int, keyPath string, a if len(attrIters) == 0 { return parquetquery.NewJoinIterator( oneLevelUp(definitionLevel), - []parquetquery.Iterator{makeIter(keyPath, newDistinctKeyPredicate(), "key")}, + []parquetquery.Iterator{makeIter(keyPath, nil, "key")}, newDistinctAttrCollector(scopeFromDefinitionLevel(definitionLevel), true), ), nil } @@ -563,7 +560,7 @@ func keyNameIterator(makeIter makeIterFn, definitionLevel int, keyPath string, a return parquetquery.NewLeftJoinIterator( oneLevelUp(definitionLevel), attrIters, - []parquetquery.Iterator{makeIter(keyPath, newDistinctKeyPredicate(), "key")}, + []parquetquery.Iterator{makeIter(keyPath, nil, "key")}, newDistinctAttrCollector(scopeFromDefinitionLevel(definitionLevel), true), ) } @@ -808,51 +805,6 @@ func (d *distinctAttrCollector) KeepGroup(result *parquetquery.IteratorResult) b return true } -// distinctKeyPredicate skips chunks that only contain keys that have been seen. -// jpe - this is approximate! document it as such -type distinctKeyPredicate struct { - sentKeys map[string]struct{} -} - -func newDistinctKeyPredicate() *distinctKeyPredicate { - return &distinctKeyPredicate{ - sentKeys: make(map[string]struct{}), - } -} - -func (d *distinctKeyPredicate) String() string { - return fmt.Sprintf("distinctKeyPredicate{}") -} - -func (d *distinctKeyPredicate) KeepColumnChunk(cc *parquetquery.ColumnChunkHelper) bool { - // foundNew := false - - // if dict := cc.Dictionary(); dict != nil { - // l := dict.Len() - // for i := 0; i < l; i++ { - // v := dict.Index(int32(i)) - // key := unsafeToString(v.ByteArray()) - - // if _, ok := d.sentKeys[key]; !ok { - // d.sentKeys[key] = struct{}{} - // foundNew = true - // break // jpe - noodle on this. it's a perf boost somehow? - // } - // } - // } - - // return foundNew - return true -} - -func (d *distinctKeyPredicate) KeepPage(pq parquet.Page) bool { - return true -} - -func (d *distinctKeyPredicate) KeepValue(v parquet.Value) bool { - return true -} - type entry struct { Key string Value parquet.Value diff --git a/tempodb/encoding/vparquet3/block_autocomplete_test.go b/tempodb/encoding/vparquet3/block_autocomplete_test.go index 15d5164c291..d1ec0438aef 100644 --- a/tempodb/encoding/vparquet3/block_autocomplete_test.go +++ b/tempodb/encoding/vparquet3/block_autocomplete_test.go @@ -26,7 +26,7 @@ func TestFetchTagNames(t *testing.T) { expectedResourceValues []string }{ { - name: "no query - fall back to old search", // jpe - not working. not falling back to old search due to span start time cond? + name: "no query - fall back to old search", query: "{}", expectedSpanValues: []string{ "generic-01-01", diff --git a/tempodb/encoding/vparquet3/wal_block.go b/tempodb/encoding/vparquet3/wal_block.go index b735d45fc44..eaa22c59c96 100644 --- a/tempodb/encoding/vparquet3/wal_block.go +++ b/tempodb/encoding/vparquet3/wal_block.go @@ -755,7 +755,7 @@ func (b *walBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsReque return err } - if len(req.Conditions) < 1 || mingledConditions { // jpe - fetch with {} puts in a condition for span start time? + if len(req.Conditions) < 1 || mingledConditions { return b.SearchTags(ctx, traceql.AttributeScopeResource, func(t string, scope traceql.AttributeScope) { cb(t, scope) }, opts) From 912f28314926ed3aab1e04aa789f2f15a200ba67 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 28 Jun 2024 09:34:25 -0400 Subject: [PATCH 09/21] vp4 Signed-off-by: Joe Elliott --- .../encoding/vparquet4/block_autocomplete.go | 328 ++++++++++++++---- .../vparquet4/block_autocomplete_test.go | 325 +++++++++++++++++ tempodb/encoding/vparquet4/wal_block.go | 69 +++- 3 files changed, 650 insertions(+), 72 deletions(-) diff --git a/tempodb/encoding/vparquet4/block_autocomplete.go b/tempodb/encoding/vparquet4/block_autocomplete.go index 30b04cee126..e9b126a27a7 100644 --- a/tempodb/encoding/vparquet4/block_autocomplete.go +++ b/tempodb/encoding/vparquet4/block_autocomplete.go @@ -14,6 +14,139 @@ import ( "github.com/pkg/errors" ) +type tagRequest struct { + // applies to tag names and tag values. the conditions by which to return the filtered data + conditions []traceql.Condition + // scope requested. only used for tag names. A scope of None means all scopes. + scope traceql.AttributeScope + // tag requested. only used for tag values. if populated then return tag values for this tag, otherwise return tag names. + tag traceql.Attribute +} + +func (r tagRequest) keysRequested(scope traceql.AttributeScope) bool { + if r.tag != (traceql.Attribute{}) { + return false + } + + // none scope means return all scopes + if r.scope == traceql.AttributeScopeNone { + return true + } + + return r.scope == scope +} + +func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsRequest, cb traceql.FetchTagsCallback, opts common.SearchOptions) error { + err := checkConditions(req.Conditions) + if err != nil { + return errors.Wrap(err, "conditions invalid") + } + + _, mingledConditions, err := categorizeConditions(req.Conditions) + if err != nil { + return err + } + + // Last check. No conditions, use old path. It's much faster. + if len(req.Conditions) < 1 || mingledConditions { + return b.SearchTags(ctx, traceql.AttributeScopeResource, func(t string, scope traceql.AttributeScope) { + cb(t, scope) + }, opts) + } + + pf, _, err := b.openForSearch(ctx, opts) + if err != nil { + return err + } + + tr := tagRequest{ + conditions: req.Conditions, + scope: req.Scope, + } + + iter, err := autocompleteIter(ctx, tr, pf, opts, b.meta.DedicatedColumns) + if err != nil { + return errors.Wrap(err, "creating fetch iter") + } + defer iter.Close() + + for { + // Exhaust the iterator + res, err := iter.Next() + if err != nil { + return err + } + if res == nil { + break + } + for _, oe := range res.OtherEntries { + if cb(oe.Key, oe.Value.(traceql.AttributeScope)) { + return nil // We have enough values + } + } + } + + tagNamesForSpecialColumns(req.Scope, pf, b.meta.DedicatedColumns, cb) + + return nil +} + +func tagNamesForSpecialColumns(scope traceql.AttributeScope, pf *parquet.File, dcs backend.DedicatedColumns, cb traceql.FetchTagsCallback) { + // currently just seeing if any row groups have values. future improvements: + // - only check those row groups that otherwise have a match in the iterators above + // - use rep/def levels to determine if a value exists at a row w/o actually testing values. + // atm i believe this requires reading the pages themselves b/c the rep/def lvls come w/ the page + hasValues := func(path string, pf *parquet.File) bool { + idx, _ := parquetquery.GetColumnIndexByPath(pf, path) + md := pf.Metadata() + for _, rg := range md.RowGroups { + col := rg.Columns[idx] + if col.MetaData.NumValues-col.MetaData.Statistics.NullCount > 0 { + return true + } + } + + return false + } + + // add all well known columns that have values + for name, entry := range wellKnownColumnLookups { + if entry.level != scope && scope != traceql.AttributeScopeNone { + continue + } + + if hasValues(entry.columnPath, pf) { + if cb(name, entry.level) { + return + } + } + } + + // add all span dedicated columns that have values + if scope == traceql.AttributeScopeNone || scope == traceql.AttributeScopeSpan { + dedCols := dedicatedColumnsToColumnMapping(dcs, backend.DedicatedColumnScopeSpan) + for name, col := range dedCols.mapping { + if hasValues(col.ColumnPath, pf) { + if cb(name, traceql.AttributeScopeSpan) { + return + } + } + } + } + + // add all resource dedicated columns that have values + if scope == traceql.AttributeScopeNone || scope == traceql.AttributeScopeResource { + dedCols := dedicatedColumnsToColumnMapping(dcs, backend.DedicatedColumnScopeResource) + for name, col := range dedCols.mapping { + if hasValues(col.ColumnPath, pf) { + if cb(name, traceql.AttributeScopeResource) { + return + } + } + } + } +} + func (b *backendBlock) FetchTagValues(ctx context.Context, req traceql.FetchTagValuesRequest, cb traceql.FetchTagValuesCallback, opts common.SearchOptions) error { err := checkConditions(req.Conditions) if err != nil { @@ -35,7 +168,12 @@ func (b *backendBlock) FetchTagValues(ctx context.Context, req traceql.FetchTagV return err } - iter, err := autocompleteIter(ctx, req, pf, opts, b.meta.DedicatedColumns) + tr := tagRequest{ + conditions: req.Conditions, + tag: req.TagName, + } + + iter, err := autocompleteIter(ctx, tr, pf, opts, b.meta.DedicatedColumns) if err != nil { return errors.Wrap(err, "creating fetch iter") } @@ -62,25 +200,9 @@ func (b *backendBlock) FetchTagValues(ctx context.Context, req traceql.FetchTagV } // autocompleteIter creates an iterator that will collect values for a given attribute/tag. -func autocompleteIter(ctx context.Context, req traceql.FetchTagValuesRequest, pf *parquet.File, opts common.SearchOptions, dc backend.DedicatedColumns) (parquetquery.Iterator, error) { - iter, err := createDistinctIterator(ctx, req.Conditions, req.TagName, pf, opts, dc) - if err != nil { - return nil, fmt.Errorf("error creating iterator: %w", err) - } - - return iter, nil -} - -func createDistinctIterator( - ctx context.Context, - conds []traceql.Condition, - tag traceql.Attribute, - pf *parquet.File, - opts common.SearchOptions, - dc backend.DedicatedColumns, -) (parquetquery.Iterator, error) { - // categorize conditions by scope - catConditions, _, err := categorizeConditions(conds) +func autocompleteIter(ctx context.Context, tr tagRequest, pf *parquet.File, opts common.SearchOptions, dc backend.DedicatedColumns) (parquetquery.Iterator, error) { + // categorizeConditions conditions into span-level or resource-level + catConditions, _, err := categorizeConditions(tr.conditions) if err != nil { return nil, err } @@ -90,22 +212,22 @@ func createDistinctIterator( var currentIter parquetquery.Iterator - if len(catConditions.span) > 0 { - currentIter, err = createDistinctSpanIterator(makeIter, tag, currentIter, catConditions.span, dc) + if len(catConditions.span) > 0 || tr.keysRequested(traceql.AttributeScopeSpan) { + currentIter, err = createDistinctSpanIterator(makeIter, tr, catConditions.span, dc) if err != nil { return nil, errors.Wrap(err, "creating span iterator") } } - if len(catConditions.resource) > 0 { - currentIter, err = createDistinctResourceIterator(makeIter, tag, currentIter, catConditions.resource, dc) + if len(catConditions.resource) > 0 || tr.keysRequested(traceql.AttributeScopeResource) { + currentIter, err = createDistinctResourceIterator(makeIter, tr, currentIter, catConditions.resource, dc) if err != nil { return nil, errors.Wrap(err, "creating resource iterator") } } if len(catConditions.trace) > 0 { - currentIter, err = createDistinctTraceIterator(makeIter, currentIter, catConditions.trace) + currentIter, err = createDistinctTraceIterator(makeIter, tr, currentIter, catConditions.trace) if err != nil { return nil, errors.Wrap(err, "creating trace iterator") } @@ -118,8 +240,7 @@ func createDistinctIterator( // one span each. Spans are returned that match any of the given conditions. func createDistinctSpanIterator( makeIter makeIterFn, - tag traceql.Attribute, - primaryIter parquetquery.Iterator, + tr tagRequest, conditions []traceql.Condition, dedicatedColumns backend.DedicatedColumns, ) (parquetquery.Iterator, error) { @@ -133,8 +254,8 @@ func createDistinctSpanIterator( // TODO: Potentially problematic when wanted attribute is also part of a condition // e.g. { span.foo =~ ".*" && span.foo = } - addSelectAs := func(attr traceql.Attribute, columnPath, selectAs string) { - if attr == tag { + addSelectAs := func(attr traceql.Attribute, columnPath string, selectAs string) { + if attr == tr.tag { columnSelectAs[columnPath] = selectAs } else { columnSelectAs[columnPath] = "" // Don't select, just filter @@ -200,14 +321,37 @@ func createDistinctSpanIterator( addSelectAs(cond.Attribute, columnPathSpanStatusMessage, columnPathSpanStatusMessage) continue + case traceql.IntrinsicNestedSetLeft: + pred, err := createIntPredicate(cond.Op, cond.Operands) + if err != nil { + return nil, err + } + addPredicate(columnPathSpanNestedSetLeft, pred) + addSelectAs(cond.Attribute, columnPathSpanNestedSetLeft, columnPathSpanNestedSetLeft) + continue + + case traceql.IntrinsicNestedSetRight: + pred, err := createIntPredicate(cond.Op, cond.Operands) + if err != nil { + return nil, err + } + addPredicate(columnPathSpanNestedSetRight, pred) + addSelectAs(cond.Attribute, columnPathSpanNestedSetRight, columnPathSpanNestedSetRight) + continue + + case traceql.IntrinsicNestedSetParent: + pred, err := createIntPredicate(cond.Op, cond.Operands) + if err != nil { + return nil, err + } + addPredicate(columnPathSpanParentID, pred) + addSelectAs(cond.Attribute, columnPathSpanParentID, columnPathSpanParentID) + continue + // TODO: Support structural operators case traceql.IntrinsicStructuralDescendant, traceql.IntrinsicStructuralChild, - traceql.IntrinsicStructuralSibling, - // nested set intrinsics should not be considered when autocompleting - traceql.IntrinsicNestedSetLeft, - traceql.IntrinsicNestedSetRight, - traceql.IntrinsicNestedSetParent: + traceql.IntrinsicStructuralSibling: continue } @@ -260,7 +404,7 @@ func createDistinctSpanIterator( iters = append(iters, makeIter(columnPath, orIfNeeded(predicates), columnSelectAs[columnPath])) } - attrIter, err := createDistinctAttributeIterator(makeIter, tag, genericConditions, DefinitionLevelResourceSpansILSSpanAttrs, + attrIter, err := createDistinctAttributeIterator(makeIter, tr, genericConditions, DefinitionLevelResourceSpansILSSpanAttrs, columnPathSpanAttrKey, columnPathSpanAttrString, columnPathSpanAttrInt, columnPathSpanAttrDouble, columnPathSpanAttrBool) if err != nil { return nil, errors.Wrap(err, "creating span attribute iterator") @@ -269,17 +413,13 @@ func createDistinctSpanIterator( iters = append(iters, attrIter) } - if primaryIter != nil { - iters = append([]parquetquery.Iterator{primaryIter}, iters...) - } - if len(columnPredicates) == 0 { // If no special+intrinsic+dedicated columns are being searched, // we can iterate over the generic attributes directly. return attrIter, nil } - spanCol := newDistinctValueCollector(mapSpanAttr) + spanCol := newDistinctValueCollector(mapSpanAttr, "span") // Left join here means the span id/start/end iterators + 1 are required, // and all other conditions are optional. Whatever matches is returned. @@ -288,7 +428,7 @@ func createDistinctSpanIterator( func createDistinctAttributeIterator( makeIter makeIterFn, - tag traceql.Attribute, + tr tagRequest, conditions []traceql.Condition, definitionLevel int, keyPath, strPath, intPath, floatPath, boolPath string, @@ -300,17 +440,16 @@ func createDistinctAttributeIterator( ) selectAs := func(key string, attr traceql.Attribute) string { - if tag == attr { + if tr.tag == attr { return key } return "" } for _, cond := range conditions { - if cond.Op == traceql.OpNone { // This means we have to scan all values, we don't know what type to expect - if tag == cond.Attribute { + if tr.tag == cond.Attribute { // If it's not the tag we're looking for, we can skip it attrKeys = append(attrKeys, cond.Attribute.Name) attrStringPreds = append(attrStringPreds, nil) @@ -380,19 +519,25 @@ func createDistinctAttributeIterator( valueIters = append(valueIters, makeIter(boolPath, orIfNeeded(boolPreds), "bool")) } - if len(valueIters) > 0 || len(iters) > 0 { + scope := scopeFromDefinitionLevel(definitionLevel) + if len(valueIters) > 0 || len(iters) > 0 || tr.keysRequested(scope) { if len(valueIters) > 0 { tagIter, err := parquetquery.NewLeftJoinIterator( definitionLevel, []parquetquery.Iterator{makeIter(keyPath, parquetquery.NewStringInPredicate(attrKeys), "key")}, valueIters, - newDistinctAttrCollector(scopeFromDefinitionLevel(definitionLevel)), + newDistinctAttrCollector(scope, false), ) if err != nil { return nil, fmt.Errorf("creating left join iterator: %w", err) } iters = append(iters, tagIter) } + + if tr.keysRequested(scope) { + return keyNameIterator(makeIter, definitionLevel, keyPath, iters) + } + return parquetquery.NewJoinIterator( oneLevelUp(definitionLevel), iters, @@ -403,6 +548,23 @@ func createDistinctAttributeIterator( return nil, nil } +func keyNameIterator(makeIter makeIterFn, definitionLevel int, keyPath string, attrIters []parquetquery.Iterator) (parquetquery.Iterator, error) { + if len(attrIters) == 0 { + return parquetquery.NewJoinIterator( + oneLevelUp(definitionLevel), + []parquetquery.Iterator{makeIter(keyPath, nil, "key")}, + newDistinctAttrCollector(scopeFromDefinitionLevel(definitionLevel), true), + ), nil + } + + return parquetquery.NewLeftJoinIterator( + oneLevelUp(definitionLevel), + attrIters, + []parquetquery.Iterator{makeIter(keyPath, nil, "key")}, + newDistinctAttrCollector(scopeFromDefinitionLevel(definitionLevel), true), + ) +} + func oneLevelUp(definitionLevel int) int { switch definitionLevel { case DefinitionLevelResourceSpansILSSpanAttrs: @@ -415,7 +577,7 @@ func oneLevelUp(definitionLevel int) int { func createDistinctResourceIterator( makeIter makeIterFn, - tag traceql.Attribute, + tr tagRequest, spanIterator parquetquery.Iterator, conditions []traceql.Condition, dedicatedColumns backend.DedicatedColumns, @@ -432,8 +594,8 @@ func createDistinctResourceIterator( columnPredicates[columnPath] = append(columnPredicates[columnPath], p) } - addSelectAs := func(attr traceql.Attribute, columnPath, selectAs string) { - if attr == tag { + addSelectAs := func(attr traceql.Attribute, columnPath string, selectAs string) { + if attr == tr.tag { columnSelectAs[columnPath] = selectAs } else { columnSelectAs[columnPath] = "" // Don't select, just filter @@ -456,7 +618,7 @@ func createDistinctResourceIterator( return nil, errors.Wrap(err, "creating predicate") } selectAs := cond.Attribute.Name - if tag != cond.Attribute { + if tr.tag != cond.Attribute { selectAs = "" } iters = append(iters, makeIter(entry.columnPath, pred, selectAs)) @@ -493,7 +655,7 @@ func createDistinctResourceIterator( iters = append(iters, makeIter(columnPath, orIfNeeded(predicates), columnSelectAs[columnPath])) } - attrIter, err := createDistinctAttributeIterator(makeIter, tag, genericConditions, DefinitionLevelResourceAttrs, + attrIter, err := createDistinctAttributeIterator(makeIter, tr, genericConditions, DefinitionLevelResourceAttrs, columnPathResourceAttrKey, columnPathResourceAttrString, columnPathResourceAttrInt, columnPathResourceAttrDouble, columnPathResourceAttrBool) if err != nil { return nil, errors.Wrap(err, "creating span attribute iterator") @@ -502,7 +664,7 @@ func createDistinctResourceIterator( iters = append(iters, attrIter) } - batchCol := newDistinctValueCollector(mapResourceAttr) + batchCol := newDistinctValueCollector(mapResourceAttr, "resource") // Put span iterator last, so it is only read when // the resource conditions are met. @@ -515,12 +677,20 @@ func createDistinctResourceIterator( func createDistinctTraceIterator( makeIter makeIterFn, + tr tagRequest, resourceIter parquetquery.Iterator, conds []traceql.Condition, ) (parquetquery.Iterator, error) { var err error traceIters := make([]parquetquery.Iterator, 0, 3) + selectAs := func(attr traceql.Attribute, columnPath string) string { + if attr == tr.tag { + return columnPath + } + return "" + } + // add conditional iterators first. this way if someone searches for { traceDuration > 1s && span.foo = "bar"} the query will // be sped up by searching for traceDuration first. note that we can only set the predicates if all conditions is true. // otherwise we just pass the info up to the engine to make a choice @@ -535,7 +705,7 @@ func createDistinctTraceIterator( if err != nil { return nil, err } - traceIters = append(traceIters, makeIter(columnPathDurationNanos, pred, columnPathDurationNanos)) + traceIters = append(traceIters, makeIter(columnPathDurationNanos, pred, selectAs(cond.Attribute, columnPathDurationNanos))) case traceql.IntrinsicTraceRootSpan: var pred parquetquery.Predicate @@ -543,7 +713,7 @@ func createDistinctTraceIterator( if err != nil { return nil, err } - traceIters = append(traceIters, makeIter(columnPathRootSpanName, pred, columnPathRootSpanName)) + traceIters = append(traceIters, makeIter(columnPathRootSpanName, pred, selectAs(cond.Attribute, columnPathRootSpanName))) case traceql.IntrinsicTraceRootService: var pred parquetquery.Predicate @@ -551,7 +721,7 @@ func createDistinctTraceIterator( if err != nil { return nil, err } - traceIters = append(traceIters, makeIter(columnPathRootServiceName, pred, columnPathRootServiceName)) + traceIters = append(traceIters, makeIter(columnPathRootServiceName, pred, selectAs(cond.Attribute, columnPathRootServiceName))) } } @@ -564,21 +734,25 @@ func createDistinctTraceIterator( // Final trace iterator // Join iterator means it requires matching resources to have been found // TraceCollor adds trace-level data to the spansets - return parquetquery.NewJoinIterator(DefinitionLevelTrace, traceIters, newDistinctValueCollector(mapTraceAttr)), nil + return parquetquery.NewJoinIterator(DefinitionLevelTrace, traceIters, newDistinctValueCollector(mapTraceAttr, "trace")), nil } var _ parquetquery.GroupPredicate = (*distinctAttrCollector)(nil) type distinctAttrCollector struct { - scope traceql.AttributeScope + scope traceql.AttributeScope + attrNames bool sentVals map[traceql.Static]struct{} + sentKeys map[string]struct{} } -func newDistinctAttrCollector(scope traceql.AttributeScope) *distinctAttrCollector { +func newDistinctAttrCollector(scope traceql.AttributeScope, attrNames bool) *distinctAttrCollector { return &distinctAttrCollector{ - scope: scope, - sentVals: make(map[traceql.Static]struct{}), + scope: scope, + sentVals: make(map[traceql.Static]struct{}), + sentKeys: make(map[string]struct{}), + attrNames: attrNames, } } @@ -596,15 +770,25 @@ func (d *distinctAttrCollector) KeepGroup(result *parquetquery.IteratorResult) b continue } - switch e.Key { - case "string": - val = traceql.NewStaticString(unsafeToString(e.Value.ByteArray())) - case "int": - val = traceql.NewStaticInt(int(e.Value.Int64())) - case "float": - val = traceql.NewStaticFloat(e.Value.Double()) - case "bool": - val = traceql.NewStaticBool(e.Value.Boolean()) + if d.attrNames { + if e.Key == "key" { + key := unsafeToString(e.Value.ByteArray()) + if _, ok := d.sentKeys[key]; !ok { + result.AppendOtherValue(key, d.scope) + d.sentKeys[key] = struct{}{} + } + } + } else { + switch e.Key { + case "string": + val = traceql.NewStaticString(unsafeToString(e.Value.ByteArray())) + case "int": + val = traceql.NewStaticInt(int(e.Value.Int64())) + case "float": + val = traceql.NewStaticFloat(e.Value.Double()) + case "bool": + val = traceql.NewStaticBool(e.Value.Boolean()) + } } } @@ -631,16 +815,18 @@ var _ parquetquery.GroupPredicate = (*distinctValueCollector)(nil) type distinctValueCollector struct { mapToStatic func(entry) traceql.Static sentVals map[traceql.Static]struct{} + name string } -func newDistinctValueCollector(mapToStatic func(entry) traceql.Static) *distinctValueCollector { +func newDistinctValueCollector(mapToStatic func(entry) traceql.Static, name string) *distinctValueCollector { return &distinctValueCollector{ mapToStatic: mapToStatic, sentVals: make(map[traceql.Static]struct{}), + name: name, } } -func (d distinctValueCollector) String() string { return "distinctValueCollector" } +func (d distinctValueCollector) String() string { return "distinctValueCollector(" + d.name + ")" } func (d distinctValueCollector) KeepGroup(result *parquetquery.IteratorResult) bool { for _, e := range result.Entries { diff --git a/tempodb/encoding/vparquet4/block_autocomplete_test.go b/tempodb/encoding/vparquet4/block_autocomplete_test.go index 4f50e93c043..ae9cc7c84b9 100644 --- a/tempodb/encoding/vparquet4/block_autocomplete_test.go +++ b/tempodb/encoding/vparquet4/block_autocomplete_test.go @@ -11,12 +11,238 @@ import ( "github.com/grafana/tempo/pkg/collector" "github.com/grafana/tempo/pkg/tempopb" "github.com/grafana/tempo/pkg/traceql" + "github.com/grafana/tempo/pkg/util/test" "github.com/grafana/tempo/tempodb/backend" "github.com/grafana/tempo/tempodb/backend/local" "github.com/grafana/tempo/tempodb/encoding/common" "github.com/stretchr/testify/require" ) +func TestFetchTagNames(t *testing.T) { + testCases := []struct { + name string + query string + expectedSpanValues []string + expectedResourceValues []string + }{ + { + name: "no query - fall back to old search", + query: "{}", + expectedSpanValues: []string{ + "generic-01-01", + "generic-01-02", + "generic-02-01", + "span-same", + }, + expectedResourceValues: []string{"generic-01", "generic-02", "resource-same"}, + }, + { + name: "matches nothing", + query: "{span.generic-01-01=`bar`}", + expectedSpanValues: []string{}, + expectedResourceValues: []string{}, + }, + // span + { + name: "intrinsic span", + query: "{statusMessage=`msg-01-01`}", + expectedSpanValues: []string{"generic-01-01", "span-same"}, + expectedResourceValues: []string{"generic-01", "resource-same"}, + }, + { + name: "well known span", + query: "{span.http.method=`method-01-01`}", + expectedSpanValues: []string{"generic-01-01", "span-same"}, + expectedResourceValues: []string{"generic-01", "resource-same"}, + }, + { + name: "generic span", + query: "{span.generic-01-01=`foo`}", + expectedSpanValues: []string{"generic-01-01", "span-same"}, + expectedResourceValues: []string{"generic-01", "resource-same"}, + }, + { + name: "match two spans", + query: "{span.span-same=`foo`}", + expectedSpanValues: []string{"generic-01-01", "span-same", "generic-02-01"}, + expectedResourceValues: []string{"generic-01", "resource-same", "generic-02"}, + }, + // resource + { + name: "well known resource", + query: "{resource.cluster=`cluster-01`}", + expectedSpanValues: []string{"generic-01-01", "generic-01-02", "span-same"}, + expectedResourceValues: []string{"generic-01", "resource-same"}, + }, + { + name: "generic resource", + query: "{resource.generic-01=`bar`}", + expectedSpanValues: []string{"generic-01-01", "generic-01-02", "span-same"}, + expectedResourceValues: []string{"generic-01", "resource-same"}, + }, + { + name: "match two resources", + query: "{resource.resource-same=`foo`}", + expectedSpanValues: []string{"generic-01-01", "generic-01-02", "span-same", "generic-02-01"}, + expectedResourceValues: []string{"generic-01", "resource-same", "generic-02"}, + }, + // trace level match + { + name: "trace", + query: "{rootName=`root` }", + expectedSpanValues: []string{"generic-01-01", "generic-01-02", "span-same", "generic-02-01"}, + expectedResourceValues: []string{"generic-01", "resource-same", "generic-02"}, + }, + } + + strPtr := func(s string) *string { return &s } + tr := &Trace{ + TraceID: test.ValidTraceID(nil), + RootServiceName: "tr", + RootSpanName: "root", + ResourceSpans: []ResourceSpans{ + { + Resource: Resource{ + ServiceName: "svc-01", + Cluster: strPtr("cluster-01"), // well known + Attrs: []Attribute{ + {Key: "generic-01", Value: []string{"bar"}}, // generic + {Key: "resource-same", Value: []string{"foo"}}, + }, + DedicatedAttributes: DedicatedAttributes{ + String01: strPtr("dedicated-01"), + }, + }, + ScopeSpans: []ScopeSpans{ + { + Spans: []Span{ + { + SpanID: []byte("0101"), + Name: "span-01-01", + HttpMethod: strPtr("method-01-01"), // well known + StatusMessage: "msg-01-01", // intrinsic + Attrs: []Attribute{ + {Key: "generic-01-01", Value: []string{"foo"}}, // generic + {Key: "span-same", Value: []string{"foo"}}, // generic + }, + DedicatedAttributes: DedicatedAttributes{ + String01: strPtr("dedicated-01-01"), + }, + }, + { + SpanID: []byte("0102"), + Name: "span-01-02", + HttpMethod: strPtr("method-01-02"), // well known + StatusMessage: "msg-01-02", // intrinsic + Attrs: []Attribute{ + {Key: "generic-01-02", Value: []string{"foo"}}, // generic + }, + DedicatedAttributes: DedicatedAttributes{ + String01: strPtr("dedicated-01-02"), + }, + }, + }, + }, + }, + }, + { + Resource: Resource{ + ServiceName: "svc-02", + Cluster: strPtr("cluster-02"), // well known + Attrs: []Attribute{ + {Key: "generic-02", Value: []string{"bar"}}, // generic + {Key: "resource-same", Value: []string{"foo"}}, + }, + DedicatedAttributes: DedicatedAttributes{ + String01: strPtr("dedicated-02"), + }, + }, + ScopeSpans: []ScopeSpans{ + { + Spans: []Span{ + { + SpanID: []byte("0201"), + Name: "span-02-01", + HttpMethod: strPtr("method-02-01"), // well known + StatusMessage: "msg-02-01", // intrinsic + Attrs: []Attribute{ + {Key: "generic-02-01", Value: []string{"foo"}}, // generic + {Key: "span-same", Value: []string{"foo"}}, // generic + }, + DedicatedAttributes: DedicatedAttributes{ + String01: strPtr("dedicated-02-01"), + }, + }, + }, + }, + }, + }, + }, + } + + ctx := context.TODO() + block := makeBackendBlockWithTraces(t, []*Trace{tr}) + + opts := common.DefaultSearchOptions() + + for _, tc := range testCases { + for _, scope := range []traceql.AttributeScope{traceql.AttributeScopeSpan, traceql.AttributeScopeResource, traceql.AttributeScopeNone} { + expectedSpanValues := tc.expectedSpanValues + expectedResourceValues := tc.expectedResourceValues + + // add dedicated and well known columns to expected values. the code currently does not + // attempt to perfectly filter these, but instead adds them to the return if any values are present + dedicatedSpanValues := []string{"dedicated.span.1"} + dedicatedResourceValues := []string{"dedicated.resource.1"} + + wellKnownSpanValues := []string{"http.method"} + wellKnownResourceValues := []string{"cluster", "service.name"} + + expectedValues := map[string][]string{} + if scope == traceql.AttributeScopeSpan || scope == traceql.AttributeScopeNone { + expectedValues["span"] = append(expectedValues["span"], expectedSpanValues...) + expectedValues["span"] = append(expectedValues["span"], wellKnownSpanValues...) + expectedValues["span"] = append(expectedValues["span"], dedicatedSpanValues...) + } + if scope == traceql.AttributeScopeResource || scope == traceql.AttributeScopeNone { + expectedValues["resource"] = append(expectedValues["resource"], expectedResourceValues...) + expectedValues["resource"] = append(expectedValues["resource"], wellKnownResourceValues...) + expectedValues["resource"] = append(expectedValues["resource"], dedicatedResourceValues...) + } + + t.Run(fmt.Sprintf("query: %s %s-%s", tc.name, tc.query, scope), func(t *testing.T) { + distinctAttrNames := collector.NewScopedDistinctString(0) + req, err := traceql.ExtractFetchSpansRequest(tc.query) + require.NoError(t, err) + + // Build autocomplete request + autocompleteReq := traceql.FetchTagsRequest{ + Conditions: req.Conditions, + Scope: scope, + } + + err = block.FetchTagNames(ctx, autocompleteReq, func(t string, scope traceql.AttributeScope) bool { + distinctAttrNames.Collect(scope.String(), t) + return false + }, opts) + require.NoError(t, err) + + actualValues := distinctAttrNames.Strings() + + require.Equal(t, len(expectedValues), len(actualValues)) + for k := range expectedValues { + actual := actualValues[k] + sort.Strings(actual) + expected := expectedValues[k] + sort.Strings(expected) + + require.Equal(t, expected, actual, "scope: %s", k) + } + }) + } + } +} + func TestFetchTagValues(t *testing.T) { testCases := []struct { name string @@ -324,6 +550,29 @@ func BenchmarkFetchTagValues(b *testing.B) { tag: "resource.namespace", query: `{span.http.status_code=200}`, }, + { + tag: "resource.namespace", + query: `{span.http.status_code=200}`, + }, + // pathologic cases + /* + { + tag: "resource.k8s.node.name", + query: `{span.http.method="GET"}`, + }, + { + tag: "span.sampler.type", + query: `{span.http.method="GET"}`, + }, + { + tag: "span.sampler.type", + query: `{resource.k8s.node.name>"aaa"}`, + }, + { + tag: "resource.k8s.node.name", + query: `{span.sampler.type>"aaa"}`, + }, + */ } ctx := context.TODO() @@ -372,3 +621,79 @@ func BenchmarkFetchTagValues(b *testing.B) { }) } } + +func BenchmarkFetchTags(b *testing.B) { + testCases := []struct { + query string + }{ + { + query: `{resource.namespace="tempo-ops"}`, // well known/dedicated column + }, + { + query: `{resource.k8s.node.name>"h"}`, // generic attribute + }, + { + query: `{span.http.status_code=200}`, // well known/dedicated column + }, + { + query: `{nestedSetParent=-1}`, // generic attribute + }, + { + query: `{rootName="Memcache.Put"}`, // trace level + }, + // pathological cases + /* + { + query: `{resource.k8s.node.name>"aaa"}`, // generic attribute + }, + { + query: `{span.http.method="GET"}`, // well known/dedicated column + }, + { + query: `{span.sampler.type>"aaa"}`, // generic attribute + }, + */ + } + + ctx := context.TODO() + tenantID := "1" + // blockID := uuid.MustParse("3685ee3d-cbbf-4f36-bf28-93447a19dea6") + blockID := uuid.MustParse("00145f38-6058-4e57-b1ba-334db8edce23") + + r, _, _, err := local.New(&local.Config{ + // Path: path.Join("/Users/marty/src/tmp/"), + Path: path.Join("/Users/joe/testblock"), + }) + require.NoError(b, err) + + rr := backend.NewReader(r) + meta, err := rr.BlockMeta(ctx, blockID, tenantID) + require.NoError(b, err) + + block := newBackendBlock(meta, rr) + opts := common.DefaultSearchOptions() + + for _, tc := range testCases { + for _, scope := range []traceql.AttributeScope{traceql.AttributeScopeSpan, traceql.AttributeScopeResource, traceql.AttributeScopeNone} { + b.Run(fmt.Sprintf("query: %s %s", tc.query, scope), func(b *testing.B) { + distinctStrings := collector.NewScopedDistinctString(1_000_000) + req, err := traceql.ExtractFetchSpansRequest(tc.query) + require.NoError(b, err) + + autocompleteReq := traceql.FetchTagsRequest{ + Conditions: req.Conditions, + Scope: scope, + } + b.ResetTimer() + + for i := 0; i < b.N; i++ { + err := block.FetchTagNames(ctx, autocompleteReq, func(t string, scope traceql.AttributeScope) bool { + distinctStrings.Collect(scope.String(), t) + return false + }, opts) + require.NoError(b, err) + } + }) + } + } +} diff --git a/tempodb/encoding/vparquet4/wal_block.go b/tempodb/encoding/vparquet4/wal_block.go index c88a3af576c..e4d3678af03 100644 --- a/tempodb/encoding/vparquet4/wal_block.go +++ b/tempodb/encoding/vparquet4/wal_block.go @@ -710,7 +710,12 @@ func (b *walBlock) FetchTagValues(ctx context.Context, req traceql.FetchTagValue pf := file.parquetFile - iter, err := autocompleteIter(ctx, req, pf, opts, b.meta.DedicatedColumns) + tr := tagRequest{ + conditions: req.Conditions, + tag: req.TagName, + } + + iter, err := autocompleteIter(ctx, tr, pf, opts, b.meta.DedicatedColumns) if err != nil { return fmt.Errorf("creating fetch iter: %w", err) } @@ -741,6 +746,68 @@ func (b *walBlock) FetchTagValues(ctx context.Context, req traceql.FetchTagValue return nil } +func (b *walBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsRequest, cb traceql.FetchTagsCallback, opts common.SearchOptions) error { + err := checkConditions(req.Conditions) + if err != nil { + return fmt.Errorf("conditions invalid: %w", err) + } + + _, mingledConditions, err := categorizeConditions(req.Conditions) + if err != nil { + return err + } + + if len(req.Conditions) < 1 || mingledConditions { + return b.SearchTags(ctx, traceql.AttributeScopeResource, func(t string, scope traceql.AttributeScope) { + cb(t, scope) + }, opts) + } + + blockFlushes := b.readFlushes() + for _, page := range blockFlushes { + file, err := page.file(ctx) + if err != nil { + return fmt.Errorf("error opening file %s: %w", page.path, err) + } + defer file.Close() + + tr := tagRequest{ + conditions: req.Conditions, + scope: req.Scope, + } + + iter, err := autocompleteIter(ctx, tr, file.parquetFile, opts, b.meta.DedicatedColumns) + if err != nil { + return fmt.Errorf("creating fetch iter: %w", err) + } + + for { + // Exhaust the iterator + res, err := iter.Next() + if err != nil { + iter.Close() + return err + } + if res == nil { + break + } + for _, oe := range res.OtherEntries { + if cb(oe.Key, oe.Value.(traceql.AttributeScope)) { + iter.Close() + return nil // We have enough values + } + } + } + iter.Close() + + // add well known + tagNamesForSpecialColumns(req.Scope, file.parquetFile, b.meta.DedicatedColumns, cb) + } + + // combine iters? + return nil +} + func (b *walBlock) walPath() string { filename := fmt.Sprintf("%s+%s+%s", b.meta.BlockID, b.meta.TenantID, VersionString) return filepath.Join(b.path, filename) From cdf14ea910bfef213c663bdd814d6e88dbaec848 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 28 Jun 2024 10:22:35 -0400 Subject: [PATCH 10/21] pipe through Signed-off-by: Joe Elliott --- integration/e2e/api_test.go | 1 + modules/ingester/instance_search.go | 29 ++++++++++++++----- modules/querier/querier.go | 44 +++++++++++++++++++++++++---- pkg/traceql/engine.go | 32 +++++++++++++++++++++ pkg/traceql/storage.go | 18 ++++++++++++ tempodb/tempodb.go | 11 ++++++++ tempodb/tempodb_search_test.go | 2 +- 7 files changed, 124 insertions(+), 13 deletions(-) diff --git a/integration/e2e/api_test.go b/integration/e2e/api_test.go index 178f5ac0c82..c42febc89b4 100644 --- a/integration/e2e/api_test.go +++ b/integration/e2e/api_test.go @@ -24,6 +24,7 @@ const ( resourceX = "resource.xx" ) +// jpe - add here func TestSearchTagValuesV2(t *testing.T) { s, err := e2e.NewScenario("tempo_e2e") require.NoError(t, err) diff --git a/modules/ingester/instance_search.go b/modules/ingester/instance_search.go index c7890dd525f..9ede1a42724 100644 --- a/modules/ingester/instance_search.go +++ b/modules/ingester/instance_search.go @@ -230,6 +230,9 @@ func (i *instance) SearchTagsV2(ctx context.Context, scope string) (*tempopb.Sea limit := i.limiter.limits.MaxBytesPerTagValuesQuery(userID) distinctValues := collector.NewScopedDistinctString(limit) + engine := traceql.NewEngine() + query := traceql.ExtractMatchers("{}") // jpe - where is the query? + searchBlock := func(ctx context.Context, s common.Searcher, spanName string) error { span, ctx := opentracing.StartSpanFromContext(ctx, "instance.SearchTags."+spanName) defer span.Finish() @@ -240,14 +243,26 @@ func (i *instance) SearchTagsV2(ctx context.Context, scope string) (*tempopb.Sea if distinctValues.Exceeded() { return nil } - err = s.SearchTags(ctx, attributeScope, func(t string, scope traceql.AttributeScope) { - distinctValues.Collect(scope.String(), t) - }, common.DefaultSearchOptions()) - if err != nil && !errors.Is(err, common.ErrUnsupported) { - return fmt.Errorf("unexpected error searching tags: %w", err) + + // if the query is empty, use the old search + if traceql.IsEmptyQuery(query) { + err = s.SearchTags(ctx, attributeScope, func(t string, scope traceql.AttributeScope) { + distinctValues.Collect(scope.String(), t) + }, common.DefaultSearchOptions()) + if err != nil && !errors.Is(err, common.ErrUnsupported) { + return fmt.Errorf("unexpected error searching tags: %w", err) + } } - return nil + // otherwise use the filtered search + fetcher := traceql.NewTagNamesFetcherWrapper(func(ctx context.Context, req traceql.FetchTagsRequest, cb traceql.FetchTagsCallback) error { + return s.FetchTagNames(ctx, req, cb, common.DefaultSearchOptions()) + }) + + return engine.ExecuteTagNames(ctx, attributeScope, query, func(tag string, scope traceql.AttributeScope) bool { + distinctValues.Collect(scope.String(), tag) + return distinctValues.Exceeded() + }, fetcher) } i.headBlockMtx.RLock() @@ -402,7 +417,7 @@ func (i *instance) SearchTagValuesV2(ctx context.Context, req *tempopb.SearchTag return nil } - // if the query is empty or autocomplete filtering is disabled, use the old search + // if the query is empty, use the old search if traceql.IsEmptyQuery(query) { return s.SearchTagValuesV2(ctx, tag, traceql.MakeCollectTagValueFunc(valueCollector.Collect), common.DefaultSearchOptions()) } diff --git a/modules/querier/querier.go b/modules/querier/querier.go index 55267642270..65e35732ecd 100644 --- a/modules/querier/querier.go +++ b/modules/querier/querier.go @@ -892,16 +892,50 @@ func (q *Querier) internalTagsSearchBlockV2(ctx context.Context, req *tempopb.Se opts.StartPage = int(req.StartPage) opts.TotalPages = int(req.PagesToSearch) - resp, err := q.store.SearchTags(ctx, meta, req.SearchReq.Scope, opts) + query := traceql.ExtractMatchers("{}") // jpe query + if traceql.IsEmptyQuery(query) { + resp, err := q.store.SearchTags(ctx, meta, req.SearchReq.Scope, opts) + if err != nil { + return nil, err + } + + // add intrinsic tags if scope is none + if req.SearchReq.Scope == "" { + resp.Scopes = append(resp.Scopes, &tempopb.SearchTagsV2Scope{ + Name: api.ParamScopeIntrinsic, + Tags: search.GetVirtualIntrinsicValues(), + }) + } + + return resp, nil + } + + fetcher := traceql.NewTagNamesFetcherWrapper(func(ctx context.Context, req traceql.FetchTagsRequest, cb traceql.FetchTagsCallback) error { + return q.store.FetchTagNames(ctx, meta, req, cb, common.DefaultSearchOptions()) + }) + + scope := traceql.AttributeScopeFromString(req.SearchReq.Scope) + if scope == traceql.AttributeScopeUnknown { + return nil, fmt.Errorf("unknown scope: %s", req.SearchReq.Scope) + } + + valueCollector := collector.NewScopedDistinctString(q.limits.MaxBytesPerTagValuesQuery(tenantID)) + err = q.engine.ExecuteTagNames(ctx, scope, query, func(tag string, scope traceql.AttributeScope) bool { + valueCollector.Collect(scope.String(), tag) + return valueCollector.Exceeded() + }, fetcher) if err != nil { return nil, err } - // add intrinsic tags if scope is none - if req.SearchReq.Scope == "" { + scopedVals := valueCollector.Strings() + resp := &tempopb.SearchTagsV2Response{ + Scopes: make([]*tempopb.SearchTagsV2Scope, 0, len(scopedVals)), + } + for scope, vals := range scopedVals { resp.Scopes = append(resp.Scopes, &tempopb.SearchTagsV2Scope{ - Name: api.ParamScopeIntrinsic, - Tags: search.GetVirtualIntrinsicValues(), + Name: scope, + Tags: vals, }) } diff --git a/pkg/traceql/engine.go b/pkg/traceql/engine.go index 166b5356fef..fd874f4de51 100644 --- a/pkg/traceql/engine.go +++ b/pkg/traceql/engine.go @@ -171,6 +171,38 @@ func (e *Engine) ExecuteTagValues( return fetcher.Fetch(ctx, autocompleteReq, cb) } +func (e *Engine) ExecuteTagNames( + ctx context.Context, + scope AttributeScope, + query string, + cb FetchTagsCallback, + fetcher TagNamesFetcher, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "traceql.Engine.ExecuteTagNames") + defer span.Finish() + + span.SetTag("sanitized query", query) + + var conditions []Condition + rootExpr, err := Parse(query) + // if the parse succeeded then use those conditions, otherwise pass in none. the next layer will handle it + if err == nil { + req := &FetchSpansRequest{} + rootExpr.Pipeline.extractConditions(req) + conditions = req.Conditions + } + + autocompleteReq := FetchTagsRequest{ + Conditions: conditions, + Scope: scope, + } + + span.SetTag("pipeline", rootExpr.Pipeline) + span.SetTag("autocompleteReq", autocompleteReq) + + return fetcher.Fetch(ctx, autocompleteReq, cb) +} + func (e *Engine) parseQuery(searchReq *tempopb.SearchRequest) (*RootExpr, error) { r, err := Parse(searchReq.Query) if err != nil { diff --git a/pkg/traceql/storage.go b/pkg/traceql/storage.go index 6efe3cdf876..54a96278848 100644 --- a/pkg/traceql/storage.go +++ b/pkg/traceql/storage.go @@ -270,3 +270,21 @@ type FetchTagsRequest struct { Scope AttributeScope // TODO: Add start and end time? } + +type TagNamesFetcher interface { + Fetch(context.Context, FetchTagsRequest, FetchTagsCallback) error +} + +type TagNamesFetcherWrapper struct { + f func(context.Context, FetchTagsRequest, FetchTagsCallback) error +} + +var _ TagNamesFetcher = (*TagNamesFetcherWrapper)(nil) + +func NewTagNamesFetcherWrapper(f func(context.Context, FetchTagsRequest, FetchTagsCallback) error) TagNamesFetcher { + return TagNamesFetcherWrapper{f} +} + +func (s TagNamesFetcherWrapper) Fetch(ctx context.Context, request FetchTagsRequest, callback FetchTagsCallback) error { + return s.f(ctx, request, callback) +} diff --git a/tempodb/tempodb.go b/tempodb/tempodb.go index ebd941034b4..6396499c390 100644 --- a/tempodb/tempodb.go +++ b/tempodb/tempodb.go @@ -85,6 +85,7 @@ type Reader interface { Fetch(ctx context.Context, meta *backend.BlockMeta, req traceql.FetchSpansRequest, opts common.SearchOptions) (traceql.FetchSpansResponse, error) FetchTagValues(ctx context.Context, meta *backend.BlockMeta, req traceql.FetchTagValuesRequest, cb traceql.FetchTagValuesCallback, opts common.SearchOptions) error + FetchTagNames(ctx context.Context, meta *backend.BlockMeta, req traceql.FetchTagsRequest, cb traceql.FetchTagsCallback, opts common.SearchOptions) error BlockMetas(tenantID string) []*backend.BlockMeta EnablePolling(ctx context.Context, sharder blocklist.JobSharder) @@ -461,6 +462,16 @@ func (rw *readerWriter) FetchTagValues(ctx context.Context, meta *backend.BlockM return block.FetchTagValues(ctx, req, cb, opts) } +func (rw *readerWriter) FetchTagNames(ctx context.Context, meta *backend.BlockMeta, req traceql.FetchTagsRequest, cb traceql.FetchTagsCallback, opts common.SearchOptions) error { + block, err := encoding.OpenBlock(meta, rw.r) + if err != nil { + return err + } + + rw.cfg.Search.ApplyToOptions(&opts) + return block.FetchTagNames(ctx, req, cb, opts) +} + func (rw *readerWriter) Shutdown() { // todo: stop blocklist poll rw.pool.Shutdown() diff --git a/tempodb/tempodb_search_test.go b/tempodb/tempodb_search_test.go index 4e657c7b039..b00887ba38f 100644 --- a/tempodb/tempodb_search_test.go +++ b/tempodb/tempodb_search_test.go @@ -1290,7 +1290,7 @@ func traceQLExistence(t *testing.T, _ *tempopb.Trace, _ *tempopb.TraceSearchMeta } } -// autoComplete! +// autoComplete! - jpe - add here func autoComplete(t *testing.T, _ *tempopb.Trace, _ *tempopb.TraceSearchMetadata, _, _ []*tempopb.SearchRequest, _ *backend.BlockMeta, _ Reader, bb common.BackendBlock) { ctx := context.Background() e := traceql.NewEngine() From 21502873ad5835713ebafa578b6ef35abf2f38d7 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 28 Jun 2024 10:58:01 -0400 Subject: [PATCH 11/21] add to request Signed-off-by: Joe Elliott --- modules/frontend/search_sharder_test.go | 4 + modules/ingester/ingester_search.go | 2 +- modules/ingester/instance_search.go | 7 +- modules/querier/querier.go | 2 +- pkg/tempopb/tempo.pb.go | 379 ++++++++++++++---------- pkg/tempopb/tempo.proto | 1 + 6 files changed, 226 insertions(+), 169 deletions(-) diff --git a/modules/frontend/search_sharder_test.go b/modules/frontend/search_sharder_test.go index 8d6bf37b247..a78ddcbb4aa 100644 --- a/modules/frontend/search_sharder_test.go +++ b/modules/frontend/search_sharder_test.go @@ -74,6 +74,10 @@ func (m *mockReader) Fetch(context.Context, *backend.BlockMeta, traceql.FetchSpa return traceql.FetchSpansResponse{}, nil } +func (m *mockReader) FetchTagNames(ctx context.Context, meta *backend.BlockMeta, req traceql.FetchTagsRequest, cb traceql.FetchTagsCallback, opts common.SearchOptions) error { + return nil +} + func (m *mockReader) EnablePolling(context.Context, blocklist.JobSharder) {} func (m *mockReader) Shutdown() {} diff --git a/modules/ingester/ingester_search.go b/modules/ingester/ingester_search.go index 06f53519bd4..d87af8b429b 100644 --- a/modules/ingester/ingester_search.go +++ b/modules/ingester/ingester_search.go @@ -78,7 +78,7 @@ func (i *Ingester) SearchTagsV2(ctx context.Context, req *tempopb.SearchTagsRequ return &tempopb.SearchTagsV2Response{}, nil } - res, err = inst.SearchTagsV2(ctx, req.Scope) + res, err = inst.SearchTagsV2(ctx, req) if err != nil { return nil, err } diff --git a/modules/ingester/instance_search.go b/modules/ingester/instance_search.go index 9ede1a42724..0d6b85124ba 100644 --- a/modules/ingester/instance_search.go +++ b/modules/ingester/instance_search.go @@ -175,7 +175,7 @@ func (i *instance) Search(ctx context.Context, req *tempopb.SearchRequest) (*tem } func (i *instance) SearchTags(ctx context.Context, scope string) (*tempopb.SearchTagsResponse, error) { - v2Response, err := i.SearchTagsV2(ctx, scope) + v2Response, err := i.SearchTagsV2(ctx, &tempopb.SearchTagsRequest{Scope: scope}) if err != nil { return nil, err } @@ -200,7 +200,7 @@ func (i *instance) SearchTags(ctx context.Context, scope string) (*tempopb.Searc } // SearchTagsV2 calls SearchTags for each scope and returns the results. -func (i *instance) SearchTagsV2(ctx context.Context, scope string) (*tempopb.SearchTagsV2Response, error) { +func (i *instance) SearchTagsV2(ctx context.Context, req *tempopb.SearchTagsRequest) (*tempopb.SearchTagsV2Response, error) { span, ctx := opentracing.StartSpanFromContext(ctx, "instance.SearchTagsV2") defer span.Finish() @@ -209,6 +209,7 @@ func (i *instance) SearchTagsV2(ctx context.Context, scope string) (*tempopb.Sea return nil, err } + scope := req.Scope // check if it's the special intrinsic scope if scope == api.ParamScopeIntrinsic { return &tempopb.SearchTagsV2Response{ @@ -231,7 +232,7 @@ func (i *instance) SearchTagsV2(ctx context.Context, scope string) (*tempopb.Sea distinctValues := collector.NewScopedDistinctString(limit) engine := traceql.NewEngine() - query := traceql.ExtractMatchers("{}") // jpe - where is the query? + query := traceql.ExtractMatchers(req.Query) searchBlock := func(ctx context.Context, s common.Searcher, spanName string) error { span, ctx := opentracing.StartSpanFromContext(ctx, "instance.SearchTags."+spanName) diff --git a/modules/querier/querier.go b/modules/querier/querier.go index 65e35732ecd..9b232dca48b 100644 --- a/modules/querier/querier.go +++ b/modules/querier/querier.go @@ -892,7 +892,7 @@ func (q *Querier) internalTagsSearchBlockV2(ctx context.Context, req *tempopb.Se opts.StartPage = int(req.StartPage) opts.TotalPages = int(req.PagesToSearch) - query := traceql.ExtractMatchers("{}") // jpe query + query := traceql.ExtractMatchers(req.SearchReq.Query) if traceql.IsEmptyQuery(query) { resp, err := q.store.SearchTags(ctx, meta, req.SearchReq.Scope, opts) if err != nil { diff --git a/pkg/tempopb/tempo.pb.go b/pkg/tempopb/tempo.pb.go index f4a2e4c10a2..4245165ad8e 100644 --- a/pkg/tempopb/tempo.pb.go +++ b/pkg/tempopb/tempo.pb.go @@ -997,6 +997,7 @@ func (m *SearchMetrics) GetInspectedSpans() uint64 { type SearchTagsRequest struct { Scope string `protobuf:"bytes,1,opt,name=scope,proto3" json:"scope,omitempty"` + Query string `protobuf:"bytes,2,opt,name=query,proto3" json:"query,omitempty"` Start uint32 `protobuf:"varint,3,opt,name=start,proto3" json:"start,omitempty"` End uint32 `protobuf:"varint,4,opt,name=end,proto3" json:"end,omitempty"` } @@ -1041,6 +1042,13 @@ func (m *SearchTagsRequest) GetScope() string { return "" } +func (m *SearchTagsRequest) GetQuery() string { + if m != nil { + return m.Query + } + return "" +} + func (m *SearchTagsRequest) GetStart() uint32 { if m != nil { return m.Start @@ -3122,173 +3130,173 @@ func init() { func init() { proto.RegisterFile("pkg/tempopb/tempo.proto", fileDescriptor_f22805646f4f62b6) } var fileDescriptor_f22805646f4f62b6 = []byte{ - // 2646 bytes of a gzipped FileDescriptorProto + // 2648 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5a, 0xcd, 0x6e, 0x23, 0xc7, 0xf1, 0xd7, 0x88, 0x1f, 0x22, 0x8b, 0x94, 0x44, 0xf5, 0xae, 0x65, 0x2e, 0xd7, 0xd6, 0xea, 0x3f, 0x5e, 0xf8, 0xaf, 0xf8, 0x43, 0xd2, 0xd2, 0xbb, 0xb0, 0xd7, 0x4e, 0x1c, 0xac, 0x56, 0xca, 0x5a, - 0xb6, 0xbe, 0xdc, 0x94, 0x65, 0x23, 0x30, 0x20, 0x8c, 0xc8, 0x5e, 0xee, 0x40, 0xe4, 0x0c, 0x3d, - 0x33, 0x54, 0x96, 0x39, 0x06, 0x48, 0x80, 0x00, 0x39, 0xe4, 0x90, 0x1c, 0x7c, 0xcc, 0x29, 0xc8, - 0x39, 0x8f, 0x10, 0x20, 0x30, 0x10, 0xc4, 0x30, 0x90, 0x8b, 0x91, 0x83, 0x11, 0xd8, 0x87, 0x3c, - 0x40, 0x5e, 0x20, 0xa8, 0xea, 0xee, 0x99, 0x1e, 0x72, 0xa4, 0xf5, 0x26, 0x6b, 0xc4, 0x07, 0x9f, - 0xd4, 0x55, 0x5d, 0x5d, 0x5d, 0xdd, 0x55, 0xfd, 0xab, 0xaa, 0xa1, 0xe0, 0xe9, 0xc1, 0x69, 0x77, - 0x2d, 0x12, 0xfd, 0x81, 0x3f, 0x38, 0x91, 0x7f, 0x57, 0x07, 0x81, 0x1f, 0xf9, 0x6c, 0x46, 0x31, - 0x1b, 0x8b, 0x6d, 0xbf, 0xdf, 0xf7, 0xbd, 0xb5, 0xb3, 0x1b, 0x6b, 0x72, 0x24, 0x05, 0x1a, 0x2f, - 0x77, 0xdd, 0xe8, 0xc1, 0xf0, 0x64, 0xb5, 0xed, 0xf7, 0xd7, 0xba, 0x7e, 0xd7, 0x5f, 0x23, 0xf6, - 0xc9, 0xf0, 0x3e, 0x51, 0x44, 0xd0, 0x48, 0x89, 0x5f, 0x8e, 0x02, 0xa7, 0x2d, 0x50, 0x0b, 0x0d, - 0x24, 0xd7, 0xfe, 0x85, 0x05, 0xb5, 0x43, 0xa4, 0x37, 0x46, 0xdb, 0x9b, 0x5c, 0x7c, 0x34, 0x14, - 0x61, 0xc4, 0xea, 0x30, 0x43, 0x32, 0xdb, 0x9b, 0x75, 0x6b, 0xd9, 0x5a, 0xa9, 0x72, 0x4d, 0xb2, - 0x25, 0x80, 0x93, 0x9e, 0xdf, 0x3e, 0x6d, 0x45, 0x4e, 0x10, 0xd5, 0xa7, 0x97, 0xad, 0x95, 0x32, - 0x37, 0x38, 0xac, 0x01, 0x25, 0xa2, 0xb6, 0xbc, 0x4e, 0x3d, 0x47, 0xb3, 0x31, 0xcd, 0x9e, 0x81, - 0xf2, 0x47, 0x43, 0x11, 0x8c, 0x76, 0xfd, 0x8e, 0xa8, 0x17, 0x68, 0x32, 0x61, 0xd8, 0x1e, 0x2c, - 0x18, 0x76, 0x84, 0x03, 0xdf, 0x0b, 0x05, 0xbb, 0x0e, 0x05, 0xda, 0x99, 0xcc, 0xa8, 0x34, 0xe7, - 0x56, 0xd5, 0x9d, 0xac, 0x92, 0x28, 0x97, 0x93, 0xec, 0x15, 0x98, 0xe9, 0x8b, 0x28, 0x70, 0xdb, - 0x21, 0x59, 0x54, 0x69, 0x5e, 0x49, 0xcb, 0xa1, 0xca, 0x5d, 0x29, 0xc0, 0xb5, 0xa4, 0xcd, 0x8c, - 0x73, 0xab, 0x49, 0xfb, 0xd3, 0x69, 0x98, 0x6d, 0x09, 0x27, 0x68, 0x3f, 0xd0, 0x37, 0xf1, 0x3a, - 0xe4, 0x0f, 0x9d, 0x6e, 0x58, 0xb7, 0x96, 0x73, 0x2b, 0x95, 0xe6, 0x72, 0xac, 0x37, 0x25, 0xb5, - 0x8a, 0x22, 0x5b, 0x5e, 0x14, 0x8c, 0x36, 0xf2, 0x9f, 0x7c, 0x71, 0x6d, 0x8a, 0xd3, 0x1a, 0x76, - 0x1d, 0x66, 0x77, 0x5d, 0x6f, 0x73, 0x18, 0x38, 0x91, 0xeb, 0x7b, 0xbb, 0xd2, 0xb8, 0x59, 0x9e, - 0x66, 0x92, 0x94, 0xf3, 0xd0, 0x90, 0xca, 0x29, 0x29, 0x93, 0xc9, 0x2e, 0x43, 0x61, 0xc7, 0xed, - 0xbb, 0x51, 0x3d, 0x4f, 0xb3, 0x92, 0x40, 0x6e, 0x48, 0x8e, 0x28, 0x48, 0x2e, 0x11, 0xac, 0x06, - 0x39, 0xe1, 0x75, 0xea, 0x45, 0xe2, 0xe1, 0x10, 0xe5, 0xde, 0xc5, 0x8b, 0xae, 0x97, 0xe8, 0xd6, - 0x25, 0xc1, 0x56, 0x60, 0xbe, 0x35, 0x70, 0xbc, 0xf0, 0x40, 0x04, 0xf8, 0xb7, 0x25, 0xa2, 0x7a, - 0x99, 0xd6, 0x8c, 0xb3, 0x1b, 0xaf, 0x42, 0x39, 0x3e, 0x22, 0xaa, 0x3f, 0x15, 0x23, 0xf2, 0x48, - 0x99, 0xe3, 0x10, 0xd5, 0x9f, 0x39, 0xbd, 0xa1, 0x50, 0xf1, 0x20, 0x89, 0xd7, 0xa7, 0x5f, 0xb3, - 0xec, 0x3f, 0xe7, 0x80, 0xc9, 0xab, 0xda, 0xc0, 0x28, 0xd0, 0xb7, 0x7a, 0x13, 0xca, 0xa1, 0xbe, - 0x40, 0xe5, 0xda, 0xc5, 0xec, 0xab, 0xe5, 0x89, 0x20, 0x46, 0x25, 0xc5, 0xd2, 0xf6, 0xa6, 0xda, - 0x48, 0x93, 0x18, 0x59, 0x74, 0xf4, 0x03, 0xa7, 0x2b, 0xd4, 0xfd, 0x25, 0x0c, 0xbc, 0xe1, 0x81, - 0xd3, 0x15, 0xe1, 0xa1, 0x2f, 0x55, 0xab, 0x3b, 0x4c, 0x33, 0x31, 0x72, 0x85, 0xd7, 0xf6, 0x3b, - 0xae, 0xd7, 0x55, 0xc1, 0x19, 0xd3, 0xa8, 0xc1, 0xf5, 0x3a, 0xe2, 0x21, 0xaa, 0x6b, 0xb9, 0x3f, - 0x15, 0xea, 0x6e, 0xd3, 0x4c, 0x66, 0x43, 0x35, 0xf2, 0x23, 0xa7, 0xc7, 0x45, 0xdb, 0x0f, 0x3a, - 0x61, 0x7d, 0x86, 0x84, 0x52, 0x3c, 0x94, 0xe9, 0x38, 0x91, 0xb3, 0xa5, 0x77, 0x92, 0x0e, 0x49, - 0xf1, 0xf0, 0x9c, 0x67, 0x22, 0x08, 0x5d, 0xdf, 0x23, 0x7f, 0x94, 0xb9, 0x26, 0x19, 0x83, 0x7c, - 0x88, 0xdb, 0xc3, 0xb2, 0xb5, 0x92, 0xe7, 0x34, 0xc6, 0x17, 0x79, 0xdf, 0xf7, 0x23, 0x11, 0x90, - 0x61, 0x15, 0xda, 0xd3, 0xe0, 0xb0, 0x4d, 0xa8, 0x75, 0x44, 0xc7, 0x6d, 0x3b, 0x91, 0xe8, 0xdc, - 0xf5, 0x7b, 0xc3, 0xbe, 0x17, 0xd6, 0xab, 0x14, 0xcd, 0xf5, 0xf8, 0xca, 0x37, 0xd3, 0x02, 0x7c, - 0x62, 0x85, 0xfd, 0x27, 0x0b, 0xe6, 0xc7, 0xa4, 0xd8, 0x4d, 0x28, 0x84, 0x6d, 0x7f, 0x20, 0x6f, - 0x7c, 0xae, 0xb9, 0x74, 0x9e, 0xba, 0xd5, 0x16, 0x4a, 0x71, 0x29, 0x8c, 0x67, 0xf0, 0x9c, 0xbe, - 0x8e, 0x15, 0x1a, 0xb3, 0x1b, 0x90, 0x8f, 0x46, 0x03, 0xf9, 0xca, 0xe7, 0x9a, 0xcf, 0x9e, 0xab, - 0xe8, 0x70, 0x34, 0x10, 0x9c, 0x44, 0xed, 0x6b, 0x50, 0x20, 0xb5, 0xac, 0x04, 0xf9, 0xd6, 0xc1, - 0x9d, 0xbd, 0xda, 0x14, 0xab, 0x42, 0x89, 0x6f, 0xb5, 0xf6, 0xdf, 0xe3, 0x77, 0xb7, 0x6a, 0x96, - 0xcd, 0x20, 0x8f, 0xe2, 0x0c, 0xa0, 0xd8, 0x3a, 0xe4, 0xdb, 0x7b, 0xf7, 0x6a, 0x53, 0xf6, 0x43, - 0x98, 0xd3, 0xd1, 0xa5, 0x00, 0xe6, 0x26, 0x14, 0x09, 0x43, 0xf4, 0x0b, 0x7f, 0x26, 0x8d, 0x1c, - 0x52, 0x7a, 0x57, 0x44, 0x0e, 0x7a, 0x88, 0x2b, 0x59, 0xb6, 0x3e, 0x0e, 0x38, 0xe3, 0xd1, 0x3b, - 0x81, 0x36, 0x7f, 0xcb, 0xc1, 0xa5, 0x0c, 0x8d, 0xe3, 0x48, 0x5b, 0x4e, 0x90, 0x76, 0x05, 0xe6, - 0x03, 0xdf, 0x8f, 0x5a, 0x22, 0x38, 0x73, 0xdb, 0x62, 0x2f, 0xb9, 0xb2, 0x71, 0x36, 0x46, 0x27, - 0xb2, 0x48, 0x3d, 0xc9, 0x49, 0xe0, 0x4d, 0x33, 0xd9, 0x4b, 0xb0, 0x40, 0x4f, 0xe2, 0xd0, 0xed, - 0x8b, 0xf7, 0x3c, 0xf7, 0xe1, 0x9e, 0xe3, 0xf9, 0xf4, 0x12, 0xf2, 0x7c, 0x72, 0x02, 0xa3, 0xaa, - 0x93, 0x40, 0x92, 0x84, 0x17, 0x83, 0xc3, 0x5e, 0x80, 0x99, 0x50, 0x61, 0x46, 0x91, 0x6e, 0xa0, - 0x96, 0xdc, 0x80, 0xe4, 0x73, 0x2d, 0xc0, 0x5e, 0x82, 0x92, 0x1a, 0xe2, 0x9b, 0xc8, 0x65, 0x0a, - 0xc7, 0x12, 0x8c, 0x43, 0x35, 0x94, 0x87, 0x6b, 0x45, 0x4e, 0x14, 0xd6, 0x4b, 0xb4, 0x62, 0xf5, - 0x22, 0xbf, 0xac, 0xb6, 0x8c, 0x05, 0x04, 0x52, 0x3c, 0xa5, 0xa3, 0x71, 0x04, 0x0b, 0x13, 0x22, - 0x19, 0x38, 0xf6, 0xa2, 0x89, 0x63, 0x95, 0xe6, 0x53, 0x86, 0x53, 0x93, 0xc5, 0x26, 0xbc, 0xed, - 0x40, 0xd5, 0x9c, 0x22, 0x1c, 0x1a, 0x38, 0xde, 0x5d, 0x7f, 0xe8, 0x45, 0xa4, 0x18, 0x71, 0x48, - 0x33, 0xf0, 0x4e, 0x45, 0x10, 0xf8, 0x81, 0x9c, 0x96, 0xc9, 0xc0, 0xe0, 0xd8, 0x3f, 0xb7, 0x60, - 0x46, 0xdd, 0x07, 0x7b, 0x0e, 0x0a, 0xb8, 0x50, 0x87, 0xe5, 0x6c, 0xea, 0xc2, 0xb8, 0x9c, 0xc3, - 0xe0, 0xe9, 0x3b, 0x51, 0xfb, 0x81, 0xe8, 0x28, 0x6d, 0x9a, 0x64, 0x6f, 0x00, 0x38, 0x51, 0x14, - 0xb8, 0x27, 0xc3, 0x48, 0x60, 0x46, 0x41, 0x1d, 0x57, 0x63, 0x1d, 0xaa, 0x8a, 0x38, 0xbb, 0xb1, - 0xfa, 0x8e, 0x18, 0x1d, 0xe1, 0x69, 0xb8, 0x21, 0x8e, 0x6f, 0x3d, 0x8f, 0xdb, 0xb0, 0x45, 0x28, - 0xe2, 0x46, 0x71, 0x6c, 0x2a, 0x2a, 0xf3, 0x09, 0x67, 0x86, 0x57, 0xee, 0xbc, 0xf0, 0xba, 0x0e, - 0xb3, 0x3a, 0x98, 0x90, 0x0e, 0x55, 0x20, 0xa6, 0x99, 0x63, 0xa7, 0x28, 0x3c, 0xde, 0x29, 0x3e, - 0x8e, 0x73, 0xb9, 0x7a, 0x8c, 0xf8, 0xa2, 0x5c, 0x2f, 0x1c, 0x88, 0x76, 0x24, 0x3a, 0x87, 0xfa, - 0xd1, 0x53, 0xbe, 0x1b, 0x63, 0xb3, 0xe7, 0x61, 0x2e, 0x66, 0x6d, 0x8c, 0x70, 0xf3, 0x69, 0xb2, - 0x6f, 0x8c, 0xcb, 0x96, 0xa1, 0x42, 0xe8, 0x4e, 0xc9, 0x4d, 0x67, 0x6e, 0x93, 0x85, 0x07, 0x6d, - 0xfb, 0xfd, 0x41, 0x4f, 0x44, 0xa2, 0xf3, 0xb6, 0x7f, 0x12, 0xea, 0xdc, 0x93, 0x62, 0x62, 0xdc, - 0xd0, 0x22, 0x92, 0x90, 0x8f, 0x2d, 0x61, 0xa0, 0xdd, 0x89, 0x4a, 0x69, 0x4e, 0x91, 0xcc, 0x19, - 0x67, 0xa7, 0xec, 0xa6, 0x1c, 0x4e, 0x39, 0xc8, 0xb4, 0x9b, 0xb8, 0xf6, 0xbb, 0xf8, 0x1e, 0xf0, - 0x6a, 0x30, 0xab, 0xeb, 0xa4, 0x7c, 0x59, 0xc3, 0xb9, 0x74, 0xb6, 0x82, 0xeb, 0xb8, 0xc4, 0xc8, - 0x65, 0x94, 0x18, 0xf9, 0xb8, 0xc4, 0xb0, 0x3f, 0xcd, 0xc1, 0x62, 0xa2, 0x33, 0x95, 0xed, 0x5f, - 0x9b, 0xcc, 0xf6, 0x8d, 0x31, 0xbc, 0x34, 0xec, 0xf8, 0x2e, 0xe3, 0x7f, 0x3b, 0x32, 0xfe, 0xe7, - 0x39, 0xb8, 0x1a, 0x3b, 0x87, 0x9e, 0x57, 0xda, 0xab, 0x3f, 0x98, 0xf4, 0xea, 0xb5, 0x49, 0xaf, - 0xca, 0x85, 0xdf, 0xb9, 0xf6, 0x5b, 0xe5, 0xda, 0x75, 0x5d, 0x94, 0xcb, 0x67, 0xa7, 0x4a, 0xa1, - 0x06, 0x94, 0x22, 0xa7, 0x8b, 0xb5, 0x82, 0xcc, 0x3a, 0x65, 0x1e, 0xd3, 0xf6, 0xdb, 0x70, 0x39, - 0x59, 0x71, 0xd4, 0x8c, 0xd7, 0x34, 0xa1, 0x48, 0x30, 0xa1, 0xf3, 0x54, 0xd6, 0xbb, 0x3e, 0x6a, - 0xca, 0xfa, 0x4f, 0x49, 0xda, 0x6f, 0x98, 0xe0, 0xa3, 0x26, 0xe3, 0x94, 0x62, 0x19, 0x29, 0x85, - 0x41, 0x3e, 0xc2, 0xde, 0x6b, 0x9a, 0x8c, 0xa1, 0xb1, 0x3d, 0x30, 0x50, 0x26, 0x15, 0x5b, 0x54, - 0x49, 0x49, 0x73, 0xe3, 0x4a, 0x4a, 0x92, 0x08, 0x61, 0xd4, 0x66, 0xea, 0xf6, 0x84, 0x88, 0x04, - 0xd8, 0xf2, 0x19, 0xc0, 0x56, 0x48, 0x80, 0xed, 0x55, 0x78, 0x7a, 0x62, 0x47, 0x75, 0x7a, 0x84, - 0x6d, 0xcd, 0x54, 0x57, 0x96, 0x30, 0xec, 0x9b, 0x50, 0xd2, 0x4b, 0xe8, 0x28, 0xa3, 0x18, 0x5a, - 0x69, 0x9c, 0xdd, 0x35, 0xd9, 0x3b, 0x70, 0x65, 0x6c, 0x3b, 0xe3, 0xba, 0xd7, 0xc6, 0x37, 0xac, - 0x34, 0x17, 0x92, 0xc2, 0x48, 0xcd, 0x98, 0x36, 0x6c, 0x40, 0x81, 0x52, 0x1a, 0xbb, 0x0d, 0x33, - 0x27, 0x54, 0x1b, 0xe8, 0x75, 0xc9, 0x5b, 0x95, 0x5f, 0x03, 0xce, 0x6e, 0xac, 0x72, 0x11, 0xfa, - 0xc3, 0xa0, 0x2d, 0x28, 0x47, 0x70, 0x2d, 0x6f, 0xef, 0x41, 0xf5, 0x60, 0x18, 0x26, 0x25, 0xf3, - 0x9b, 0x30, 0x4b, 0x45, 0x4b, 0xb8, 0x31, 0x3a, 0x54, 0xbd, 0x79, 0x6e, 0x65, 0xce, 0x08, 0x40, - 0x94, 0xde, 0x42, 0x09, 0x2e, 0x9c, 0xd0, 0xf7, 0x78, 0x5a, 0xdc, 0xfe, 0x9d, 0x05, 0x35, 0x14, - 0xa1, 0x94, 0xa5, 0xbd, 0xf7, 0x72, 0x5c, 0x87, 0xa3, 0xb7, 0xab, 0x1b, 0x4f, 0x61, 0x1f, 0xfd, - 0xf7, 0x2f, 0xae, 0xcd, 0x1e, 0x04, 0xc2, 0xe9, 0xf5, 0xfc, 0xb6, 0x94, 0xd6, 0x05, 0xf8, 0xff, - 0x43, 0xce, 0xed, 0xc8, 0xc2, 0xe6, 0x5c, 0x59, 0x94, 0x60, 0xb7, 0x00, 0x24, 0xe6, 0x6c, 0x3a, - 0x91, 0x53, 0xcf, 0x5f, 0x24, 0x6f, 0x08, 0xda, 0xbb, 0xd2, 0x44, 0x79, 0x13, 0xca, 0xc4, 0xff, - 0xe2, 0x0a, 0xaf, 0x03, 0xa8, 0x6f, 0x0d, 0x98, 0xa5, 0x17, 0x53, 0x3d, 0x47, 0x55, 0x1f, 0xca, - 0x7e, 0x13, 0xca, 0x3b, 0xae, 0x77, 0xda, 0xea, 0xb9, 0x6d, 0x6c, 0x89, 0x0a, 0x3d, 0xd7, 0x3b, - 0xd5, 0x7b, 0x5d, 0x9d, 0xdc, 0x0b, 0xf7, 0x58, 0xc5, 0x05, 0x5c, 0x4a, 0xda, 0x3f, 0xb3, 0x80, - 0x21, 0x53, 0x37, 0x1f, 0x49, 0x5e, 0x97, 0xe1, 0x6f, 0x99, 0xe1, 0x5f, 0x87, 0x99, 0x6e, 0xe0, - 0x0f, 0x07, 0x1b, 0xfa, 0x59, 0x68, 0x12, 0xe5, 0x7b, 0xf4, 0xa9, 0x41, 0x56, 0x6f, 0x92, 0xf8, - 0xda, 0xcf, 0xe5, 0x97, 0x16, 0x5c, 0x31, 0x8c, 0x68, 0x0d, 0xfb, 0x7d, 0x27, 0x18, 0xfd, 0x6f, - 0x6c, 0xf9, 0x83, 0x05, 0x97, 0x52, 0x17, 0x92, 0xbc, 0x5b, 0x11, 0x46, 0x6e, 0x1f, 0x31, 0x91, - 0x2c, 0x29, 0xf1, 0x84, 0x91, 0x2e, 0xe2, 0x65, 0xdd, 0x67, 0x14, 0xf1, 0xcf, 0xc3, 0x1c, 0x85, - 0x73, 0x2b, 0x16, 0x91, 0xa6, 0x8d, 0x71, 0xd9, 0x6a, 0xd2, 0x22, 0xe6, 0xc9, 0x83, 0x97, 0x53, - 0x25, 0xfc, 0x44, 0x83, 0xf8, 0x7d, 0xa8, 0x72, 0xe7, 0x27, 0x6f, 0xb9, 0x61, 0xe4, 0x77, 0x03, - 0xa7, 0x8f, 0x41, 0x72, 0x32, 0x6c, 0x9f, 0x0a, 0xd9, 0x47, 0xe4, 0xb9, 0xa2, 0xf0, 0xec, 0x6d, - 0xc3, 0x32, 0x49, 0xd8, 0x6f, 0x43, 0x49, 0x17, 0xc1, 0x19, 0x7d, 0xcd, 0x4b, 0xe9, 0xbe, 0x66, - 0x31, 0xdd, 0x4b, 0xbd, 0xbb, 0x83, 0xcd, 0x8b, 0xdb, 0xd6, 0x08, 0xf4, 0x1b, 0x0b, 0x2a, 0x86, - 0x89, 0x6c, 0x03, 0x16, 0x7a, 0x4e, 0x24, 0xbc, 0xf6, 0xe8, 0xf8, 0x81, 0x36, 0x4f, 0x45, 0x65, - 0xd2, 0x21, 0x99, 0xb6, 0xf3, 0x9a, 0x92, 0x4f, 0x4e, 0xf3, 0x3d, 0x28, 0x86, 0x22, 0x70, 0xd5, - 0xf3, 0x36, 0x51, 0x2b, 0xae, 0xdd, 0x95, 0x00, 0x1e, 0x5c, 0xe2, 0x85, 0xba, 0x58, 0x45, 0xd9, - 0x7f, 0x4d, 0x47, 0xb7, 0x0a, 0xac, 0xc9, 0x96, 0xeb, 0x11, 0xde, 0x9a, 0xce, 0xf4, 0x56, 0x62, - 0x5f, 0xee, 0x51, 0xf6, 0xd5, 0x20, 0x37, 0xb8, 0x7d, 0x5b, 0x35, 0x2c, 0x38, 0x94, 0x9c, 0x5b, - 0x14, 0x78, 0xc4, 0xb9, 0x25, 0x39, 0xeb, 0xaa, 0x4a, 0xc7, 0x21, 0x71, 0x6e, 0xad, 0xab, 0x72, - 0x1c, 0x87, 0xf6, 0xfb, 0xd0, 0xc8, 0x7a, 0x27, 0x2a, 0x44, 0x6f, 0x43, 0x39, 0x24, 0x96, 0x2b, - 0x26, 0x21, 0x20, 0x63, 0x5d, 0x22, 0x6d, 0xff, 0xd6, 0x82, 0xd9, 0x94, 0x63, 0x53, 0xd9, 0xa7, - 0xa0, 0xb2, 0x4f, 0x15, 0x2c, 0x8f, 0x2e, 0x23, 0xc7, 0x2d, 0x0f, 0xa9, 0xfb, 0x74, 0xdf, 0x16, - 0xb7, 0xee, 0x23, 0x25, 0x1b, 0x95, 0x32, 0xb7, 0x42, 0xa4, 0x4e, 0xe8, 0x70, 0x25, 0x6e, 0x9d, - 0x20, 0xd5, 0x51, 0x07, 0xb3, 0x3a, 0xd4, 0x21, 0x46, 0x4e, 0x34, 0x94, 0xf5, 0x51, 0x81, 0x2b, - 0x0a, 0x77, 0x3c, 0x75, 0xbd, 0x0e, 0x55, 0x44, 0x05, 0x4e, 0x63, 0x5b, 0xc8, 0xcf, 0x8d, 0xca, - 0x70, 0x84, 0x59, 0x2c, 0x77, 0x02, 0x11, 0x0e, 0x7b, 0xd1, 0x61, 0x92, 0x1c, 0x0d, 0x0e, 0x96, - 0x17, 0x92, 0x52, 0x61, 0xd3, 0xc8, 0x7c, 0x43, 0x24, 0xc1, 0x95, 0x24, 0xa2, 0xe0, 0xc2, 0xc4, - 0x2c, 0x86, 0x49, 0xcf, 0x39, 0x11, 0x3d, 0xa3, 0x3e, 0x48, 0x18, 0x68, 0x07, 0x11, 0x47, 0x46, - 0x3e, 0x36, 0x38, 0x6c, 0x0d, 0xa6, 0x23, 0x1d, 0x1a, 0xd7, 0xce, 0xb7, 0xe1, 0xc0, 0x77, 0xbd, - 0x88, 0x4f, 0x47, 0x21, 0xbe, 0xa1, 0xc5, 0xec, 0x69, 0x72, 0x86, 0xab, 0x8c, 0x98, 0xe5, 0x34, - 0xc6, 0xe8, 0x38, 0x73, 0x7a, 0xb4, 0xb1, 0xc5, 0x71, 0x88, 0x3d, 0x9f, 0x78, 0x28, 0xfa, 0x83, - 0x9e, 0x13, 0x1c, 0xaa, 0xef, 0x43, 0x39, 0xfa, 0x12, 0x3f, 0xce, 0x66, 0x2f, 0x40, 0x4d, 0xb3, - 0xf4, 0xf7, 0x62, 0x15, 0x9c, 0x13, 0x7c, 0xfb, 0x2f, 0x39, 0x58, 0xa0, 0x6f, 0xbf, 0xdc, 0xf1, - 0xba, 0xe2, 0x62, 0x50, 0x8e, 0x41, 0x56, 0x01, 0x4d, 0x0a, 0x64, 0xe5, 0xd3, 0xa4, 0x6f, 0xcb, - 0x58, 0xc6, 0x46, 0x62, 0xa0, 0xf6, 0xa4, 0x31, 0x02, 0x7a, 0xf8, 0xc0, 0x09, 0x3a, 0xdb, 0x9b, - 0x0a, 0x8e, 0x35, 0x89, 0x37, 0x4d, 0x43, 0xf9, 0x18, 0x65, 0xe5, 0x6d, 0x70, 0xd2, 0xbf, 0x11, - 0xcc, 0x8c, 0xfd, 0x46, 0x60, 0x36, 0x0d, 0xa5, 0x0b, 0x9a, 0x86, 0xf2, 0x23, 0x9b, 0x06, 0xc8, - 0x6a, 0x1a, 0x8c, 0x52, 0xbd, 0x92, 0x2e, 0xd5, 0xcd, 0x76, 0xa2, 0x3a, 0xd6, 0x4e, 0xe8, 0x32, - 0x7e, 0xf6, 0xdc, 0x32, 0x7e, 0xee, 0x6b, 0x95, 0xf1, 0xf3, 0x8f, 0x5d, 0xc6, 0x87, 0xc0, 0x4c, - 0x67, 0x2a, 0xe4, 0x78, 0x31, 0x86, 0x32, 0x09, 0x1b, 0x97, 0x12, 0xb4, 0x77, 0xfb, 0xa2, 0x45, - 0x53, 0x31, 0x98, 0x3d, 0xfe, 0x87, 0xcc, 0x3b, 0x50, 0x6c, 0x39, 0xfd, 0x41, 0x4f, 0xb0, 0xff, - 0x83, 0x2a, 0x06, 0x6f, 0x18, 0x39, 0xfd, 0xc1, 0x71, 0x3f, 0x54, 0x60, 0x52, 0x89, 0x79, 0xf2, - 0x57, 0x0b, 0x99, 0x78, 0x2c, 0x8a, 0x6c, 0x95, 0x60, 0x3e, 0xb6, 0x00, 0x12, 0x5b, 0xd8, 0x6d, - 0x28, 0xd2, 0x53, 0x9b, 0xc4, 0xb9, 0xc9, 0x2f, 0x3c, 0xea, 0xf7, 0x15, 0xb5, 0x80, 0xad, 0xc1, - 0x4c, 0x48, 0xc6, 0xe8, 0xbc, 0x32, 0x9f, 0x98, 0x4f, 0x7c, 0x25, 0xaf, 0xa5, 0xd8, 0x35, 0xa8, - 0x0c, 0x02, 0xbf, 0x7f, 0xac, 0x36, 0x94, 0x1f, 0x4a, 0x01, 0x59, 0x3b, 0xc4, 0x79, 0xe1, 0x43, - 0x98, 0x1f, 0x2b, 0x5f, 0x59, 0x15, 0x4a, 0x7b, 0xfb, 0xc7, 0x5b, 0x9c, 0xef, 0xf3, 0xda, 0x14, - 0xbb, 0x04, 0xf3, 0xbb, 0x77, 0x3e, 0x38, 0xde, 0xd9, 0x3e, 0xda, 0x3a, 0x3e, 0xe4, 0x77, 0xee, - 0x6e, 0xb5, 0x6a, 0x16, 0x32, 0x69, 0x7c, 0x7c, 0xb8, 0xbf, 0x7f, 0xbc, 0x73, 0x87, 0xdf, 0xdb, - 0xaa, 0x4d, 0xb3, 0x05, 0x98, 0x7d, 0x6f, 0xef, 0x9d, 0xbd, 0xfd, 0xf7, 0xf7, 0xd4, 0xe2, 0x5c, - 0xf3, 0x57, 0x16, 0x14, 0x51, 0xbd, 0x08, 0xd8, 0x0f, 0xa1, 0x1c, 0x17, 0xc1, 0xec, 0x4a, 0xaa, - 0x76, 0x36, 0x0b, 0xe3, 0xc6, 0x53, 0xa9, 0x29, 0xed, 0x65, 0x7b, 0x8a, 0xdd, 0x81, 0x4a, 0x2c, - 0x7c, 0xd4, 0xfc, 0x4f, 0x54, 0x34, 0xff, 0x69, 0x41, 0x4d, 0x39, 0xf8, 0x9e, 0xf0, 0x44, 0xe0, - 0x44, 0x7e, 0x6c, 0x18, 0x55, 0xb0, 0x63, 0x5a, 0xcd, 0x72, 0xf8, 0x7c, 0xc3, 0xb6, 0x01, 0xee, - 0x89, 0x48, 0x57, 0x0f, 0x57, 0xb3, 0xe1, 0x52, 0xea, 0x78, 0xe6, 0x1c, 0x2c, 0xd5, 0xaa, 0xee, - 0x01, 0x24, 0x11, 0xce, 0x12, 0xf4, 0x9f, 0xc0, 0xb0, 0xc6, 0xd5, 0xcc, 0xb9, 0xf8, 0xa4, 0xbf, - 0xcf, 0xc3, 0x0c, 0x4e, 0xb8, 0x22, 0x60, 0x6f, 0xc1, 0xec, 0x8f, 0x5c, 0xaf, 0x13, 0xff, 0xf8, - 0xc7, 0x32, 0x7e, 0x2d, 0xd4, 0x6a, 0x1b, 0x59, 0x53, 0x86, 0x0b, 0xaa, 0xfa, 0xe7, 0x84, 0xb6, - 0xf0, 0x22, 0x76, 0xce, 0x6f, 0x58, 0x8d, 0xa7, 0x27, 0xf8, 0xb1, 0x8a, 0x2d, 0xa8, 0x18, 0xbf, - 0x8f, 0x99, 0xb7, 0x35, 0xf1, 0xab, 0xd9, 0x45, 0x6a, 0xee, 0x01, 0x24, 0x3d, 0x35, 0xbb, 0xe0, - 0xeb, 0x5a, 0xe3, 0x6a, 0xe6, 0x5c, 0xac, 0xe8, 0x1d, 0x7d, 0x24, 0xd9, 0x9c, 0x5f, 0xa8, 0xea, - 0xd9, 0xcc, 0x66, 0xdf, 0x50, 0x76, 0x04, 0xf3, 0x63, 0xbd, 0x2c, 0x7b, 0xd4, 0x27, 0xa2, 0xc6, - 0xf2, 0xf9, 0x02, 0xb1, 0xde, 0x1f, 0x1b, 0x5f, 0x10, 0x74, 0x8f, 0xfc, 0x68, 0xcd, 0xf6, 0x79, - 0x02, 0xa6, 0xcd, 0xcd, 0x7f, 0xe5, 0xa0, 0xd6, 0x8a, 0x02, 0xe1, 0xf4, 0x5d, 0xaf, 0xab, 0x43, - 0xe6, 0x0d, 0x28, 0xaa, 0x1c, 0xf1, 0xb8, 0x2e, 0x5e, 0xb7, 0xf0, 0x3d, 0x3c, 0x11, 0xdf, 0xac, - 0x5b, 0x6c, 0xf7, 0x09, 0x7a, 0x67, 0xdd, 0x62, 0x1f, 0x7c, 0x33, 0xfe, 0x59, 0xb7, 0xd8, 0x87, - 0xdf, 0x9c, 0x87, 0xd6, 0x2d, 0x76, 0x00, 0x0b, 0x0a, 0x2b, 0x9e, 0x08, 0x3a, 0xac, 0x5b, 0xcd, - 0x3f, 0x5a, 0x30, 0xa3, 0x11, 0xeb, 0x38, 0xb3, 0xcf, 0xb0, 0x2f, 0xaa, 0xbe, 0xd5, 0x36, 0xcf, - 0x5d, 0x28, 0xf3, 0xc4, 0x51, 0x6d, 0xa3, 0xfe, 0xc9, 0x97, 0x4b, 0xd6, 0x67, 0x5f, 0x2e, 0x59, - 0xff, 0xf8, 0x72, 0xc9, 0xfa, 0xf5, 0x57, 0x4b, 0x53, 0x9f, 0x7d, 0xb5, 0x34, 0xf5, 0xf9, 0x57, - 0x4b, 0x53, 0x27, 0x45, 0xfa, 0xe7, 0x8e, 0x57, 0xfe, 0x1d, 0x00, 0x00, 0xff, 0xff, 0x35, 0x3c, - 0x12, 0xda, 0x5d, 0x22, 0x00, 0x00, + 0xb6, 0xa4, 0x95, 0x9b, 0xb2, 0x6c, 0x04, 0x06, 0x84, 0x11, 0xd9, 0xcb, 0x1d, 0x88, 0x9c, 0xa1, + 0x67, 0x86, 0xca, 0x32, 0xc7, 0x00, 0x09, 0x10, 0x20, 0x87, 0x1c, 0x92, 0x83, 0x8f, 0x39, 0x05, + 0x39, 0xe7, 0x11, 0x02, 0x04, 0x06, 0x82, 0x18, 0x06, 0x72, 0x31, 0x72, 0x30, 0x02, 0xfb, 0x90, + 0x07, 0xc8, 0x0b, 0x04, 0x55, 0xdd, 0x3d, 0xd3, 0x43, 0x8e, 0x24, 0x6f, 0xb2, 0x46, 0x7c, 0xf0, + 0x49, 0x5d, 0xd5, 0xd5, 0xd5, 0xd5, 0x55, 0xd5, 0xbf, 0xae, 0x1a, 0x0a, 0x9e, 0x1e, 0x9c, 0x74, + 0xd7, 0x22, 0xd1, 0x1f, 0xf8, 0x83, 0x63, 0xf9, 0x77, 0x75, 0x10, 0xf8, 0x91, 0xcf, 0x66, 0x14, + 0xb3, 0xb1, 0xd8, 0xf6, 0xfb, 0x7d, 0xdf, 0x5b, 0x3b, 0xbd, 0xb1, 0x26, 0x47, 0x52, 0xa0, 0xf1, + 0x72, 0xd7, 0x8d, 0x1e, 0x0e, 0x8f, 0x57, 0xdb, 0x7e, 0x7f, 0xad, 0xeb, 0x77, 0xfd, 0x35, 0x62, + 0x1f, 0x0f, 0x1f, 0x10, 0x45, 0x04, 0x8d, 0x94, 0xf8, 0xe5, 0x28, 0x70, 0xda, 0x02, 0xb5, 0xd0, + 0x40, 0x72, 0xed, 0x5f, 0x58, 0x50, 0x3b, 0x40, 0x7a, 0x63, 0xb4, 0xbd, 0xc9, 0xc5, 0x47, 0x43, + 0x11, 0x46, 0xac, 0x0e, 0x33, 0x24, 0xb3, 0xbd, 0x59, 0xb7, 0x96, 0xad, 0x95, 0x2a, 0xd7, 0x24, + 0x5b, 0x02, 0x38, 0xee, 0xf9, 0xed, 0x93, 0x56, 0xe4, 0x04, 0x51, 0x7d, 0x7a, 0xd9, 0x5a, 0x29, + 0x73, 0x83, 0xc3, 0x1a, 0x50, 0x22, 0x6a, 0xcb, 0xeb, 0xd4, 0x73, 0x34, 0x1b, 0xd3, 0xec, 0x19, + 0x28, 0x7f, 0x34, 0x14, 0xc1, 0x68, 0xd7, 0xef, 0x88, 0x7a, 0x81, 0x26, 0x13, 0x86, 0xed, 0xc1, + 0x82, 0x61, 0x47, 0x38, 0xf0, 0xbd, 0x50, 0xb0, 0xeb, 0x50, 0xa0, 0x9d, 0xc9, 0x8c, 0x4a, 0x73, + 0x6e, 0x55, 0xf9, 0x64, 0x95, 0x44, 0xb9, 0x9c, 0x64, 0xaf, 0xc0, 0x4c, 0x5f, 0x44, 0x81, 0xdb, + 0x0e, 0xc9, 0xa2, 0x4a, 0xf3, 0x4a, 0x5a, 0x0e, 0x55, 0xee, 0x4a, 0x01, 0xae, 0x25, 0x6d, 0x66, + 0x9c, 0x5b, 0x4d, 0xda, 0x9f, 0x4e, 0xc3, 0x6c, 0x4b, 0x38, 0x41, 0xfb, 0xa1, 0xf6, 0xc4, 0xeb, + 0x90, 0x3f, 0x70, 0xba, 0x61, 0xdd, 0x5a, 0xce, 0xad, 0x54, 0x9a, 0xcb, 0xb1, 0xde, 0x94, 0xd4, + 0x2a, 0x8a, 0x6c, 0x79, 0x51, 0x30, 0xda, 0xc8, 0x7f, 0xf2, 0xc5, 0xb5, 0x29, 0x4e, 0x6b, 0xd8, + 0x75, 0x98, 0xdd, 0x75, 0xbd, 0xcd, 0x61, 0xe0, 0x44, 0xae, 0xef, 0xed, 0x4a, 0xe3, 0x66, 0x79, + 0x9a, 0x49, 0x52, 0xce, 0x23, 0x43, 0x2a, 0xa7, 0xa4, 0x4c, 0x26, 0xbb, 0x0c, 0x85, 0x1d, 0xb7, + 0xef, 0x46, 0xf5, 0x3c, 0xcd, 0x4a, 0x02, 0xb9, 0x21, 0x05, 0xa2, 0x20, 0xb9, 0x44, 0xb0, 0x1a, + 0xe4, 0x84, 0xd7, 0xa9, 0x17, 0x89, 0x87, 0x43, 0x94, 0x7b, 0x17, 0x1d, 0x5d, 0x2f, 0x91, 0xd7, + 0x25, 0xc1, 0x56, 0x60, 0xbe, 0x35, 0x70, 0xbc, 0x70, 0x5f, 0x04, 0xf8, 0xb7, 0x25, 0xa2, 0x7a, + 0x99, 0xd6, 0x8c, 0xb3, 0x1b, 0xaf, 0x42, 0x39, 0x3e, 0x22, 0xaa, 0x3f, 0x11, 0x23, 0x8a, 0x48, + 0x99, 0xe3, 0x10, 0xd5, 0x9f, 0x3a, 0xbd, 0xa1, 0x50, 0xf9, 0x20, 0x89, 0xd7, 0xa7, 0x5f, 0xb3, + 0xec, 0x3f, 0xe7, 0x80, 0x49, 0x57, 0x6d, 0x60, 0x16, 0x68, 0xaf, 0xde, 0x84, 0x72, 0xa8, 0x1d, + 0xa8, 0x42, 0xbb, 0x98, 0xed, 0x5a, 0x9e, 0x08, 0x62, 0x56, 0x52, 0x2e, 0x6d, 0x6f, 0xaa, 0x8d, + 0x34, 0x89, 0x99, 0x45, 0x47, 0xdf, 0x77, 0xba, 0x42, 0xf9, 0x2f, 0x61, 0xa0, 0x87, 0x07, 0x4e, + 0x57, 0x84, 0x07, 0xbe, 0x54, 0xad, 0x7c, 0x98, 0x66, 0x62, 0xe6, 0x0a, 0xaf, 0xed, 0x77, 0x5c, + 0xaf, 0xab, 0x92, 0x33, 0xa6, 0x51, 0x83, 0xeb, 0x75, 0xc4, 0x23, 0x54, 0xd7, 0x72, 0x7f, 0x2a, + 0x94, 0x6f, 0xd3, 0x4c, 0x66, 0x43, 0x35, 0xf2, 0x23, 0xa7, 0xc7, 0x45, 0xdb, 0x0f, 0x3a, 0x61, + 0x7d, 0x86, 0x84, 0x52, 0x3c, 0x94, 0xe9, 0x38, 0x91, 0xb3, 0xa5, 0x77, 0x92, 0x01, 0x49, 0xf1, + 0xf0, 0x9c, 0xa7, 0x22, 0x08, 0x5d, 0xdf, 0xa3, 0x78, 0x94, 0xb9, 0x26, 0x19, 0x83, 0x7c, 0x88, + 0xdb, 0xc3, 0xb2, 0xb5, 0x92, 0xe7, 0x34, 0xc6, 0x1b, 0xf9, 0xc0, 0xf7, 0x23, 0x11, 0x90, 0x61, + 0x15, 0xda, 0xd3, 0xe0, 0xb0, 0x4d, 0xa8, 0x75, 0x44, 0xc7, 0x6d, 0x3b, 0x91, 0xe8, 0xdc, 0xf5, + 0x7b, 0xc3, 0xbe, 0x17, 0xd6, 0xab, 0x94, 0xcd, 0xf5, 0xd8, 0xe5, 0x9b, 0x69, 0x01, 0x3e, 0xb1, + 0xc2, 0xfe, 0x93, 0x05, 0xf3, 0x63, 0x52, 0xec, 0x26, 0x14, 0xc2, 0xb6, 0x3f, 0x90, 0x1e, 0x9f, + 0x6b, 0x2e, 0x9d, 0xa5, 0x6e, 0xb5, 0x85, 0x52, 0x5c, 0x0a, 0xe3, 0x19, 0x3c, 0xa7, 0xaf, 0x73, + 0x85, 0xc6, 0xec, 0x06, 0xe4, 0xa3, 0xd1, 0x40, 0xde, 0xf2, 0xb9, 0xe6, 0xb3, 0x67, 0x2a, 0x3a, + 0x18, 0x0d, 0x04, 0x27, 0x51, 0xfb, 0x1a, 0x14, 0x48, 0x2d, 0x2b, 0x41, 0xbe, 0xb5, 0x7f, 0x67, + 0xaf, 0x36, 0xc5, 0xaa, 0x50, 0xe2, 0x5b, 0xad, 0xfb, 0xef, 0xf1, 0xbb, 0x5b, 0x35, 0xcb, 0x66, + 0x90, 0x47, 0x71, 0x06, 0x50, 0x6c, 0x1d, 0xf0, 0xed, 0xbd, 0x7b, 0xb5, 0x29, 0xfb, 0x11, 0xcc, + 0xe9, 0xec, 0x52, 0x00, 0x73, 0x13, 0x8a, 0x84, 0x21, 0xfa, 0x86, 0x3f, 0x93, 0x46, 0x0e, 0x29, + 0xbd, 0x2b, 0x22, 0x07, 0x23, 0xc4, 0x95, 0x2c, 0x5b, 0x1f, 0x07, 0x9c, 0xf1, 0xec, 0x9d, 0x40, + 0x9b, 0xbf, 0xe5, 0xe0, 0x52, 0x86, 0xc6, 0x71, 0xa4, 0x2d, 0x27, 0x48, 0xbb, 0x02, 0xf3, 0x81, + 0xef, 0x47, 0x2d, 0x11, 0x9c, 0xba, 0x6d, 0xb1, 0x97, 0xb8, 0x6c, 0x9c, 0x8d, 0xd9, 0x89, 0x2c, + 0x52, 0x4f, 0x72, 0x12, 0x78, 0xd3, 0x4c, 0xf6, 0x12, 0x2c, 0xd0, 0x95, 0x38, 0x70, 0xfb, 0xe2, + 0x3d, 0xcf, 0x7d, 0xb4, 0xe7, 0x78, 0x3e, 0xdd, 0x84, 0x3c, 0x9f, 0x9c, 0xc0, 0xac, 0xea, 0x24, + 0x90, 0x24, 0xe1, 0xc5, 0xe0, 0xb0, 0x17, 0x60, 0x26, 0x54, 0x98, 0x51, 0x24, 0x0f, 0xd4, 0x12, + 0x0f, 0x48, 0x3e, 0xd7, 0x02, 0xec, 0x25, 0x28, 0xa9, 0x21, 0xde, 0x89, 0x5c, 0xa6, 0x70, 0x2c, + 0xc1, 0x38, 0x54, 0x43, 0x79, 0xb8, 0x56, 0xe4, 0x44, 0x61, 0xbd, 0x44, 0x2b, 0x56, 0xcf, 0x8b, + 0xcb, 0x6a, 0xcb, 0x58, 0x40, 0x20, 0xc5, 0x53, 0x3a, 0x1a, 0x87, 0xb0, 0x30, 0x21, 0x92, 0x81, + 0x63, 0x2f, 0x9a, 0x38, 0x56, 0x69, 0x3e, 0x65, 0x04, 0x35, 0x59, 0x6c, 0xc2, 0xdb, 0x0e, 0x54, + 0xcd, 0x29, 0xc2, 0xa1, 0x81, 0xe3, 0xdd, 0xf5, 0x87, 0x5e, 0x44, 0x8a, 0x11, 0x87, 0x34, 0x03, + 0x7d, 0x2a, 0x82, 0xc0, 0x0f, 0xe4, 0xb4, 0x7c, 0x0c, 0x0c, 0x8e, 0xfd, 0x73, 0x0b, 0x66, 0x94, + 0x3f, 0xd8, 0x73, 0x50, 0xc0, 0x85, 0x3a, 0x2d, 0x67, 0x53, 0x0e, 0xe3, 0x72, 0x0e, 0x93, 0xa7, + 0xef, 0x44, 0xed, 0x87, 0xa2, 0xa3, 0xb4, 0x69, 0x92, 0xbd, 0x01, 0xe0, 0x44, 0x51, 0xe0, 0x1e, + 0x0f, 0x23, 0x81, 0x2f, 0x0a, 0xea, 0xb8, 0x1a, 0xeb, 0x50, 0x55, 0xc4, 0xe9, 0x8d, 0xd5, 0x77, + 0xc4, 0xe8, 0x10, 0x4f, 0xc3, 0x0d, 0x71, 0xbc, 0xeb, 0x79, 0xdc, 0x86, 0x2d, 0x42, 0x11, 0x37, + 0x8a, 0x73, 0x53, 0x51, 0x99, 0x57, 0x38, 0x33, 0xbd, 0x72, 0x67, 0xa5, 0xd7, 0x75, 0x98, 0xd5, + 0xc9, 0x84, 0x74, 0xa8, 0x12, 0x31, 0xcd, 0x1c, 0x3b, 0x45, 0xe1, 0xf1, 0x4e, 0xf1, 0x71, 0xfc, + 0x96, 0xab, 0xcb, 0x88, 0x37, 0xca, 0xf5, 0xc2, 0x81, 0x68, 0x47, 0xa2, 0x73, 0xa0, 0x2f, 0x3d, + 0xbd, 0x77, 0x63, 0x6c, 0xf6, 0x3c, 0xcc, 0xc5, 0xac, 0x8d, 0x11, 0x6e, 0x3e, 0x4d, 0xf6, 0x8d, + 0x71, 0xd9, 0x32, 0x54, 0x08, 0xdd, 0xe9, 0x71, 0xd3, 0x2f, 0xb7, 0xc9, 0xc2, 0x83, 0xb6, 0xfd, + 0xfe, 0xa0, 0x27, 0x22, 0xd1, 0x79, 0xdb, 0x3f, 0x0e, 0xf5, 0xdb, 0x93, 0x62, 0x62, 0xde, 0xd0, + 0x22, 0x92, 0x90, 0x97, 0x2d, 0x61, 0xa0, 0xdd, 0x89, 0x4a, 0x69, 0x4e, 0x91, 0xcc, 0x19, 0x67, + 0xa7, 0xec, 0xa6, 0x37, 0x9c, 0xde, 0x20, 0xd3, 0x6e, 0xe2, 0xda, 0x5d, 0xbc, 0x0f, 0xe8, 0x1a, + 0x7c, 0xd5, 0xf5, 0xa3, 0x7c, 0x59, 0xc3, 0xb9, 0x0c, 0xb6, 0x82, 0xeb, 0xcb, 0x50, 0xa0, 0x1a, + 0x4d, 0xbf, 0xed, 0x44, 0x24, 0x85, 0x47, 0x2e, 0xa3, 0xf0, 0xc8, 0xc7, 0x85, 0x87, 0xfd, 0x69, + 0x0e, 0x16, 0x93, 0x9d, 0x52, 0x35, 0xc0, 0x6b, 0x93, 0x35, 0x40, 0x63, 0x0c, 0x45, 0x0d, 0xeb, + 0xbe, 0xab, 0x03, 0xbe, 0x1d, 0x75, 0xc0, 0xe7, 0x39, 0xb8, 0x1a, 0x07, 0x87, 0x2e, 0x5d, 0x3a, + 0xaa, 0x3f, 0x98, 0x8c, 0xea, 0xb5, 0xc9, 0xa8, 0xca, 0x85, 0xdf, 0x85, 0xf6, 0x5b, 0x15, 0xda, + 0x75, 0x5d, 0xaa, 0xcb, 0x6b, 0xa7, 0x0a, 0xa4, 0x06, 0x94, 0x22, 0xa7, 0x8b, 0x15, 0x84, 0x7c, + 0x8b, 0xca, 0x3c, 0xa6, 0xed, 0xb7, 0xe1, 0x72, 0xb2, 0xe2, 0xb0, 0x19, 0xaf, 0x69, 0x42, 0x91, + 0xc0, 0x43, 0xbf, 0x5e, 0x59, 0xf7, 0xfa, 0xb0, 0x29, 0xab, 0x42, 0x25, 0x69, 0xbf, 0x61, 0x42, + 0x92, 0x9a, 0x8c, 0x1f, 0x1a, 0xcb, 0x78, 0x68, 0x18, 0xe4, 0x23, 0xec, 0xc8, 0xa6, 0xc9, 0x18, + 0x1a, 0xdb, 0x03, 0x03, 0x65, 0x52, 0xb9, 0x45, 0xf5, 0x95, 0x34, 0x37, 0xae, 0xaf, 0x24, 0x79, + 0x11, 0xb0, 0xe5, 0x33, 0x80, 0xad, 0x90, 0x00, 0xdb, 0xab, 0xf0, 0xf4, 0xc4, 0x8e, 0xea, 0xf4, + 0x08, 0xe6, 0x9a, 0xa9, 0x5c, 0x96, 0x30, 0xec, 0x9b, 0x50, 0xd2, 0x4b, 0xe8, 0x28, 0xa3, 0x18, + 0x70, 0x69, 0x9c, 0xdd, 0x4b, 0xd9, 0x3b, 0x70, 0x65, 0x6c, 0x3b, 0xc3, 0xdd, 0x6b, 0xe3, 0x1b, + 0x56, 0x9a, 0x0b, 0x49, 0xb9, 0xa4, 0x66, 0x4c, 0x1b, 0x36, 0xa0, 0x40, 0x0f, 0x1d, 0xbb, 0x0d, + 0x33, 0xc7, 0x54, 0x31, 0xe8, 0x75, 0xc9, 0x5d, 0x95, 0xdf, 0x08, 0x4e, 0x6f, 0xac, 0x72, 0x11, + 0xfa, 0xc3, 0xa0, 0x2d, 0xe8, 0xe5, 0xe0, 0x5a, 0xde, 0xde, 0x83, 0xea, 0xfe, 0x30, 0x4c, 0x0a, + 0xe9, 0x37, 0x61, 0x96, 0x4a, 0x99, 0x70, 0x63, 0x74, 0xa0, 0x3a, 0xf6, 0xdc, 0xca, 0x9c, 0x91, + 0x80, 0x28, 0xbd, 0x85, 0x12, 0x5c, 0x38, 0xa1, 0xef, 0xf1, 0xb4, 0xb8, 0xfd, 0x3b, 0x0b, 0x6a, + 0x28, 0x42, 0x0f, 0x99, 0x8e, 0xde, 0xcb, 0x71, 0x75, 0x8e, 0xd1, 0xae, 0x6e, 0x3c, 0x85, 0xdd, + 0xf5, 0xdf, 0xbf, 0xb8, 0x36, 0xbb, 0x1f, 0x08, 0xa7, 0xd7, 0xf3, 0xdb, 0x52, 0x5a, 0x97, 0xe5, + 0xff, 0x0f, 0x39, 0xb7, 0x23, 0xcb, 0x9d, 0x33, 0x65, 0x51, 0x82, 0xdd, 0x02, 0x90, 0x98, 0xb3, + 0xe9, 0x44, 0x4e, 0x3d, 0x7f, 0x9e, 0xbc, 0x21, 0x68, 0xef, 0x4a, 0x13, 0xa5, 0x27, 0x94, 0x89, + 0xff, 0x85, 0x0b, 0xaf, 0x03, 0xa8, 0x2f, 0x10, 0xf8, 0x76, 0x2f, 0xa6, 0x3a, 0x91, 0xaa, 0x3e, + 0x94, 0xfd, 0x26, 0x94, 0x77, 0x5c, 0xef, 0xa4, 0xd5, 0x73, 0xdb, 0xd8, 0x28, 0x15, 0x7a, 0xae, + 0x77, 0xa2, 0xf7, 0xba, 0x3a, 0xb9, 0x17, 0xee, 0xb1, 0x8a, 0x0b, 0xb8, 0x94, 0xb4, 0x7f, 0x66, + 0x01, 0x43, 0xa6, 0x6e, 0x49, 0x92, 0xd7, 0x5e, 0xa6, 0xbf, 0x65, 0xa6, 0x7f, 0x1d, 0x66, 0xba, + 0x81, 0x3f, 0x1c, 0x6c, 0xe8, 0x6b, 0xa1, 0x49, 0x94, 0xef, 0xd1, 0x07, 0x08, 0x59, 0xd3, 0x49, + 0xe2, 0x6b, 0x5f, 0x97, 0x5f, 0x5a, 0x70, 0xc5, 0x30, 0xa2, 0x35, 0xec, 0xf7, 0x9d, 0x60, 0xf4, + 0xbf, 0xb1, 0xe5, 0x0f, 0x16, 0x5c, 0x4a, 0x39, 0x24, 0xb9, 0xb7, 0x22, 0x8c, 0xdc, 0x3e, 0x62, + 0x22, 0x59, 0x52, 0xe2, 0x09, 0x23, 0x5d, 0xda, 0xcb, 0x6a, 0xd0, 0x28, 0xed, 0x9f, 0x87, 0x39, + 0x4a, 0xe7, 0x56, 0x2c, 0x22, 0x4d, 0x1b, 0xe3, 0xb2, 0xd5, 0xa4, 0x71, 0xcc, 0x53, 0x04, 0x2f, + 0xa7, 0x0a, 0xfb, 0x89, 0xb6, 0xf1, 0xfb, 0x50, 0xe5, 0xce, 0x4f, 0xde, 0x72, 0xc3, 0xc8, 0xef, + 0x06, 0x4e, 0x1f, 0x93, 0xe4, 0x78, 0xd8, 0x3e, 0x11, 0xb2, 0xbb, 0xc8, 0x73, 0x45, 0xe1, 0xd9, + 0xdb, 0x86, 0x65, 0x92, 0xb0, 0xdf, 0x86, 0x92, 0x2e, 0x8d, 0x33, 0xba, 0x9d, 0x97, 0xd2, 0xdd, + 0xce, 0x62, 0xba, 0xc3, 0x7a, 0x77, 0x07, 0x5b, 0x1a, 0xb7, 0xad, 0x11, 0xe8, 0x37, 0x16, 0x54, + 0x0c, 0x13, 0xd9, 0x06, 0x2c, 0xf4, 0x9c, 0x48, 0x78, 0xed, 0xd1, 0xd1, 0x43, 0x6d, 0x9e, 0xca, + 0xca, 0xa4, 0x6f, 0x32, 0x6d, 0xe7, 0x35, 0x25, 0x9f, 0x9c, 0xe6, 0x7b, 0x50, 0x0c, 0x45, 0xe0, + 0xaa, 0xeb, 0x6d, 0xa2, 0x56, 0x5c, 0xd1, 0x2b, 0x01, 0x3c, 0xb8, 0xc4, 0x0b, 0xe5, 0x58, 0x45, + 0xd9, 0x7f, 0x4d, 0x67, 0xb7, 0x4a, 0xac, 0xc9, 0x46, 0xec, 0x82, 0x68, 0x4d, 0x67, 0x46, 0x2b, + 0xb1, 0x2f, 0x77, 0x91, 0x7d, 0x35, 0xc8, 0x0d, 0x6e, 0xdf, 0x56, 0x6d, 0x0c, 0x0e, 0x25, 0xe7, + 0x16, 0x25, 0x1e, 0x71, 0x6e, 0x49, 0xce, 0xba, 0xaa, 0xdd, 0x71, 0x48, 0x9c, 0x5b, 0xeb, 0xaa, + 0x48, 0xc7, 0xa1, 0xfd, 0x3e, 0x34, 0xb2, 0xee, 0x89, 0x4a, 0xd1, 0xdb, 0x50, 0x0e, 0x89, 0xe5, + 0x8a, 0x49, 0x08, 0xc8, 0x58, 0x97, 0x48, 0xdb, 0xbf, 0xb5, 0x60, 0x36, 0x15, 0xd8, 0xd4, 0xeb, + 0x53, 0x50, 0xaf, 0x4f, 0x15, 0x2c, 0x8f, 0x9c, 0x91, 0xe3, 0x96, 0x87, 0xd4, 0x03, 0xf2, 0xb7, + 0xc5, 0xad, 0x07, 0x48, 0xc9, 0xf6, 0xa5, 0xcc, 0xad, 0x10, 0xa9, 0x63, 0x3a, 0x5c, 0x89, 0x5b, + 0xc7, 0x48, 0x75, 0xd4, 0xc1, 0xac, 0x0e, 0xf5, 0x8d, 0x91, 0x13, 0x0d, 0x65, 0x7d, 0x54, 0xe0, + 0x8a, 0xc2, 0x1d, 0x4f, 0x5c, 0xaf, 0x43, 0x15, 0x51, 0x81, 0xd3, 0xd8, 0x16, 0xf2, 0x23, 0xa4, + 0x32, 0x1c, 0x61, 0x16, 0xcb, 0x9d, 0x40, 0x84, 0xc3, 0x5e, 0x74, 0x90, 0x3c, 0x8e, 0x06, 0x07, + 0xcb, 0x0b, 0x49, 0xa9, 0xb4, 0x69, 0x64, 0xde, 0x21, 0x92, 0xe0, 0x4a, 0x12, 0x51, 0x70, 0x61, + 0x62, 0x16, 0xd3, 0xa4, 0xe7, 0x1c, 0x8b, 0x9e, 0x51, 0x1f, 0x24, 0x0c, 0xb4, 0x83, 0x88, 0x43, + 0xe3, 0x3d, 0x36, 0x38, 0x6c, 0x0d, 0xa6, 0x23, 0x9d, 0x1a, 0xd7, 0xce, 0xb6, 0x61, 0xdf, 0x77, + 0xbd, 0x88, 0x4f, 0x47, 0x21, 0xde, 0xa1, 0xc5, 0xec, 0x69, 0x0a, 0x86, 0xab, 0x8c, 0x98, 0xe5, + 0x34, 0xc6, 0xec, 0x38, 0x75, 0x7a, 0xb4, 0xb1, 0xc5, 0x71, 0x88, 0x9d, 0xa0, 0x78, 0x24, 0xfa, + 0x83, 0x9e, 0x13, 0x1c, 0xa8, 0xaf, 0x46, 0x39, 0xfa, 0x3e, 0x3f, 0xce, 0x66, 0x2f, 0x40, 0x4d, + 0xb3, 0xf4, 0x57, 0x64, 0x95, 0x9c, 0x13, 0x7c, 0xfb, 0x2f, 0x39, 0x58, 0xa0, 0x2f, 0xc2, 0xdc, + 0xf1, 0xba, 0xe2, 0x7c, 0x50, 0x8e, 0x41, 0x56, 0x01, 0x4d, 0x0a, 0x64, 0xe5, 0xd5, 0xa4, 0x2f, + 0xce, 0x58, 0xc6, 0x46, 0x62, 0xa0, 0xf6, 0xa4, 0x31, 0x02, 0x7a, 0xf8, 0xd0, 0x09, 0x3a, 0xdb, + 0x9b, 0x0a, 0x8e, 0x35, 0x89, 0x9e, 0xa6, 0xa1, 0xbc, 0x8c, 0xb2, 0xf2, 0x36, 0x38, 0xe9, 0x5f, + 0x0e, 0x66, 0xc6, 0x7e, 0x39, 0x30, 0x9b, 0x86, 0xd2, 0x39, 0x4d, 0x43, 0xf9, 0xc2, 0xa6, 0x01, + 0xb2, 0x9a, 0x06, 0xa3, 0x54, 0xaf, 0xa4, 0x4b, 0x75, 0xb3, 0x9d, 0xa8, 0x8e, 0xb5, 0x13, 0xba, + 0x8c, 0x9f, 0x3d, 0xb3, 0x8c, 0x9f, 0xfb, 0x5a, 0x65, 0xfc, 0xfc, 0x63, 0x97, 0xf1, 0x21, 0x30, + 0x33, 0x98, 0x0a, 0x39, 0x5e, 0x8c, 0xa1, 0x4c, 0xc2, 0xc6, 0xa5, 0x04, 0xed, 0xdd, 0xbe, 0x68, + 0xd1, 0x54, 0x0c, 0x66, 0x8f, 0xff, 0x79, 0xf3, 0x0e, 0x14, 0x5b, 0x4e, 0x7f, 0xd0, 0x13, 0xec, + 0xff, 0xa0, 0x8a, 0xc9, 0x1b, 0x46, 0x4e, 0x7f, 0x70, 0xd4, 0x0f, 0x15, 0x98, 0x54, 0x62, 0x9e, + 0xfc, 0x2d, 0x43, 0x3e, 0x3c, 0x16, 0x65, 0xb6, 0x7a, 0x60, 0x3e, 0xb6, 0x00, 0x12, 0x5b, 0xd8, + 0x6d, 0x28, 0xd2, 0x55, 0x9b, 0xc4, 0xb9, 0xc9, 0xef, 0x3e, 0xea, 0x57, 0x17, 0xb5, 0x80, 0xad, + 0xc1, 0x4c, 0x48, 0xc6, 0xe8, 0x77, 0x65, 0x3e, 0x31, 0x9f, 0xf8, 0x4a, 0x5e, 0x4b, 0xb1, 0x6b, + 0x50, 0x19, 0x04, 0x7e, 0xff, 0x48, 0x6d, 0x28, 0x3f, 0x9f, 0x02, 0xb2, 0x76, 0x88, 0xf3, 0xc2, + 0x87, 0x30, 0x3f, 0x56, 0xbe, 0xb2, 0x2a, 0x94, 0xf6, 0xee, 0x1f, 0x6d, 0x71, 0x7e, 0x9f, 0xd7, + 0xa6, 0xd8, 0x25, 0x98, 0xdf, 0xbd, 0xf3, 0xc1, 0xd1, 0xce, 0xf6, 0xe1, 0xd6, 0xd1, 0x01, 0xbf, + 0x73, 0x77, 0xab, 0x55, 0xb3, 0x90, 0x49, 0xe3, 0xa3, 0x83, 0xfb, 0xf7, 0x8f, 0x76, 0xee, 0xf0, + 0x7b, 0x5b, 0xb5, 0x69, 0xb6, 0x00, 0xb3, 0xef, 0xed, 0xbd, 0xb3, 0x77, 0xff, 0xfd, 0x3d, 0xb5, + 0x38, 0xd7, 0xfc, 0x95, 0x05, 0x45, 0x54, 0x2f, 0x02, 0xf6, 0x43, 0x28, 0xc7, 0x45, 0x30, 0xbb, + 0x92, 0xaa, 0x9d, 0xcd, 0xc2, 0xb8, 0xf1, 0x54, 0x6a, 0x4a, 0x47, 0xd9, 0x9e, 0x62, 0x77, 0xa0, + 0x12, 0x0b, 0x1f, 0x36, 0xff, 0x13, 0x15, 0xcd, 0x7f, 0x5a, 0x50, 0x53, 0x01, 0xbe, 0x27, 0x3c, + 0x11, 0x38, 0x91, 0x1f, 0x1b, 0x46, 0x15, 0xec, 0x98, 0x56, 0xb3, 0x1c, 0x3e, 0xdb, 0xb0, 0x6d, + 0x80, 0x7b, 0x22, 0xd2, 0xd5, 0xc3, 0xd5, 0x6c, 0xb8, 0x94, 0x3a, 0x9e, 0x39, 0x03, 0x4b, 0xb5, + 0xaa, 0x7b, 0x00, 0x49, 0x86, 0xb3, 0x04, 0xfd, 0x27, 0x30, 0xac, 0x71, 0x35, 0x73, 0x2e, 0x3e, + 0xe9, 0xef, 0xf3, 0x30, 0x83, 0x13, 0xae, 0x08, 0xd8, 0x5b, 0x30, 0xfb, 0x23, 0xd7, 0xeb, 0xc4, + 0x3f, 0x09, 0xb2, 0x8c, 0xdf, 0x10, 0xb5, 0xda, 0x46, 0xd6, 0x94, 0x11, 0x82, 0xaa, 0xfe, 0x91, + 0xa1, 0x2d, 0xbc, 0x88, 0x9d, 0xf1, 0xcb, 0x56, 0xe3, 0xe9, 0x09, 0x7e, 0xac, 0x62, 0x0b, 0x2a, + 0xc6, 0xaf, 0x66, 0xa6, 0xb7, 0x26, 0x7e, 0x4b, 0x3b, 0x4f, 0xcd, 0x3d, 0x80, 0xa4, 0xa7, 0x66, + 0xe7, 0x7c, 0x5d, 0x6b, 0x5c, 0xcd, 0x9c, 0x8b, 0x15, 0xbd, 0xa3, 0x8f, 0x24, 0x9b, 0xf3, 0x73, + 0x55, 0x3d, 0x9b, 0xd9, 0xec, 0x1b, 0xca, 0x0e, 0x61, 0x7e, 0xac, 0x97, 0x65, 0x17, 0x7d, 0x22, + 0x6a, 0x2c, 0x9f, 0x2d, 0x10, 0xeb, 0xfd, 0xb1, 0xf1, 0x05, 0x41, 0xf7, 0xc8, 0x17, 0x6b, 0xb6, + 0xcf, 0x12, 0x30, 0x6d, 0x6e, 0xfe, 0x2b, 0x07, 0xb5, 0x56, 0x14, 0x08, 0xa7, 0xef, 0x7a, 0x5d, + 0x9d, 0x32, 0x6f, 0x40, 0x51, 0xbd, 0x11, 0x8f, 0x1b, 0xe2, 0x75, 0x0b, 0xef, 0xc3, 0x13, 0x89, + 0xcd, 0xba, 0xc5, 0x76, 0x9f, 0x60, 0x74, 0xd6, 0x2d, 0xf6, 0xc1, 0x37, 0x13, 0x9f, 0x75, 0x8b, + 0x7d, 0xf8, 0xcd, 0x45, 0x68, 0xdd, 0x62, 0xfb, 0xb0, 0xa0, 0xb0, 0xe2, 0x89, 0xa0, 0xc3, 0xba, + 0xd5, 0xfc, 0xa3, 0x05, 0x33, 0x1a, 0xb1, 0x8e, 0x32, 0xfb, 0x0c, 0xfb, 0xbc, 0xea, 0x5b, 0x6d, + 0xf3, 0xdc, 0xb9, 0x32, 0x4f, 0x1c, 0xd5, 0x36, 0xea, 0x9f, 0x7c, 0xb9, 0x64, 0x7d, 0xf6, 0xe5, + 0x92, 0xf5, 0x8f, 0x2f, 0x97, 0xac, 0x5f, 0x7f, 0xb5, 0x34, 0xf5, 0xd9, 0x57, 0x4b, 0x53, 0x9f, + 0x7f, 0xb5, 0x34, 0x75, 0x5c, 0xa4, 0x7f, 0xf9, 0x78, 0xe5, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, + 0xef, 0x6a, 0xd7, 0x76, 0x73, 0x22, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -5105,6 +5113,13 @@ func (m *SearchTagsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x18 } + if len(m.Query) > 0 { + i -= len(m.Query) + copy(dAtA[i:], m.Query) + i = encodeVarintTempo(dAtA, i, uint64(len(m.Query))) + i-- + dAtA[i] = 0x12 + } if len(m.Scope) > 0 { i -= len(m.Scope) copy(dAtA[i:], m.Scope) @@ -7055,6 +7070,10 @@ func (m *SearchTagsRequest) Size() (n int) { if l > 0 { n += 1 + l + sovTempo(uint64(l)) } + l = len(m.Query) + if l > 0 { + n += 1 + l + sovTempo(uint64(l)) + } if m.Start != 0 { n += 1 + sovTempo(uint64(m.Start)) } @@ -10112,6 +10131,38 @@ func (m *SearchTagsRequest) Unmarshal(dAtA []byte) error { } m.Scope = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Query", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTempo + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthTempo + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthTempo + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Query = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Start", wireType) diff --git a/pkg/tempopb/tempo.proto b/pkg/tempopb/tempo.proto index 253ea1b8b89..6ecf005dd9e 100644 --- a/pkg/tempopb/tempo.proto +++ b/pkg/tempopb/tempo.proto @@ -152,6 +152,7 @@ message SearchMetrics { message SearchTagsRequest { string scope = 1; + string query = 2; uint32 start = 3; uint32 end = 4; } From a2040b0ccc9b254423217bcbbbc99e8fe709ecc1 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 28 Jun 2024 11:07:11 -0400 Subject: [PATCH 12/21] parse query Signed-off-by: Joe Elliott --- pkg/api/search_tags.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/api/search_tags.go b/pkg/api/search_tags.go index be653ab0562..530454daf4a 100644 --- a/pkg/api/search_tags.go +++ b/pkg/api/search_tags.go @@ -422,14 +422,17 @@ func parseSearchTagValuesRequest(r *http.Request, enforceTraceQL bool) (*tempopb func ParseSearchTagsRequest(r *http.Request) (*tempopb.SearchTagsRequest, error) { scope, _ := extractQueryParam(r, urlParamScope) + query, _ := extractQueryParam(r, urlParamQuery) attScope := traceql.AttributeScopeFromString(scope) if attScope == traceql.AttributeScopeUnknown && scope != ParamScopeIntrinsic { return nil, fmt.Errorf("invalid scope: %s", scope) } - req := &tempopb.SearchTagsRequest{} - req.Scope = scope + req := &tempopb.SearchTagsRequest{ + Query: query, + Scope: scope, + } if s, ok := extractQueryParam(r, urlParamStart); ok { start, err := strconv.ParseInt(s, 10, 32) @@ -464,6 +467,7 @@ func BuildSearchTagsRequest(req *http.Request, searchReq *tempopb.SearchTagsRequ q.Set(urlParamStart, strconv.FormatUint(uint64(searchReq.Start), 10)) q.Set(urlParamEnd, strconv.FormatUint(uint64(searchReq.End), 10)) q.Set(urlParamScope, searchReq.Scope) + q.Set(urlParamQuery, searchReq.Query) req.URL.RawQuery = q.Encode() From 941ab945267087f3fc3cbb2bafcbd760940a9c1b Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 28 Jun 2024 11:30:19 -0400 Subject: [PATCH 13/21] tempodb tests Signed-off-by: Joe Elliott --- tempodb/tempodb_search_test.go | 79 ++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 3 deletions(-) diff --git a/tempodb/tempodb_search_test.go b/tempodb/tempodb_search_test.go index b00887ba38f..9dbf6a41811 100644 --- a/tempodb/tempodb_search_test.go +++ b/tempodb/tempodb_search_test.go @@ -6,6 +6,7 @@ import ( "fmt" "math/rand" "path" + "slices" "sort" "strings" "testing" @@ -53,7 +54,8 @@ func TestSearchCompleteBlock(t *testing.T) { traceQLStructural, traceQLExistence, nestedSet, - autoComplete, + tagValuesRunner, + tagNamesRunner, ) }) } @@ -1290,8 +1292,8 @@ func traceQLExistence(t *testing.T, _ *tempopb.Trace, _ *tempopb.TraceSearchMeta } } -// autoComplete! - jpe - add here -func autoComplete(t *testing.T, _ *tempopb.Trace, _ *tempopb.TraceSearchMetadata, _, _ []*tempopb.SearchRequest, _ *backend.BlockMeta, _ Reader, bb common.BackendBlock) { +// tagValuesRunner! +func tagValuesRunner(t *testing.T, _ *tempopb.Trace, _ *tempopb.TraceSearchMetadata, _, _ []*tempopb.SearchRequest, _ *backend.BlockMeta, _ Reader, bb common.BackendBlock) { ctx := context.Background() e := traceql.NewEngine() @@ -1401,6 +1403,77 @@ func autoComplete(t *testing.T, _ *tempopb.Trace, _ *tempopb.TraceSearchMetadata } } +// tagNamesRunner! +func tagNamesRunner(t *testing.T, _ *tempopb.Trace, _ *tempopb.TraceSearchMetadata, _, _ []*tempopb.SearchRequest, _ *backend.BlockMeta, _ Reader, bb common.BackendBlock) { + ctx := context.Background() + e := traceql.NewEngine() + + tcs := []struct { + name string + scope string + query string + expected map[string][]string + }{ + { + name: "no matches", + scope: "none", + query: "{ span.dne = `123456`}", + expected: map[string][]string{ + // even with no matches, we still return dedicated and intrinsic attributes that have values + "span": {"http.method", "http.status_code", "http.url", "span-dedicated.01", "span-dedicated.02"}, + "resource": {"cluster", "container", "k8s.cluster.name", "k8s.container.name", "k8s.namespace.name", "k8s.pod.name", "namespace", "pod", "res-dedicated.01", "res-dedicated.02", "service.name"}, + }, + }, + { + name: "resource match", + scope: "none", + query: "{ resource.cluster = `MyCluster` }", + expected: map[string][]string{ + "span": {"child", "foo", "http.method", "http.status_code", "http.url", "span-dedicated.01", "span-dedicated.02"}, + "resource": {"bat", "{ } ( ) = ~ ! < > & | ^", "cluster", "container", "k8s.cluster.name", "k8s.container.name", "k8s.namespace.name", "k8s.pod.name", "namespace", "pod", "res-dedicated.01", "res-dedicated.02", "service.name"}, + }, + }, + { + name: "span match", + scope: "none", + query: "{ span.foo = `Bar` }", + expected: map[string][]string{ + "span": {"child", "parent", "{ } ( ) = ~ ! < > & | ^", "foo", "http.method", "http.status_code", "http.url", "span-dedicated.01", "span-dedicated.02"}, + "resource": {"bat", "{ } ( ) = ~ ! < > & | ^", "cluster", "container", "k8s.cluster.name", "k8s.container.name", "k8s.namespace.name", "k8s.pod.name", "namespace", "pod", "res-dedicated.01", "res-dedicated.02", "service.name"}, + }, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + fetcher := traceql.NewTagNamesFetcherWrapper(func(ctx context.Context, req traceql.FetchTagsRequest, cb traceql.FetchTagsCallback) error { + return bb.FetchTagNames(ctx, req, cb, common.DefaultSearchOptions()) + }) + + valueCollector := collector.NewScopedDistinctString(0) + err := e.ExecuteTagNames(ctx, traceql.AttributeScopeFromString(tc.scope), tc.query, func(tag string, scope traceql.AttributeScope) bool { + valueCollector.Collect(scope.String(), tag) + return valueCollector.Exceeded() + }, fetcher) + if errors.Is(err, common.ErrUnsupported) { + return + } + require.NoError(t, err, "autocomplete request: %+v", tc) + + actualMap := valueCollector.Strings() + require.Equal(t, len(tc.expected), len(actualMap)) + + for k, expected := range tc.expected { + actual := actualMap[k] + + slices.Sort(actual) + slices.Sort(expected) + require.Equal(t, expected, actual, "key: %s", k) + } + }) + } +} + // oneQueryRunner is a good place to place a single query for debugging // func oneQueryRunner(t *testing.T, wantTr *tempopb.Trace, wantMeta *tempopb.TraceSearchMetadata, _, _ []*tempopb.SearchRequest, meta *backend.BlockMeta, r Reader) { // ctx := context.Background() From 15faed550f05ef1b16683394b495efcb45d6041b Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 28 Jun 2024 11:54:07 -0400 Subject: [PATCH 14/21] docs Signed-off-by: Joe Elliott --- docs/sources/tempo/api_docs/_index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/sources/tempo/api_docs/_index.md b/docs/sources/tempo/api_docs/_index.md index 265c4f0ced2..d503ac3d590 100644 --- a/docs/sources/tempo/api_docs/_index.md +++ b/docs/sources/tempo/api_docs/_index.md @@ -326,6 +326,8 @@ Parameters: - `scope = (resource|span|intrinsic)` Specifies the scope of the tags, this is an optional parameter, if not specified it means all scopes. Default = `all` +- `q = (traceql query)` + Optional. A TraceQL query to filter tag names by. Currently only works for a single spanset of &&ed conditions. ie. `{ span.foo = "bar" && resource.baz = "bat" ...}`. See also [Filtered tag values](#filtered-tag-values). - `start = (unix epoch seconds)` Optional. Along with `end` define a time range from which tags should be returned. - `end = (unix epoch seconds)` From 4ba971bed36231462b8656a5d36be9cef185c9f2 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 28 Jun 2024 15:48:34 -0400 Subject: [PATCH 15/21] integration Signed-off-by: Joe Elliott --- integration/e2e/api_test.go | 340 +++++++++++++++++- integration/e2e/e2e_test.go | 12 +- .../encoding/vparquet4/block_autocomplete.go | 2 +- 3 files changed, 344 insertions(+), 10 deletions(-) diff --git a/integration/e2e/api_test.go b/integration/e2e/api_test.go index c42febc89b4..7fc66dbe1e3 100644 --- a/integration/e2e/api_test.go +++ b/integration/e2e/api_test.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/url" + "slices" "sort" "strconv" "testing" @@ -24,7 +25,249 @@ const ( resourceX = "resource.xx" ) -// jpe - add here +func TestSearchTagsV2(t *testing.T) { + s, err := e2e.NewScenario("tempo_e2e") + require.NoError(t, err) + defer s.Close() + + require.NoError(t, util.CopyFileToSharedDir(s, configAllInOneLocal, "config.yaml")) + tempo := util.NewTempoAllInOne() + require.NoError(t, s.StartAndWaitReady(tempo)) + + jaegerClient, err := util.NewJaegerGRPCClient(tempo.Endpoint(14250)) + require.NoError(t, err) + require.NotNil(t, jaegerClient) + + type batchTmpl struct { + spanCount int + name string + resourceAttVal, spanAttVal string + resourceName, spanName string + } + + firstBatch := batchTmpl{spanCount: 2, name: "foo", resourceAttVal: "bar", spanAttVal: "bar", resourceName: "firstRes", spanName: "firstSpan"} + secondBatch := batchTmpl{spanCount: 2, name: "baz", resourceAttVal: "qux", spanAttVal: "qux", resourceName: "secondRes", spanName: "secondSpan"} + + batch := makeThriftBatchWithSpanCountAttributeAndName(firstBatch.spanCount, firstBatch.name, firstBatch.resourceAttVal, firstBatch.spanAttVal, firstBatch.resourceName, firstBatch.spanName) + require.NoError(t, jaegerClient.EmitBatch(context.Background(), batch)) + + batch = makeThriftBatchWithSpanCountAttributeAndName(secondBatch.spanCount, secondBatch.name, secondBatch.resourceAttVal, secondBatch.spanAttVal, secondBatch.resourceName, secondBatch.spanName) + require.NoError(t, jaegerClient.EmitBatch(context.Background(), batch)) + + // Wait for the traces to be written to the WAL + time.Sleep(time.Second * 3) + + testCases := []struct { + name string + query string + scope string + expected tempopb.SearchTagsV2Response + }{ + { + name: "no filtering", + query: "", + scope: "none", + expected: tempopb.SearchTagsV2Response{ + Scopes: []*tempopb.SearchTagsV2Scope{ + { + Name: "span", + Tags: []string{firstBatch.spanName, secondBatch.spanName}, + }, + { + Name: "resource", + Tags: []string{firstBatch.resourceName, secondBatch.resourceName, "service.name"}, + }, + }, + }, + }, + { + name: "first batch - resource", + query: fmt.Sprintf(`{ name="%s" }`, firstBatch.name), + scope: "resource", + expected: tempopb.SearchTagsV2Response{ + Scopes: []*tempopb.SearchTagsV2Scope{ + { + Name: "resource", + Tags: []string{firstBatch.resourceName, "service.name"}, + }, + }, + }, + }, + { + name: "second batch with incomplete query - span", + query: fmt.Sprintf(`{ name="%s" && span.x = }`, secondBatch.name), + scope: "span", + expected: tempopb.SearchTagsV2Response{ + Scopes: []*tempopb.SearchTagsV2Scope{ + { + Name: "span", + Tags: []string{secondBatch.spanName}, + }, + }, + }, + }, + { + name: "first batch - resource att - span", + query: fmt.Sprintf(`{ resource.%s="%s" }`, firstBatch.resourceName, firstBatch.resourceAttVal), + scope: "span", + expected: tempopb.SearchTagsV2Response{ + Scopes: []*tempopb.SearchTagsV2Scope{ + { + Name: "span", + Tags: []string{firstBatch.spanName}, + }, + }, + }, + }, + { + name: "first batch - resource att - resource", + query: fmt.Sprintf(`{ resource.%s="%s" }`, firstBatch.resourceName, firstBatch.resourceAttVal), + scope: "resource", + expected: tempopb.SearchTagsV2Response{ + Scopes: []*tempopb.SearchTagsV2Scope{ + { + Name: "resource", + Tags: []string{firstBatch.resourceName, "service.name"}, + }, + }, + }, + }, + { + name: "second batch - resource attribute - span", + query: fmt.Sprintf(`{ resource.%s="%s" }`, secondBatch.resourceName, secondBatch.resourceAttVal), + scope: "span", + expected: tempopb.SearchTagsV2Response{ + Scopes: []*tempopb.SearchTagsV2Scope{ + { + Name: "span", + Tags: []string{secondBatch.spanName}, + }, + }, + }, + }, + { + name: "too restrictive query", + query: fmt.Sprintf(`{ resource.%s="%s" && resource.y="%s" }`, firstBatch.resourceName, firstBatch.resourceAttVal, secondBatch.resourceAttVal), + scope: "none", + expected: tempopb.SearchTagsV2Response{ + Scopes: []*tempopb.SearchTagsV2Scope{ + { + Name: "resource", + Tags: []string{"service.name"}, // well known column so included + }, + }, + }, + }, + // Unscoped not supported, unfiltered results. + { + name: "unscoped span attribute", + query: fmt.Sprintf(`{ .x="%s" }`, firstBatch.spanAttVal), + scope: "none", + expected: tempopb.SearchTagsV2Response{ + Scopes: []*tempopb.SearchTagsV2Scope{ + { + Name: "span", + Tags: []string{firstBatch.spanName, secondBatch.spanName}, + }, + { + Name: "resource", + Tags: []string{firstBatch.resourceName, secondBatch.resourceName, "service.name"}, + }, + }, + }, + }, + { + name: "unscoped res attribute", + query: fmt.Sprintf(`{ .xx="%s" }`, firstBatch.resourceAttVal), + scope: "none", + expected: tempopb.SearchTagsV2Response{ + Scopes: []*tempopb.SearchTagsV2Scope{ + { + Name: "span", + Tags: []string{firstBatch.spanName, secondBatch.spanName}, + }, + { + Name: "resource", + Tags: []string{firstBatch.resourceName, secondBatch.resourceName, "service.name"}, + }, + }, + }, + }, + { + name: "both batches - name and resource attribute", + query: `{ resource.service.name="my-service"}`, + scope: "none", + expected: tempopb.SearchTagsV2Response{ + Scopes: []*tempopb.SearchTagsV2Scope{ + { + Name: "span", + Tags: []string{firstBatch.spanName, secondBatch.spanName}, + }, + { + Name: "resource", + Tags: []string{firstBatch.resourceName, secondBatch.resourceName, "service.name"}, + }, + }, + }, + }, + { + name: "bad query - unfiltered results", + query: fmt.Sprintf("%s = bar", spanX), // bad query, missing quotes + scope: "none", + expected: tempopb.SearchTagsV2Response{ + Scopes: []*tempopb.SearchTagsV2Scope{ + { + Name: "span", + Tags: []string{firstBatch.spanName, secondBatch.spanName}, + }, + { + Name: "resource", + Tags: []string{firstBatch.resourceName, secondBatch.resourceName, "service.name"}, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + callSearchTagsV2AndAssert(t, tempo, tc.scope, tc.query, tc.expected, 0, 0) + }) + } + + // Wait to block flushed to backend, 20 seconds is the complete_block_timeout configuration on all in one, we add + // 2s for security. + callFlush(t, tempo) + time.Sleep(time.Second * 22) + callFlush(t, tempo) + + // test metrics + require.NoError(t, tempo.WaitSumMetrics(e2e.Equals(1), "tempo_ingester_blocks_flushed_total")) + require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"tempodb_blocklist_length"}, e2e.WaitMissingMetrics)) + require.NoError(t, tempo.WaitSumMetrics(e2e.Equals(1), "tempo_ingester_blocks_cleared_total")) + + // Assert no more on the ingester + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + callSearchTagsV2AndAssert(t, tempo, tc.scope, tc.query, tempopb.SearchTagsV2Response{}, 0, 0) + }) + } + + // Wait to blocklist_poll to be completed + require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"tempodb_blocklist_length"}, e2e.WaitMissingMetrics)) + + // Assert tags on storage backend + now := time.Now() + start := now.Add(-2 * time.Hour) + end := now.Add(2 * time.Hour) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + callSearchTagsV2AndAssert(t, tempo, tc.scope, tc.query, tc.expected, start.Unix(), end.Unix()) + }) + } +} + func TestSearchTagValuesV2(t *testing.T) { s, err := e2e.NewScenario("tempo_e2e") require.NoError(t, err) @@ -47,10 +290,10 @@ func TestSearchTagValuesV2(t *testing.T) { firstBatch := batchTmpl{spanCount: 2, name: "foo", resourceAttVal: "bar", spanAttVal: "bar"} secondBatch := batchTmpl{spanCount: 2, name: "baz", resourceAttVal: "qux", spanAttVal: "qux"} - batch := makeThriftBatchWithSpanCountAttributeAndName(firstBatch.spanCount, firstBatch.name, firstBatch.resourceAttVal, firstBatch.spanAttVal) + batch := makeThriftBatchWithSpanCountAttributeAndName(firstBatch.spanCount, firstBatch.name, firstBatch.resourceAttVal, firstBatch.spanAttVal, "xx", "x") require.NoError(t, jaegerClient.EmitBatch(context.Background(), batch)) - batch = makeThriftBatchWithSpanCountAttributeAndName(secondBatch.spanCount, secondBatch.name, secondBatch.resourceAttVal, secondBatch.spanAttVal) + batch = makeThriftBatchWithSpanCountAttributeAndName(secondBatch.spanCount, secondBatch.name, secondBatch.resourceAttVal, secondBatch.spanAttVal, "xx", "x") require.NoError(t, jaegerClient.EmitBatch(context.Background(), batch)) // Wait for the traces to be written to the WAL @@ -346,6 +589,97 @@ func callSearchTagValuesV2AndAssert(t *testing.T, svc *e2e.HTTPService, tagName, require.Equal(t, expected, actualGrpcResp) } +func callSearchTagsV2AndAssert(t *testing.T, svc *e2e.HTTPService, scope, query string, expected tempopb.SearchTagsV2Response, start, end int64) { + urlPath := fmt.Sprintf(`/api/v2/search/tags?scope=%s&q=%s`, scope, url.QueryEscape(query)) + + // expected will not have the intrinsic scope since it's the same every time, add it here. + if scope == "none" || scope == "" || scope == "intrinsic" { + expected.Scopes = append(expected.Scopes, &tempopb.SearchTagsV2Scope{ + Name: "intrinsic", + Tags: []string{"duration", "event:name", "kind", "name", "rootName", "rootServiceName", "span:duration", "span:kind", "span:name", "span:status", "span:statusMessage", "status", "statusMessage", "trace:duration", "trace:rootName", "trace:rootService", "traceDuration"}, + }) + } + sort.Slice(expected.Scopes, func(i, j int) bool { return expected.Scopes[i].Name < expected.Scopes[j].Name }) + for _, scope := range expected.Scopes { + slices.Sort(scope.Tags) + } + + // search for tag values + req, err := http.NewRequest(http.MethodGet, "http://"+svc.Endpoint(3200)+urlPath, nil) + require.NoError(t, err) + + q := req.URL.Query() + + if start != 0 { + q.Set("start", strconv.Itoa(int(start))) + } + + if end != 0 { + q.Set("end", strconv.Itoa(int(end))) + } + + req.URL.RawQuery = q.Encode() + + res, err := http.DefaultClient.Do(req) + require.NoError(t, err) + require.Equal(t, http.StatusOK, res.StatusCode) + + // read body and print it + body, err := io.ReadAll(res.Body) + require.NoError(t, err) + defer res.Body.Close() + + // parse response + var response tempopb.SearchTagsV2Response + require.NoError(t, json.Unmarshal(body, &response)) + + prepTagsResponse(&response) + require.Equal(t, expected, response) + + // streaming + grpcReq := &tempopb.SearchTagsRequest{ + Scope: scope, + Query: query, + Start: uint32(start), + End: uint32(end), + } + + grpcClient, err := util.NewSearchGRPCClient(context.Background(), svc.Endpoint(3200)) + require.NoError(t, err) + + respTagsValuesV2, err := grpcClient.SearchTagsV2(context.Background(), grpcReq) + require.NoError(t, err) + var grpcResp *tempopb.SearchTagsV2Response + for { + resp, err := respTagsValuesV2.Recv() + if resp != nil { + grpcResp = resp + } + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + } + require.NotNil(t, grpcResp) + + prepTagsResponse(&response) + require.Equal(t, expected, response) +} + +func prepTagsResponse(resp *tempopb.SearchTagsV2Response) { + if len(resp.Scopes) == 0 { + resp.Scopes = nil + } + sort.Slice(resp.Scopes, func(i, j int) bool { return resp.Scopes[i].Name < resp.Scopes[j].Name }) + for _, scope := range resp.Scopes { + if len(scope.Tags) == 0 { + scope.Tags = nil + } + + slices.Sort(scope.Tags) + } +} + func callSearchTagsAndAssert(t *testing.T, svc *e2e.HTTPService, expected searchTagsResponse, start, end int64) { urlPath := "/api/search/tags" // search for tag values diff --git a/integration/e2e/e2e_test.go b/integration/e2e/e2e_test.go index 81ecb2832f9..794c42bc0a1 100644 --- a/integration/e2e/e2e_test.go +++ b/integration/e2e/e2e_test.go @@ -491,10 +491,10 @@ func makeThriftBatch() *thrift.Batch { } func makeThriftBatchWithSpanCount(n int) *thrift.Batch { - return makeThriftBatchWithSpanCountAttributeAndName(n, "my operation", "", "y") + return makeThriftBatchWithSpanCountAttributeAndName(n, "my operation", "", "y", "xx", "x") } -func makeThriftBatchWithSpanCountAttributeAndName(n int, name, resourceTag, spanTag string) *thrift.Batch { +func makeThriftBatchWithSpanCountAttributeAndName(n int, name, resourceValue, spanValue, resourceTag, spanTag string) *thrift.Batch { var spans []*thrift.Span traceIDLow := rand.Int63() @@ -512,8 +512,8 @@ func makeThriftBatchWithSpanCountAttributeAndName(n int, name, resourceTag, span Duration: 1, Tags: []*thrift.Tag{ { - Key: "x", - VStr: &spanTag, + Key: spanTag, + VStr: &spanValue, }, }, Logs: nil, @@ -525,9 +525,9 @@ func makeThriftBatchWithSpanCountAttributeAndName(n int, name, resourceTag, span ServiceName: "my-service", Tags: []*thrift.Tag{ { - Key: "xx", + Key: resourceTag, VType: thrift.TagType_STRING, - VStr: &resourceTag, + VStr: &resourceValue, }, }, }, diff --git a/tempodb/encoding/vparquet4/block_autocomplete.go b/tempodb/encoding/vparquet4/block_autocomplete.go index e9b126a27a7..e4a3d18c0a3 100644 --- a/tempodb/encoding/vparquet4/block_autocomplete.go +++ b/tempodb/encoding/vparquet4/block_autocomplete.go @@ -49,7 +49,7 @@ func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsR // Last check. No conditions, use old path. It's much faster. if len(req.Conditions) < 1 || mingledConditions { - return b.SearchTags(ctx, traceql.AttributeScopeResource, func(t string, scope traceql.AttributeScope) { + return b.SearchTags(ctx, req.Scope, func(t string, scope traceql.AttributeScope) { cb(t, scope) }, opts) } From d0081ebe2dbee3683ede9c86cc4b9263b22d0628 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 28 Jun 2024 15:52:48 -0400 Subject: [PATCH 16/21] changelog Signed-off-by: Joe Elliott --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 640fa6cc0cb..51905e78091 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ * [FEATURE] TraceQL support for event scope and event:name intrinsic [#3708](https://github.com/grafana/tempo/pull/3708) (@stoewer) * [FEATURE] Flush and query RF1 blocks for TraceQL metric queries [#3628](https://github.com/grafana/tempo/pull/3628) [#3691](https://github.com/grafana/tempo/pull/3691) [#3723](https://github.com/grafana/tempo/pull/3723) (@mapno) * [FEATURE] Add new compare() metrics function [#3695](https://github.com/grafana/tempo/pull/3695) (@mdisibio) +* [FEATURE] Add a `q` parameter to `/api/v2/serach/tags` for tag name filtering [#3822](https://github.com/grafana/tempo/pull/3822) (@joe-elliott) * [ENHANCEMENT] Tag value lookup use protobuf internally for improved latency [#3731](https://github.com/grafana/tempo/pull/3731) (@mdisibio) * [ENHANCEMENT] TraceQL metrics queries use protobuf internally for improved latency [#3745](https://github.com/grafana/tempo/pull/3745) (@mdisibio) * [ENHANCEMENT] Add local disk caching of metrics queries in local-blocks processor [#3799](https://github.com/grafana/tempo/pull/3799) (@mdisibio) From d9ffab27ea60f56cc43faa07421940db962073b9 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 28 Jun 2024 15:53:49 -0400 Subject: [PATCH 17/21] vendor check Signed-off-by: Joe Elliott --- vendor/modules.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/vendor/modules.txt b/vendor/modules.txt index 7de3e31bdb5..88323ab95a1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1201,8 +1201,6 @@ github.com/shirou/gopsutil/v3/process # github.com/shoenig/go-m1cpu v0.1.6 ## explicit; go 1.20 github.com/shoenig/go-m1cpu -# github.com/sirupsen/logrus v1.9.3 -## explicit # github.com/soheilhy/cmux v0.1.5 ## explicit; go 1.11 github.com/soheilhy/cmux From e6310c27d234d9a5009884f989ef23f12d606dec Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 28 Jun 2024 15:55:29 -0400 Subject: [PATCH 18/21] lint Signed-off-by: Joe Elliott --- modules/frontend/search_sharder_test.go | 2 +- tempodb/encoding/v2/wal_block.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/frontend/search_sharder_test.go b/modules/frontend/search_sharder_test.go index a78ddcbb4aa..7c83884d9c9 100644 --- a/modules/frontend/search_sharder_test.go +++ b/modules/frontend/search_sharder_test.go @@ -74,7 +74,7 @@ func (m *mockReader) Fetch(context.Context, *backend.BlockMeta, traceql.FetchSpa return traceql.FetchSpansResponse{}, nil } -func (m *mockReader) FetchTagNames(ctx context.Context, meta *backend.BlockMeta, req traceql.FetchTagsRequest, cb traceql.FetchTagsCallback, opts common.SearchOptions) error { +func (m *mockReader) FetchTagNames(_ context.Context, _ *backend.BlockMeta, _ traceql.FetchTagsRequest, _ traceql.FetchTagsCallback, _ common.SearchOptions) error { return nil } diff --git a/tempodb/encoding/v2/wal_block.go b/tempodb/encoding/v2/wal_block.go index d1a29da2c76..31b9c0ffed7 100644 --- a/tempodb/encoding/v2/wal_block.go +++ b/tempodb/encoding/v2/wal_block.go @@ -312,7 +312,7 @@ func (a *walBlock) FetchTagValues(context.Context, traceql.FetchTagValuesRequest } // FetchTagNames implements traceql.Searcher -func (b *walBlock) FetchTagNames(context.Context, traceql.FetchTagsRequest, traceql.FetchTagsCallback, common.SearchOptions) error { +func (a *walBlock) FetchTagNames(context.Context, traceql.FetchTagsRequest, traceql.FetchTagsCallback, common.SearchOptions) error { return common.ErrUnsupported } From bf74bb487405666d89b0539fadf491b9661ec0e9 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 28 Jun 2024 16:48:34 -0400 Subject: [PATCH 19/21] remove hardcoded scopes Signed-off-by: Joe Elliott --- tempodb/encoding/vparquet3/block_autocomplete.go | 2 +- tempodb/encoding/vparquet3/wal_block.go | 2 +- tempodb/encoding/vparquet4/wal_block.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tempodb/encoding/vparquet3/block_autocomplete.go b/tempodb/encoding/vparquet3/block_autocomplete.go index 0b0f527e648..a8d08174380 100644 --- a/tempodb/encoding/vparquet3/block_autocomplete.go +++ b/tempodb/encoding/vparquet3/block_autocomplete.go @@ -49,7 +49,7 @@ func (b *backendBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsR // Last check. No conditions, use old path. It's much faster. if len(req.Conditions) < 1 || mingledConditions { - return b.SearchTags(ctx, traceql.AttributeScopeResource, func(t string, scope traceql.AttributeScope) { + return b.SearchTags(ctx, req.Scope, func(t string, scope traceql.AttributeScope) { cb(t, scope) }, opts) } diff --git a/tempodb/encoding/vparquet3/wal_block.go b/tempodb/encoding/vparquet3/wal_block.go index eaa22c59c96..003e0cde897 100644 --- a/tempodb/encoding/vparquet3/wal_block.go +++ b/tempodb/encoding/vparquet3/wal_block.go @@ -756,7 +756,7 @@ func (b *walBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsReque } if len(req.Conditions) < 1 || mingledConditions { - return b.SearchTags(ctx, traceql.AttributeScopeResource, func(t string, scope traceql.AttributeScope) { + return b.SearchTags(ctx, req.Scope, func(t string, scope traceql.AttributeScope) { cb(t, scope) }, opts) } diff --git a/tempodb/encoding/vparquet4/wal_block.go b/tempodb/encoding/vparquet4/wal_block.go index e4d3678af03..eadef6d3ef0 100644 --- a/tempodb/encoding/vparquet4/wal_block.go +++ b/tempodb/encoding/vparquet4/wal_block.go @@ -758,7 +758,7 @@ func (b *walBlock) FetchTagNames(ctx context.Context, req traceql.FetchTagsReque } if len(req.Conditions) < 1 || mingledConditions { - return b.SearchTags(ctx, traceql.AttributeScopeResource, func(t string, scope traceql.AttributeScope) { + return b.SearchTags(ctx, req.Scope, func(t string, scope traceql.AttributeScope) { cb(t, scope) }, opts) } From 6ba02b4b88cd5fe4377669b2c4e475ca27cb3d28 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Wed, 10 Jul 2024 15:34:32 -0400 Subject: [PATCH 20/21] Update docs/sources/tempo/api_docs/_index.md Co-authored-by: Kim Nylander <104772500+knylander-grafana@users.noreply.github.com> --- docs/sources/tempo/api_docs/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/tempo/api_docs/_index.md b/docs/sources/tempo/api_docs/_index.md index d503ac3d590..15fb8e3c23e 100644 --- a/docs/sources/tempo/api_docs/_index.md +++ b/docs/sources/tempo/api_docs/_index.md @@ -327,7 +327,7 @@ Parameters: Specifies the scope of the tags, this is an optional parameter, if not specified it means all scopes. Default = `all` - `q = (traceql query)` - Optional. A TraceQL query to filter tag names by. Currently only works for a single spanset of &&ed conditions. ie. `{ span.foo = "bar" && resource.baz = "bat" ...}`. See also [Filtered tag values](#filtered-tag-values). + Optional. A TraceQL query to filter tag names by. Currently only works for a single spanset of `&&`ed conditions. For example: `{ span.foo = "bar" && resource.baz = "bat" ...}`. See also [Filtered tag values](#filtered-tag-values). - `start = (unix epoch seconds)` Optional. Along with `end` define a time range from which tags should be returned. - `end = (unix epoch seconds)` From 9cddee60407427ca313f13099188cff6ee14c707 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Wed, 10 Jul 2024 15:38:18 -0400 Subject: [PATCH 21/21] review Signed-off-by: Joe Elliott --- integration/e2e/api_test.go | 48 +++++++++---------- .../vparquet3/block_autocomplete_test.go | 4 -- .../vparquet4/block_autocomplete_test.go | 4 -- 3 files changed, 24 insertions(+), 32 deletions(-) diff --git a/integration/e2e/api_test.go b/integration/e2e/api_test.go index 7fc66dbe1e3..bc033027cfb 100644 --- a/integration/e2e/api_test.go +++ b/integration/e2e/api_test.go @@ -42,16 +42,16 @@ func TestSearchTagsV2(t *testing.T) { spanCount int name string resourceAttVal, spanAttVal string - resourceName, spanName string + resourceAttr, SpanAttr string } - firstBatch := batchTmpl{spanCount: 2, name: "foo", resourceAttVal: "bar", spanAttVal: "bar", resourceName: "firstRes", spanName: "firstSpan"} - secondBatch := batchTmpl{spanCount: 2, name: "baz", resourceAttVal: "qux", spanAttVal: "qux", resourceName: "secondRes", spanName: "secondSpan"} + firstBatch := batchTmpl{spanCount: 2, name: "foo", resourceAttVal: "bar", spanAttVal: "bar", resourceAttr: "firstRes", SpanAttr: "firstSpan"} + secondBatch := batchTmpl{spanCount: 2, name: "baz", resourceAttVal: "qux", spanAttVal: "qux", resourceAttr: "secondRes", SpanAttr: "secondSpan"} - batch := makeThriftBatchWithSpanCountAttributeAndName(firstBatch.spanCount, firstBatch.name, firstBatch.resourceAttVal, firstBatch.spanAttVal, firstBatch.resourceName, firstBatch.spanName) + batch := makeThriftBatchWithSpanCountAttributeAndName(firstBatch.spanCount, firstBatch.name, firstBatch.resourceAttVal, firstBatch.spanAttVal, firstBatch.resourceAttr, firstBatch.SpanAttr) require.NoError(t, jaegerClient.EmitBatch(context.Background(), batch)) - batch = makeThriftBatchWithSpanCountAttributeAndName(secondBatch.spanCount, secondBatch.name, secondBatch.resourceAttVal, secondBatch.spanAttVal, secondBatch.resourceName, secondBatch.spanName) + batch = makeThriftBatchWithSpanCountAttributeAndName(secondBatch.spanCount, secondBatch.name, secondBatch.resourceAttVal, secondBatch.spanAttVal, secondBatch.resourceAttr, secondBatch.SpanAttr) require.NoError(t, jaegerClient.EmitBatch(context.Background(), batch)) // Wait for the traces to be written to the WAL @@ -71,11 +71,11 @@ func TestSearchTagsV2(t *testing.T) { Scopes: []*tempopb.SearchTagsV2Scope{ { Name: "span", - Tags: []string{firstBatch.spanName, secondBatch.spanName}, + Tags: []string{firstBatch.SpanAttr, secondBatch.SpanAttr}, }, { Name: "resource", - Tags: []string{firstBatch.resourceName, secondBatch.resourceName, "service.name"}, + Tags: []string{firstBatch.resourceAttr, secondBatch.resourceAttr, "service.name"}, }, }, }, @@ -88,7 +88,7 @@ func TestSearchTagsV2(t *testing.T) { Scopes: []*tempopb.SearchTagsV2Scope{ { Name: "resource", - Tags: []string{firstBatch.resourceName, "service.name"}, + Tags: []string{firstBatch.resourceAttr, "service.name"}, }, }, }, @@ -101,53 +101,53 @@ func TestSearchTagsV2(t *testing.T) { Scopes: []*tempopb.SearchTagsV2Scope{ { Name: "span", - Tags: []string{secondBatch.spanName}, + Tags: []string{secondBatch.SpanAttr}, }, }, }, }, { name: "first batch - resource att - span", - query: fmt.Sprintf(`{ resource.%s="%s" }`, firstBatch.resourceName, firstBatch.resourceAttVal), + query: fmt.Sprintf(`{ resource.%s="%s" }`, firstBatch.resourceAttr, firstBatch.resourceAttVal), scope: "span", expected: tempopb.SearchTagsV2Response{ Scopes: []*tempopb.SearchTagsV2Scope{ { Name: "span", - Tags: []string{firstBatch.spanName}, + Tags: []string{firstBatch.SpanAttr}, }, }, }, }, { name: "first batch - resource att - resource", - query: fmt.Sprintf(`{ resource.%s="%s" }`, firstBatch.resourceName, firstBatch.resourceAttVal), + query: fmt.Sprintf(`{ resource.%s="%s" }`, firstBatch.resourceAttr, firstBatch.resourceAttVal), scope: "resource", expected: tempopb.SearchTagsV2Response{ Scopes: []*tempopb.SearchTagsV2Scope{ { Name: "resource", - Tags: []string{firstBatch.resourceName, "service.name"}, + Tags: []string{firstBatch.resourceAttr, "service.name"}, }, }, }, }, { name: "second batch - resource attribute - span", - query: fmt.Sprintf(`{ resource.%s="%s" }`, secondBatch.resourceName, secondBatch.resourceAttVal), + query: fmt.Sprintf(`{ resource.%s="%s" }`, secondBatch.resourceAttr, secondBatch.resourceAttVal), scope: "span", expected: tempopb.SearchTagsV2Response{ Scopes: []*tempopb.SearchTagsV2Scope{ { Name: "span", - Tags: []string{secondBatch.spanName}, + Tags: []string{secondBatch.SpanAttr}, }, }, }, }, { name: "too restrictive query", - query: fmt.Sprintf(`{ resource.%s="%s" && resource.y="%s" }`, firstBatch.resourceName, firstBatch.resourceAttVal, secondBatch.resourceAttVal), + query: fmt.Sprintf(`{ resource.%s="%s" && resource.y="%s" }`, firstBatch.resourceAttr, firstBatch.resourceAttVal, secondBatch.resourceAttVal), scope: "none", expected: tempopb.SearchTagsV2Response{ Scopes: []*tempopb.SearchTagsV2Scope{ @@ -167,11 +167,11 @@ func TestSearchTagsV2(t *testing.T) { Scopes: []*tempopb.SearchTagsV2Scope{ { Name: "span", - Tags: []string{firstBatch.spanName, secondBatch.spanName}, + Tags: []string{firstBatch.SpanAttr, secondBatch.SpanAttr}, }, { Name: "resource", - Tags: []string{firstBatch.resourceName, secondBatch.resourceName, "service.name"}, + Tags: []string{firstBatch.resourceAttr, secondBatch.resourceAttr, "service.name"}, }, }, }, @@ -184,11 +184,11 @@ func TestSearchTagsV2(t *testing.T) { Scopes: []*tempopb.SearchTagsV2Scope{ { Name: "span", - Tags: []string{firstBatch.spanName, secondBatch.spanName}, + Tags: []string{firstBatch.SpanAttr, secondBatch.SpanAttr}, }, { Name: "resource", - Tags: []string{firstBatch.resourceName, secondBatch.resourceName, "service.name"}, + Tags: []string{firstBatch.resourceAttr, secondBatch.resourceAttr, "service.name"}, }, }, }, @@ -201,11 +201,11 @@ func TestSearchTagsV2(t *testing.T) { Scopes: []*tempopb.SearchTagsV2Scope{ { Name: "span", - Tags: []string{firstBatch.spanName, secondBatch.spanName}, + Tags: []string{firstBatch.SpanAttr, secondBatch.SpanAttr}, }, { Name: "resource", - Tags: []string{firstBatch.resourceName, secondBatch.resourceName, "service.name"}, + Tags: []string{firstBatch.resourceAttr, secondBatch.resourceAttr, "service.name"}, }, }, }, @@ -218,11 +218,11 @@ func TestSearchTagsV2(t *testing.T) { Scopes: []*tempopb.SearchTagsV2Scope{ { Name: "span", - Tags: []string{firstBatch.spanName, secondBatch.spanName}, + Tags: []string{firstBatch.SpanAttr, secondBatch.SpanAttr}, }, { Name: "resource", - Tags: []string{firstBatch.resourceName, secondBatch.resourceName, "service.name"}, + Tags: []string{firstBatch.resourceAttr, secondBatch.resourceAttr, "service.name"}, }, }, }, diff --git a/tempodb/encoding/vparquet3/block_autocomplete_test.go b/tempodb/encoding/vparquet3/block_autocomplete_test.go index d1ec0438aef..5b6ef06e30f 100644 --- a/tempodb/encoding/vparquet3/block_autocomplete_test.go +++ b/tempodb/encoding/vparquet3/block_autocomplete_test.go @@ -550,10 +550,6 @@ func BenchmarkFetchTagValues(b *testing.B) { tag: "resource.namespace", query: `{span.http.status_code=200}`, }, - { - tag: "resource.namespace", - query: `{span.http.status_code=200}`, - }, // pathologic cases /* { diff --git a/tempodb/encoding/vparquet4/block_autocomplete_test.go b/tempodb/encoding/vparquet4/block_autocomplete_test.go index ae9cc7c84b9..1c18bff90aa 100644 --- a/tempodb/encoding/vparquet4/block_autocomplete_test.go +++ b/tempodb/encoding/vparquet4/block_autocomplete_test.go @@ -550,10 +550,6 @@ func BenchmarkFetchTagValues(b *testing.B) { tag: "resource.namespace", query: `{span.http.status_code=200}`, }, - { - tag: "resource.namespace", - query: `{span.http.status_code=200}`, - }, // pathologic cases /* {