From 61026e15672619309d3845fdc90d14bd870f44e3 Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Wed, 31 Jan 2024 13:23:27 -0500 Subject: [PATCH 01/13] Update traceql metrics to use the trace-level timestamp columns conditionally --- .../processor/localblocks/processor.go | 8 ++- modules/querier/querier_query_range.go | 2 +- pkg/traceql/engine_metrics.go | 67 +++++++++++++++---- pkg/traceql/engine_metrics_test.go | 19 ++++++ 4 files changed, 79 insertions(+), 17 deletions(-) diff --git a/modules/generator/processor/localblocks/processor.go b/modules/generator/processor/localblocks/processor.go index 6a55d81ab74..6dd80c2f92b 100644 --- a/modules/generator/processor/localblocks/processor.go +++ b/modules/generator/processor/localblocks/processor.go @@ -472,9 +472,11 @@ func (p *Processor) QueryRange(ctx context.Context, req *tempopb.QueryRangeReque go func(b common.BackendBlock) { defer wg.Done() + m := b.BlockMeta() + span, ctx := opentracing.StartSpanFromContext(ctx, "Processor.QueryRange.Block", opentracing.Tags{ - "block": b.BlockMeta().BlockID, - "blockSize": b.BlockMeta().Size, + "block": m.BlockID, + "blockSize": m.Size, }) defer span.Finish() @@ -483,7 +485,7 @@ func (p *Processor) QueryRange(ctx context.Context, req *tempopb.QueryRangeReque return b.Fetch(ctx, req, common.DefaultSearchOptions()) }) - err := eval.Do(ctx, f) + err := eval.Do(ctx, f, uint64(m.StartTime.UnixNano()), uint64(m.EndTime.UnixNano())) if err != nil { jobErr.Store(err) } diff --git a/modules/querier/querier_query_range.go b/modules/querier/querier_query_range.go index 6e1f5b85f2a..81e2092ebec 100644 --- a/modules/querier/querier_query_range.go +++ b/modules/querier/querier_query_range.go @@ -102,7 +102,7 @@ func (q *Querier) queryBackend(ctx context.Context, req *tempopb.QueryRangeReque }) // TODO handle error - err := eval.Do(ctx, f) + err := eval.Do(ctx, f, uint64(m.StartTime.UnixNano()), uint64(m.EndTime.UnixNano())) if err != nil { jobErr.Store(err) } diff --git a/pkg/traceql/engine_metrics.go b/pkg/traceql/engine_metrics.go index 81b6a80f242..b9f45ffde35 100644 --- a/pkg/traceql/engine_metrics.go +++ b/pkg/traceql/engine_metrics.go @@ -324,7 +324,7 @@ func (e *Engine) ExecuteMetricsQueryRange(ctx context.Context, req *tempopb.Quer return nil, err } - err = eval.Do(ctx, fetcher) + err = eval.Do(ctx, fetcher, 0, 0) if err != nil { return nil, err } @@ -393,11 +393,9 @@ func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, dedupe // Timestamp filtering // (1) Include any overlapping trace - // TODO - It can be faster to skip the trace-level timestamp check + // It can be faster to skip the trace-level timestamp check // when all or most of the traces overlap the window. - // Maybe it can be dynamic. - // storageReq.StartTimeUnixNanos = req.Start - // storageReq.EndTimeUnixNanos = req.End // Should this be exclusive? + // So this is done dynamically on a per-fetcher basis in Do() // (2) Only include spans that started in this time frame. // This is checked outside the fetch layer in the evaluator. Timestamp // is only checked on the spans that are the final results. @@ -470,13 +468,49 @@ type MetricsEvalulator struct { deduper *SpanDeduper2 storageReq *FetchSpansRequest metricsPipeline metricsFirstStageElement - count int - deduped int + spansTotal uint64 + spansDeduped uint64 + bytes uint64 mtx sync.Mutex } -func (e *MetricsEvalulator) Do(ctx context.Context, f SpansetFetcher) error { - fetch, err := f.Fetch(ctx, *e.storageReq) +func timeRangeOverlap(reqStart, reqEnd, dataStart, dataEnd uint64) float64 { + st := max(reqStart, dataStart) + end := min(reqEnd, dataEnd) + + if end <= st { + return 0 + } + + return float64(end-st) / float64(dataEnd-dataStart) +} + +// Do metrics on the given source of data and merge the results into the working set. Optionally, if provided, +// uses the known time range of the data for last-minute optimizations. Time range is unix nanos +func (e *MetricsEvalulator) Do(ctx context.Context, f SpansetFetcher, fetcherStart, fetcherEnd uint64) error { + // Make a copy of the request so we can modify it. + storageReq := *e.storageReq + + if fetcherStart > 0 && fetcherEnd > 0 { + // Dynamically decide whether to use the trace-level timestamp columns + // for filtering. Our heuristic is if the overlap between the given fetcher (i.e. block) + // and the request is less than 80%, use them. In the last 90-100%, the cost of loading + // them doesn't outweight the benefits. 80% was measured in local benchmarking + overlap := timeRangeOverlap(e.start, e.end, fetcherStart, fetcherEnd) + + if overlap == 0.0 { + // This shouldn't happen but might as well check. + // No overlap == nothing to do + return nil + } + + if overlap < 0.8 { + storageReq.StartTimeUnixNanos = e.start + storageReq.EndTimeUnixNanos = e.end // Should this be exclusive? + } + } + + fetch, err := f.Fetch(ctx, storageReq) if errors.Is(err, util.ErrUnsupported) { return nil } @@ -510,11 +544,11 @@ func (e *MetricsEvalulator) Do(ctx context.Context, f SpansetFetcher) error { } if e.dedupeSpans && e.deduper.Skip(ss.TraceID, s.StartTimeUnixNanos()) { - e.deduped++ + e.spansDeduped++ continue } - e.count++ + e.spansTotal++ e.metricsPipeline.observe(s) } @@ -522,11 +556,18 @@ func (e *MetricsEvalulator) Do(ctx context.Context, f SpansetFetcher) error { ss.Release() } + e.mtx.Lock() + defer e.mtx.Unlock() + e.bytes += fetch.Bytes() + return nil } -func (e *MetricsEvalulator) SpanCount() { - fmt.Println(e.count, e.deduped) +func (e *MetricsEvalulator) Metrics() (uint64, uint64, uint64) { + e.mtx.Lock() + defer e.mtx.Unlock() + + return e.bytes, e.spansTotal, e.spansDeduped } func (e *MetricsEvalulator) Results() (SeriesSet, error) { diff --git a/pkg/traceql/engine_metrics_test.go b/pkg/traceql/engine_metrics_test.go index 1b439f29829..bda614d1dc6 100644 --- a/pkg/traceql/engine_metrics_test.go +++ b/pkg/traceql/engine_metrics_test.go @@ -93,6 +93,25 @@ func TestIntervalOf(t *testing.T) { } } +func TestTimeRangeOverlap(t *testing.T) { + tc := []struct { + reqStart, reqEnd, dataStart, dataEnd uint64 + expected float64 + }{ + {1, 2, 3, 4, 0.0}, // No overlap + {0, 10, 0, 10, 1.0}, // Perfect overlap + {0, 10, 1, 2, 1.0}, // Request covers 100% of data + {3, 8, 0, 10, 0.5}, // 50% in the middle + {0, 10, 5, 15, 0.5}, // 50% of the start + {5, 15, 0, 10, 0.5}, // 50% of the end + } + + for _, c := range tc { + actual := timeRangeOverlap(c.reqStart, c.reqEnd, c.dataStart, c.dataEnd) + require.Equal(t, c.expected, actual) + } +} + func TestCompileMetricsQueryRange(t *testing.T) { tc := map[string]struct { q string From a33e9648dfa848e64c6ae465e699b625187259d1 Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Wed, 31 Jan 2024 14:18:55 -0500 Subject: [PATCH 02/13] comments --- pkg/traceql/engine_metrics.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/traceql/engine_metrics.go b/pkg/traceql/engine_metrics.go index b9f45ffde35..d1daba905b2 100644 --- a/pkg/traceql/engine_metrics.go +++ b/pkg/traceql/engine_metrics.go @@ -493,9 +493,7 @@ func (e *MetricsEvalulator) Do(ctx context.Context, f SpansetFetcher, fetcherSta if fetcherStart > 0 && fetcherEnd > 0 { // Dynamically decide whether to use the trace-level timestamp columns - // for filtering. Our heuristic is if the overlap between the given fetcher (i.e. block) - // and the request is less than 80%, use them. In the last 90-100%, the cost of loading - // them doesn't outweight the benefits. 80% was measured in local benchmarking + // for filtering. overlap := timeRangeOverlap(e.start, e.end, fetcherStart, fetcherEnd) if overlap == 0.0 { @@ -504,6 +502,9 @@ func (e *MetricsEvalulator) Do(ctx context.Context, f SpansetFetcher, fetcherSta return nil } + // Our heuristic is if the overlap between the given fetcher (i.e. block) + // and the request is less than 80%, use them. In the last 80-100%, the cost of loading + // them doesn't outweight the benefits. 80% was measured in local benchmarking. if overlap < 0.8 { storageReq.StartTimeUnixNanos = e.start storageReq.EndTimeUnixNanos = e.end // Should this be exclusive? From 06242ac309460321052247f47562e0ac2ef1803f Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Thu, 1 Feb 2024 07:31:17 -0500 Subject: [PATCH 03/13] Update benchmark, comment --- pkg/traceql/engine_metrics.go | 1 + .../encoding/vparquet3/block_traceql_test.go | 51 +++++++++++++------ 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/pkg/traceql/engine_metrics.go b/pkg/traceql/engine_metrics.go index d1daba905b2..90002121b2f 100644 --- a/pkg/traceql/engine_metrics.go +++ b/pkg/traceql/engine_metrics.go @@ -505,6 +505,7 @@ func (e *MetricsEvalulator) Do(ctx context.Context, f SpansetFetcher, fetcherSta // Our heuristic is if the overlap between the given fetcher (i.e. block) // and the request is less than 80%, use them. In the last 80-100%, the cost of loading // them doesn't outweight the benefits. 80% was measured in local benchmarking. + // TODO - Make configurable or a query hint? if overlap < 0.8 { storageReq.StartTimeUnixNanos = e.start storageReq.EndTimeUnixNanos = e.end // Should this be exclusive? diff --git a/tempodb/encoding/vparquet3/block_traceql_test.go b/tempodb/encoding/vparquet3/block_traceql_test.go index 6cffd39d81b..ccf079b19a0 100644 --- a/tempodb/encoding/vparquet3/block_traceql_test.go +++ b/tempodb/encoding/vparquet3/block_traceql_test.go @@ -6,6 +6,7 @@ import ( "fmt" "math/rand" "path" + "strconv" "testing" "time" @@ -694,7 +695,8 @@ func BenchmarkBackendBlockQueryRange(b *testing.B) { opts = common.DefaultSearchOptions() tenantID = "1" blockID = uuid.MustParse("06ebd383-8d4e-4289-b0e9-cf2197d611d5") - path = "/Users/marty/src/tmp/" + // blockID = uuid.MustParse("18364616-f80d-45a6-b2a3-cb63e203edff") + path = "/Users/marty/src/tmp/" ) r, _, _, err := local.New(&local.Config{ @@ -717,20 +719,39 @@ func BenchmarkBackendBlockQueryRange(b *testing.B) { for _, tc := range testCases { b.Run(tc, func(b *testing.B) { - req := &tempopb.QueryRangeRequest{ - Query: tc, - Step: uint64(time.Minute), - Start: uint64(meta.StartTime.UnixNano()), - End: uint64(meta.EndTime.UnixNano()), - } - - eval, err := e.CompileMetricsQueryRange(req, false) - require.NoError(b, err) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - err := eval.Do(ctx, f) - require.NoError(b, err) + for _, minutes := range []int{1, 2, 3, 4, 5, 6, 7, 8, 9} { + b.Run(strconv.Itoa(minutes), func(b *testing.B) { + st := meta.StartTime + end := st.Add(time.Duration(minutes) * time.Minute) + + if end.After(meta.EndTime) { + b.SkipNow() + return + } + + req := &tempopb.QueryRangeRequest{ + Query: tc, + Step: uint64(time.Minute), + Start: uint64(st.UnixNano()), + End: uint64(end.UnixNano()), + ShardID: 30, + ShardCount: 65, + } + + eval, err := e.CompileMetricsQueryRange(req, false) + require.NoError(b, err) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := eval.Do(ctx, f, uint64(block.meta.StartTime.UnixNano()), uint64(block.meta.EndTime.UnixNano())) + require.NoError(b, err) + } + + bytes, spansTotal, _ := eval.Metrics() + b.ReportMetric(float64(bytes)/float64(b.N)/1024.0/1024.0, "MB_IO/op") + b.ReportMetric(float64(spansTotal)/float64(b.N), "spans/op") + b.ReportMetric(float64(spansTotal)/float64(b.Elapsed().Seconds()), "spans/s") + }) } }) } From 6a01e39d1c302f19d791024dc710125ccf06fc28 Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Fri, 2 Feb 2024 09:43:38 -0500 Subject: [PATCH 04/13] lint --- tempodb/encoding/vparquet3/block_traceql_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tempodb/encoding/vparquet3/block_traceql_test.go b/tempodb/encoding/vparquet3/block_traceql_test.go index ccf079b19a0..83d5c3f3784 100644 --- a/tempodb/encoding/vparquet3/block_traceql_test.go +++ b/tempodb/encoding/vparquet3/block_traceql_test.go @@ -750,7 +750,7 @@ func BenchmarkBackendBlockQueryRange(b *testing.B) { bytes, spansTotal, _ := eval.Metrics() b.ReportMetric(float64(bytes)/float64(b.N)/1024.0/1024.0, "MB_IO/op") b.ReportMetric(float64(spansTotal)/float64(b.N), "spans/op") - b.ReportMetric(float64(spansTotal)/float64(b.Elapsed().Seconds()), "spans/s") + b.ReportMetric(float64(spansTotal)/b.Elapsed().Seconds(), "spans/s") }) } }) From 106cb14ad93e160cd21ea9b3fda837a8d2e710e2 Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Fri, 2 Feb 2024 14:26:54 -0500 Subject: [PATCH 05/13] Change overlap cutoff to 20% --- pkg/traceql/engine_metrics.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/traceql/engine_metrics.go b/pkg/traceql/engine_metrics.go index 90002121b2f..79699404ba7 100644 --- a/pkg/traceql/engine_metrics.go +++ b/pkg/traceql/engine_metrics.go @@ -503,10 +503,10 @@ func (e *MetricsEvalulator) Do(ctx context.Context, f SpansetFetcher, fetcherSta } // Our heuristic is if the overlap between the given fetcher (i.e. block) - // and the request is less than 80%, use them. In the last 80-100%, the cost of loading - // them doesn't outweight the benefits. 80% was measured in local benchmarking. + // and the request is less than 20%, use them. Above 20%, the cost of loading + // them doesn't outweight the benefits. 20% was measured in local benchmarking. // TODO - Make configurable or a query hint? - if overlap < 0.8 { + if overlap < 0.2 { storageReq.StartTimeUnixNanos = e.start storageReq.EndTimeUnixNanos = e.end // Should this be exclusive? } From 88aec48bb005baf8ca7e7bd7ec8cab4af53df396 Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Tue, 13 Feb 2024 14:21:58 -0500 Subject: [PATCH 06/13] add more instrumentation and a little cleanup --- modules/frontend/query_range_sharding.go | 5 +- modules/querier/http.go | 5 + modules/querier/querier_query_range.go | 28 +- pkg/tempopb/tempo.pb.go | 348 +++++++++++++---------- pkg/tempopb/tempo.proto | 1 + pkg/traceql/combine.go | 2 + pkg/traceql/engine_metrics.go | 44 +-- pkg/traceql/enum_attributes.go | 2 + 8 files changed, 254 insertions(+), 181 deletions(-) diff --git a/modules/frontend/query_range_sharding.go b/modules/frontend/query_range_sharding.go index d69bf4d799f..05a68402d56 100644 --- a/modules/frontend/query_range_sharding.go +++ b/modules/frontend/query_range_sharding.go @@ -217,15 +217,18 @@ func (s queryRangeSharder) RoundTrip(r *http.Request) (*http.Response, error) { res.Metrics.TotalBlockBytes = uint64(totalBlockBytes) reqTime := time.Since(now) - throughput := float64(res.Metrics.InspectedBytes) / reqTime.Seconds() + throughput := math.Round(float64(res.Metrics.InspectedBytes) / reqTime.Seconds()) + spanThroughput := math.Round(float64(res.Metrics.InspectedSpans) / reqTime.Seconds()) span.SetTag("totalBlocks", res.Metrics.TotalBlocks) span.SetTag("inspectedBytes", res.Metrics.InspectedBytes) span.SetTag("inspectedTraces", res.Metrics.InspectedTraces) + span.SetTag("inspectedSpans", res.Metrics.InspectedSpans) span.SetTag("totalBlockBytes", res.Metrics.TotalBlockBytes) span.SetTag("totalJobs", res.Metrics.TotalJobs) span.SetTag("finishedJobs", res.Metrics.CompletedJobs) span.SetTag("requestThroughput", throughput) + span.SetTag("spanThroughput", spanThroughput) if jErr := jobErr.Load(); jErr != nil { return s.respErrHandler(isProm, jErr) diff --git a/modules/querier/http.go b/modules/querier/http.go index 9efcd063c61..057bacc97bc 100644 --- a/modules/querier/http.go +++ b/modules/querier/http.go @@ -456,6 +456,11 @@ func (q *Querier) QueryRangeHandler(w http.ResponseWriter, r *http.Request) { errHandler(ctx, span, err) return } + + if resp != nil && resp.Metrics != nil { + span.SetTag("inspectedBytes", resp.Metrics.InspectedBytes) + span.SetTag("inspectedSpans", resp.Metrics.InspectedSpans) + } } func handleError(w http.ResponseWriter, err error) { diff --git a/modules/querier/querier_query_range.go b/modules/querier/querier_query_range.go index 81e2092ebec..2e22053a4a9 100644 --- a/modules/querier/querier_query_range.go +++ b/modules/querier/querier_query_range.go @@ -64,11 +64,6 @@ func (q *Querier) queryBackend(ctx context.Context, req *tempopb.QueryRangeReque return nil, err } - eval, err := traceql.NewEngine().CompileMetricsQueryRange(req, true) - if err != nil { - return nil, err - } - // Get blocks that overlap this time range metas := q.store.BlockMetas(tenantID) withinTimeRange := metas[:0] @@ -78,6 +73,19 @@ func (q *Querier) queryBackend(ctx context.Context, req *tempopb.QueryRangeReque } } + if len(withinTimeRange) == 0 { + return nil, nil + } + + // Optimization + // If there's only 1 block then dedupe not needed. + dedupe := len(withinTimeRange) > 1 + + eval, err := traceql.NewEngine().CompileMetricsQueryRange(req, dedupe) + if err != nil { + return nil, err + } + wg := boundedwaitgroup.New(2) jobErr := atomic.Error{} @@ -119,7 +127,15 @@ func (q *Querier) queryBackend(ctx context.Context, req *tempopb.QueryRangeReque return nil, err } - return &tempopb.QueryRangeResponse{Series: queryRangeTraceQLToProto(res, req)}, nil + inspectedBytes, spansTotal, _ := eval.Metrics() + + return &tempopb.QueryRangeResponse{ + Series: queryRangeTraceQLToProto(res, req), + Metrics: &tempopb.SearchMetrics{ + InspectedBytes: inspectedBytes, + InspectedSpans: spansTotal, + }, + }, nil } func queryRangeTraceQLToProto(set traceql.SeriesSet, req *tempopb.QueryRangeRequest) []*tempopb.TimeSeries { diff --git a/pkg/tempopb/tempo.pb.go b/pkg/tempopb/tempo.pb.go index 31248579f85..b21a2c7ae97 100644 --- a/pkg/tempopb/tempo.pb.go +++ b/pkg/tempopb/tempo.pb.go @@ -850,6 +850,7 @@ type SearchMetrics struct { CompletedJobs uint32 `protobuf:"varint,4,opt,name=completedJobs,proto3" json:"completedJobs,omitempty"` TotalJobs uint32 `protobuf:"varint,5,opt,name=totalJobs,proto3" json:"totalJobs,omitempty"` TotalBlockBytes uint64 `protobuf:"varint,6,opt,name=totalBlockBytes,proto3" json:"totalBlockBytes,omitempty"` + InspectedSpans uint64 `protobuf:"varint,7,opt,name=inspectedSpans,proto3" json:"inspectedSpans,omitempty"` } func (m *SearchMetrics) Reset() { *m = SearchMetrics{} } @@ -927,6 +928,13 @@ func (m *SearchMetrics) GetTotalBlockBytes() uint64 { return 0 } +func (m *SearchMetrics) GetInspectedSpans() uint64 { + if m != nil { + return m.InspectedSpans + } + return 0 +} + type SearchTagsRequest struct { Scope string `protobuf:"bytes,1,opt,name=scope,proto3" json:"scope,omitempty"` Start uint32 `protobuf:"varint,3,opt,name=start,proto3" json:"start,omitempty"` @@ -2990,162 +2998,163 @@ func init() { func init() { proto.RegisterFile("pkg/tempopb/tempo.proto", fileDescriptor_f22805646f4f62b6) } var fileDescriptor_f22805646f4f62b6 = []byte{ - // 2478 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x59, 0xcd, 0x6f, 0x64, 0x47, - 0x11, 0xf7, 0xf3, 0x7c, 0x79, 0xca, 0x63, 0x7b, 0xdc, 0xbb, 0x71, 0x66, 0xc7, 0x89, 0x6d, 0x5e, - 0x56, 0x60, 0x42, 0x62, 0x7b, 0x27, 0xbb, 0x4a, 0x36, 0x81, 0x20, 0x7b, 0x6d, 0x1c, 0x27, 0xfe, - 0x4a, 0xcf, 0xc4, 0x41, 0x08, 0xc9, 0x7a, 0x33, 0xd3, 0x3b, 0xfb, 0xe4, 0x99, 0xf7, 0x26, 0xef, - 0xf5, 0x98, 0x1d, 0x8e, 0x48, 0x20, 0x21, 0x71, 0xe0, 0x00, 0x07, 0x8e, 0x9c, 0x22, 0xce, 0x9c, - 0x39, 0x21, 0xa1, 0x5c, 0x88, 0x72, 0x8c, 0x38, 0x44, 0x28, 0x7b, 0xe0, 0xcc, 0x7f, 0x80, 0xaa, - 0xba, 0xfb, 0x7d, 0xcc, 0x8c, 0xbd, 0x2c, 0x20, 0x91, 0x43, 0x4e, 0xd3, 0xf5, 0xeb, 0x7a, 0xd5, - 0xd5, 0x55, 0xd5, 0x55, 0xd5, 0x3d, 0xf0, 0x7c, 0xff, 0xa2, 0xb3, 0x29, 0x45, 0xaf, 0xef, 0xf7, - 0x9b, 0xea, 0x77, 0xa3, 0x1f, 0xf8, 0xd2, 0x67, 0x05, 0x0d, 0x56, 0x97, 0x5a, 0x7e, 0xaf, 0xe7, - 0x7b, 0x9b, 0x97, 0x77, 0x36, 0xd5, 0x48, 0x31, 0x54, 0x5f, 0xed, 0xb8, 0xf2, 0xd1, 0xa0, 0xb9, - 0xd1, 0xf2, 0x7b, 0x9b, 0x1d, 0xbf, 0xe3, 0x6f, 0x12, 0xdc, 0x1c, 0x3c, 0x24, 0x8a, 0x08, 0x1a, - 0x69, 0xf6, 0x9b, 0x32, 0x70, 0x5a, 0x02, 0xa5, 0xd0, 0x40, 0xa1, 0xf6, 0x2f, 0x2c, 0x28, 0x37, - 0x90, 0xde, 0x19, 0x1e, 0xec, 0x72, 0xf1, 0xd1, 0x40, 0x84, 0x92, 0x55, 0xa0, 0x40, 0x3c, 0x07, - 0xbb, 0x15, 0x6b, 0xcd, 0x5a, 0x2f, 0x71, 0x43, 0xb2, 0x15, 0x80, 0x66, 0xd7, 0x6f, 0x5d, 0xd4, - 0xa5, 0x13, 0xc8, 0xca, 0xf4, 0x9a, 0xb5, 0x5e, 0xe4, 0x09, 0x84, 0x55, 0x61, 0x86, 0xa8, 0x3d, - 0xaf, 0x5d, 0xc9, 0xd0, 0x6c, 0x44, 0xb3, 0x17, 0xa0, 0xf8, 0xd1, 0x40, 0x04, 0xc3, 0x23, 0xbf, - 0x2d, 0x2a, 0x39, 0x9a, 0x8c, 0x01, 0xdb, 0x83, 0xc5, 0x84, 0x1e, 0x61, 0xdf, 0xf7, 0x42, 0xc1, - 0x6e, 0x43, 0x8e, 0x56, 0x26, 0x35, 0x66, 0x6b, 0xf3, 0x1b, 0xda, 0x26, 0x1b, 0xc4, 0xca, 0xd5, - 0x24, 0x7b, 0x0d, 0x0a, 0x3d, 0x21, 0x03, 0xb7, 0x15, 0x92, 0x46, 0xb3, 0xb5, 0x5b, 0x69, 0x3e, - 0x14, 0x79, 0xa4, 0x18, 0xb8, 0xe1, 0xb4, 0x59, 0x62, 0xdf, 0x7a, 0xd2, 0xfe, 0x74, 0x1a, 0xe6, - 0xea, 0xc2, 0x09, 0x5a, 0x8f, 0x8c, 0x25, 0xde, 0x84, 0x6c, 0xc3, 0xe9, 0x84, 0x15, 0x6b, 0x2d, - 0xb3, 0x3e, 0x5b, 0x5b, 0x8b, 0xe4, 0xa6, 0xb8, 0x36, 0x90, 0x65, 0xcf, 0x93, 0xc1, 0x70, 0x27, - 0xfb, 0xc9, 0x17, 0xab, 0x53, 0x9c, 0xbe, 0x61, 0xb7, 0x61, 0xee, 0xc8, 0xf5, 0x76, 0x07, 0x81, - 0x23, 0x5d, 0xdf, 0x3b, 0x52, 0xca, 0xcd, 0xf1, 0x34, 0x48, 0x5c, 0xce, 0xe3, 0x04, 0x57, 0x46, - 0x73, 0x25, 0x41, 0x76, 0x13, 0x72, 0x87, 0x6e, 0xcf, 0x95, 0x95, 0x2c, 0xcd, 0x2a, 0x02, 0xd1, - 0x90, 0x1c, 0x91, 0x53, 0x28, 0x11, 0xac, 0x0c, 0x19, 0xe1, 0xb5, 0x2b, 0x79, 0xc2, 0x70, 0x88, - 0x7c, 0xef, 0xa3, 0xa1, 0x2b, 0x33, 0x64, 0x75, 0x45, 0xb0, 0x75, 0x58, 0xa8, 0xf7, 0x1d, 0x2f, - 0x3c, 0x15, 0x01, 0xfe, 0xd6, 0x85, 0xac, 0x14, 0xe9, 0x9b, 0x51, 0xb8, 0xfa, 0x3a, 0x14, 0xa3, - 0x2d, 0xa2, 0xf8, 0x0b, 0x31, 0x24, 0x8f, 0x14, 0x39, 0x0e, 0x51, 0xfc, 0xa5, 0xd3, 0x1d, 0x08, - 0x1d, 0x0f, 0x8a, 0x78, 0x73, 0xfa, 0x0d, 0xcb, 0xfe, 0x4b, 0x06, 0x98, 0x32, 0xd5, 0x0e, 0x46, - 0x81, 0xb1, 0xea, 0x5d, 0x28, 0x86, 0xc6, 0x80, 0xda, 0xb5, 0x4b, 0x93, 0x4d, 0xcb, 0x63, 0x46, - 0x8c, 0x4a, 0x8a, 0xa5, 0x83, 0x5d, 0xbd, 0x90, 0x21, 0x31, 0xb2, 0x68, 0xeb, 0xa7, 0x4e, 0x47, - 0x68, 0xfb, 0xc5, 0x00, 0x5a, 0xb8, 0xef, 0x74, 0x44, 0xd8, 0xf0, 0x95, 0x68, 0x6d, 0xc3, 0x34, - 0x88, 0x91, 0x2b, 0xbc, 0x96, 0xdf, 0x76, 0xbd, 0x8e, 0x0e, 0xce, 0x88, 0x46, 0x09, 0xae, 0xd7, - 0x16, 0x8f, 0x51, 0x5c, 0xdd, 0xfd, 0xa9, 0xd0, 0xb6, 0x4d, 0x83, 0xcc, 0x86, 0x92, 0xf4, 0xa5, - 0xd3, 0xe5, 0xa2, 0xe5, 0x07, 0xed, 0xb0, 0x52, 0x20, 0xa6, 0x14, 0x86, 0x3c, 0x6d, 0x47, 0x3a, - 0x7b, 0x66, 0x25, 0xe5, 0x90, 0x14, 0x86, 0xfb, 0xbc, 0x14, 0x41, 0xe8, 0xfa, 0x1e, 0xf9, 0xa3, - 0xc8, 0x0d, 0xc9, 0x18, 0x64, 0x43, 0x5c, 0x1e, 0xd6, 0xac, 0xf5, 0x2c, 0xa7, 0x31, 0x9e, 0xc8, - 0x87, 0xbe, 0x2f, 0x45, 0x40, 0x8a, 0xcd, 0xd2, 0x9a, 0x09, 0x84, 0xed, 0x42, 0xb9, 0x2d, 0xda, - 0x6e, 0xcb, 0x91, 0xa2, 0xfd, 0xc0, 0xef, 0x0e, 0x7a, 0x5e, 0x58, 0x29, 0x51, 0x34, 0x57, 0x22, - 0x93, 0xef, 0xa6, 0x19, 0xf8, 0xd8, 0x17, 0xf6, 0x9f, 0x2d, 0x58, 0x18, 0xe1, 0x62, 0x77, 0x21, - 0x17, 0xb6, 0xfc, 0xbe, 0xb2, 0xf8, 0x7c, 0x6d, 0xe5, 0x2a, 0x71, 0x1b, 0x75, 0xe4, 0xe2, 0x8a, - 0x19, 0xf7, 0xe0, 0x39, 0x3d, 0x13, 0x2b, 0x34, 0x66, 0x77, 0x20, 0x2b, 0x87, 0x7d, 0x75, 0xca, - 0xe7, 0x6b, 0x2f, 0x5e, 0x29, 0xa8, 0x31, 0xec, 0x0b, 0x4e, 0xac, 0xf6, 0x2a, 0xe4, 0x48, 0x2c, - 0x9b, 0x81, 0x6c, 0xfd, 0x74, 0xfb, 0xb8, 0x3c, 0xc5, 0x4a, 0x30, 0xc3, 0xf7, 0xea, 0x27, 0x1f, - 0xf0, 0x07, 0x7b, 0x65, 0xcb, 0x66, 0x90, 0x45, 0x76, 0x06, 0x90, 0xaf, 0x37, 0xf8, 0xc1, 0xf1, - 0x7e, 0x79, 0xca, 0x7e, 0x0c, 0xf3, 0x26, 0xba, 0x74, 0x82, 0xb9, 0x0b, 0x79, 0xca, 0x21, 0xe6, - 0x84, 0xbf, 0x90, 0xce, 0x1c, 0x8a, 0xfb, 0x48, 0x48, 0x07, 0x3d, 0xc4, 0x35, 0x2f, 0xdb, 0x1a, - 0x4d, 0x38, 0xa3, 0xd1, 0x3b, 0x96, 0x6d, 0x3e, 0x9e, 0x86, 0x1b, 0x13, 0x24, 0x8e, 0x66, 0xda, - 0x62, 0x9c, 0x69, 0xd7, 0x61, 0x21, 0xf0, 0x7d, 0x59, 0x17, 0xc1, 0xa5, 0xdb, 0x12, 0xc7, 0xb1, - 0xc9, 0x46, 0x61, 0x8c, 0x4e, 0x84, 0x48, 0x3c, 0xf1, 0xa9, 0xc4, 0x9b, 0x06, 0xd9, 0x2b, 0xb0, - 0x48, 0x47, 0xa2, 0xe1, 0xf6, 0xc4, 0x07, 0x9e, 0xfb, 0xf8, 0xd8, 0xf1, 0x7c, 0x3a, 0x09, 0x59, - 0x3e, 0x3e, 0x81, 0x51, 0xd5, 0x8e, 0x53, 0x92, 0x4a, 0x2f, 0x09, 0x84, 0xbd, 0x0c, 0x85, 0x50, - 0xe7, 0x8c, 0x3c, 0x59, 0xa0, 0x1c, 0x5b, 0x40, 0xe1, 0xdc, 0x30, 0xb0, 0x57, 0x60, 0x46, 0x0f, - 0xf1, 0x4c, 0x64, 0x26, 0x32, 0x47, 0x1c, 0xf6, 0xcf, 0x2d, 0x28, 0x68, 0x94, 0xbd, 0x04, 0x39, - 0xc4, 0x8d, 0x73, 0xe6, 0x52, 0x9f, 0x71, 0x35, 0x87, 0x26, 0xec, 0x39, 0xb2, 0xf5, 0x48, 0xb4, - 0x75, 0x82, 0x35, 0x24, 0x7b, 0x0b, 0xc0, 0x91, 0x32, 0x70, 0x9b, 0x03, 0x29, 0x30, 0xaf, 0xa2, - 0x8c, 0xe5, 0x48, 0x86, 0xae, 0xa5, 0x97, 0x77, 0x36, 0xde, 0x13, 0xc3, 0x33, 0x4c, 0x59, 0x3c, - 0xc1, 0x8e, 0x11, 0x9f, 0xc5, 0x65, 0xd8, 0x12, 0xe4, 0x71, 0xa1, 0xc8, 0x43, 0x9a, 0x9a, 0x18, - 0xc8, 0x13, 0x8d, 0x9c, 0xb9, 0xca, 0xc8, 0xb7, 0x61, 0xce, 0x98, 0x14, 0xe9, 0x50, 0xbb, 0x23, - 0x0d, 0x8e, 0xec, 0x22, 0xf7, 0x6c, 0xbb, 0xf8, 0xa7, 0x65, 0x2a, 0x9a, 0x0e, 0x49, 0x8c, 0x2b, - 0xd7, 0x0b, 0xfb, 0xa2, 0x25, 0x45, 0xbb, 0x61, 0x42, 0x9f, 0xb2, 0xfe, 0x08, 0xcc, 0xbe, 0x09, - 0xf3, 0x11, 0xb4, 0x33, 0xc4, 0xc5, 0xa7, 0x49, 0xbf, 0x11, 0x94, 0xad, 0xc1, 0x2c, 0xe5, 0x38, - 0x4a, 0xf1, 0xa6, 0x7e, 0x25, 0x21, 0xdc, 0x68, 0xcb, 0xef, 0xf5, 0xbb, 0x42, 0x8a, 0xf6, 0xbb, - 0x7e, 0x33, 0x34, 0x19, 0x38, 0x05, 0x62, 0x16, 0xa7, 0x8f, 0x88, 0x43, 0x85, 0x5c, 0x0c, 0xa0, - 0xde, 0xb1, 0x48, 0xa5, 0x4e, 0x9e, 0xd4, 0x19, 0x85, 0xed, 0xf7, 0x61, 0x51, 0x6d, 0x19, 0x6b, - 0x96, 0x29, 0x39, 0x37, 0x4d, 0xb2, 0x52, 0x4e, 0xd4, 0xc9, 0x28, 0x2a, 0xa0, 0x99, 0x09, 0x05, - 0x34, 0x1b, 0x15, 0x50, 0xfb, 0xd3, 0x0c, 0x2c, 0xc5, 0x32, 0x53, 0xb5, 0xec, 0x8d, 0xf1, 0x5a, - 0x56, 0x1d, 0xc9, 0x06, 0x09, 0x3d, 0xbe, 0xae, 0x67, 0x5f, 0x8d, 0x7a, 0xf6, 0x79, 0x06, 0x96, - 0x23, 0xe7, 0xd0, 0xb1, 0x49, 0x7b, 0xf5, 0x7b, 0xe3, 0x5e, 0x5d, 0x1d, 0xf7, 0xaa, 0xfa, 0xf0, - 0x6b, 0xd7, 0x7e, 0xa5, 0x5c, 0xbb, 0x65, 0x5a, 0x4e, 0x75, 0xec, 0x74, 0xa1, 0xaf, 0xc2, 0x8c, - 0x74, 0x3a, 0x58, 0x09, 0x55, 0x35, 0x29, 0xf2, 0x88, 0xb6, 0xdf, 0x85, 0x9b, 0xf1, 0x17, 0x67, - 0xb5, 0xe8, 0x9b, 0x1a, 0xe4, 0x29, 0x4d, 0x98, 0xfa, 0x33, 0xe9, 0x5c, 0x9f, 0xd5, 0x54, 0x77, - 0xa3, 0x39, 0xed, 0xb7, 0x92, 0xc9, 0x47, 0x4f, 0x46, 0xa5, 0xc2, 0x4a, 0x94, 0x0a, 0x06, 0x59, - 0x89, 0x37, 0x8b, 0x69, 0x52, 0x86, 0xc6, 0x76, 0x3f, 0x91, 0x65, 0x52, 0xb1, 0x45, 0x7d, 0x82, - 0x52, 0x37, 0xea, 0x13, 0x14, 0x89, 0x29, 0x8c, 0x2e, 0x51, 0xa6, 0xf9, 0x26, 0x22, 0x4e, 0x6c, - 0xd9, 0x09, 0x89, 0x2d, 0x17, 0x27, 0xb6, 0xd7, 0xe1, 0xf9, 0xb1, 0x15, 0xf5, 0xee, 0x31, 0x1d, - 0x1b, 0x50, 0x9b, 0x2c, 0x06, 0xec, 0xbb, 0x30, 0x63, 0x3e, 0xa1, 0xad, 0x0c, 0xa3, 0xd4, 0x4a, - 0xe3, 0xc9, 0x77, 0x02, 0xfb, 0x10, 0x6e, 0x8d, 0x2c, 0x97, 0x30, 0xf7, 0xe6, 0xe8, 0x82, 0xb3, - 0xb5, 0xc5, 0xb8, 0x1d, 0xd3, 0x33, 0x49, 0x1d, 0x76, 0x20, 0x47, 0xa5, 0x8a, 0xdd, 0x87, 0x42, - 0x93, 0x6a, 0xbe, 0xf9, 0x2e, 0x3e, 0xab, 0xea, 0xae, 0x7b, 0x79, 0x67, 0x83, 0x8b, 0xd0, 0x1f, - 0x04, 0x2d, 0x41, 0x77, 0x1b, 0x6e, 0xf8, 0xed, 0x63, 0x28, 0x9d, 0x0e, 0xc2, 0xb8, 0x21, 0x7c, - 0x1b, 0xe6, 0x44, 0x10, 0xf8, 0x41, 0xb8, 0x33, 0x6c, 0xe8, 0x9b, 0x67, 0x66, 0x7d, 0x3e, 0x11, - 0x80, 0xc8, 0xbd, 0x87, 0x1c, 0x5c, 0x38, 0xa1, 0xef, 0xf1, 0x34, 0xbb, 0xfd, 0x7b, 0x0b, 0xca, - 0xc8, 0x42, 0xa5, 0xc8, 0x78, 0xef, 0xd5, 0xa8, 0xcb, 0x44, 0x6f, 0x97, 0x76, 0x9e, 0xc3, 0x5b, - 0xe2, 0xdf, 0xbe, 0x58, 0x9d, 0x3b, 0x0d, 0x84, 0xd3, 0xed, 0xfa, 0x2d, 0xc5, 0x6d, 0xda, 0xcb, - 0x6f, 0x41, 0xc6, 0x6d, 0xab, 0x86, 0xe5, 0x4a, 0x5e, 0xe4, 0x60, 0xf7, 0x00, 0x54, 0xce, 0xd9, - 0x75, 0xa4, 0x53, 0xc9, 0x5e, 0xc7, 0x9f, 0x60, 0xb4, 0x8f, 0x94, 0x8a, 0xca, 0x12, 0x5a, 0xc5, - 0xff, 0xc2, 0x84, 0xb7, 0x01, 0xf4, 0x4d, 0x1a, 0xbb, 0x81, 0xa5, 0x54, 0x47, 0x5d, 0x32, 0x9b, - 0xb2, 0xdf, 0x86, 0xe2, 0xa1, 0xeb, 0x5d, 0xd4, 0xbb, 0x6e, 0x0b, 0x1b, 0xfe, 0x5c, 0xd7, 0xf5, - 0x2e, 0xcc, 0x5a, 0xcb, 0xe3, 0x6b, 0xe1, 0x1a, 0x1b, 0xf8, 0x01, 0x57, 0x9c, 0xf6, 0xcf, 0x2c, - 0x60, 0x08, 0x9a, 0xd6, 0x3a, 0xae, 0xeb, 0x2a, 0xfc, 0xad, 0x64, 0xf8, 0x57, 0xa0, 0xd0, 0x09, - 0xfc, 0x41, 0x7f, 0xc7, 0x1c, 0x0b, 0x43, 0x22, 0x7f, 0x97, 0x2e, 0xd2, 0xaa, 0x2b, 0x53, 0xc4, - 0xbf, 0x7d, 0x5c, 0x7e, 0x69, 0xc1, 0xad, 0x84, 0x12, 0xf5, 0x41, 0xaf, 0xe7, 0x04, 0xc3, 0xff, - 0x8f, 0x2e, 0x7f, 0xb0, 0xe0, 0x46, 0xca, 0x20, 0xf1, 0xb9, 0x15, 0xa1, 0x74, 0x7b, 0x98, 0x13, - 0x49, 0x93, 0x19, 0x1e, 0x03, 0x54, 0x7f, 0xfa, 0x8e, 0xf7, 0xc0, 0x1f, 0x78, 0x52, 0xf7, 0x73, - 0x31, 0x80, 0x2d, 0x1f, 0x85, 0x73, 0x3d, 0x62, 0x51, 0xaa, 0x8d, 0xa0, 0x6c, 0x23, 0xbe, 0x00, - 0x65, 0xc9, 0x83, 0x37, 0x53, 0xad, 0xf9, 0xd8, 0xf5, 0xe7, 0xbb, 0x50, 0xe2, 0xce, 0x4f, 0xde, - 0x71, 0x43, 0xe9, 0x77, 0x02, 0xa7, 0x87, 0x41, 0xd2, 0x1c, 0xb4, 0x2e, 0x84, 0x24, 0x05, 0xb3, - 0x5c, 0x53, 0xb8, 0xf7, 0x56, 0x42, 0x33, 0x45, 0xd8, 0xef, 0xc2, 0x8c, 0x69, 0x6e, 0x27, 0xbc, - 0x3e, 0xbc, 0x92, 0xcc, 0x34, 0xc9, 0xab, 0x18, 0x05, 0xe5, 0xfb, 0x87, 0x75, 0xe9, 0x48, 0xb7, - 0x65, 0x32, 0xd0, 0x6f, 0x2c, 0x98, 0x4d, 0xa8, 0xc8, 0x76, 0x60, 0xb1, 0xeb, 0x48, 0xe1, 0xb5, - 0x86, 0xe7, 0x8f, 0x8c, 0x7a, 0x3a, 0x2a, 0x9f, 0x8b, 0x24, 0x25, 0x75, 0xe7, 0x65, 0xcd, 0x1f, - 0xef, 0xe6, 0xdb, 0x90, 0x0f, 0x45, 0xe0, 0xea, 0xe3, 0x9d, 0xcc, 0x5a, 0x51, 0x4f, 0xae, 0x19, - 0x70, 0xe3, 0x2a, 0x5f, 0x68, 0xc3, 0x6a, 0xca, 0xfe, 0x6b, 0x3a, 0xba, 0x75, 0x60, 0xa5, 0xbd, - 0x65, 0x3d, 0xdd, 0x5b, 0xd3, 0x13, 0xbd, 0x15, 0xeb, 0x97, 0x79, 0x9a, 0x7e, 0x65, 0xc8, 0xf4, - 0xef, 0xdf, 0xd7, 0x17, 0x11, 0x1c, 0x2a, 0xe4, 0x1e, 0x05, 0x1e, 0x21, 0xf7, 0x14, 0xb2, 0xa5, - 0xbb, 0x6f, 0x1c, 0x12, 0x72, 0x6f, 0x8b, 0xba, 0x08, 0x44, 0xee, 0x6d, 0xd9, 0x1f, 0x42, 0x75, - 0xd2, 0x39, 0xd1, 0x21, 0x7a, 0x1f, 0x8a, 0x21, 0x41, 0xae, 0x18, 0x4f, 0x01, 0x13, 0xbe, 0x8b, - 0xb9, 0xed, 0xdf, 0x5a, 0x30, 0x97, 0x72, 0x6c, 0xaa, 0xfa, 0xe4, 0x74, 0xf5, 0x29, 0x81, 0xe5, - 0x91, 0x31, 0x32, 0xdc, 0xf2, 0x90, 0x7a, 0x48, 0xf6, 0xb6, 0xb8, 0xf5, 0x10, 0x29, 0x75, 0x01, - 0x29, 0x72, 0x2b, 0x44, 0xaa, 0x49, 0x9b, 0x9b, 0xe1, 0x56, 0x13, 0xa9, 0xb6, 0xde, 0x98, 0xd5, - 0xa6, 0x9b, 0x9f, 0x74, 0xe4, 0x40, 0xf5, 0x47, 0x39, 0xae, 0x29, 0x5c, 0xf1, 0xc2, 0xf5, 0xda, - 0xd4, 0x11, 0xe5, 0x38, 0x8d, 0x6d, 0xa1, 0x1e, 0xd3, 0xb4, 0xe2, 0x98, 0x66, 0xb1, 0xdd, 0x09, - 0x44, 0x38, 0xe8, 0xca, 0x46, 0x5c, 0x1c, 0x13, 0x08, 0xb6, 0x17, 0x8a, 0xd2, 0x61, 0x53, 0x9d, - 0x78, 0x86, 0x88, 0x83, 0x6b, 0x4e, 0xcc, 0x82, 0x8b, 0x63, 0xb3, 0x18, 0x26, 0x5d, 0xa7, 0x29, - 0xba, 0x89, 0xfe, 0x20, 0x06, 0x50, 0x0f, 0x22, 0xce, 0x12, 0xf5, 0x38, 0x81, 0xb0, 0x4d, 0x98, - 0x96, 0x26, 0x34, 0x56, 0xaf, 0xd6, 0xe1, 0xd4, 0x77, 0x3d, 0xc9, 0xa7, 0x65, 0x88, 0x67, 0x68, - 0x69, 0xf2, 0x34, 0x39, 0xc3, 0xd5, 0x4a, 0xcc, 0x71, 0x1a, 0x63, 0x74, 0x5c, 0x3a, 0x5d, 0x5a, - 0xd8, 0xe2, 0x38, 0xc4, 0xbb, 0x9c, 0x78, 0x2c, 0x7a, 0xfd, 0xae, 0x13, 0x34, 0xf4, 0xeb, 0x47, - 0x86, 0xde, 0x99, 0x47, 0x61, 0xf6, 0x32, 0x94, 0x0d, 0x64, 0x5e, 0x43, 0x75, 0x70, 0x8e, 0xe1, - 0xf6, 0x9f, 0x2c, 0x58, 0xa4, 0x97, 0x4d, 0xee, 0x78, 0x1d, 0x71, 0x7d, 0x52, 0x8e, 0x92, 0xac, - 0x4e, 0x34, 0xa9, 0x24, 0xab, 0x8e, 0x26, 0xbd, 0x9c, 0x62, 0x1b, 0x2b, 0x45, 0x5f, 0xaf, 0x49, - 0x63, 0x4c, 0xe8, 0xe1, 0x23, 0x27, 0x68, 0x1f, 0xec, 0xea, 0x74, 0x6c, 0x48, 0xb4, 0x34, 0x0d, - 0xd5, 0x61, 0x54, 0x9d, 0x77, 0x02, 0x49, 0xbf, 0x80, 0x17, 0x46, 0x5f, 0xc0, 0x43, 0x60, 0x49, - 0xf5, 0xf5, 0x59, 0xf9, 0x4e, 0x74, 0x78, 0xd5, 0x41, 0xb9, 0x11, 0xe7, 0x37, 0xb7, 0x27, 0xea, - 0x34, 0x15, 0x1d, 0xdf, 0x67, 0x7f, 0x98, 0xda, 0x86, 0x7c, 0xdd, 0xc1, 0x5b, 0x38, 0xfb, 0x06, - 0x94, 0xd0, 0x5d, 0xa1, 0x74, 0x7a, 0xfd, 0xf3, 0x5e, 0xa8, 0x8f, 0xcf, 0x6c, 0x84, 0xa9, 0x57, - 0x68, 0x95, 0x6a, 0x2d, 0xf2, 0xa5, 0x4e, 0xa9, 0xbf, 0xb3, 0x00, 0x62, 0x5d, 0xd8, 0x7d, 0xc8, - 0x53, 0x70, 0x8d, 0x9f, 0xec, 0xf1, 0xb7, 0x0a, 0xfd, 0x5e, 0xae, 0x3f, 0x60, 0x9b, 0x50, 0x08, - 0x49, 0x19, 0x93, 0x49, 0x17, 0x62, 0xf5, 0x09, 0xd7, 0xfc, 0x86, 0x8b, 0xad, 0xc2, 0x6c, 0x3f, - 0xf0, 0x7b, 0xe7, 0x7a, 0x41, 0xf5, 0xf0, 0x05, 0x08, 0x1d, 0x12, 0xf2, 0xf2, 0x8f, 0x61, 0x61, - 0xa4, 0x61, 0x63, 0x25, 0x98, 0x39, 0x3e, 0x39, 0xdf, 0xe3, 0xfc, 0x84, 0x97, 0xa7, 0xd8, 0x0d, - 0x58, 0x38, 0xda, 0xfe, 0xe1, 0xf9, 0xe1, 0xc1, 0xd9, 0xde, 0x79, 0x83, 0x6f, 0x3f, 0xd8, 0xab, - 0x97, 0x2d, 0x04, 0x69, 0x7c, 0xde, 0x38, 0x39, 0x39, 0x3f, 0xdc, 0xe6, 0xfb, 0x7b, 0xe5, 0x69, - 0xb6, 0x08, 0x73, 0x1f, 0x1c, 0xbf, 0x77, 0x7c, 0xf2, 0xe1, 0xb1, 0xfe, 0x38, 0x53, 0xfb, 0x95, - 0x05, 0x79, 0x14, 0x2f, 0x02, 0xf6, 0x7d, 0x28, 0x46, 0x6d, 0x1f, 0xbb, 0x95, 0xea, 0x16, 0x93, - 0xad, 0x60, 0xf5, 0xb9, 0xd4, 0x94, 0xf1, 0xb2, 0x3d, 0xc5, 0xb6, 0x61, 0x36, 0x62, 0x3e, 0xab, - 0xfd, 0x27, 0x22, 0x6a, 0xff, 0xb0, 0xa0, 0xac, 0x1d, 0xbc, 0x2f, 0x3c, 0x11, 0x38, 0xd2, 0x8f, - 0x14, 0xa3, 0x9e, 0x6d, 0x44, 0x6a, 0xb2, 0x01, 0xbc, 0x5a, 0xb1, 0x03, 0x80, 0x7d, 0x21, 0x4d, - 0xbd, 0x5c, 0x9e, 0x9c, 0x20, 0x94, 0x8c, 0x17, 0xae, 0xc8, 0x1e, 0x46, 0xd4, 0x3e, 0x40, 0x1c, - 0xe1, 0x2c, 0xce, 0x77, 0x63, 0xa7, 0xb6, 0xba, 0x3c, 0x71, 0x2e, 0xda, 0xe9, 0xc7, 0x59, 0x28, - 0xe0, 0x84, 0x2b, 0x02, 0xf6, 0x0e, 0xcc, 0xfd, 0xc0, 0xf5, 0xda, 0xd1, 0x9f, 0x39, 0x6c, 0xc2, - 0xbf, 0x3f, 0x46, 0x6c, 0x75, 0xd2, 0x54, 0xc2, 0x05, 0x25, 0xf3, 0x3c, 0xdc, 0x12, 0x9e, 0x64, - 0x57, 0xfc, 0x27, 0x51, 0x7d, 0x7e, 0x0c, 0x8f, 0x44, 0xec, 0xc1, 0x6c, 0xe2, 0xff, 0x8e, 0xa4, - 0xb5, 0xc6, 0xfe, 0x05, 0xb9, 0x4e, 0xcc, 0x3e, 0x40, 0x7c, 0x8b, 0x64, 0xd7, 0xbc, 0x27, 0x55, - 0x97, 0x27, 0xce, 0x45, 0x82, 0xde, 0x33, 0x5b, 0x52, 0xd7, 0xd1, 0x6b, 0x45, 0xbd, 0x38, 0xf1, - 0x7a, 0x9b, 0x10, 0x76, 0x06, 0x0b, 0x23, 0xb7, 0x37, 0xf6, 0xb4, 0x47, 0x91, 0xea, 0xda, 0xd5, - 0x0c, 0x91, 0xdc, 0x1f, 0x25, 0xee, 0xcc, 0xe6, 0x56, 0xf8, 0x74, 0xc9, 0xf6, 0x55, 0x0c, 0x49, - 0x9d, 0x6b, 0x27, 0x50, 0xae, 0xcb, 0x40, 0x38, 0x3d, 0xd7, 0xeb, 0x98, 0x88, 0x79, 0x0b, 0xf2, - 0xfa, 0x25, 0xe5, 0x59, 0x3d, 0xbc, 0x65, 0xd5, 0xfe, 0x68, 0x41, 0xc1, 0x1c, 0x86, 0xf3, 0x89, - 0x4d, 0x9b, 0x7d, 0x5d, 0x2b, 0xa3, 0x17, 0x78, 0xe9, 0x5a, 0x9e, 0xff, 0xf9, 0x81, 0xd9, 0xa9, - 0x7c, 0xf2, 0xe5, 0x8a, 0xf5, 0xd9, 0x97, 0x2b, 0xd6, 0xdf, 0xbf, 0x5c, 0xb1, 0x7e, 0xfd, 0x64, - 0x65, 0xea, 0xb3, 0x27, 0x2b, 0x53, 0x9f, 0x3f, 0x59, 0x99, 0x6a, 0xe6, 0xe9, 0x7f, 0xe0, 0xd7, - 0xfe, 0x15, 0x00, 0x00, 0xff, 0xff, 0x93, 0xad, 0x1c, 0x84, 0x88, 0x1e, 0x00, 0x00, + // 2484 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x59, 0xcd, 0x6f, 0x5b, 0xc7, + 0x11, 0xd7, 0x13, 0xbf, 0xc4, 0x11, 0x25, 0x51, 0x6b, 0x47, 0xa1, 0xa9, 0x44, 0x52, 0x5f, 0x8c, + 0x56, 0x4d, 0x13, 0xc9, 0x66, 0x6c, 0x24, 0x4e, 0xda, 0x14, 0x92, 0xa5, 0x2a, 0x4a, 0xf4, 0x95, + 0x25, 0xad, 0x14, 0x45, 0x01, 0xe1, 0x91, 0x5c, 0xd3, 0x0f, 0x22, 0xdf, 0x63, 0xde, 0x5b, 0xaa, + 0x66, 0x8f, 0x05, 0x5a, 0xa0, 0x40, 0x0f, 0x3d, 0xb4, 0x87, 0x1c, 0x7b, 0x0a, 0x7a, 0xee, 0xb9, + 0xa7, 0x02, 0x45, 0x2e, 0x0d, 0x72, 0x0c, 0x7a, 0x08, 0x0a, 0xfb, 0xd0, 0x7f, 0xa3, 0x98, 0xd9, + 0xdd, 0xf7, 0x41, 0x52, 0x72, 0xdd, 0x16, 0x68, 0x0e, 0x39, 0x71, 0xe7, 0xb7, 0xf3, 0x66, 0x67, + 0x67, 0x66, 0x67, 0x66, 0x97, 0xf0, 0x62, 0xff, 0xbc, 0xb3, 0x29, 0x45, 0xaf, 0xef, 0xf7, 0x9b, + 0xea, 0x77, 0xa3, 0x1f, 0xf8, 0xd2, 0x67, 0x05, 0x0d, 0x56, 0x97, 0x5a, 0x7e, 0xaf, 0xe7, 0x7b, + 0x9b, 0x17, 0xb7, 0x37, 0xd5, 0x48, 0x31, 0x54, 0x5f, 0xef, 0xb8, 0xf2, 0xd1, 0xa0, 0xb9, 0xd1, + 0xf2, 0x7b, 0x9b, 0x1d, 0xbf, 0xe3, 0x6f, 0x12, 0xdc, 0x1c, 0x3c, 0x24, 0x8a, 0x08, 0x1a, 0x69, + 0xf6, 0xeb, 0x32, 0x70, 0x5a, 0x02, 0xa5, 0xd0, 0x40, 0xa1, 0xf6, 0xaf, 0x2c, 0x28, 0x37, 0x90, + 0xde, 0x1e, 0xee, 0xef, 0x70, 0xf1, 0xf1, 0x40, 0x84, 0x92, 0x55, 0xa0, 0x40, 0x3c, 0xfb, 0x3b, + 0x15, 0x6b, 0xcd, 0x5a, 0x2f, 0x71, 0x43, 0xb2, 0x15, 0x80, 0x66, 0xd7, 0x6f, 0x9d, 0xd7, 0xa5, + 0x13, 0xc8, 0xca, 0xf4, 0x9a, 0xb5, 0x5e, 0xe4, 0x09, 0x84, 0x55, 0x61, 0x86, 0xa8, 0x5d, 0xaf, + 0x5d, 0xc9, 0xd0, 0x6c, 0x44, 0xb3, 0x97, 0xa0, 0xf8, 0xf1, 0x40, 0x04, 0xc3, 0x43, 0xbf, 0x2d, + 0x2a, 0x39, 0x9a, 0x8c, 0x01, 0xdb, 0x83, 0xc5, 0x84, 0x1e, 0x61, 0xdf, 0xf7, 0x42, 0xc1, 0x6e, + 0x42, 0x8e, 0x56, 0x26, 0x35, 0x66, 0x6b, 0xf3, 0x1b, 0xda, 0x26, 0x1b, 0xc4, 0xca, 0xd5, 0x24, + 0x7b, 0x03, 0x0a, 0x3d, 0x21, 0x03, 0xb7, 0x15, 0x92, 0x46, 0xb3, 0xb5, 0x1b, 0x69, 0x3e, 0x14, + 0x79, 0xa8, 0x18, 0xb8, 0xe1, 0xb4, 0x59, 0x62, 0xdf, 0x7a, 0xd2, 0xfe, 0x7c, 0x1a, 0xe6, 0xea, + 0xc2, 0x09, 0x5a, 0x8f, 0x8c, 0x25, 0xde, 0x86, 0x6c, 0xc3, 0xe9, 0x84, 0x15, 0x6b, 0x2d, 0xb3, + 0x3e, 0x5b, 0x5b, 0x8b, 0xe4, 0xa6, 0xb8, 0x36, 0x90, 0x65, 0xd7, 0x93, 0xc1, 0x70, 0x3b, 0xfb, + 0xd9, 0x57, 0xab, 0x53, 0x9c, 0xbe, 0x61, 0x37, 0x61, 0xee, 0xd0, 0xf5, 0x76, 0x06, 0x81, 0x23, + 0x5d, 0xdf, 0x3b, 0x54, 0xca, 0xcd, 0xf1, 0x34, 0x48, 0x5c, 0xce, 0xe3, 0x04, 0x57, 0x46, 0x73, + 0x25, 0x41, 0x76, 0x1d, 0x72, 0x07, 0x6e, 0xcf, 0x95, 0x95, 0x2c, 0xcd, 0x2a, 0x02, 0xd1, 0x90, + 0x1c, 0x91, 0x53, 0x28, 0x11, 0xac, 0x0c, 0x19, 0xe1, 0xb5, 0x2b, 0x79, 0xc2, 0x70, 0x88, 0x7c, + 0x1f, 0xa2, 0xa1, 0x2b, 0x33, 0x64, 0x75, 0x45, 0xb0, 0x75, 0x58, 0xa8, 0xf7, 0x1d, 0x2f, 0x3c, + 0x11, 0x01, 0xfe, 0xd6, 0x85, 0xac, 0x14, 0xe9, 0x9b, 0x51, 0xb8, 0xfa, 0x26, 0x14, 0xa3, 0x2d, + 0xa2, 0xf8, 0x73, 0x31, 0x24, 0x8f, 0x14, 0x39, 0x0e, 0x51, 0xfc, 0x85, 0xd3, 0x1d, 0x08, 0x1d, + 0x0f, 0x8a, 0x78, 0x7b, 0xfa, 0x2d, 0xcb, 0xfe, 0x6b, 0x06, 0x98, 0x32, 0xd5, 0x36, 0x46, 0x81, + 0xb1, 0xea, 0x1d, 0x28, 0x86, 0xc6, 0x80, 0xda, 0xb5, 0x4b, 0x93, 0x4d, 0xcb, 0x63, 0x46, 0x8c, + 0x4a, 0x8a, 0xa5, 0xfd, 0x1d, 0xbd, 0x90, 0x21, 0x31, 0xb2, 0x68, 0xeb, 0x27, 0x4e, 0x47, 0x68, + 0xfb, 0xc5, 0x00, 0x5a, 0xb8, 0xef, 0x74, 0x44, 0xd8, 0xf0, 0x95, 0x68, 0x6d, 0xc3, 0x34, 0x88, + 0x91, 0x2b, 0xbc, 0x96, 0xdf, 0x76, 0xbd, 0x8e, 0x0e, 0xce, 0x88, 0x46, 0x09, 0xae, 0xd7, 0x16, + 0x8f, 0x51, 0x5c, 0xdd, 0xfd, 0xb9, 0xd0, 0xb6, 0x4d, 0x83, 0xcc, 0x86, 0x92, 0xf4, 0xa5, 0xd3, + 0xe5, 0xa2, 0xe5, 0x07, 0xed, 0xb0, 0x52, 0x20, 0xa6, 0x14, 0x86, 0x3c, 0x6d, 0x47, 0x3a, 0xbb, + 0x66, 0x25, 0xe5, 0x90, 0x14, 0x86, 0xfb, 0xbc, 0x10, 0x41, 0xe8, 0xfa, 0x1e, 0xf9, 0xa3, 0xc8, + 0x0d, 0xc9, 0x18, 0x64, 0x43, 0x5c, 0x1e, 0xd6, 0xac, 0xf5, 0x2c, 0xa7, 0x31, 0x9e, 0xc8, 0x87, + 0xbe, 0x2f, 0x45, 0x40, 0x8a, 0xcd, 0xd2, 0x9a, 0x09, 0x84, 0xed, 0x40, 0xb9, 0x2d, 0xda, 0x6e, + 0xcb, 0x91, 0xa2, 0x7d, 0xdf, 0xef, 0x0e, 0x7a, 0x5e, 0x58, 0x29, 0x51, 0x34, 0x57, 0x22, 0x93, + 0xef, 0xa4, 0x19, 0xf8, 0xd8, 0x17, 0xf6, 0x5f, 0x2c, 0x58, 0x18, 0xe1, 0x62, 0x77, 0x20, 0x17, + 0xb6, 0xfc, 0xbe, 0xb2, 0xf8, 0x7c, 0x6d, 0xe5, 0x32, 0x71, 0x1b, 0x75, 0xe4, 0xe2, 0x8a, 0x19, + 0xf7, 0xe0, 0x39, 0x3d, 0x13, 0x2b, 0x34, 0x66, 0xb7, 0x21, 0x2b, 0x87, 0x7d, 0x75, 0xca, 0xe7, + 0x6b, 0x2f, 0x5f, 0x2a, 0xa8, 0x31, 0xec, 0x0b, 0x4e, 0xac, 0xf6, 0x2a, 0xe4, 0x48, 0x2c, 0x9b, + 0x81, 0x6c, 0xfd, 0x64, 0xeb, 0xa8, 0x3c, 0xc5, 0x4a, 0x30, 0xc3, 0x77, 0xeb, 0xc7, 0x0f, 0xf8, + 0xfd, 0xdd, 0xb2, 0x65, 0x33, 0xc8, 0x22, 0x3b, 0x03, 0xc8, 0xd7, 0x1b, 0x7c, 0xff, 0x68, 0xaf, + 0x3c, 0x65, 0x3f, 0x86, 0x79, 0x13, 0x5d, 0x3a, 0xc1, 0xdc, 0x81, 0x3c, 0xe5, 0x10, 0x73, 0xc2, + 0x5f, 0x4a, 0x67, 0x0e, 0xc5, 0x7d, 0x28, 0xa4, 0x83, 0x1e, 0xe2, 0x9a, 0x97, 0xdd, 0x1a, 0x4d, + 0x38, 0xa3, 0xd1, 0x3b, 0x96, 0x6d, 0x3e, 0x9d, 0x86, 0x6b, 0x13, 0x24, 0x8e, 0x66, 0xda, 0x62, + 0x9c, 0x69, 0xd7, 0x61, 0x21, 0xf0, 0x7d, 0x59, 0x17, 0xc1, 0x85, 0xdb, 0x12, 0x47, 0xb1, 0xc9, + 0x46, 0x61, 0x8c, 0x4e, 0x84, 0x48, 0x3c, 0xf1, 0xa9, 0xc4, 0x9b, 0x06, 0xd9, 0x6b, 0xb0, 0x48, + 0x47, 0xa2, 0xe1, 0xf6, 0xc4, 0x03, 0xcf, 0x7d, 0x7c, 0xe4, 0x78, 0x3e, 0x9d, 0x84, 0x2c, 0x1f, + 0x9f, 0xc0, 0xa8, 0x6a, 0xc7, 0x29, 0x49, 0xa5, 0x97, 0x04, 0xc2, 0x5e, 0x85, 0x42, 0xa8, 0x73, + 0x46, 0x9e, 0x2c, 0x50, 0x8e, 0x2d, 0xa0, 0x70, 0x6e, 0x18, 0xd8, 0x6b, 0x30, 0xa3, 0x87, 0x78, + 0x26, 0x32, 0x13, 0x99, 0x23, 0x0e, 0xfb, 0x97, 0x16, 0x14, 0x34, 0xca, 0x5e, 0x81, 0x1c, 0xe2, + 0xc6, 0x39, 0x73, 0xa9, 0xcf, 0xb8, 0x9a, 0x43, 0x13, 0xf6, 0x1c, 0xd9, 0x7a, 0x24, 0xda, 0x3a, + 0xc1, 0x1a, 0x92, 0xbd, 0x03, 0xe0, 0x48, 0x19, 0xb8, 0xcd, 0x81, 0x14, 0x98, 0x57, 0x51, 0xc6, + 0x72, 0x24, 0x43, 0xd7, 0xd2, 0x8b, 0xdb, 0x1b, 0x1f, 0x88, 0xe1, 0x29, 0xa6, 0x2c, 0x9e, 0x60, + 0xc7, 0x88, 0xcf, 0xe2, 0x32, 0x6c, 0x09, 0xf2, 0xb8, 0x50, 0xe4, 0x21, 0x4d, 0x4d, 0x0c, 0xe4, + 0x89, 0x46, 0xce, 0x5c, 0x66, 0xe4, 0x9b, 0x30, 0x67, 0x4c, 0x8a, 0x74, 0xa8, 0xdd, 0x91, 0x06, + 0x47, 0x76, 0x91, 0x7b, 0xbe, 0x5d, 0x7c, 0x12, 0x55, 0x34, 0x1d, 0x92, 0x18, 0x57, 0xae, 0x17, + 0xf6, 0x45, 0x4b, 0x8a, 0x76, 0xc3, 0x84, 0x3e, 0x65, 0xfd, 0x11, 0x98, 0x7d, 0x1b, 0xe6, 0x23, + 0x68, 0x7b, 0x88, 0x8b, 0x4f, 0x93, 0x7e, 0x23, 0x28, 0x5b, 0x83, 0x59, 0xca, 0x71, 0x94, 0xe2, + 0x4d, 0xfd, 0x4a, 0x42, 0xb8, 0xd1, 0x96, 0xdf, 0xeb, 0x77, 0x85, 0x14, 0xed, 0xf7, 0xfd, 0x66, + 0x68, 0x32, 0x70, 0x0a, 0xc4, 0x2c, 0x4e, 0x1f, 0x11, 0x87, 0x0a, 0xb9, 0x18, 0x40, 0xbd, 0x63, + 0x91, 0x4a, 0x9d, 0x3c, 0xa9, 0x33, 0x0a, 0xa7, 0xf4, 0xa6, 0x4a, 0x46, 0x99, 0x38, 0xa9, 0x37, + 0xa1, 0xf6, 0x87, 0xb0, 0xa8, 0x4c, 0x83, 0xb5, 0xcd, 0x94, 0xa6, 0xeb, 0x26, 0xa9, 0x29, 0x67, + 0xeb, 0xa4, 0x15, 0x15, 0xda, 0xcc, 0x84, 0x42, 0x9b, 0x8d, 0x0a, 0xad, 0xfd, 0x79, 0x06, 0x96, + 0x62, 0x99, 0xa9, 0x9a, 0xf7, 0xd6, 0x78, 0xcd, 0xab, 0x8e, 0x64, 0x8d, 0x84, 0x1e, 0xdf, 0xd4, + 0xbd, 0xaf, 0x47, 0xdd, 0xfb, 0x32, 0x03, 0xcb, 0x91, 0x73, 0xe8, 0x78, 0xa5, 0xbd, 0xfa, 0x83, + 0x71, 0xaf, 0xae, 0x8e, 0x7b, 0x55, 0x7d, 0xf8, 0x8d, 0x6b, 0xbf, 0x56, 0xae, 0xbd, 0x65, 0x5a, + 0x53, 0x75, 0xec, 0x74, 0x43, 0x50, 0x85, 0x19, 0xe9, 0x74, 0xb0, 0x62, 0xaa, 0xaa, 0x53, 0xe4, + 0x11, 0x6d, 0xbf, 0x0f, 0xd7, 0xe3, 0x2f, 0x4e, 0x6b, 0xd1, 0x37, 0x35, 0xc8, 0x53, 0x9a, 0x30, + 0x75, 0x6a, 0xd2, 0xb9, 0x3e, 0xad, 0xa9, 0x2e, 0x48, 0x73, 0xda, 0xef, 0x24, 0x93, 0x8f, 0x9e, + 0x8c, 0x4a, 0x8a, 0x95, 0x28, 0x29, 0x0c, 0xb2, 0x12, 0x6f, 0x20, 0xd3, 0xa4, 0x0c, 0x8d, 0xed, + 0x7e, 0x22, 0xcb, 0xa4, 0x62, 0x8b, 0xfa, 0x09, 0xa5, 0x6e, 0xd4, 0x4f, 0x28, 0x12, 0x53, 0x18, + 0x5d, 0xb6, 0x4c, 0x93, 0x4e, 0x44, 0x9c, 0xd8, 0xb2, 0x13, 0x12, 0x5b, 0x2e, 0x4e, 0x6c, 0x6f, + 0xc2, 0x8b, 0x63, 0x2b, 0xea, 0xdd, 0x63, 0xda, 0x36, 0xa0, 0x36, 0x59, 0x0c, 0xd8, 0x77, 0x60, + 0xc6, 0x7c, 0x42, 0x5b, 0x19, 0x46, 0xa9, 0x95, 0xc6, 0x93, 0xef, 0x0e, 0xf6, 0x01, 0xdc, 0x18, + 0x59, 0x2e, 0x61, 0xee, 0xcd, 0xd1, 0x05, 0x67, 0x6b, 0x8b, 0x71, 0xdb, 0xa6, 0x67, 0x92, 0x3a, + 0x6c, 0x43, 0x8e, 0x4a, 0x1a, 0xbb, 0x07, 0x85, 0x26, 0xf5, 0x06, 0xe6, 0xbb, 0xf8, 0xac, 0xaa, + 0x3b, 0xf1, 0xc5, 0xed, 0x0d, 0x2e, 0x42, 0x7f, 0x10, 0xb4, 0x04, 0xd5, 0x08, 0x6e, 0xf8, 0xed, + 0x23, 0x28, 0x9d, 0x0c, 0xc2, 0xb8, 0x71, 0x7c, 0x17, 0xe6, 0x44, 0x10, 0xf8, 0x41, 0xb8, 0x3d, + 0x6c, 0xe8, 0x1b, 0x6a, 0x66, 0x7d, 0x3e, 0x11, 0x80, 0xc8, 0xbd, 0x8b, 0x1c, 0x5c, 0x38, 0xa1, + 0xef, 0xf1, 0x34, 0xbb, 0xfd, 0x07, 0x0b, 0xca, 0xc8, 0x42, 0x25, 0xcb, 0x78, 0xef, 0xf5, 0xa8, + 0x1b, 0x45, 0x6f, 0x97, 0xb6, 0x5f, 0xc0, 0xdb, 0xe4, 0xdf, 0xbf, 0x5a, 0x9d, 0x3b, 0x09, 0x84, + 0xd3, 0xed, 0xfa, 0x2d, 0xc5, 0x6d, 0xda, 0xd0, 0xef, 0x40, 0xc6, 0x6d, 0xab, 0xc6, 0xe6, 0x52, + 0x5e, 0xe4, 0x60, 0x77, 0x01, 0x54, 0xce, 0xd9, 0x71, 0xa4, 0x53, 0xc9, 0x5e, 0xc5, 0x9f, 0x60, + 0xb4, 0x0f, 0x95, 0x8a, 0xca, 0x12, 0x5a, 0xc5, 0xff, 0xc2, 0x84, 0x37, 0x01, 0xf4, 0x8d, 0x1b, + 0xab, 0xf4, 0x52, 0xaa, 0xf3, 0x2e, 0x99, 0x4d, 0xd9, 0xef, 0x42, 0xf1, 0xc0, 0xf5, 0xce, 0xeb, + 0x5d, 0xb7, 0x85, 0x17, 0x83, 0x5c, 0xd7, 0xf5, 0xce, 0xcd, 0x5a, 0xcb, 0xe3, 0x6b, 0xe1, 0x1a, + 0x1b, 0xf8, 0x01, 0x57, 0x9c, 0xf6, 0x2f, 0x2c, 0x60, 0x08, 0x9a, 0x16, 0x3c, 0xae, 0xeb, 0x2a, + 0xfc, 0xad, 0x64, 0xf8, 0x57, 0xa0, 0xd0, 0x09, 0xfc, 0x41, 0x7f, 0xdb, 0x1c, 0x0b, 0x43, 0x22, + 0x7f, 0x97, 0x2e, 0xdc, 0xaa, 0x7b, 0x53, 0xc4, 0xbf, 0x7d, 0x5c, 0x7e, 0x6d, 0xc1, 0x8d, 0x84, + 0x12, 0xf5, 0x41, 0xaf, 0xe7, 0x04, 0xc3, 0xff, 0x8f, 0x2e, 0x7f, 0xb4, 0xe0, 0x5a, 0xca, 0x20, + 0xf1, 0xb9, 0x15, 0xa1, 0x74, 0x7b, 0x98, 0x13, 0x49, 0x93, 0x19, 0x1e, 0x03, 0x54, 0x7f, 0xfa, + 0x8e, 0x77, 0xdf, 0x1f, 0x78, 0x52, 0xf7, 0x7d, 0x31, 0x80, 0x2d, 0x16, 0x85, 0x73, 0x3d, 0x62, + 0x51, 0xaa, 0x8d, 0xa0, 0x6c, 0x23, 0xbe, 0x28, 0x65, 0xc9, 0x83, 0xd7, 0x53, 0x2d, 0xfc, 0xd8, + 0x35, 0xe9, 0xfb, 0x50, 0xe2, 0xce, 0xcf, 0xde, 0x73, 0x43, 0xe9, 0x77, 0x02, 0xa7, 0x87, 0x41, + 0xd2, 0x1c, 0xb4, 0xce, 0x85, 0x24, 0x05, 0xb3, 0x5c, 0x53, 0xb8, 0xf7, 0x56, 0x42, 0x33, 0x45, + 0xd8, 0xef, 0xc3, 0x8c, 0x69, 0x82, 0x27, 0xbc, 0x52, 0xbc, 0x96, 0xcc, 0x34, 0xc9, 0x2b, 0x1b, + 0x05, 0xe5, 0x87, 0x07, 0x75, 0xe9, 0x48, 0xb7, 0x65, 0x32, 0xd0, 0xef, 0x2c, 0x98, 0x4d, 0xa8, + 0xc8, 0xb6, 0x61, 0xb1, 0xeb, 0x48, 0xe1, 0xb5, 0x86, 0x67, 0x8f, 0x8c, 0x7a, 0x3a, 0x2a, 0x5f, + 0x88, 0x24, 0x25, 0x75, 0xe7, 0x65, 0xcd, 0x1f, 0xef, 0xe6, 0xbb, 0x90, 0x0f, 0x45, 0xe0, 0xea, + 0xe3, 0x9d, 0xcc, 0x5a, 0x51, 0xef, 0xae, 0x19, 0x70, 0xe3, 0x2a, 0x5f, 0x68, 0xc3, 0x6a, 0xca, + 0xfe, 0x5b, 0x3a, 0xba, 0x75, 0x60, 0xa5, 0xbd, 0x65, 0x3d, 0xdb, 0x5b, 0xd3, 0x13, 0xbd, 0x15, + 0xeb, 0x97, 0x79, 0x96, 0x7e, 0x65, 0xc8, 0xf4, 0xef, 0xdd, 0xd3, 0x17, 0x16, 0x1c, 0x2a, 0xe4, + 0x2e, 0x05, 0x1e, 0x21, 0x77, 0x15, 0x72, 0x4b, 0x77, 0xe9, 0x38, 0x24, 0xe4, 0xee, 0x2d, 0xdd, + 0x8e, 0xe3, 0xd0, 0xfe, 0x08, 0xaa, 0x93, 0xce, 0x89, 0x0e, 0xd1, 0x7b, 0x50, 0x0c, 0x09, 0x72, + 0xc5, 0x78, 0x0a, 0x98, 0xf0, 0x5d, 0xcc, 0x6d, 0xff, 0xde, 0x82, 0xb9, 0x94, 0x63, 0x53, 0xd5, + 0x27, 0xa7, 0xab, 0x4f, 0x09, 0x2c, 0x8f, 0x8c, 0x91, 0xe1, 0x96, 0x87, 0xd4, 0x43, 0xb2, 0xb7, + 0xc5, 0xad, 0x87, 0x48, 0xa9, 0x8b, 0x4a, 0x91, 0x5b, 0x21, 0x52, 0x4d, 0xda, 0xdc, 0x0c, 0xb7, + 0x9a, 0x48, 0xb5, 0xf5, 0xc6, 0xac, 0x36, 0xdd, 0x10, 0xa5, 0x23, 0x07, 0xaa, 0x3f, 0xca, 0x71, + 0x4d, 0xe1, 0x8a, 0xe7, 0xae, 0xd7, 0xa6, 0x8e, 0x28, 0xc7, 0x69, 0x6c, 0x0b, 0xf5, 0xe8, 0xa6, + 0x15, 0xc7, 0x34, 0x8b, 0xed, 0x4e, 0x20, 0xc2, 0x41, 0x57, 0x36, 0xe2, 0xe2, 0x98, 0x40, 0xb0, + 0xbd, 0x50, 0x94, 0x0e, 0x9b, 0xea, 0xc4, 0x33, 0x44, 0x1c, 0x5c, 0x73, 0x62, 0x16, 0x5c, 0x1c, + 0x9b, 0xc5, 0x30, 0xe9, 0x3a, 0x4d, 0xd1, 0x4d, 0xf4, 0x07, 0x31, 0x80, 0x7a, 0x10, 0x71, 0x9a, + 0xa8, 0xc7, 0x09, 0x84, 0x6d, 0xc2, 0xb4, 0x34, 0xa1, 0xb1, 0x7a, 0xb9, 0x0e, 0x27, 0xbe, 0xeb, + 0x49, 0x3e, 0x2d, 0x43, 0x3c, 0x43, 0x4b, 0x93, 0xa7, 0xc9, 0x19, 0xae, 0x56, 0x62, 0x8e, 0xd3, + 0x18, 0xa3, 0xe3, 0xc2, 0xe9, 0xd2, 0xc2, 0x16, 0xc7, 0x21, 0xde, 0xf9, 0xc4, 0x63, 0xd1, 0xeb, + 0x77, 0x9d, 0xa0, 0xa1, 0x5f, 0x49, 0x32, 0xf4, 0x1e, 0x3d, 0x0a, 0xb3, 0x57, 0xa1, 0x6c, 0x20, + 0xf3, 0x6a, 0xaa, 0x83, 0x73, 0x0c, 0xb7, 0xff, 0x6c, 0xc1, 0x22, 0xbd, 0x80, 0x72, 0xc7, 0xeb, + 0x88, 0xab, 0x93, 0x72, 0x94, 0x64, 0x75, 0xa2, 0x49, 0x25, 0x59, 0x75, 0x34, 0xe9, 0x85, 0x15, + 0xdb, 0x58, 0x29, 0xfa, 0x7a, 0x4d, 0x1a, 0x63, 0x42, 0x0f, 0x1f, 0x39, 0x41, 0x7b, 0x7f, 0x47, + 0xa7, 0x63, 0x43, 0xa2, 0xa5, 0x69, 0xa8, 0x0e, 0xa3, 0xea, 0xbc, 0x13, 0x48, 0xfa, 0xa5, 0xbc, + 0x30, 0xfa, 0x52, 0x1e, 0x02, 0x4b, 0xaa, 0xaf, 0xcf, 0xca, 0xf7, 0xa2, 0xc3, 0xab, 0x0e, 0xca, + 0xb5, 0x38, 0xbf, 0xb9, 0x3d, 0x51, 0xa7, 0xa9, 0xe8, 0xf8, 0x3e, 0xff, 0x03, 0xd6, 0x16, 0xe4, + 0xeb, 0x0e, 0xde, 0xd6, 0xd9, 0xb7, 0xa0, 0x84, 0xee, 0x0a, 0xa5, 0xd3, 0xeb, 0x9f, 0xf5, 0x42, + 0x7d, 0x7c, 0x66, 0x23, 0x4c, 0xbd, 0x56, 0xab, 0x54, 0x6b, 0x91, 0x2f, 0x75, 0x4a, 0xfd, 0xc4, + 0x02, 0x88, 0x75, 0x61, 0xf7, 0x20, 0x4f, 0xc1, 0x35, 0x7e, 0xb2, 0xc7, 0xdf, 0x34, 0xf4, 0xbb, + 0xba, 0xfe, 0x80, 0x6d, 0x42, 0x21, 0x24, 0x65, 0x4c, 0x26, 0x5d, 0x88, 0xd5, 0x27, 0x5c, 0xf3, + 0x1b, 0x2e, 0xb6, 0x0a, 0xb3, 0xfd, 0xc0, 0xef, 0x9d, 0xe9, 0x05, 0xd5, 0x03, 0x19, 0x20, 0x74, + 0x40, 0xc8, 0xab, 0x3f, 0x85, 0x85, 0x91, 0x86, 0x8d, 0x95, 0x60, 0xe6, 0xe8, 0xf8, 0x6c, 0x97, + 0xf3, 0x63, 0x5e, 0x9e, 0x62, 0xd7, 0x60, 0xe1, 0x70, 0xeb, 0xc7, 0x67, 0x07, 0xfb, 0xa7, 0xbb, + 0x67, 0x0d, 0xbe, 0x75, 0x7f, 0xb7, 0x5e, 0xb6, 0x10, 0xa4, 0xf1, 0x59, 0xe3, 0xf8, 0xf8, 0xec, + 0x60, 0x8b, 0xef, 0xed, 0x96, 0xa7, 0xd9, 0x22, 0xcc, 0x3d, 0x38, 0xfa, 0xe0, 0xe8, 0xf8, 0xa3, + 0x23, 0xfd, 0x71, 0xa6, 0xf6, 0x1b, 0x0b, 0xf2, 0x28, 0x5e, 0x04, 0xec, 0x87, 0x50, 0x8c, 0xda, + 0x3e, 0x76, 0x23, 0xd5, 0x2d, 0x26, 0x5b, 0xc1, 0xea, 0x0b, 0xa9, 0x29, 0xe3, 0x65, 0x7b, 0x8a, + 0x6d, 0xc1, 0x6c, 0xc4, 0x7c, 0x5a, 0xfb, 0x4f, 0x44, 0xd4, 0xfe, 0x69, 0x41, 0x59, 0x3b, 0x78, + 0x4f, 0x78, 0x22, 0x70, 0xa4, 0x1f, 0x29, 0x46, 0x3d, 0xdb, 0x88, 0xd4, 0x64, 0x03, 0x78, 0xb9, + 0x62, 0xfb, 0x00, 0x7b, 0x42, 0x9a, 0x7a, 0xb9, 0x3c, 0x39, 0x41, 0x28, 0x19, 0x2f, 0x5d, 0x92, + 0x3d, 0x8c, 0xa8, 0x3d, 0x80, 0x38, 0xc2, 0x59, 0x9c, 0xef, 0xc6, 0x4e, 0x6d, 0x75, 0x79, 0xe2, + 0x5c, 0xb4, 0xd3, 0x4f, 0xb3, 0x50, 0xc0, 0x09, 0x57, 0x04, 0xec, 0x3d, 0x98, 0xfb, 0x91, 0xeb, + 0xb5, 0xa3, 0x3f, 0x7d, 0xd8, 0x84, 0x7f, 0x89, 0x8c, 0xd8, 0xea, 0xa4, 0xa9, 0x84, 0x0b, 0x4a, + 0xe6, 0x19, 0xb9, 0x25, 0x3c, 0xc9, 0x2e, 0xf9, 0xef, 0xa2, 0xfa, 0xe2, 0x18, 0x1e, 0x89, 0xd8, + 0x85, 0xd9, 0xc4, 0xff, 0x22, 0x49, 0x6b, 0x8d, 0xfd, 0x5b, 0x72, 0x95, 0x98, 0x3d, 0x80, 0xf8, + 0x16, 0xc9, 0xae, 0x78, 0x4f, 0xaa, 0x2e, 0x4f, 0x9c, 0x8b, 0x04, 0x7d, 0x60, 0xb6, 0xa4, 0xae, + 0xa3, 0x57, 0x8a, 0x7a, 0x79, 0xe2, 0xf5, 0x36, 0x21, 0xec, 0x14, 0x16, 0x46, 0x6e, 0x6f, 0xec, + 0x59, 0x8f, 0x22, 0xd5, 0xb5, 0xcb, 0x19, 0x22, 0xb9, 0x3f, 0x49, 0xdc, 0x99, 0xcd, 0xad, 0xf0, + 0xd9, 0x92, 0xed, 0xcb, 0x18, 0x92, 0x3a, 0xd7, 0x8e, 0xa1, 0x5c, 0x97, 0x81, 0x70, 0x7a, 0xae, + 0xd7, 0x31, 0x11, 0xf3, 0x0e, 0xe4, 0xf5, 0x4b, 0xca, 0xf3, 0x7a, 0xf8, 0x96, 0x55, 0xfb, 0x93, + 0x05, 0x05, 0x73, 0x18, 0xce, 0x26, 0x36, 0x6d, 0xf6, 0x55, 0xad, 0x8c, 0x5e, 0xe0, 0x95, 0x2b, + 0x79, 0xfe, 0xe7, 0x07, 0x66, 0xbb, 0xf2, 0xd9, 0x93, 0x15, 0xeb, 0x8b, 0x27, 0x2b, 0xd6, 0x3f, + 0x9e, 0xac, 0x58, 0xbf, 0x7d, 0xba, 0x32, 0xf5, 0xc5, 0xd3, 0x95, 0xa9, 0x2f, 0x9f, 0xae, 0x4c, + 0x35, 0xf3, 0xf4, 0x7f, 0xf1, 0x1b, 0xff, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x4e, 0xb9, 0xe0, 0x30, + 0xb0, 0x1e, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -4520,6 +4529,11 @@ func (m *SearchMetrics) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.InspectedSpans != 0 { + i = encodeVarintTempo(dAtA, i, uint64(m.InspectedSpans)) + i-- + dAtA[i] = 0x38 + } if m.TotalBlockBytes != 0 { i = encodeVarintTempo(dAtA, i, uint64(m.TotalBlockBytes)) i-- @@ -6434,6 +6448,9 @@ func (m *SearchMetrics) Size() (n int) { if m.TotalBlockBytes != 0 { n += 1 + sovTempo(uint64(m.TotalBlockBytes)) } + if m.InspectedSpans != 0 { + n += 1 + sovTempo(uint64(m.InspectedSpans)) + } return n } @@ -9156,6 +9173,25 @@ func (m *SearchMetrics) Unmarshal(dAtA []byte) error { break } } + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field InspectedSpans", wireType) + } + m.InspectedSpans = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTempo + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.InspectedSpans |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipTempo(dAtA[iNdEx:]) diff --git a/pkg/tempopb/tempo.proto b/pkg/tempopb/tempo.proto index 9ad9ca8e822..10c2bc4f459 100644 --- a/pkg/tempopb/tempo.proto +++ b/pkg/tempopb/tempo.proto @@ -136,6 +136,7 @@ message SearchMetrics { uint32 completedJobs = 4; uint32 totalJobs = 5; uint64 totalBlockBytes = 6; + uint64 inspectedSpans = 7; } message SearchTagsRequest { diff --git a/pkg/traceql/combine.go b/pkg/traceql/combine.go index 7b2effb5b55..56867d23d08 100644 --- a/pkg/traceql/combine.go +++ b/pkg/traceql/combine.go @@ -154,6 +154,8 @@ func (q *QueryRangeCombiner) Combine(resp *tempopb.QueryRangeResponse) { q.metrics.TotalBlocks += resp.Metrics.TotalBlocks q.metrics.TotalBlockBytes += resp.Metrics.TotalBlockBytes q.metrics.InspectedBytes += resp.Metrics.InspectedBytes + q.metrics.InspectedTraces += resp.Metrics.InspectedTraces + q.metrics.InspectedSpans += resp.Metrics.InspectedSpans } } diff --git a/pkg/traceql/engine_metrics.go b/pkg/traceql/engine_metrics.go index 79699404ba7..aa094c78a81 100644 --- a/pkg/traceql/engine_metrics.go +++ b/pkg/traceql/engine_metrics.go @@ -365,32 +365,32 @@ func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, dedupe dedupeSpans: dedupeSpans, } - // TraceID (not always required) - if req.ShardCount > 1 || dedupeSpans { + // TraceID (optional) + if req.ShardCount > 1 { // For sharding it must be in the first pass so that we only evalulate our traces. storageReq.ShardID = req.ShardID storageReq.ShardCount = req.ShardCount - traceID := NewIntrinsic(IntrinsicTraceID) - if !storageReq.HasAttribute(traceID) { - storageReq.Conditions = append(storageReq.Conditions, Condition{Attribute: traceID}) + if !storageReq.HasAttribute(IntrinsicTraceIDAttribute) { + storageReq.Conditions = append(storageReq.Conditions, Condition{Attribute: IntrinsicTraceIDAttribute}) } } - // Span start time (always required) - startTime := NewIntrinsic(IntrinsicSpanStartTime) - if !storageReq.HasAttribute(startTime) { - if storageReq.AllConditions { - // The most efficient case. We can add it to the primary pass - // without affecting correctness. And this lets us avoid the - // entire second pass. - storageReq.Conditions = append(storageReq.Conditions, Condition{Attribute: startTime}) - } else { - // Complex query with a second pass. In this case it is better to - // add it to the second pass so that it's only returned for the matches. - storageReq.SecondPassConditions = append(storageReq.SecondPassConditions, Condition{Attribute: startTime}) + if dedupeSpans { + // For dedupe we only need the trace ID on matching spans, so it can go in the second pass. + // This is a no-op if we are already sharding and it's in the first pass. + // Finally, this is often optimized back to the first pass when it lets us avoid a second pass altogether. + if !storageReq.HasAttribute(IntrinsicTraceIDAttribute) { + storageReq.SecondPassConditions = append(storageReq.SecondPassConditions, Condition{Attribute: IntrinsicTraceIDAttribute}) } } + // Span start time (always required) + if !storageReq.HasAttribute(IntrinsicSpanStartTimeAttribute) { + // Technically we only need the start time of matching spans, so we add it to the second pass. + // However this is often optimized back to the first pass when it lets us avoid a second pass altogether. + storageReq.SecondPassConditions = append(storageReq.SecondPassConditions, Condition{Attribute: IntrinsicSpanStartTimeAttribute}) + } + // Timestamp filtering // (1) Include any overlapping trace // It can be faster to skip the trace-level timestamp check @@ -399,6 +399,11 @@ func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, dedupe // (2) Only include spans that started in this time frame. // This is checked outside the fetch layer in the evaluator. Timestamp // is only checked on the spans that are the final results. + // TODO - I think there are cases where we can push this down. + // Queries like {status=error} | rate() don't assert inter-span conditions + // and we could filter on span start time without affecting correctness. + // Queries where we can't are like: {A} >> {B} | rate() because only require + // that {B} occurs within our time range but {A} is allowed to occur any time. me.checkTime = true me.start = req.Start me.end = req.End @@ -422,7 +427,10 @@ func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, dedupe // optimize numerous things within the request that is specific to metrics. func optimize(req *FetchSpansRequest) { - // Special optimization for queries like {} | rate() by (rootName) + // Special optimization for queries like: + // {} | rate() + // {} | rate() by (rootName) + // {} | rate() by (resource.service.name) // When the second pass consists only of intrinsics, then it's possible to // move them to the first pass and increase performance. It avoids the second pass/bridge // layer and doesn't alter the correctness of the query. diff --git a/pkg/traceql/enum_attributes.go b/pkg/traceql/enum_attributes.go index e68d9d550da..dbbcfac3f7a 100644 --- a/pkg/traceql/enum_attributes.go +++ b/pkg/traceql/enum_attributes.go @@ -86,9 +86,11 @@ var ( IntrinsicStatusMessageAttribute = NewIntrinsic(IntrinsicStatusMessage) IntrinsicKindAttribute = NewIntrinsic(IntrinsicKind) IntrinsicChildCountAttribute = NewIntrinsic(IntrinsicChildCount) + IntrinsicTraceIDAttribute = NewIntrinsic(IntrinsicTraceID) IntrinsicTraceRootServiceAttribute = NewIntrinsic(IntrinsicTraceRootService) IntrinsicTraceRootSpanAttribute = NewIntrinsic(IntrinsicTraceRootSpan) IntrinsicTraceDurationAttribute = NewIntrinsic(IntrinsicTraceDuration) + IntrinsicSpanStartTimeAttribute = NewIntrinsic(IntrinsicSpanStartTime) ) func (i Intrinsic) String() string { From 9c1b4509437e45bd98bbf0a552d9c6cd6ed59454 Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Wed, 14 Feb 2024 10:35:03 -0500 Subject: [PATCH 07/13] Add many new hints, with the unsafe hints enabled by per-tenant flag --- modules/frontend/query_range_sharding.go | 71 +++++++++++++++---- modules/generator/overrides.go | 1 + modules/generator/overrides_test.go | 5 ++ .../processor/localblocks/processor.go | 3 +- .../processor/localblocks/processor_test.go | 4 ++ modules/overrides/config.go | 2 + modules/overrides/config_legacy.go | 3 + modules/overrides/interface.go | 1 + modules/overrides/runtime_config_overrides.go | 4 ++ modules/querier/querier_query_range.go | 17 ++++- pkg/traceql/engine.go | 10 +-- pkg/traceql/engine_metrics.go | 69 +++++++++--------- pkg/traceql/engine_metrics_test.go | 4 +- pkg/traceql/enum_hints.go | 33 +++++++-- pkg/traceqlmetrics/metrics.go | 2 +- .../encoding/vparquet3/block_traceql_test.go | 2 +- 16 files changed, 168 insertions(+), 63 deletions(-) diff --git a/modules/frontend/query_range_sharding.go b/modules/frontend/query_range_sharding.go index 05a68402d56..d0ad2e16ff1 100644 --- a/modules/frontend/query_range_sharding.go +++ b/modules/frontend/query_range_sharding.go @@ -113,13 +113,9 @@ func (s queryRangeSharder) RoundTrip(r *http.Request) (*http.Response, error) { }, nil } - // Check sampling rate hint - samplingRate := 1.0 - if ok, v := expr.Hints.GetFloat(traceql.HintSample); ok { - if v > 0 && v < 1.0 { - samplingRate = v - } - } + samplingRate := s.samplingRate(expr) + targetBytesPerRequest := s.jobSize(tenantID, expr, samplingRate) + interval := s.jobInterval(tenantID, expr) generatorReq = s.generatorRequest(*queryRangeReq, samplingRate) @@ -131,7 +127,7 @@ func (s queryRangeSharder) RoundTrip(r *http.Request) (*http.Response, error) { reqCh <- generatorReq } - totalBlocks, totalBlockBytes := s.backendRequests(tenantID, queryRangeReq, now, samplingRate, reqCh, stopCh) + totalBlocks, totalBlockBytes := s.backendRequests(tenantID, queryRangeReq, now, samplingRate, targetBytesPerRequest, interval, reqCh, stopCh) wg := boundedwaitgroup.New(uint(s.cfg.ConcurrentRequests)) jobErr := atomic.Error{} @@ -277,7 +273,7 @@ func (s *queryRangeSharder) blockMetas(start, end int64, tenantID string) []*bac return metas } -func (s *queryRangeSharder) backendRequests(tenantID string, searchReq *tempopb.QueryRangeRequest, now time.Time, samplingRate float64, reqCh chan *queryRangeJob, stopCh <-chan struct{}) (totalBlocks, totalBlockBytes int) { +func (s *queryRangeSharder) backendRequests(tenantID string, searchReq *tempopb.QueryRangeRequest, now time.Time, samplingRate float64, targetBytesPerRequest int, interval time.Duration, reqCh chan *queryRangeJob, stopCh <-chan struct{}) (totalBlocks, totalBlockBytes int) { // request without start or end, search only in generator if searchReq.Start == 0 || searchReq.End == 0 { close(reqCh) @@ -313,16 +309,16 @@ func (s *queryRangeSharder) backendRequests(tenantID string, searchReq *tempopb. } go func() { - s.buildBackendRequests(tenantID, searchReq, start, end, samplingRate, reqCh, stopCh) + s.buildBackendRequests(tenantID, searchReq, start, end, samplingRate, targetBytesPerRequest, interval, reqCh, stopCh) }() return } -func (s *queryRangeSharder) buildBackendRequests(tenantID string, searchReq *tempopb.QueryRangeRequest, start, end uint64, samplingRate float64, reqCh chan *queryRangeJob, stopCh <-chan struct{}) { +func (s *queryRangeSharder) buildBackendRequests(tenantID string, searchReq *tempopb.QueryRangeRequest, start, end uint64, samplingRate float64, targetBytesPerRequest int, interval time.Duration, reqCh chan *queryRangeJob, stopCh <-chan struct{}) { defer close(reqCh) - timeWindowSize := uint64(s.cfg.Interval.Nanoseconds()) + timeWindowSize := uint64(interval.Nanoseconds()) for start < end { @@ -343,7 +339,7 @@ func (s *queryRangeSharder) buildBackendRequests(tenantID string, searchReq *tem totalBlockSize += b.Size } - shards := uint32(math.Ceil(float64(totalBlockSize) / float64(s.cfg.TargetBytesPerRequest))) + shards := uint32(math.Ceil(float64(totalBlockSize) / float64(targetBytesPerRequest))) for i := uint32(1); i <= shards; i++ { shardR := *searchReq @@ -441,6 +437,55 @@ func (s *queryRangeSharder) maxDuration(tenantID string) time.Duration { return s.cfg.MaxDuration } +func (s *queryRangeSharder) samplingRate(expr *traceql.RootExpr) float64 { + samplingRate := 1.0 + if ok, v := expr.Hints.GetFloat(traceql.HintSample); ok { + if v > 0 && v < 1.0 { + samplingRate = v + } + } + return samplingRate +} + +func (s *queryRangeSharder) jobSize(userID string, expr *traceql.RootExpr, samplingRate float64) int { + // If we have a query hint then use it + if s.overrides.UnsafeQueryHints(userID) { + if ok, v := expr.Hints.GetInt("target_bytes_per_request"); ok && v > 0 { + return v + } + } + + // Else use configured value. + size := s.cfg.TargetBytesPerRequest + + // Automatically scale job size when sampling less than 100% + // This improves performance. + if samplingRate < 1.0 { + factor := 1.0 / samplingRate + + // Keep it within reason + if factor > 10.0 { + factor = 10.0 + } + + size = int(float64(size) * factor) + } + + return size +} + +func (s *queryRangeSharder) jobInterval(userID string, expr *traceql.RootExpr) time.Duration { + // If we have a query hint then use it + if s.overrides.UnsafeQueryHints(userID) { + if ok, v := expr.Hints.GetDuration("interval"); ok && v > 0 { + return v + } + } + + // Else use configured value + return s.cfg.Interval +} + func (s *queryRangeSharder) convertToPromFormat(resp *tempopb.QueryRangeResponse) PromResponse { // Sort series alphabetically so they are stable in the UI sort.Slice(resp.Series, func(i, j int) bool { diff --git a/modules/generator/overrides.go b/modules/generator/overrides.go index 1aea3c9f780..f84ff4b04bc 100644 --- a/modules/generator/overrides.go +++ b/modules/generator/overrides.go @@ -34,6 +34,7 @@ type metricsGeneratorOverrides interface { MetricsGeneratorProcessorSpanMetricsTargetInfoExcludedDimensions(userID string) []string DedicatedColumns(userID string) backend.DedicatedColumns MaxBytesPerTrace(userID string) int + UnsafeQueryHints(userID string) bool } var _ metricsGeneratorOverrides = (overrides.Interface)(nil) diff --git a/modules/generator/overrides_test.go b/modules/generator/overrides_test.go index 6cdecc391b2..38e9799e629 100644 --- a/modules/generator/overrides_test.go +++ b/modules/generator/overrides_test.go @@ -29,6 +29,7 @@ type mockOverrides struct { localBlocksCompleteBlockTimeout time.Duration dedicatedColumns backend.DedicatedColumns maxBytesPerTrace int + unsafeQueryHints bool } var _ metricsGeneratorOverrides = (*mockOverrides)(nil) @@ -134,3 +135,7 @@ func (m *mockOverrides) DedicatedColumns(string) backend.DedicatedColumns { func (m *mockOverrides) MaxBytesPerTrace(string) int { return m.maxBytesPerTrace } + +func (m *mockOverrides) UnsafeQueryHints(string) bool { + return m.unsafeQueryHints +} diff --git a/modules/generator/processor/localblocks/processor.go b/modules/generator/processor/localblocks/processor.go index 6dd80c2f92b..d88ed823a05 100644 --- a/modules/generator/processor/localblocks/processor.go +++ b/modules/generator/processor/localblocks/processor.go @@ -35,6 +35,7 @@ const timeBuffer = 5 * time.Minute type ProcessorOverrides interface { DedicatedColumns(string) backend.DedicatedColumns MaxBytesPerTrace(string) int + UnsafeQueryHints(string) bool } type Processor struct { @@ -445,7 +446,7 @@ func (p *Processor) QueryRange(ctx context.Context, req *tempopb.QueryRangeReque blocks = append(blocks, b) } - eval, err := traceql.NewEngine().CompileMetricsQueryRange(req, false) + eval, err := traceql.NewEngine().CompileMetricsQueryRange(req, false, p.overrides.UnsafeQueryHints(p.tenant)) if err != nil { return nil, err } diff --git a/modules/generator/processor/localblocks/processor_test.go b/modules/generator/processor/localblocks/processor_test.go index 31ed90b241e..ed0365b9863 100644 --- a/modules/generator/processor/localblocks/processor_test.go +++ b/modules/generator/processor/localblocks/processor_test.go @@ -29,6 +29,10 @@ func (m *mockOverrides) MaxBytesPerTrace(string) int { return 0 } +func (m *mockOverrides) UnsafeQueryHints(string) bool { + return false +} + func TestProcessorDoesNotRace(t *testing.T) { wal, err := wal.New(&wal.Config{ Filepath: t.TempDir(), diff --git a/modules/overrides/config.go b/modules/overrides/config.go index 8787536190d..87592a8c53e 100644 --- a/modules/overrides/config.go +++ b/modules/overrides/config.go @@ -128,6 +128,8 @@ type ReadOverrides struct { // QueryFrontend enforced overrides MaxSearchDuration model.Duration `yaml:"max_search_duration,omitempty" json:"max_search_duration,omitempty"` + + UnsafeQueryHints bool `yaml:"unsafe_query_hints,omitempty" json:"unsafe_query_hints,omitempty"` } type CompactionOverrides struct { diff --git a/modules/overrides/config_legacy.go b/modules/overrides/config_legacy.go index 6e4685061f0..c22f5ce6e3c 100644 --- a/modules/overrides/config_legacy.go +++ b/modules/overrides/config_legacy.go @@ -55,6 +55,7 @@ func (c *Overrides) toLegacy() LegacyOverrides { MaxBytesPerTagValuesQuery: c.Read.MaxBytesPerTagValuesQuery, MaxBlocksPerTagValuesQuery: c.Read.MaxBlocksPerTagValuesQuery, MaxSearchDuration: c.Read.MaxSearchDuration, + UnsafeQueryHints: c.Read.UnsafeQueryHints, MaxBytesPerTrace: c.Global.MaxBytesPerTrace, @@ -115,6 +116,7 @@ type LegacyOverrides struct { // QueryFrontend enforced limits MaxSearchDuration model.Duration `yaml:"max_search_duration" json:"max_search_duration"` + UnsafeQueryHints bool `yaml:"unsafe_query_hints" json:"unsafe_query_hints"` // MaxBytesPerTrace is enforced in the Ingester, Compactor, Querier (Search) and Serverless (Search). It // is not used when doing a trace by id lookup. @@ -137,6 +139,7 @@ func (l *LegacyOverrides) toNewLimits() Overrides { MaxBytesPerTagValuesQuery: l.MaxBytesPerTagValuesQuery, MaxBlocksPerTagValuesQuery: l.MaxBlocksPerTagValuesQuery, MaxSearchDuration: l.MaxSearchDuration, + UnsafeQueryHints: l.UnsafeQueryHints, }, Compaction: CompactionOverrides{ BlockRetention: l.BlockRetention, diff --git a/modules/overrides/interface.go b/modules/overrides/interface.go index 650697e02cb..9842fa8143d 100644 --- a/modules/overrides/interface.go +++ b/modules/overrides/interface.go @@ -65,6 +65,7 @@ type Interface interface { BlockRetention(userID string) time.Duration MaxSearchDuration(userID string) time.Duration DedicatedColumns(userID string) backend.DedicatedColumns + UnsafeQueryHints(userID string) bool // Management API WriteStatusRuntimeConfig(w io.Writer, r *http.Request) error diff --git a/modules/overrides/runtime_config_overrides.go b/modules/overrides/runtime_config_overrides.go index 9708abbab28..db9c53a9ab3 100644 --- a/modules/overrides/runtime_config_overrides.go +++ b/modules/overrides/runtime_config_overrides.go @@ -312,6 +312,10 @@ func (o *runtimeConfigOverridesManager) MaxBlocksPerTagValuesQuery(userID string return o.getOverridesForUser(userID).Read.MaxBlocksPerTagValuesQuery } +func (o *runtimeConfigOverridesManager) UnsafeQueryHints(userID string) bool { + return o.getOverridesForUser(userID).Read.UnsafeQueryHints +} + // MaxSearchDuration is the duration of the max search duration for this tenant. func (o *runtimeConfigOverridesManager) MaxSearchDuration(userID string) time.Duration { return time.Duration(o.getOverridesForUser(userID).Read.MaxSearchDuration) diff --git a/modules/querier/querier_query_range.go b/modules/querier/querier_query_range.go index 2e22053a4a9..a23aba78b36 100644 --- a/modules/querier/querier_query_range.go +++ b/modules/querier/querier_query_range.go @@ -77,16 +77,29 @@ func (q *Querier) queryBackend(ctx context.Context, req *tempopb.QueryRangeReque return nil, nil } + unsafe := q.limits.UnsafeQueryHints(tenantID) + // Optimization // If there's only 1 block then dedupe not needed. dedupe := len(withinTimeRange) > 1 - eval, err := traceql.NewEngine().CompileMetricsQueryRange(req, dedupe) + eval, err := traceql.NewEngine().CompileMetricsQueryRange(req, dedupe, unsafe) if err != nil { return nil, err } - wg := boundedwaitgroup.New(2) + concurrency := 2 + if unsafe { + expr, err := traceql.Parse(req.Query) + if err != nil { + return nil, err + } + if ok, v := expr.Hints.GetInt("block_concurrency"); ok && v > 0 && v < 100 { + concurrency = v + } + } + + wg := boundedwaitgroup.New(uint(concurrency)) jobErr := atomic.Error{} for _, m := range withinTimeRange { diff --git a/pkg/traceql/engine.go b/pkg/traceql/engine.go index 7686470f585..8bab04952ea 100644 --- a/pkg/traceql/engine.go +++ b/pkg/traceql/engine.go @@ -18,16 +18,18 @@ const ( DefaultSpansPerSpanSet int = 3 ) +type SpansetFilterFunc func(input []*Spanset) (result []*Spanset, err error) + type Engine struct{} func NewEngine() *Engine { return &Engine{} } -func (e *Engine) Compile(query string) (func(input []*Spanset) (result []*Spanset, err error), metricsFirstStageElement, *FetchSpansRequest, error) { +func (e *Engine) Compile(query string) (*RootExpr, SpansetFilterFunc, metricsFirstStageElement, *FetchSpansRequest, error) { expr, err := Parse(query) if err != nil { - return nil, nil, nil, err + return nil, nil, nil, nil, err } req := &FetchSpansRequest{ @@ -37,10 +39,10 @@ func (e *Engine) Compile(query string) (func(input []*Spanset) (result []*Spanse err = expr.validate() if err != nil { - return nil, nil, nil, err + return nil, nil, nil, nil, err } - return expr.Pipeline.evaluate, expr.MetricsPipeline, req, nil + return expr, expr.Pipeline.evaluate, expr.MetricsPipeline, req, nil } func (e *Engine) ExecuteSearch(ctx context.Context, searchReq *tempopb.SearchRequest, spanSetFetcher SpansetFetcher) (*tempopb.SearchResponse, error) { diff --git a/pkg/traceql/engine_metrics.go b/pkg/traceql/engine_metrics.go index aa094c78a81..fbf329a3d31 100644 --- a/pkg/traceql/engine_metrics.go +++ b/pkg/traceql/engine_metrics.go @@ -317,23 +317,11 @@ func (u *UngroupedAggregator) Series() SeriesSet { } } -// ExecuteMetricsQueryRange - Execute the given metrics query. Just a wrapper around CompileMetricsQueryRange -func (e *Engine) ExecuteMetricsQueryRange(ctx context.Context, req *tempopb.QueryRangeRequest, fetcher SpansetFetcher) (results SeriesSet, err error) { - eval, err := e.CompileMetricsQueryRange(req, false) - if err != nil { - return nil, err - } - - err = eval.Do(ctx, fetcher, 0, 0) - if err != nil { - return nil, err - } - - return eval.Results() -} - // CompileMetricsQueryRange returns an evalulator that can be reused across multiple data sources. -func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, dedupeSpans bool) (*MetricsEvalulator, error) { +// Dedupe spans parameter is an indicator of whether to expect duplicates in the datasource. For +// example if the datasource is replication factor=1 or only a single block then we know there +// aren't duplicates and we can make some optimizations. +func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, dedupeSpans bool, allowUnsafeQueryHints bool) (*MetricsEvalulator, error) { if req.Start <= 0 { return nil, fmt.Errorf("start required") } @@ -347,7 +335,7 @@ func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, dedupe return nil, fmt.Errorf("step required") } - eval, metricsPipeline, storageReq, err := e.Compile(req.Query) + expr, eval, metricsPipeline, storageReq, err := e.Compile(req.Query) if err != nil { return nil, fmt.Errorf("compiling query: %w", err) } @@ -356,13 +344,25 @@ func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, dedupe return nil, fmt.Errorf("not a metrics query") } + timeOverlapCutoff := 0.2 + if allowUnsafeQueryHints { + if ok, v := expr.Hints.GetFloat("time_overlap_cutoff"); ok && v >= 0 && v <= 1.0 { + timeOverlapCutoff = v + } + } + + if ok, v := expr.Hints.GetBool(HintDedupe); ok { + dedupeSpans = v + } + // This initializes all step buffers, counters, etc metricsPipeline.init(req) me := &MetricsEvalulator{ - storageReq: storageReq, - metricsPipeline: metricsPipeline, - dedupeSpans: dedupeSpans, + storageReq: storageReq, + metricsPipeline: metricsPipeline, + dedupeSpans: dedupeSpans, + timeOverlapCutoff: timeOverlapCutoff, } // TraceID (optional) @@ -470,16 +470,17 @@ func lookup(needles []Attribute, haystack Span) Static { } type MetricsEvalulator struct { - start, end uint64 - checkTime bool - dedupeSpans bool - deduper *SpanDeduper2 - storageReq *FetchSpansRequest - metricsPipeline metricsFirstStageElement - spansTotal uint64 - spansDeduped uint64 - bytes uint64 - mtx sync.Mutex + start, end uint64 + checkTime bool + dedupeSpans bool + deduper *SpanDeduper2 + timeOverlapCutoff float64 + storageReq *FetchSpansRequest + metricsPipeline metricsFirstStageElement + spansTotal uint64 + spansDeduped uint64 + bytes uint64 + mtx sync.Mutex } func timeRangeOverlap(reqStart, reqEnd, dataStart, dataEnd uint64) float64 { @@ -511,10 +512,10 @@ func (e *MetricsEvalulator) Do(ctx context.Context, f SpansetFetcher, fetcherSta } // Our heuristic is if the overlap between the given fetcher (i.e. block) - // and the request is less than 20%, use them. Above 20%, the cost of loading - // them doesn't outweight the benefits. 20% was measured in local benchmarking. - // TODO - Make configurable or a query hint? - if overlap < 0.2 { + // and the request is less than X%, use them. Above X%, the cost of loading + // them doesn't outweight the benefits. The default 20% was measured in + // local benchmarking. + if overlap < e.timeOverlapCutoff { storageReq.StartTimeUnixNanos = e.start storageReq.EndTimeUnixNanos = e.end // Should this be exclusive? } diff --git a/pkg/traceql/engine_metrics_test.go b/pkg/traceql/engine_metrics_test.go index bda614d1dc6..b0cd00a1635 100644 --- a/pkg/traceql/engine_metrics_test.go +++ b/pkg/traceql/engine_metrics_test.go @@ -165,7 +165,7 @@ func TestCompileMetricsQueryRange(t *testing.T) { Start: c.start, End: c.end, Step: c.step, - }, false) + }, false, false) if c.expectedErr != nil { require.EqualError(t, err, c.expectedErr.Error()) @@ -280,7 +280,7 @@ func TestCompileMetricsQueryRangeFetchSpansRequest(t *testing.T) { Start: 1, End: 2, Step: 3, - }, tc.dedupe) + }, tc.dedupe, false) require.NoError(t, err) // Nil out func to Equal works diff --git a/pkg/traceql/enum_hints.go b/pkg/traceql/enum_hints.go index 2ac6f467720..2c773447608 100644 --- a/pkg/traceql/enum_hints.go +++ b/pkg/traceql/enum_hints.go @@ -1,6 +1,13 @@ package traceql -const HintSample = "sample" +import ( + "time" +) + +const ( + HintSample = "sample" + HintDedupe = "dedupe" +) type Hint struct { Name string @@ -20,15 +27,31 @@ func newHints(h []*Hint) *Hints { } func (h *Hints) GetFloat(k string) (bool, float64) { + return get(h, k, TypeFloat, func(s Static) float64 { return s.F }) +} + +func (h *Hints) GetInt(k string) (bool, int) { + return get(h, k, TypeInt, func(s Static) int { return s.N }) +} + +func (h *Hints) GetDuration(k string) (bool, time.Duration) { + return get(h, k, TypeDuration, func(s Static) time.Duration { return s.D }) +} + +func (h *Hints) GetBool(k string) (bool, bool) { + return get(h, k, TypeBoolean, func(s Static) bool { return s.B }) +} + +func get[T any](h *Hints, k string, t StaticType, f func(Static) T) (ok bool, value T) { if h == nil { - return false, 0 + return } for _, hh := range h.Hints { - if hh.Name == k && hh.Value.Type == TypeFloat { - return true, hh.Value.F + if hh.Name == k && hh.Value.Type == t { + return true, f(hh.Value) } } - return false, 0 + return } diff --git a/pkg/traceqlmetrics/metrics.go b/pkg/traceqlmetrics/metrics.go index dd343ee39e3..d5cd16122fa 100644 --- a/pkg/traceqlmetrics/metrics.go +++ b/pkg/traceqlmetrics/metrics.go @@ -203,7 +203,7 @@ func GetMetrics(ctx context.Context, query, groupBy string, spanLimit int, start groupByKeys[i] = groupBys[i][0].String() } - eval, _, req, err := traceql.NewEngine().Compile(query) + _, eval, _, req, err := traceql.NewEngine().Compile(query) if err != nil { return nil, fmt.Errorf("compiling query: %w", err) } diff --git a/tempodb/encoding/vparquet3/block_traceql_test.go b/tempodb/encoding/vparquet3/block_traceql_test.go index 83d5c3f3784..10805f32f57 100644 --- a/tempodb/encoding/vparquet3/block_traceql_test.go +++ b/tempodb/encoding/vparquet3/block_traceql_test.go @@ -738,7 +738,7 @@ func BenchmarkBackendBlockQueryRange(b *testing.B) { ShardCount: 65, } - eval, err := e.CompileMetricsQueryRange(req, false) + eval, err := e.CompileMetricsQueryRange(req, false, false) require.NoError(b, err) b.ResetTimer() From d1839bd9f3530af4c31d211ba63d0d26e928651b Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Mon, 4 Mar 2024 11:57:48 -0500 Subject: [PATCH 08/13] Collect all hints in one place, decomplicate code --- modules/frontend/query_range_sharding.go | 27 +++++------ modules/querier/querier_query_range.go | 15 +++--- pkg/traceql/engine_metrics.go | 8 ++- pkg/traceql/enum_hints.go | 62 +++++++++++++++++++----- 4 files changed, 72 insertions(+), 40 deletions(-) diff --git a/modules/frontend/query_range_sharding.go b/modules/frontend/query_range_sharding.go index 66506044f1c..e79dedaa707 100644 --- a/modules/frontend/query_range_sharding.go +++ b/modules/frontend/query_range_sharding.go @@ -112,9 +112,10 @@ func (s queryRangeSharder) RoundTrip(r *http.Request) (*http.Response, error) { }, nil } - samplingRate := s.samplingRate(expr) - targetBytesPerRequest := s.jobSize(tenantID, expr, samplingRate) - interval := s.jobInterval(tenantID, expr) + allowUnsafe := s.overrides.UnsafeQueryHints(tenantID) + samplingRate := s.samplingRate(expr, allowUnsafe) + targetBytesPerRequest := s.jobSize(expr, samplingRate, allowUnsafe) + interval := s.jobInterval(expr, allowUnsafe) generatorReq = s.generatorRequest(*queryRangeReq, samplingRate) @@ -441,9 +442,9 @@ func (s *queryRangeSharder) maxDuration(tenantID string) time.Duration { return s.cfg.MaxDuration } -func (s *queryRangeSharder) samplingRate(expr *traceql.RootExpr) float64 { +func (s *queryRangeSharder) samplingRate(expr *traceql.RootExpr, allowUnsafe bool) float64 { samplingRate := 1.0 - if ok, v := expr.Hints.GetFloat(traceql.HintSample); ok { + if ok, v := expr.Hints.GetFloat(traceql.HintSample, allowUnsafe); ok { if v > 0 && v < 1.0 { samplingRate = v } @@ -451,12 +452,10 @@ func (s *queryRangeSharder) samplingRate(expr *traceql.RootExpr) float64 { return samplingRate } -func (s *queryRangeSharder) jobSize(userID string, expr *traceql.RootExpr, samplingRate float64) int { +func (s *queryRangeSharder) jobSize(expr *traceql.RootExpr, samplingRate float64, allowUnsafe bool) int { // If we have a query hint then use it - if s.overrides.UnsafeQueryHints(userID) { - if ok, v := expr.Hints.GetInt("target_bytes_per_request"); ok && v > 0 { - return v - } + if ok, v := expr.Hints.GetInt(traceql.HintJobSize, allowUnsafe); ok && v > 0 { + return v } // Else use configured value. @@ -478,12 +477,10 @@ func (s *queryRangeSharder) jobSize(userID string, expr *traceql.RootExpr, sampl return size } -func (s *queryRangeSharder) jobInterval(userID string, expr *traceql.RootExpr) time.Duration { +func (s *queryRangeSharder) jobInterval(expr *traceql.RootExpr, allowUnsafe bool) time.Duration { // If we have a query hint then use it - if s.overrides.UnsafeQueryHints(userID) { - if ok, v := expr.Hints.GetDuration("interval"); ok && v > 0 { - return v - } + if ok, v := expr.Hints.GetDuration(traceql.HintJobInterval, allowUnsafe); ok && v > 0 { + return v } // Else use configured value diff --git a/modules/querier/querier_query_range.go b/modules/querier/querier_query_range.go index 8b904afb240..f72de4e254d 100644 --- a/modules/querier/querier_query_range.go +++ b/modules/querier/querier_query_range.go @@ -88,15 +88,14 @@ func (q *Querier) queryBackend(ctx context.Context, req *tempopb.QueryRangeReque return nil, err } + expr, err := traceql.Parse(req.Query) + if err != nil { + return nil, err + } + concurrency := 2 - if unsafe { - expr, err := traceql.Parse(req.Query) - if err != nil { - return nil, err - } - if ok, v := expr.Hints.GetInt("block_concurrency"); ok && v > 0 && v < 100 { - concurrency = v - } + if ok, v := expr.Hints.GetInt(traceql.HintBlockConcurrency, unsafe); ok && v > 0 && v < 100 { + concurrency = v } wg := boundedwaitgroup.New(uint(concurrency)) diff --git a/pkg/traceql/engine_metrics.go b/pkg/traceql/engine_metrics.go index 7f5bbe9511d..e9d8b0f0407 100644 --- a/pkg/traceql/engine_metrics.go +++ b/pkg/traceql/engine_metrics.go @@ -359,13 +359,11 @@ func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, dedupe } timeOverlapCutoff := 0.2 - if allowUnsafeQueryHints { - if ok, v := expr.Hints.GetFloat("time_overlap_cutoff"); ok && v >= 0 && v <= 1.0 { - timeOverlapCutoff = v - } + if ok, v := expr.Hints.GetFloat(HintTimeOverlapCutoff, allowUnsafeQueryHints); ok && v >= 0 && v <= 1.0 { + timeOverlapCutoff = v } - if ok, v := expr.Hints.GetBool(HintDedupe); ok { + if ok, v := expr.Hints.GetBool(HintDedupe, allowUnsafeQueryHints); ok { dedupeSpans = v } diff --git a/pkg/traceql/enum_hints.go b/pkg/traceql/enum_hints.go index 2c773447608..65b3ea196c0 100644 --- a/pkg/traceql/enum_hints.go +++ b/pkg/traceql/enum_hints.go @@ -5,10 +5,23 @@ import ( ) const ( - HintSample = "sample" - HintDedupe = "dedupe" + HintSample = "sample" + HintDedupe = "dedupe" + HintJobInterval = "job_interval" + HintJobSize = "job_size" + HintTimeOverlapCutoff = "time_overlap_cutoff" + HintBlockConcurrency = "block_concurrency" ) +func isUnsafe(h string) bool { + switch h { + case HintSample: + return false + default: + return true + } +} + type Hint struct { Name string Value Static @@ -26,30 +39,55 @@ func newHints(h []*Hint) *Hints { return &Hints{h} } -func (h *Hints) GetFloat(k string) (bool, float64) { - return get(h, k, TypeFloat, func(s Static) float64 { return s.F }) +func (h *Hints) GetFloat(k string, allowUnsafe bool) (ok bool, v float64) { + if ok, v := h.Get(k, TypeFloat, allowUnsafe); ok { + return ok, v.F + } + + // If float not found, then try integer. + if ok, v := h.Get(k, TypeInt, allowUnsafe); ok { + return ok, float64(v.N) + } + + return } -func (h *Hints) GetInt(k string) (bool, int) { - return get(h, k, TypeInt, func(s Static) int { return s.N }) +func (h *Hints) GetInt(k string, allowUnsafe bool) (ok bool, v int) { + if ok, v := h.Get(k, TypeInt, allowUnsafe); ok { + return ok, v.N + } + + return } -func (h *Hints) GetDuration(k string) (bool, time.Duration) { - return get(h, k, TypeDuration, func(s Static) time.Duration { return s.D }) +func (h *Hints) GetDuration(k string, allowUnsafe bool) (ok bool, v time.Duration) { + if ok, v := h.Get(k, TypeDuration, allowUnsafe); ok { + return ok, v.D + } + + return } -func (h *Hints) GetBool(k string) (bool, bool) { - return get(h, k, TypeBoolean, func(s Static) bool { return s.B }) +func (h *Hints) GetBool(k string, allowUnsafe bool) (ok bool, v bool) { + if ok, v := h.Get(k, TypeBoolean, allowUnsafe); ok { + return ok, v.B + } + + return } -func get[T any](h *Hints, k string, t StaticType, f func(Static) T) (ok bool, value T) { +func (h *Hints) Get(k string, t StaticType, allowUnsafe bool) (ok bool, v Static) { if h == nil { return } + if isUnsafe(k) && !allowUnsafe { + return + } + for _, hh := range h.Hints { if hh.Name == k && hh.Value.Type == t { - return true, f(hh.Value) + return true, hh.Value } } From 8218c746c31dc5876332266c8a434184c4668e8c Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Mon, 4 Mar 2024 12:02:51 -0500 Subject: [PATCH 09/13] Make func signature golang convention --- modules/frontend/query_range_sharding.go | 6 ++--- modules/querier/querier_query_range.go | 2 +- pkg/traceql/engine_metrics.go | 4 +-- pkg/traceql/enum_hints.go | 32 ++++++++++++------------ 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/modules/frontend/query_range_sharding.go b/modules/frontend/query_range_sharding.go index e79dedaa707..bcf36f03c0f 100644 --- a/modules/frontend/query_range_sharding.go +++ b/modules/frontend/query_range_sharding.go @@ -444,7 +444,7 @@ func (s *queryRangeSharder) maxDuration(tenantID string) time.Duration { func (s *queryRangeSharder) samplingRate(expr *traceql.RootExpr, allowUnsafe bool) float64 { samplingRate := 1.0 - if ok, v := expr.Hints.GetFloat(traceql.HintSample, allowUnsafe); ok { + if v, ok := expr.Hints.GetFloat(traceql.HintSample, allowUnsafe); ok { if v > 0 && v < 1.0 { samplingRate = v } @@ -454,7 +454,7 @@ func (s *queryRangeSharder) samplingRate(expr *traceql.RootExpr, allowUnsafe boo func (s *queryRangeSharder) jobSize(expr *traceql.RootExpr, samplingRate float64, allowUnsafe bool) int { // If we have a query hint then use it - if ok, v := expr.Hints.GetInt(traceql.HintJobSize, allowUnsafe); ok && v > 0 { + if v, ok := expr.Hints.GetInt(traceql.HintJobSize, allowUnsafe); ok && v > 0 { return v } @@ -479,7 +479,7 @@ func (s *queryRangeSharder) jobSize(expr *traceql.RootExpr, samplingRate float64 func (s *queryRangeSharder) jobInterval(expr *traceql.RootExpr, allowUnsafe bool) time.Duration { // If we have a query hint then use it - if ok, v := expr.Hints.GetDuration(traceql.HintJobInterval, allowUnsafe); ok && v > 0 { + if v, ok := expr.Hints.GetDuration(traceql.HintJobInterval, allowUnsafe); ok && v > 0 { return v } diff --git a/modules/querier/querier_query_range.go b/modules/querier/querier_query_range.go index f72de4e254d..b1d7fa1e54c 100644 --- a/modules/querier/querier_query_range.go +++ b/modules/querier/querier_query_range.go @@ -94,7 +94,7 @@ func (q *Querier) queryBackend(ctx context.Context, req *tempopb.QueryRangeReque } concurrency := 2 - if ok, v := expr.Hints.GetInt(traceql.HintBlockConcurrency, unsafe); ok && v > 0 && v < 100 { + if v, ok := expr.Hints.GetInt(traceql.HintBlockConcurrency, unsafe); ok && v > 0 && v < 100 { concurrency = v } diff --git a/pkg/traceql/engine_metrics.go b/pkg/traceql/engine_metrics.go index e9d8b0f0407..df6fdad299a 100644 --- a/pkg/traceql/engine_metrics.go +++ b/pkg/traceql/engine_metrics.go @@ -359,11 +359,11 @@ func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, dedupe } timeOverlapCutoff := 0.2 - if ok, v := expr.Hints.GetFloat(HintTimeOverlapCutoff, allowUnsafeQueryHints); ok && v >= 0 && v <= 1.0 { + if v, ok := expr.Hints.GetFloat(HintTimeOverlapCutoff, allowUnsafeQueryHints); ok && v >= 0 && v <= 1.0 { timeOverlapCutoff = v } - if ok, v := expr.Hints.GetBool(HintDedupe, allowUnsafeQueryHints); ok { + if v, ok := expr.Hints.GetBool(HintDedupe, allowUnsafeQueryHints); ok { dedupeSpans = v } diff --git a/pkg/traceql/enum_hints.go b/pkg/traceql/enum_hints.go index 65b3ea196c0..c298895ac33 100644 --- a/pkg/traceql/enum_hints.go +++ b/pkg/traceql/enum_hints.go @@ -39,44 +39,44 @@ func newHints(h []*Hint) *Hints { return &Hints{h} } -func (h *Hints) GetFloat(k string, allowUnsafe bool) (ok bool, v float64) { - if ok, v := h.Get(k, TypeFloat, allowUnsafe); ok { - return ok, v.F +func (h *Hints) GetFloat(k string, allowUnsafe bool) (v float64, ok bool) { + if v, ok := h.Get(k, TypeFloat, allowUnsafe); ok { + return v.F, ok } // If float not found, then try integer. - if ok, v := h.Get(k, TypeInt, allowUnsafe); ok { - return ok, float64(v.N) + if v, ok := h.Get(k, TypeInt, allowUnsafe); ok { + return float64(v.N), ok } return } -func (h *Hints) GetInt(k string, allowUnsafe bool) (ok bool, v int) { - if ok, v := h.Get(k, TypeInt, allowUnsafe); ok { - return ok, v.N +func (h *Hints) GetInt(k string, allowUnsafe bool) (v int, ok bool) { + if v, ok := h.Get(k, TypeInt, allowUnsafe); ok { + return v.N, ok } return } -func (h *Hints) GetDuration(k string, allowUnsafe bool) (ok bool, v time.Duration) { - if ok, v := h.Get(k, TypeDuration, allowUnsafe); ok { - return ok, v.D +func (h *Hints) GetDuration(k string, allowUnsafe bool) (v time.Duration, ok bool) { + if v, ok := h.Get(k, TypeDuration, allowUnsafe); ok { + return v.D, ok } return } -func (h *Hints) GetBool(k string, allowUnsafe bool) (ok bool, v bool) { - if ok, v := h.Get(k, TypeBoolean, allowUnsafe); ok { - return ok, v.B +func (h *Hints) GetBool(k string, allowUnsafe bool) (v, ok bool) { + if v, ok := h.Get(k, TypeBoolean, allowUnsafe); ok { + return v.B, ok } return } -func (h *Hints) Get(k string, t StaticType, allowUnsafe bool) (ok bool, v Static) { +func (h *Hints) Get(k string, t StaticType, allowUnsafe bool) (v Static, ok bool) { if h == nil { return } @@ -87,7 +87,7 @@ func (h *Hints) Get(k string, t StaticType, allowUnsafe bool) (ok bool, v Static for _, hh := range h.Hints { if hh.Name == k && hh.Value.Type == t { - return true, hh.Value + return hh.Value, true } } From 25e9e8b99a7aa086f0204a08c8ba3a3e49b861e3 Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Mon, 4 Mar 2024 12:41:06 -0500 Subject: [PATCH 10/13] make querier block_concurrency configurable --- modules/querier/config.go | 6 ++++++ modules/querier/querier_query_range.go | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/querier/config.go b/modules/querier/config.go index 2a9708e39f9..ffaae9fc43c 100644 --- a/modules/querier/config.go +++ b/modules/querier/config.go @@ -14,6 +14,7 @@ import ( type Config struct { Search SearchConfig `yaml:"search"` TraceByID TraceByIDConfig `yaml:"trace_by_id"` + Metrics MetricsConfig `yaml:"metrics"` ExtraQueryDelay time.Duration `yaml:"extra_query_delay,omitempty"` MaxConcurrentQueries int `yaml:"max_concurrent_queries"` @@ -43,6 +44,10 @@ type TraceByIDConfig struct { QueryTimeout time.Duration `yaml:"query_timeout"` } +type MetricsConfig struct { + BlockConcurrency int `yaml:"block_concurrency,omitempty"` +} + // RegisterFlagsAndApplyDefaults register flags. func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) { cfg.TraceByID.QueryTimeout = 10 * time.Second @@ -53,6 +58,7 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) cfg.Search.HedgeRequestsAt = 8 * time.Second cfg.Search.HedgeRequestsUpTo = 2 cfg.Search.QueryTimeout = 30 * time.Second + cfg.Metrics.BlockConcurrency = 2 cfg.Worker = worker.Config{ MatchMaxConcurrency: true, MaxConcurrentRequests: cfg.MaxConcurrentQueries, diff --git a/modules/querier/querier_query_range.go b/modules/querier/querier_query_range.go index b1d7fa1e54c..c27443e2f71 100644 --- a/modules/querier/querier_query_range.go +++ b/modules/querier/querier_query_range.go @@ -93,7 +93,7 @@ func (q *Querier) queryBackend(ctx context.Context, req *tempopb.QueryRangeReque return nil, err } - concurrency := 2 + concurrency := q.cfg.Metrics.BlockConcurrency if v, ok := expr.Hints.GetInt(traceql.HintBlockConcurrency, unsafe); ok && v > 0 && v < 100 { concurrency = v } From d7cb553f558f819bcc863c9aba5eb0522ebd5e7a Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Mon, 4 Mar 2024 12:43:55 -0500 Subject: [PATCH 11/13] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30de4b0ff43..298950a195e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## main / unreleased * [ENHANCEMENT] Add string interning to TraceQL queries [#3411](https://github.com/grafana/tempo/pull/3411) (@mapno) +* [ENHANCEMENT] Add new (unsafe) query hints for metrics queries [#3396](https://github.com/grafana/tempo/pull/3396) (@mdisibio) * [BUGFIX] Fix metrics query results when filtering and rating on the same attribute [#3428](https://github.com/grafana/tempo/issues/3428) (@mdisibio) * [BUGFIX] Fix metrics query results when series contain empty strings or nil values [#3429](https://github.com/grafana/tempo/issues/3429) (@mdisibio) * [BUGFIX] Return unfiltered results when a bad TraceQL query is provided in autocomplete. [#3426](https://github.com/grafana/tempo/pull/3426) (@mapno) From 6813b9e9428c3f6b7c2b8de72c76013b588b44f0 Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Mon, 4 Mar 2024 14:46:10 -0500 Subject: [PATCH 12/13] Make time_overlap_cutoff configurable. Rename block_concurrency to concurrent_blocks to match naming convention of other concurrent_* fields. --- .../generator/processor/localblocks/config.go | 18 ++++++++++++-- .../processor/localblocks/processor.go | 24 +++++++++++++++++-- .../processor/localblocks/processor_test.go | 5 +++- modules/querier/config.go | 13 ++++++++-- modules/querier/querier_query_range.go | 17 ++++++++----- pkg/traceql/engine_metrics.go | 7 +----- pkg/traceql/engine_metrics_test.go | 4 ++-- pkg/traceql/enum_hints.go | 4 +++- 8 files changed, 70 insertions(+), 22 deletions(-) diff --git a/modules/generator/processor/localblocks/config.go b/modules/generator/processor/localblocks/config.go index 6a736eae666..8cc93c8db5a 100644 --- a/modules/generator/processor/localblocks/config.go +++ b/modules/generator/processor/localblocks/config.go @@ -22,8 +22,19 @@ type Config struct { MaxBlockBytes uint64 `yaml:"max_block_bytes"` CompleteBlockTimeout time.Duration `yaml:"complete_block_timeout"` MaxLiveTraces uint64 `yaml:"max_live_traces"` - ConcurrentBlocks uint `yaml:"concurrent_blocks"` FilterServerSpans bool `yaml:"filter_server_spans"` + Metrics MetricsConfig `yaml:",inline"` +} + +type MetricsConfig struct { + ConcurrentBlocks uint `yaml:"concurrent_blocks"` + // TimeOverlapCutoff is a tuning factor that controls whether the trace-level + // timestamp columns are used in a metrics query. Loading these columns has a cost, + // so in some cases it faster to skip these columns entirely, reducing I/O but + // increasing the number of spans evalulated and thrown away. The value is a ratio + // between 0.0 and 1.0. If a block overlaps the time window by less than this value, + // then we skip the columns. A value of 1.0 will always load the columns, and 0.0 never. + TimeOverlapCutoff float64 `yaml:"time_overlap_cutoff,omitempty"` } func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) { @@ -39,6 +50,9 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) cfg.MaxBlockDuration = 1 * time.Minute cfg.MaxBlockBytes = 500_000_000 cfg.CompleteBlockTimeout = time.Hour - cfg.ConcurrentBlocks = 10 cfg.FilterServerSpans = true + cfg.Metrics = MetricsConfig{ + ConcurrentBlocks: 10, + TimeOverlapCutoff: 0.2, + } } diff --git a/modules/generator/processor/localblocks/processor.go b/modules/generator/processor/localblocks/processor.go index d88ed823a05..d19be0dadfd 100644 --- a/modules/generator/processor/localblocks/processor.go +++ b/modules/generator/processor/localblocks/processor.go @@ -445,14 +445,34 @@ func (p *Processor) QueryRange(ctx context.Context, req *tempopb.QueryRangeReque for _, b := range p.completeBlocks { blocks = append(blocks, b) } + if len(blocks) == 0 { + return nil, nil + } + + expr, err := traceql.Parse(req.Query) + if err != nil { + return nil, fmt.Errorf("compiling query: %w", err) + } + + unsafe := p.overrides.UnsafeQueryHints(p.tenant) + + timeOverlapCutoff := p.Cfg.Metrics.TimeOverlapCutoff + if v, ok := expr.Hints.GetFloat(traceql.HintTimeOverlapCutoff, unsafe); ok && v >= 0 && v <= 1.0 { + timeOverlapCutoff = v + } + + concurrency := p.Cfg.Metrics.ConcurrentBlocks + if v, ok := expr.Hints.GetInt(traceql.HintConcurrentBlocks, unsafe); ok && v > 0 && v < 100 { + concurrency = uint(v) + } - eval, err := traceql.NewEngine().CompileMetricsQueryRange(req, false, p.overrides.UnsafeQueryHints(p.tenant)) + eval, err := traceql.NewEngine().CompileMetricsQueryRange(req, false, timeOverlapCutoff, unsafe) if err != nil { return nil, err } var ( - wg = boundedwaitgroup.New(p.Cfg.ConcurrentBlocks) + wg = boundedwaitgroup.New(concurrency) jobErr = atomic.Error{} ) diff --git a/modules/generator/processor/localblocks/processor_test.go b/modules/generator/processor/localblocks/processor_test.go index ed0365b9863..26e4c41406f 100644 --- a/modules/generator/processor/localblocks/processor_test.go +++ b/modules/generator/processor/localblocks/processor_test.go @@ -47,12 +47,15 @@ func TestProcessorDoesNotRace(t *testing.T) { FlushCheckPeriod: 10 * time.Millisecond, TraceIdlePeriod: time.Second, CompleteBlockTimeout: time.Minute, - ConcurrentBlocks: 10, Block: &common.BlockConfig{ BloomShardSizeBytes: 100_000, BloomFP: 0.05, Version: encoding.DefaultEncoding().Version(), }, + Metrics: MetricsConfig{ + ConcurrentBlocks: 10, + TimeOverlapCutoff: 0.2, + }, } overrides = &mockOverrides{} ) diff --git a/modules/querier/config.go b/modules/querier/config.go index ffaae9fc43c..2d7557ec7a9 100644 --- a/modules/querier/config.go +++ b/modules/querier/config.go @@ -45,7 +45,15 @@ type TraceByIDConfig struct { } type MetricsConfig struct { - BlockConcurrency int `yaml:"block_concurrency,omitempty"` + ConcurrentBlocks int `yaml:"concurrent_blocks,omitempty"` + + // TimeOverlapCutoff is a tuning factor that controls whether the trace-level + // timestamp columns are used in a metrics query. Loading these columns has a cost, + // so in some cases it faster to skip these columns entirely, reducing I/O but + // increasing the number of spans evalulated and thrown away. The value is a ratio + // between 0.0 and 1.0. If a block overlaps the time window by less than this value, + // then we skip the columns. A value of 1.0 will always load the columns, and 0.0 never. + TimeOverlapCutoff float64 `yaml:"time_overlap_cutoff,omitempty"` } // RegisterFlagsAndApplyDefaults register flags. @@ -58,7 +66,8 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) cfg.Search.HedgeRequestsAt = 8 * time.Second cfg.Search.HedgeRequestsUpTo = 2 cfg.Search.QueryTimeout = 30 * time.Second - cfg.Metrics.BlockConcurrency = 2 + cfg.Metrics.ConcurrentBlocks = 2 + cfg.Metrics.TimeOverlapCutoff = 0.2 cfg.Worker = worker.Config{ MatchMaxConcurrency: true, MaxConcurrentRequests: cfg.MaxConcurrentQueries, diff --git a/modules/querier/querier_query_range.go b/modules/querier/querier_query_range.go index c27443e2f71..f1e7cd4722a 100644 --- a/modules/querier/querier_query_range.go +++ b/modules/querier/querier_query_range.go @@ -83,21 +83,26 @@ func (q *Querier) queryBackend(ctx context.Context, req *tempopb.QueryRangeReque // If there's only 1 block then dedupe not needed. dedupe := len(withinTimeRange) > 1 - eval, err := traceql.NewEngine().CompileMetricsQueryRange(req, dedupe, unsafe) + expr, err := traceql.Parse(req.Query) if err != nil { return nil, err } - expr, err := traceql.Parse(req.Query) - if err != nil { - return nil, err + timeOverlapCutoff := q.cfg.Metrics.TimeOverlapCutoff + if v, ok := expr.Hints.GetFloat(traceql.HintTimeOverlapCutoff, unsafe); ok && v >= 0 && v <= 1.0 { + timeOverlapCutoff = v } - concurrency := q.cfg.Metrics.BlockConcurrency - if v, ok := expr.Hints.GetInt(traceql.HintBlockConcurrency, unsafe); ok && v > 0 && v < 100 { + concurrency := q.cfg.Metrics.ConcurrentBlocks + if v, ok := expr.Hints.GetInt(traceql.HintConcurrentBlocks, unsafe); ok && v > 0 && v < 100 { concurrency = v } + eval, err := traceql.NewEngine().CompileMetricsQueryRange(req, dedupe, timeOverlapCutoff, unsafe) + if err != nil { + return nil, err + } + wg := boundedwaitgroup.New(uint(concurrency)) jobErr := atomic.Error{} diff --git a/pkg/traceql/engine_metrics.go b/pkg/traceql/engine_metrics.go index df6fdad299a..a3266eb4c84 100644 --- a/pkg/traceql/engine_metrics.go +++ b/pkg/traceql/engine_metrics.go @@ -335,7 +335,7 @@ func (u *UngroupedAggregator) Series() SeriesSet { // Dedupe spans parameter is an indicator of whether to expect duplicates in the datasource. For // example if the datasource is replication factor=1 or only a single block then we know there // aren't duplicates and we can make some optimizations. -func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, dedupeSpans bool, allowUnsafeQueryHints bool) (*MetricsEvalulator, error) { +func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, dedupeSpans bool, timeOverlapCutoff float64, allowUnsafeQueryHints bool) (*MetricsEvalulator, error) { if req.Start <= 0 { return nil, fmt.Errorf("start required") } @@ -358,11 +358,6 @@ func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, dedupe return nil, fmt.Errorf("not a metrics query") } - timeOverlapCutoff := 0.2 - if v, ok := expr.Hints.GetFloat(HintTimeOverlapCutoff, allowUnsafeQueryHints); ok && v >= 0 && v <= 1.0 { - timeOverlapCutoff = v - } - if v, ok := expr.Hints.GetBool(HintDedupe, allowUnsafeQueryHints); ok { dedupeSpans = v } diff --git a/pkg/traceql/engine_metrics_test.go b/pkg/traceql/engine_metrics_test.go index cfa74ef9ec3..8eec72934fe 100644 --- a/pkg/traceql/engine_metrics_test.go +++ b/pkg/traceql/engine_metrics_test.go @@ -165,7 +165,7 @@ func TestCompileMetricsQueryRange(t *testing.T) { Start: c.start, End: c.end, Step: c.step, - }, false, false) + }, false, 0, false) if c.expectedErr != nil { require.EqualError(t, err, c.expectedErr.Error()) @@ -281,7 +281,7 @@ func TestCompileMetricsQueryRangeFetchSpansRequest(t *testing.T) { Start: 1, End: 2, Step: 3, - }, tc.dedupe, false) + }, tc.dedupe, 0, false) require.NoError(t, err) // Nil out func to Equal works diff --git a/pkg/traceql/enum_hints.go b/pkg/traceql/enum_hints.go index c298895ac33..1006d6e39a3 100644 --- a/pkg/traceql/enum_hints.go +++ b/pkg/traceql/enum_hints.go @@ -4,13 +4,15 @@ import ( "time" ) +// The list of all traceql query hints. Although most of these are implementation-specific +// and not part of the language or engine, we organize them here in one place. const ( HintSample = "sample" HintDedupe = "dedupe" HintJobInterval = "job_interval" HintJobSize = "job_size" HintTimeOverlapCutoff = "time_overlap_cutoff" - HintBlockConcurrency = "block_concurrency" + HintConcurrentBlocks = "concurrent_blocks" ) func isUnsafe(h string) bool { From 0448023f440289fc7c5f6a8ae5a270aee6a47d85 Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Mon, 4 Mar 2024 14:58:01 -0500 Subject: [PATCH 13/13] fix test --- tempodb/encoding/vparquet3/block_traceql_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tempodb/encoding/vparquet3/block_traceql_test.go b/tempodb/encoding/vparquet3/block_traceql_test.go index 1cbf32e7299..b4ef92e1faf 100644 --- a/tempodb/encoding/vparquet3/block_traceql_test.go +++ b/tempodb/encoding/vparquet3/block_traceql_test.go @@ -746,7 +746,7 @@ func BenchmarkBackendBlockQueryRange(b *testing.B) { ShardCount: 65, } - eval, err := e.CompileMetricsQueryRange(req, false, false) + eval, err := e.CompileMetricsQueryRange(req, false, 0, false) require.NoError(b, err) b.ResetTimer()