From b80cd9d0da00abb54eb54b9e4b9152850997460d Mon Sep 17 00:00:00 2001 From: Sebastian Rabenhorst <4246554+rabenhorst@users.noreply.github.com> Date: Mon, 10 Jul 2023 17:52:16 +0200 Subject: [PATCH 1/7] receive: add float histogram support (#6323) * Added receive float histogram support Signed-off-by: Sebastian Rabenhorst Fixed imports Signed-off-by: Sebastian Rabenhorst * Added comments for fns copied from Prometheus Signed-off-by: Sebastian Rabenhorst Improved comment Signed-off-by: Sebastian Rabenhorst * Removed unnecessary if Signed-off-by: Sebastian Rabenhorst * Fixed native histogram proto conversion in remote engine Signed-off-by: Sebastian Rabenhorst * Removed unused histogram conversion from remote engine Signed-off-by: Sebastian Rabenhorst * Fix and renaming in native_histograms_test.go Signed-off-by: Sebastian Rabenhorst * Trigger Build Signed-off-by: Sebastian Rabenhorst --------- Signed-off-by: Sebastian Rabenhorst --- pkg/query/iter.go | 2 + pkg/query/remote_engine.go | 6 +- pkg/receive/writer.go | 15 ++- pkg/receive/writer_test.go | 78 ++++++------- pkg/store/storepb/prompb/custom.go | 9 ++ pkg/store/storepb/prompb/samples.go | 123 +++++++++++++++------ pkg/store/storepb/prompb/types.pb.go | 158 +++++++++++++-------------- pkg/store/storepb/prompb/types.proto | 6 +- pkg/store/storepb/types.pb.go | 84 +++++++------- pkg/store/storepb/types.proto | 1 + test/e2e/native_histograms_test.go | 105 +++++++++++++----- 11 files changed, 351 insertions(+), 236 deletions(-) create mode 100644 pkg/store/storepb/prompb/custom.go diff --git a/pkg/query/iter.go b/pkg/query/iter.go index 2a4e90b92b..8d1d043b20 100644 --- a/pkg/query/iter.go +++ b/pkg/query/iter.go @@ -178,6 +178,8 @@ func chunkEncoding(e storepb.Chunk_Encoding) chunkenc.Encoding { return chunkenc.EncXOR case storepb.Chunk_HISTOGRAM: return chunkenc.EncHistogram + case storepb.Chunk_FLOAT_HISTOGRAM: + return chunkenc.EncFloatHistogram } return 255 // Invalid. } diff --git a/pkg/query/remote_engine.go b/pkg/query/remote_engine.go index 825cdbea05..cb2447780b 100644 --- a/pkg/query/remote_engine.go +++ b/pkg/query/remote_engine.go @@ -263,10 +263,10 @@ func (r *remoteQuery) Exec(ctx context.Context) *promql.Result { F: s.Value, }) } - for _, h := range ts.Histograms { + for _, hp := range ts.Histograms { series.Histograms = append(series.Histograms, promql.HPoint{ - T: h.Timestamp, - H: prompb.HistogramProtoToFloatHistogram(h), + T: hp.Timestamp, + H: prompb.FloatHistogramProtoToFloatHistogram(hp), }) } result = append(result, series) diff --git a/pkg/receive/writer.go b/pkg/receive/writer.go index 039c3aea52..517ee5c41a 100644 --- a/pkg/receive/writer.go +++ b/pkg/receive/writer.go @@ -12,6 +12,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/exemplar" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb" @@ -168,8 +169,18 @@ func (r *Writer) Write(ctx context.Context, tenantID string, wreq *prompb.WriteR } for _, hp := range t.Histograms { - h := prompb.HistogramProtoToHistogram(hp) - ref, err = app.AppendHistogram(ref, lset, hp.Timestamp, h, nil) + var ( + h *histogram.Histogram + fh *histogram.FloatHistogram + ) + + if hp.IsFloatHistogram() { + fh = prompb.FloatHistogramProtoToFloatHistogram(hp) + } else { + h = prompb.HistogramProtoToHistogram(hp) + } + + ref, err = app.AppendHistogram(ref, lset, hp.Timestamp, h, fh) switch err { case storage.ErrOutOfOrderSample: numSamplesOutOfOrder++ diff --git a/pkg/receive/writer_test.go b/pkg/receive/writer_test.go index 596fc1e023..34613794b8 100644 --- a/pkg/receive/writer_test.go +++ b/pkg/receive/writer_test.go @@ -10,17 +10,17 @@ import ( "testing" "time" + "github.com/efficientgo/core/testutil" "github.com/go-kit/log" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/exemplar" - "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb" + "github.com/prometheus/prometheus/tsdb/tsdbutil" - "github.com/efficientgo/core/testutil" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store/labelpb" @@ -251,7 +251,7 @@ func TestWriter(t *testing.T) { { Labels: append(lbls, labelpb.ZLabel{Name: "a", Value: "1"}, labelpb.ZLabel{Name: "b", Value: "2"}), Histograms: []prompb.Histogram{ - histogramToHistogramProto(9, testHistogram()), + prompb.HistogramToHistogramProto(10, tsdbutil.GenerateTestHistogram(0)), }, }, }, @@ -262,7 +262,30 @@ func TestWriter(t *testing.T) { { Labels: append(lbls, labelpb.ZLabel{Name: "a", Value: "1"}, labelpb.ZLabel{Name: "b", Value: "2"}), Histograms: []prompb.Histogram{ - histogramToHistogramProto(10, testHistogram()), + prompb.HistogramToHistogramProto(10, tsdbutil.GenerateTestHistogram(0)), + }, + }, + }, + }, + "should succeed on float histogram with valid labels": { + reqs: []*prompb.WriteRequest{ + { + Timeseries: []prompb.TimeSeries{ + { + Labels: append(lbls, labelpb.ZLabel{Name: "a", Value: "1"}, labelpb.ZLabel{Name: "b", Value: "2"}), + Histograms: []prompb.Histogram{ + prompb.FloatHistogramToHistogramProto(10, tsdbutil.GenerateTestFloatHistogram(1)), + }, + }, + }, + }, + }, + expectedErr: nil, + expectedIngested: []prompb.TimeSeries{ + { + Labels: append(lbls, labelpb.ZLabel{Name: "a", Value: "1"}, labelpb.ZLabel{Name: "b", Value: "2"}), + Histograms: []prompb.Histogram{ + prompb.FloatHistogramToHistogramProto(10, tsdbutil.GenerateTestFloatHistogram(1)), }, }, }, @@ -274,7 +297,7 @@ func TestWriter(t *testing.T) { { Labels: append(lbls, labelpb.ZLabel{Name: "a", Value: "1"}, labelpb.ZLabel{Name: "b", Value: "2"}), Histograms: []prompb.Histogram{ - histogramToHistogramProto(10, testHistogram()), + prompb.HistogramToHistogramProto(10, tsdbutil.GenerateTestHistogram(0)), }, }, }, @@ -284,7 +307,7 @@ func TestWriter(t *testing.T) { { Labels: append(lbls, labelpb.ZLabel{Name: "a", Value: "1"}, labelpb.ZLabel{Name: "b", Value: "2"}), Histograms: []prompb.Histogram{ - histogramToHistogramProto(9, testHistogram()), + prompb.HistogramToHistogramProto(9, tsdbutil.GenerateTestHistogram(0)), }, }, }, @@ -295,7 +318,7 @@ func TestWriter(t *testing.T) { { Labels: append(lbls, labelpb.ZLabel{Name: "a", Value: "1"}, labelpb.ZLabel{Name: "b", Value: "2"}), Histograms: []prompb.Histogram{ - histogramToHistogramProto(10, testHistogram()), + prompb.HistogramToHistogramProto(10, tsdbutil.GenerateTestHistogram(0)), }, }, }, @@ -480,7 +503,7 @@ func generateLabelsAndSeries(numLabels int, numSeries int, generateHistograms bo } if generateHistograms { - ts[j].Histograms = []prompb.Histogram{histogramToHistogramProto(10, testHistogram())} + ts[j].Histograms = []prompb.Histogram{prompb.HistogramToHistogramProto(10, tsdbutil.GenerateTestHistogram(0))} continue } @@ -489,42 +512,3 @@ func generateLabelsAndSeries(numLabels int, numSeries int, generateHistograms bo return ts } - -func testHistogram() *histogram.Histogram { - return &histogram.Histogram{ - Count: 5, - ZeroCount: 2, - Sum: 18.4, - ZeroThreshold: 0.1, - Schema: 1, - PositiveSpans: []histogram.Span{ - {Offset: 0, Length: 2}, - {Offset: 1, Length: 2}, - }, - PositiveBuckets: []int64{1, 1, -1, 0}, // counts: 1, 2, 1, 1 (total 5) - } -} - -func histogramToHistogramProto(timestamp int64, h *histogram.Histogram) prompb.Histogram { - return prompb.Histogram{ - Count: &prompb.Histogram_CountInt{CountInt: h.Count}, - Sum: h.Sum, - Schema: h.Schema, - ZeroThreshold: h.ZeroThreshold, - ZeroCount: &prompb.Histogram_ZeroCountInt{ZeroCountInt: h.ZeroCount}, - NegativeSpans: spansToSpansProto(h.NegativeSpans), - NegativeDeltas: h.NegativeBuckets, - PositiveSpans: spansToSpansProto(h.PositiveSpans), - PositiveDeltas: h.PositiveBuckets, - Timestamp: timestamp, - } -} - -func spansToSpansProto(s []histogram.Span) []*prompb.BucketSpan { - spans := make([]*prompb.BucketSpan, len(s)) - for i := 0; i < len(s); i++ { - spans[i] = &prompb.BucketSpan{Offset: s[i].Offset, Length: s[i].Length} - } - - return spans -} diff --git a/pkg/store/storepb/prompb/custom.go b/pkg/store/storepb/prompb/custom.go new file mode 100644 index 0000000000..fb3b395a9a --- /dev/null +++ b/pkg/store/storepb/prompb/custom.go @@ -0,0 +1,9 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package prompb + +func (h Histogram) IsFloatHistogram() bool { + _, ok := h.GetCount().(*Histogram_CountFloat) + return ok +} diff --git a/pkg/store/storepb/prompb/samples.go b/pkg/store/storepb/prompb/samples.go index a276d9264e..050b8e912f 100644 --- a/pkg/store/storepb/prompb/samples.go +++ b/pkg/store/storepb/prompb/samples.go @@ -62,71 +62,124 @@ func SamplesFromPromqlSeries(series promql.Series) ([]Sample, []Histogram) { // HistogramProtoToHistogram extracts a (normal integer) Histogram from the // provided proto message. The caller has to make sure that the proto message // represents an interger histogram and not a float histogram. -// Taken from https://github.com/prometheus/prometheus/blob/d33eb3ab17616a54b97d9f7791c791a79823f279/storage/remote/codec.go#L529-L542. +// Copied from https://github.com/prometheus/prometheus/blob/0ab95536115adfe50af249d36d73674be694ca3f/storage/remote/codec.go#L626-L645 func HistogramProtoToHistogram(hp Histogram) *histogram.Histogram { + if hp.IsFloatHistogram() { + panic("HistogramProtoToHistogram called with a float histogram") + } return &histogram.Histogram{ - Schema: hp.Schema, - ZeroThreshold: hp.ZeroThreshold, - ZeroCount: hp.GetZeroCountInt(), - Count: hp.GetCountInt(), - Sum: hp.Sum, - PositiveSpans: spansProtoToSpans(hp.GetPositiveSpans()), - PositiveBuckets: hp.GetPositiveDeltas(), - NegativeSpans: spansProtoToSpans(hp.GetNegativeSpans()), - NegativeBuckets: hp.GetNegativeDeltas(), + CounterResetHint: histogram.CounterResetHint(hp.ResetHint), + Schema: hp.Schema, + ZeroThreshold: hp.ZeroThreshold, + ZeroCount: hp.GetZeroCountInt(), + Count: hp.GetCountInt(), + Sum: hp.Sum, + PositiveSpans: spansProtoToSpans(hp.GetPositiveSpans()), + PositiveBuckets: hp.GetPositiveDeltas(), + NegativeSpans: spansProtoToSpans(hp.GetNegativeSpans()), + NegativeBuckets: hp.GetNegativeDeltas(), } } // FloatHistogramToHistogramProto converts a float histogram to a protobuf type. -// Taken from https://github.com/prometheus/prometheus/blob/d33eb3ab17616a54b97d9f7791c791a79823f279/storage/remote/codec.go#L587-L601. -func FloatHistogramToHistogramProto(timestamp int64, fh *histogram.FloatHistogram) Histogram { - return Histogram{ - Count: &Histogram_CountFloat{CountFloat: fh.Count}, - Sum: fh.Sum, - Schema: fh.Schema, - ZeroThreshold: fh.ZeroThreshold, - ZeroCount: &Histogram_ZeroCountFloat{ZeroCountFloat: fh.ZeroCount}, - NegativeSpans: spansToSpansProto(fh.NegativeSpans), - NegativeCounts: fh.NegativeBuckets, - PositiveSpans: spansToSpansProto(fh.PositiveSpans), - PositiveCounts: fh.PositiveBuckets, - ResetHint: Histogram_ResetHint(fh.CounterResetHint), - Timestamp: timestamp, +// Copied from https://github.com/prometheus/prometheus/blob/0ab95536115adfe50af249d36d73674be694ca3f/storage/remote/codec.go#L647-L667 +func FloatHistogramProtoToFloatHistogram(hp Histogram) *histogram.FloatHistogram { + if !hp.IsFloatHistogram() { + panic("FloatHistogramProtoToFloatHistogram called with an integer histogram") + } + return &histogram.FloatHistogram{ + CounterResetHint: histogram.CounterResetHint(hp.ResetHint), + Schema: hp.Schema, + ZeroThreshold: hp.ZeroThreshold, + ZeroCount: hp.GetZeroCountFloat(), + Count: hp.GetCountFloat(), + Sum: hp.Sum, + PositiveSpans: spansProtoToSpans(hp.GetPositiveSpans()), + PositiveBuckets: hp.GetPositiveCounts(), + NegativeSpans: spansProtoToSpans(hp.GetNegativeSpans()), + NegativeBuckets: hp.GetNegativeCounts(), } } // HistogramProtoToFloatHistogram extracts a (normal integer) Histogram from the // provided proto message to a Float Histogram. The caller has to make sure that // the proto message represents an float histogram and not a integer histogram. -// Taken from https://github.com/prometheus/prometheus/blob/d33eb3ab17616a54b97d9f7791c791a79823f279/storage/remote/codec.go#L547-L560. +// Copied from https://github.com/prometheus/prometheus/blob/0ab95536115adfe50af249d36d73674be694ca3f/storage/remote/codec.go#L669-L688 func HistogramProtoToFloatHistogram(hp Histogram) *histogram.FloatHistogram { + if hp.IsFloatHistogram() { + panic("HistogramProtoToFloatHistogram called with a float histogram") + } return &histogram.FloatHistogram{ CounterResetHint: histogram.CounterResetHint(hp.ResetHint), Schema: hp.Schema, ZeroThreshold: hp.ZeroThreshold, - ZeroCount: hp.GetZeroCountFloat(), - Count: hp.GetCountFloat(), + ZeroCount: float64(hp.GetZeroCountInt()), + Count: float64(hp.GetCountInt()), Sum: hp.Sum, PositiveSpans: spansProtoToSpans(hp.GetPositiveSpans()), - PositiveBuckets: hp.GetPositiveCounts(), + PositiveBuckets: deltasToCounts(hp.GetPositiveDeltas()), NegativeSpans: spansProtoToSpans(hp.GetNegativeSpans()), - NegativeBuckets: hp.GetNegativeCounts(), + NegativeBuckets: deltasToCounts(hp.GetNegativeDeltas()), } } -func spansToSpansProto(s []histogram.Span) []*BucketSpan { - spans := make([]*BucketSpan, len(s)) +func spansProtoToSpans(s []BucketSpan) []histogram.Span { + spans := make([]histogram.Span, len(s)) for i := 0; i < len(s); i++ { - spans[i] = &BucketSpan{Offset: s[i].Offset, Length: s[i].Length} + spans[i] = histogram.Span{Offset: s[i].Offset, Length: s[i].Length} } return spans } -func spansProtoToSpans(s []*BucketSpan) []histogram.Span { - spans := make([]histogram.Span, len(s)) +func deltasToCounts(deltas []int64) []float64 { + counts := make([]float64, len(deltas)) + var cur float64 + for i, d := range deltas { + cur += float64(d) + counts[i] = cur + } + return counts +} + +// Copied from https://github.com/prometheus/prometheus/blob/0ab95536115adfe50af249d36d73674be694ca3f/storage/remote/codec.go#L709-L723 +func HistogramToHistogramProto(timestamp int64, h *histogram.Histogram) Histogram { + return Histogram{ + Count: &Histogram_CountInt{CountInt: h.Count}, + Sum: h.Sum, + Schema: h.Schema, + ZeroThreshold: h.ZeroThreshold, + ZeroCount: &Histogram_ZeroCountInt{ZeroCountInt: h.ZeroCount}, + NegativeSpans: spansToSpansProto(h.NegativeSpans), + NegativeDeltas: h.NegativeBuckets, + PositiveSpans: spansToSpansProto(h.PositiveSpans), + PositiveDeltas: h.PositiveBuckets, + ResetHint: Histogram_ResetHint(h.CounterResetHint), + Timestamp: timestamp, + } +} + +// Copied from https://github.com/prometheus/prometheus/blob/0ab95536115adfe50af249d36d73674be694ca3f/storage/remote/codec.go#L725-L739 +func FloatHistogramToHistogramProto(timestamp int64, fh *histogram.FloatHistogram) Histogram { + return Histogram{ + Count: &Histogram_CountFloat{CountFloat: fh.Count}, + Sum: fh.Sum, + Schema: fh.Schema, + ZeroThreshold: fh.ZeroThreshold, + ZeroCount: &Histogram_ZeroCountFloat{ZeroCountFloat: fh.ZeroCount}, + NegativeSpans: spansToSpansProto(fh.NegativeSpans), + NegativeCounts: fh.NegativeBuckets, + PositiveSpans: spansToSpansProto(fh.PositiveSpans), + PositiveCounts: fh.PositiveBuckets, + ResetHint: Histogram_ResetHint(fh.CounterResetHint), + Timestamp: timestamp, + } +} + +func spansToSpansProto(s []histogram.Span) []BucketSpan { + spans := make([]BucketSpan, len(s)) for i := 0; i < len(s); i++ { - spans[i] = histogram.Span{Offset: s[i].Offset, Length: s[i].Length} + spans[i] = BucketSpan{Offset: s[i].Offset, Length: s[i].Length} } return spans diff --git a/pkg/store/storepb/prompb/types.pb.go b/pkg/store/storepb/prompb/types.pb.go index f5332f345b..42c1b0fd9c 100644 --- a/pkg/store/storepb/prompb/types.pb.go +++ b/pkg/store/storepb/prompb/types.pb.go @@ -367,14 +367,14 @@ type Histogram struct { // *Histogram_ZeroCountFloat ZeroCount isHistogram_ZeroCount `protobuf_oneof:"zero_count"` // Negative Buckets. - NegativeSpans []*BucketSpan `protobuf:"bytes,8,rep,name=negative_spans,json=negativeSpans,proto3" json:"negative_spans,omitempty"` + NegativeSpans []BucketSpan `protobuf:"bytes,8,rep,name=negative_spans,json=negativeSpans,proto3" json:"negative_spans"` // Use either "negative_deltas" or "negative_counts", the former for // regular histograms with integer counts, the latter for float // histograms. NegativeDeltas []int64 `protobuf:"zigzag64,9,rep,packed,name=negative_deltas,json=negativeDeltas,proto3" json:"negative_deltas,omitempty"` NegativeCounts []float64 `protobuf:"fixed64,10,rep,packed,name=negative_counts,json=negativeCounts,proto3" json:"negative_counts,omitempty"` // Positive Buckets. - PositiveSpans []*BucketSpan `protobuf:"bytes,11,rep,name=positive_spans,json=positiveSpans,proto3" json:"positive_spans,omitempty"` + PositiveSpans []BucketSpan `protobuf:"bytes,11,rep,name=positive_spans,json=positiveSpans,proto3" json:"positive_spans"` // Use either "positive_deltas" or "positive_counts", the former for // regular histograms with integer counts, the latter for float // histograms. @@ -510,7 +510,7 @@ func (m *Histogram) GetZeroCountFloat() float64 { return 0 } -func (m *Histogram) GetNegativeSpans() []*BucketSpan { +func (m *Histogram) GetNegativeSpans() []BucketSpan { if m != nil { return m.NegativeSpans } @@ -531,7 +531,7 @@ func (m *Histogram) GetNegativeCounts() []float64 { return nil } -func (m *Histogram) GetPositiveSpans() []*BucketSpan { +func (m *Histogram) GetPositiveSpans() []BucketSpan { if m != nil { return m.PositiveSpans } @@ -988,79 +988,79 @@ func init() { func init() { proto.RegisterFile("store/storepb/prompb/types.proto", fileDescriptor_166e07899dab7c14) } var fileDescriptor_166e07899dab7c14 = []byte{ - // 1139 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x56, 0x4b, 0x8f, 0x1a, 0xc7, - 0x13, 0x67, 0x66, 0x60, 0x60, 0x6a, 0x01, 0x8f, 0x5b, 0xfe, 0xdb, 0xe3, 0xfd, 0x27, 0x2c, 0x19, - 0xe5, 0x81, 0xac, 0x04, 0x24, 0x7b, 0x95, 0x5c, 0x36, 0x51, 0x96, 0x0d, 0xfb, 0x50, 0x02, 0xc8, - 0x0d, 0xab, 0xc4, 0xbe, 0xa0, 0x06, 0x7a, 0x99, 0xd1, 0x32, 0x0f, 0x4d, 0x37, 0xd6, 0x92, 0x4f, - 0x91, 0x73, 0x6e, 0x51, 0x6e, 0xc9, 0x29, 0x1f, 0x21, 0x37, 0x9f, 0x22, 0x1f, 0xa3, 0x1c, 0xac, - 0x68, 0xf7, 0x8b, 0x44, 0xdd, 0x33, 0xc3, 0xc0, 0x12, 0x4b, 0x39, 0xf9, 0x82, 0xaa, 0x7e, 0xf5, - 0xfa, 0x4d, 0x57, 0x75, 0x35, 0x50, 0x67, 0x3c, 0x88, 0x68, 0x4b, 0xfe, 0x86, 0xe3, 0x56, 0x18, - 0x05, 0x5e, 0x38, 0x6e, 0xf1, 0x65, 0x48, 0x59, 0x33, 0x8c, 0x02, 0x1e, 0xa0, 0x3b, 0x02, 0xa3, - 0xdc, 0xa1, 0x0b, 0x36, 0x9a, 0x04, 0xe1, 0x72, 0xf7, 0xde, 0x2c, 0x98, 0x05, 0xd2, 0xd6, 0x12, - 0x52, 0xec, 0xb6, 0xfb, 0x30, 0x4e, 0x34, 0x27, 0x63, 0x3a, 0xdf, 0xcc, 0x60, 0xff, 0xa4, 0x42, - 0xb5, 0x4b, 0x79, 0xe4, 0x4e, 0xba, 0x94, 0x93, 0x29, 0xe1, 0x04, 0x7d, 0x01, 0x79, 0xe1, 0x61, - 0x29, 0x75, 0xa5, 0x51, 0x7d, 0xfc, 0xa8, 0x79, 0xab, 0x46, 0x73, 0xd3, 0x3d, 0x51, 0x87, 0xcb, - 0x90, 0x62, 0x19, 0x87, 0x3e, 0x06, 0xe4, 0x49, 0x6c, 0x74, 0x41, 0x3c, 0x77, 0xbe, 0x1c, 0xf9, - 0xc4, 0xa3, 0x96, 0x5a, 0x57, 0x1a, 0x06, 0x36, 0x63, 0xcb, 0xb1, 0x34, 0xf4, 0x88, 0x47, 0x11, - 0x82, 0xbc, 0x43, 0xe7, 0xa1, 0x95, 0x97, 0x76, 0x29, 0x0b, 0x6c, 0xe1, 0xbb, 0xdc, 0x2a, 0xc4, - 0x98, 0x90, 0xed, 0x25, 0x40, 0x56, 0x09, 0xed, 0x40, 0xf1, 0xbc, 0xf7, 0x75, 0xaf, 0xff, 0x6d, - 0xcf, 0xcc, 0x09, 0xe5, 0xa8, 0x7f, 0xde, 0x1b, 0x76, 0xb0, 0xa9, 0x20, 0x03, 0x0a, 0x27, 0x87, - 0xe7, 0x27, 0x1d, 0x53, 0x45, 0x15, 0x30, 0x4e, 0xcf, 0x06, 0xc3, 0xfe, 0x09, 0x3e, 0xec, 0x9a, - 0x1a, 0x42, 0x50, 0x95, 0x96, 0x0c, 0xcb, 0x8b, 0xd0, 0xc1, 0x79, 0xb7, 0x7b, 0x88, 0x9f, 0x99, - 0x05, 0x54, 0x82, 0xfc, 0x59, 0xef, 0xb8, 0x6f, 0xea, 0xa8, 0x0c, 0xa5, 0xc1, 0xf0, 0x70, 0xd8, - 0x19, 0x74, 0x86, 0x66, 0xd1, 0x3e, 0x00, 0x7d, 0x40, 0xbc, 0x70, 0x4e, 0xd1, 0x3d, 0x28, 0xbc, - 0x20, 0xf3, 0x45, 0x7c, 0x36, 0x0a, 0x8e, 0x15, 0xf4, 0x0e, 0x18, 0xdc, 0xf5, 0x28, 0xe3, 0xc4, - 0x0b, 0xe5, 0x77, 0x6a, 0x38, 0x03, 0xec, 0x9f, 0x15, 0x28, 0x75, 0xae, 0xa8, 0x17, 0xce, 0x49, - 0x84, 0x26, 0xa0, 0xcb, 0x2e, 0x30, 0x4b, 0xa9, 0x6b, 0x8d, 0x9d, 0xc7, 0x95, 0x26, 0x77, 0x88, - 0x1f, 0xb0, 0xe6, 0x37, 0x02, 0x6d, 0x1f, 0xbc, 0x7c, 0xbd, 0x97, 0xfb, 0xeb, 0xf5, 0xde, 0xfe, - 0xcc, 0xe5, 0xce, 0x62, 0xdc, 0x9c, 0x04, 0x5e, 0x2b, 0x76, 0xf8, 0xc4, 0x0d, 0x12, 0xa9, 0x15, - 0x5e, 0xce, 0x5a, 0x1b, 0x0d, 0x6d, 0x3e, 0x97, 0xd1, 0x38, 0x49, 0x9d, 0xb1, 0x54, 0xdf, 0xc8, - 0x52, 0xbb, 0xcd, 0xf2, 0x8f, 0x02, 0x18, 0xa7, 0x2e, 0xe3, 0xc1, 0x2c, 0x22, 0x1e, 0x7a, 0x17, - 0x8c, 0x49, 0xb0, 0xf0, 0xf9, 0xc8, 0xf5, 0xb9, 0xfc, 0xd6, 0xfc, 0x69, 0x0e, 0x97, 0x24, 0x74, - 0xe6, 0x73, 0xf4, 0x1e, 0xec, 0xc4, 0xe6, 0x8b, 0x79, 0x40, 0x78, 0x5c, 0xe6, 0x34, 0x87, 0x41, - 0x82, 0xc7, 0x02, 0x43, 0x26, 0x68, 0x6c, 0xe1, 0xc9, 0x3a, 0x0a, 0x16, 0x22, 0xba, 0x0f, 0x3a, - 0x9b, 0x38, 0xd4, 0x23, 0xb2, 0xd5, 0x77, 0x71, 0xa2, 0xa1, 0x0f, 0xa0, 0xfa, 0x3d, 0x8d, 0x82, - 0x11, 0x77, 0x22, 0xca, 0x9c, 0x60, 0x3e, 0x95, 0x6d, 0x57, 0x70, 0x45, 0xa0, 0xc3, 0x14, 0x44, - 0x1f, 0x26, 0x6e, 0x19, 0x2f, 0x5d, 0xf2, 0x52, 0x70, 0x59, 0xe0, 0x47, 0x29, 0xb7, 0x47, 0x60, - 0xae, 0xf9, 0xc5, 0x04, 0x8b, 0x92, 0xa0, 0x82, 0xab, 0x2b, 0xcf, 0x98, 0x64, 0x1b, 0xaa, 0x3e, - 0x9d, 0x11, 0xee, 0xbe, 0xa0, 0x23, 0x16, 0x12, 0x9f, 0x59, 0x25, 0xd9, 0x95, 0xff, 0x6f, 0xcd, - 0x7c, 0x7b, 0x31, 0xb9, 0xa4, 0x7c, 0x10, 0x12, 0x1f, 0x57, 0xd2, 0x10, 0xa1, 0x31, 0xf4, 0x11, - 0xdc, 0x59, 0xe5, 0x98, 0xd2, 0x39, 0x27, 0xcc, 0x32, 0xea, 0x5a, 0x03, 0xe1, 0x55, 0xea, 0xaf, - 0x24, 0xba, 0xe1, 0x28, 0xc9, 0x31, 0x0b, 0xea, 0x5a, 0x43, 0xc9, 0x1c, 0x25, 0x33, 0x26, 0x58, - 0x85, 0x01, 0x73, 0xd7, 0x58, 0xed, 0xfc, 0x07, 0x56, 0x69, 0xc8, 0x8a, 0xd5, 0x2a, 0x47, 0xc2, - 0xaa, 0x1c, 0xb3, 0x4a, 0xe1, 0x8c, 0xd5, 0xca, 0x31, 0x61, 0x55, 0x89, 0x59, 0xa5, 0x70, 0xc2, - 0xea, 0x08, 0x20, 0xa2, 0x8c, 0xf2, 0x91, 0x23, 0xce, 0xbe, 0x2a, 0x77, 0xc3, 0xfb, 0x5b, 0x8c, - 0x56, 0x23, 0xd4, 0xc4, 0xc2, 0xf9, 0xd4, 0xf5, 0x39, 0x36, 0xa2, 0x54, 0xdc, 0x9c, 0xc1, 0x3b, - 0xb7, 0x67, 0x70, 0x1f, 0x8c, 0x55, 0xd4, 0xe6, 0x0d, 0x2f, 0x82, 0xf6, 0xac, 0x33, 0x30, 0x15, - 0xa4, 0x83, 0xda, 0xeb, 0x9b, 0x6a, 0x76, 0xcb, 0xb5, 0x76, 0x11, 0x0a, 0x92, 0x78, 0xbb, 0x0c, - 0x90, 0x75, 0xde, 0x3e, 0x00, 0xc8, 0x8e, 0x47, 0x0c, 0x5f, 0x70, 0x71, 0xc1, 0x68, 0x3c, 0xcd, - 0x77, 0x71, 0xa2, 0x09, 0x7c, 0x4e, 0xfd, 0x19, 0x77, 0xe4, 0x10, 0x57, 0x70, 0xa2, 0xd9, 0xbf, - 0xaa, 0x00, 0x43, 0xd7, 0xa3, 0x03, 0x1a, 0xb9, 0x94, 0xbd, 0x9d, 0x6b, 0xfb, 0x19, 0x14, 0x99, - 0x5c, 0x33, 0xcc, 0x52, 0x65, 0x95, 0x07, 0x5b, 0xc7, 0x1b, 0xaf, 0xa1, 0x76, 0x5e, 0xd4, 0xc3, - 0xa9, 0x37, 0xfa, 0x1c, 0x0c, 0x9a, 0x2c, 0x18, 0x66, 0x69, 0x32, 0xf4, 0xe1, 0x56, 0x68, 0xba, - 0x82, 0x92, 0xe0, 0x2c, 0x02, 0x7d, 0x09, 0xe0, 0xa4, 0x6d, 0x63, 0x56, 0x5e, 0xc6, 0xef, 0xbe, - 0xb9, 0xb3, 0x49, 0x82, 0xb5, 0x18, 0xfb, 0x47, 0x05, 0xca, 0xf2, 0x5b, 0xba, 0x84, 0x4f, 0x1c, - 0x1a, 0xa1, 0x4f, 0x37, 0x9e, 0x10, 0x7b, 0x2b, 0xd9, 0xba, 0x73, 0x73, 0xed, 0xe9, 0x40, 0x90, - 0x5f, 0x7b, 0x2c, 0xa4, 0x9c, 0x6d, 0x33, 0x4d, 0x82, 0xb1, 0x62, 0x37, 0x20, 0x2f, 0x1f, 0x02, - 0x1d, 0xd4, 0xce, 0xd3, 0x78, 0x42, 0x7a, 0x9d, 0xa7, 0xf1, 0x84, 0x60, 0xb1, 0xfc, 0x05, 0x80, - 0x3b, 0xa6, 0x66, 0xff, 0xa6, 0x88, 0xb1, 0x22, 0x53, 0x31, 0x55, 0x0c, 0x3d, 0x80, 0x22, 0xe3, - 0x34, 0x1c, 0x79, 0x4c, 0x92, 0xd3, 0xb0, 0x2e, 0xd4, 0x2e, 0x13, 0xa5, 0x2f, 0x16, 0xfe, 0x24, - 0x2d, 0x2d, 0x64, 0xf4, 0x10, 0x4a, 0x8c, 0x93, 0x88, 0x0b, 0xef, 0x78, 0x63, 0x16, 0xa5, 0xde, - 0x65, 0xe8, 0x7f, 0xa0, 0x53, 0x7f, 0x3a, 0x92, 0x07, 0x26, 0x0c, 0x05, 0xea, 0x4f, 0xbb, 0x0c, - 0xed, 0x42, 0x69, 0x16, 0x05, 0x8b, 0xd0, 0xf5, 0x67, 0x56, 0xa1, 0xae, 0x35, 0x0c, 0xbc, 0xd2, - 0x51, 0x15, 0xd4, 0xf1, 0x52, 0x6e, 0xad, 0x12, 0x56, 0xc7, 0x4b, 0x91, 0x3d, 0x22, 0xfe, 0x8c, - 0x8a, 0x24, 0xc5, 0x38, 0xbb, 0xd4, 0xbb, 0xcc, 0xfe, 0x5d, 0x81, 0xc2, 0x91, 0xb3, 0xf0, 0x2f, - 0x51, 0x0d, 0x76, 0x3c, 0xd7, 0x1f, 0x89, 0x4b, 0x92, 0x71, 0x36, 0x3c, 0xd7, 0x17, 0xd3, 0xd9, - 0x65, 0xd2, 0x4e, 0xae, 0x56, 0xf6, 0xe4, 0xf5, 0xf1, 0xc8, 0x55, 0x62, 0x7f, 0x92, 0x74, 0x42, - 0x93, 0x9d, 0xd8, 0xdb, 0xea, 0x84, 0xac, 0xd2, 0xec, 0xf8, 0x93, 0x60, 0xea, 0xfa, 0xb3, 0xac, - 0x0d, 0xe2, 0x69, 0x97, 0x9f, 0x56, 0xc6, 0x52, 0xb6, 0x5b, 0x50, 0x4a, 0xbd, 0xb6, 0xee, 0xe6, - 0x77, 0x7d, 0xf1, 0xf2, 0x6e, 0x3c, 0xb7, 0xaa, 0xfd, 0x8b, 0x02, 0x15, 0x99, 0x9d, 0x4e, 0xdf, - 0xe6, 0x2d, 0xda, 0x07, 0x7d, 0x22, 0xaa, 0xa6, 0x97, 0xe8, 0xfe, 0xbf, 0x7f, 0x72, 0x32, 0xc5, - 0x89, 0x6f, 0xbb, 0xfe, 0xf2, 0xba, 0xa6, 0xbc, 0xba, 0xae, 0x29, 0x7f, 0x5f, 0xd7, 0x94, 0x1f, - 0x6e, 0x6a, 0xb9, 0x57, 0x37, 0xb5, 0xdc, 0x9f, 0x37, 0xb5, 0xdc, 0x73, 0x3d, 0xfe, 0xdb, 0x35, - 0xd6, 0xe5, 0xff, 0xa5, 0x27, 0xff, 0x04, 0x00, 0x00, 0xff, 0xff, 0x0e, 0x67, 0xfa, 0x2f, 0x95, - 0x09, 0x00, 0x00, + // 1140 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x56, 0xcf, 0x8f, 0xda, 0xc6, + 0x17, 0xc7, 0x36, 0x18, 0xfc, 0x16, 0x88, 0x33, 0xca, 0x37, 0x71, 0xf6, 0xdb, 0x12, 0x6a, 0xf5, + 0x07, 0x8a, 0x5a, 0x90, 0x92, 0xa8, 0xbd, 0x6c, 0xab, 0x2e, 0x5b, 0x76, 0x59, 0xb5, 0x80, 0x32, + 0xb0, 0x6a, 0x93, 0x0b, 0x1a, 0x60, 0x16, 0x5b, 0x8b, 0x7f, 0xc8, 0x33, 0x44, 0x4b, 0xff, 0x8a, + 0x9e, 0x7b, 0xab, 0x7a, 0x6b, 0x4f, 0xfd, 0x13, 0x7a, 0xcb, 0x31, 0xc7, 0x2a, 0x87, 0xa8, 0xda, + 0xfd, 0x47, 0xaa, 0x19, 0xdb, 0x18, 0x96, 0x46, 0xea, 0x29, 0x17, 0xf4, 0xde, 0xe7, 0xfd, 0xfa, + 0x8c, 0xdf, 0x9b, 0x79, 0x40, 0x9d, 0xf1, 0x20, 0xa2, 0x2d, 0xf9, 0x1b, 0x4e, 0x5a, 0x61, 0x14, + 0x78, 0xe1, 0xa4, 0xc5, 0x57, 0x21, 0x65, 0xcd, 0x30, 0x0a, 0x78, 0x80, 0x6e, 0x09, 0x8c, 0x72, + 0x87, 0x2e, 0xd9, 0x78, 0x1a, 0x84, 0xab, 0xfd, 0x3b, 0xf3, 0x60, 0x1e, 0x48, 0x5b, 0x4b, 0x48, + 0xb1, 0xdb, 0xfe, 0xfd, 0x38, 0xd1, 0x82, 0x4c, 0xe8, 0x62, 0x3b, 0x83, 0xfd, 0x8b, 0x0a, 0xd5, + 0x1e, 0xe5, 0x91, 0x3b, 0xed, 0x51, 0x4e, 0x66, 0x84, 0x13, 0xf4, 0x15, 0xe4, 0x85, 0x87, 0xa5, + 0xd4, 0x95, 0x46, 0xf5, 0xd1, 0xc3, 0xe6, 0x8d, 0x1a, 0xcd, 0x6d, 0xf7, 0x44, 0x1d, 0xad, 0x42, + 0x8a, 0x65, 0x1c, 0xfa, 0x14, 0x90, 0x27, 0xb1, 0xf1, 0x39, 0xf1, 0xdc, 0xc5, 0x6a, 0xec, 0x13, + 0x8f, 0x5a, 0x6a, 0x5d, 0x69, 0x18, 0xd8, 0x8c, 0x2d, 0xc7, 0xd2, 0xd0, 0x27, 0x1e, 0x45, 0x08, + 0xf2, 0x0e, 0x5d, 0x84, 0x56, 0x5e, 0xda, 0xa5, 0x2c, 0xb0, 0xa5, 0xef, 0x72, 0xab, 0x10, 0x63, + 0x42, 0xb6, 0x57, 0x00, 0x59, 0x25, 0xb4, 0x07, 0xc5, 0xb3, 0xfe, 0xb7, 0xfd, 0xc1, 0xf7, 0x7d, + 0x33, 0x27, 0x94, 0xa3, 0xc1, 0x59, 0x7f, 0xd4, 0xc1, 0xa6, 0x82, 0x0c, 0x28, 0x9c, 0x1c, 0x9e, + 0x9d, 0x74, 0x4c, 0x15, 0x55, 0xc0, 0xe8, 0x9e, 0x0e, 0x47, 0x83, 0x13, 0x7c, 0xd8, 0x33, 0x35, + 0x84, 0xa0, 0x2a, 0x2d, 0x19, 0x96, 0x17, 0xa1, 0xc3, 0xb3, 0x5e, 0xef, 0x10, 0x3f, 0x33, 0x0b, + 0xa8, 0x04, 0xf9, 0xd3, 0xfe, 0xf1, 0xc0, 0xd4, 0x51, 0x19, 0x4a, 0xc3, 0xd1, 0xe1, 0xa8, 0x33, + 0xec, 0x8c, 0xcc, 0xa2, 0x7d, 0x00, 0xfa, 0x90, 0x78, 0xe1, 0x82, 0xa2, 0x3b, 0x50, 0x78, 0x41, + 0x16, 0xcb, 0xf8, 0xdb, 0x28, 0x38, 0x56, 0xd0, 0x7b, 0x60, 0x70, 0xd7, 0xa3, 0x8c, 0x13, 0x2f, + 0x94, 0xe7, 0xd4, 0x70, 0x06, 0xd8, 0xbf, 0x2a, 0x50, 0xea, 0x5c, 0x52, 0x2f, 0x5c, 0x90, 0x08, + 0x4d, 0x41, 0x97, 0x5d, 0x60, 0x96, 0x52, 0xd7, 0x1a, 0x7b, 0x8f, 0x2a, 0x4d, 0xee, 0x10, 0x3f, + 0x60, 0xcd, 0xef, 0x04, 0xda, 0x3e, 0x78, 0xf9, 0xe6, 0x41, 0xee, 0xf5, 0x9b, 0x07, 0x4f, 0xe6, + 0x2e, 0x77, 0x96, 0x93, 0xe6, 0x34, 0xf0, 0x5a, 0xb1, 0xc3, 0x67, 0x6e, 0x90, 0x48, 0xad, 0xf0, + 0x62, 0xde, 0xda, 0x6a, 0x68, 0xf3, 0xb9, 0x8c, 0xc6, 0x49, 0xea, 0x8c, 0xa5, 0xfa, 0x56, 0x96, + 0xda, 0x4d, 0x96, 0xaf, 0x0b, 0x60, 0x74, 0x5d, 0xc6, 0x83, 0x79, 0x44, 0x3c, 0xf4, 0x3e, 0x18, + 0xd3, 0x60, 0xe9, 0xf3, 0xb1, 0xeb, 0x73, 0x79, 0xd6, 0x7c, 0x37, 0x87, 0x4b, 0x12, 0x3a, 0xf5, + 0x39, 0xfa, 0x00, 0xf6, 0x62, 0xf3, 0xf9, 0x22, 0x20, 0x3c, 0x2e, 0xd3, 0xcd, 0x61, 0x90, 0xe0, + 0xb1, 0xc0, 0x90, 0x09, 0x1a, 0x5b, 0x7a, 0xb2, 0x8e, 0x82, 0x85, 0x88, 0xee, 0x82, 0xce, 0xa6, + 0x0e, 0xf5, 0x88, 0x6c, 0xf5, 0x6d, 0x9c, 0x68, 0xe8, 0x23, 0xa8, 0xfe, 0x48, 0xa3, 0x60, 0xcc, + 0x9d, 0x88, 0x32, 0x27, 0x58, 0xcc, 0x64, 0xdb, 0x15, 0x5c, 0x11, 0xe8, 0x28, 0x05, 0xd1, 0xc7, + 0x89, 0x5b, 0xc6, 0x4b, 0x97, 0xbc, 0x14, 0x5c, 0x16, 0xf8, 0x51, 0xca, 0xed, 0x21, 0x98, 0x1b, + 0x7e, 0x31, 0xc1, 0xa2, 0x24, 0xa8, 0xe0, 0xea, 0xda, 0x33, 0x26, 0xd9, 0x85, 0xaa, 0x4f, 0xe7, + 0x84, 0xbb, 0x2f, 0xe8, 0x98, 0x85, 0xc4, 0x67, 0x56, 0x49, 0x76, 0xe5, 0xff, 0x3b, 0x33, 0xdf, + 0x5e, 0x4e, 0x2f, 0x28, 0x1f, 0x86, 0xc4, 0x6f, 0xe7, 0x45, 0x8f, 0x70, 0x25, 0x0d, 0x14, 0x18, + 0x43, 0x9f, 0xc0, 0xad, 0x75, 0xa6, 0x19, 0x5d, 0x70, 0xc2, 0x2c, 0xa3, 0xae, 0x35, 0x10, 0x5e, + 0x17, 0xf8, 0x46, 0xa2, 0x5b, 0x8e, 0x92, 0x22, 0xb3, 0xa0, 0xae, 0x35, 0x94, 0xcc, 0x51, 0xf2, + 0x63, 0x82, 0x5b, 0x18, 0x30, 0x77, 0x83, 0xdb, 0xde, 0x7f, 0xe6, 0x96, 0x06, 0xae, 0xb9, 0xad, + 0x33, 0x25, 0xdc, 0xca, 0x31, 0xb7, 0x14, 0xce, 0xb8, 0xad, 0x1d, 0x13, 0x6e, 0x95, 0x98, 0x5b, + 0x0a, 0x27, 0xdc, 0x8e, 0x00, 0x22, 0xca, 0x28, 0x1f, 0x3b, 0xa2, 0x0f, 0x55, 0xf9, 0x4e, 0x7c, + 0xb8, 0xc3, 0x6b, 0x3d, 0x4e, 0x4d, 0x2c, 0x9c, 0xbb, 0xae, 0xcf, 0xb1, 0x11, 0xa5, 0xe2, 0xf6, + 0x3c, 0xde, 0xba, 0x39, 0x8f, 0x4f, 0xc0, 0x58, 0x47, 0x6d, 0xdf, 0xf6, 0x22, 0x68, 0xcf, 0x3a, + 0x43, 0x53, 0x41, 0x3a, 0xa8, 0xfd, 0x81, 0xa9, 0x66, 0x37, 0x5e, 0x6b, 0x17, 0xa1, 0x20, 0x89, + 0xb7, 0xcb, 0x00, 0xd9, 0x14, 0xd8, 0x07, 0x00, 0xd9, 0x47, 0x12, 0x83, 0x18, 0x9c, 0x9f, 0x33, + 0x1a, 0x4f, 0xf6, 0x6d, 0x9c, 0x68, 0x02, 0x5f, 0x50, 0x7f, 0xce, 0x1d, 0x39, 0xd0, 0x15, 0x9c, + 0x68, 0xf6, 0xef, 0x2a, 0xc0, 0xc8, 0xf5, 0xe8, 0x90, 0x46, 0x2e, 0x65, 0xef, 0xe6, 0x0a, 0x7f, + 0x01, 0x45, 0x26, 0x9f, 0x1c, 0x66, 0xa9, 0xb2, 0xca, 0xbd, 0x9d, 0xcf, 0x1b, 0x3f, 0x49, 0x49, + 0xcb, 0x53, 0x6f, 0xf4, 0x25, 0x18, 0x34, 0x79, 0x6c, 0x98, 0xa5, 0xc9, 0xd0, 0xfb, 0x3b, 0xa1, + 0xe9, 0x73, 0x94, 0x04, 0x67, 0x11, 0xe8, 0x6b, 0x00, 0x27, 0x6d, 0x1b, 0xb3, 0xf2, 0x32, 0x7e, + 0xff, 0xed, 0x9d, 0x4d, 0x12, 0x6c, 0xc4, 0xd8, 0x3f, 0x2b, 0x50, 0x96, 0x67, 0xe9, 0x11, 0x3e, + 0x75, 0x68, 0x84, 0x3e, 0xdf, 0x5a, 0x27, 0xf6, 0x4e, 0xb2, 0x4d, 0xe7, 0xe6, 0xc6, 0x1a, 0x41, + 0x90, 0xdf, 0x58, 0x1c, 0x52, 0xce, 0x5e, 0x36, 0x4d, 0x82, 0xb1, 0x62, 0x37, 0x20, 0x2f, 0x97, + 0x82, 0x0e, 0x6a, 0xe7, 0x69, 0x3c, 0x21, 0xfd, 0xce, 0xd3, 0x78, 0x42, 0xb0, 0x58, 0x04, 0x02, + 0xc0, 0x1d, 0x53, 0xb3, 0xff, 0x50, 0xc4, 0x58, 0x91, 0x99, 0x98, 0x2a, 0x86, 0xee, 0x41, 0x91, + 0x71, 0x1a, 0x8e, 0x3d, 0x26, 0xc9, 0x69, 0x58, 0x17, 0x6a, 0x8f, 0x89, 0xd2, 0xe7, 0x4b, 0x7f, + 0x9a, 0x96, 0x16, 0x32, 0xba, 0x0f, 0x25, 0xc6, 0x49, 0xc4, 0x85, 0x77, 0xfc, 0x7a, 0x16, 0xa5, + 0xde, 0x63, 0xe8, 0x7f, 0xa0, 0x53, 0x7f, 0x36, 0x96, 0x1f, 0x4c, 0x18, 0x0a, 0xd4, 0x9f, 0xf5, + 0x18, 0xda, 0x87, 0xd2, 0x3c, 0x0a, 0x96, 0xa1, 0xeb, 0xcf, 0xad, 0x42, 0x5d, 0x6b, 0x18, 0x78, + 0xad, 0xa3, 0x2a, 0xa8, 0x93, 0x95, 0x7c, 0xc1, 0x4a, 0x58, 0x9d, 0xac, 0x44, 0xf6, 0x88, 0xf8, + 0x73, 0x2a, 0x92, 0x14, 0xe3, 0xec, 0x52, 0xef, 0x31, 0xfb, 0x4f, 0x05, 0x0a, 0x47, 0xce, 0xd2, + 0xbf, 0x40, 0x35, 0xd8, 0xf3, 0x5c, 0x7f, 0x2c, 0x2e, 0x49, 0xc6, 0xd9, 0xf0, 0x5c, 0x5f, 0x4c, + 0x67, 0x8f, 0x49, 0x3b, 0xb9, 0x5c, 0xdb, 0x93, 0x4d, 0xe4, 0x91, 0xcb, 0xc4, 0xfe, 0x38, 0xe9, + 0x84, 0x26, 0x3b, 0xf1, 0x60, 0xa7, 0x13, 0xb2, 0x4a, 0xb3, 0xe3, 0x4f, 0x83, 0x99, 0xeb, 0xcf, + 0xb3, 0x36, 0x88, 0x35, 0x2f, 0x8f, 0x56, 0xc6, 0x52, 0xb6, 0x5b, 0x50, 0x4a, 0xbd, 0x76, 0xee, + 0xe6, 0x0f, 0x03, 0xb1, 0x85, 0xb7, 0x56, 0xaf, 0x6a, 0xff, 0xa6, 0x40, 0x45, 0x66, 0xa7, 0xb3, + 0x77, 0x79, 0x8b, 0x9e, 0x80, 0x3e, 0x15, 0x55, 0xd3, 0x4b, 0x74, 0xf7, 0xdf, 0x8f, 0x9c, 0x4c, + 0x71, 0xe2, 0xdb, 0xae, 0xbf, 0xbc, 0xaa, 0x29, 0xaf, 0xae, 0x6a, 0xca, 0xdf, 0x57, 0x35, 0xe5, + 0xa7, 0xeb, 0x5a, 0xee, 0xd5, 0x75, 0x2d, 0xf7, 0xd7, 0x75, 0x2d, 0xf7, 0x5c, 0x8f, 0xff, 0x82, + 0x4d, 0x74, 0xf9, 0xdf, 0xe9, 0xf1, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x98, 0x09, 0x9b, 0xd5, + 0xa1, 0x09, 0x00, 0x00, } func (m *MetricMetadata) Marshal() (dAtA []byte, err error) { @@ -2559,7 +2559,7 @@ func (m *Histogram) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.NegativeSpans = append(m.NegativeSpans, &BucketSpan{}) + m.NegativeSpans = append(m.NegativeSpans, BucketSpan{}) if err := m.NegativeSpans[len(m.NegativeSpans)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } @@ -2725,7 +2725,7 @@ func (m *Histogram) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.PositiveSpans = append(m.PositiveSpans, &BucketSpan{}) + m.PositiveSpans = append(m.PositiveSpans, BucketSpan{}) if err := m.PositiveSpans[len(m.PositiveSpans)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } diff --git a/pkg/store/storepb/prompb/types.proto b/pkg/store/storepb/prompb/types.proto index 0fea825508..bf00b62e8f 100644 --- a/pkg/store/storepb/prompb/types.proto +++ b/pkg/store/storepb/prompb/types.proto @@ -94,9 +94,9 @@ message Histogram { uint64 zero_count_int = 6; double zero_count_float = 7; } - + // Negative Buckets. - repeated BucketSpan negative_spans = 8; + repeated BucketSpan negative_spans = 8 [(gogoproto.nullable) = false]; // Use either "negative_deltas" or "negative_counts", the former for // regular histograms with integer counts, the latter for float // histograms. @@ -104,7 +104,7 @@ message Histogram { repeated double negative_counts = 10; // Absolute count of each bucket. // Positive Buckets. - repeated BucketSpan positive_spans = 11; + repeated BucketSpan positive_spans = 11 [(gogoproto.nullable) = false]; // Use either "positive_deltas" or "positive_counts", the former for // regular histograms with integer counts, the latter for float // histograms. diff --git a/pkg/store/storepb/types.pb.go b/pkg/store/storepb/types.pb.go index a87a135ba0..401dcad2dc 100644 --- a/pkg/store/storepb/types.pb.go +++ b/pkg/store/storepb/types.pb.go @@ -63,18 +63,21 @@ func (PartialResponseStrategy) EnumDescriptor() ([]byte, []int) { type Chunk_Encoding int32 const ( - Chunk_XOR Chunk_Encoding = 0 - Chunk_HISTOGRAM Chunk_Encoding = 1 + Chunk_XOR Chunk_Encoding = 0 + Chunk_HISTOGRAM Chunk_Encoding = 1 + Chunk_FLOAT_HISTOGRAM Chunk_Encoding = 2 ) var Chunk_Encoding_name = map[int32]string{ 0: "XOR", 1: "HISTOGRAM", + 2: "FLOAT_HISTOGRAM", } var Chunk_Encoding_value = map[string]int32{ - "XOR": 0, - "HISTOGRAM": 1, + "XOR": 0, + "HISTOGRAM": 1, + "FLOAT_HISTOGRAM": 2, } func (x Chunk_Encoding) String() string { @@ -290,42 +293,43 @@ func init() { func init() { proto.RegisterFile("store/storepb/types.proto", fileDescriptor_121fba57de02d8e0) } var fileDescriptor_121fba57de02d8e0 = []byte{ - // 553 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x93, 0xcf, 0x6e, 0xd3, 0x4e, - 0x10, 0xc7, 0xbd, 0xb6, 0xe3, 0x24, 0xf3, 0x6b, 0x7f, 0x32, 0xab, 0x0a, 0xdc, 0x1e, 0x9c, 0xc8, - 0x08, 0x11, 0x55, 0xaa, 0x2d, 0x15, 0x8e, 0x5c, 0x12, 0x14, 0x01, 0x12, 0x6d, 0xe8, 0x26, 0x12, - 0xa8, 0x17, 0xb4, 0x71, 0x57, 0xb6, 0xd5, 0xf8, 0x8f, 0xbc, 0x6b, 0x48, 0x1e, 0x80, 0x3b, 0x88, - 0x3b, 0xcf, 0x93, 0x63, 0x8f, 0x88, 0x43, 0x04, 0xc9, 0x8b, 0x20, 0xaf, 0x1d, 0x20, 0x52, 0x2e, - 0xd6, 0x78, 0x3e, 0xdf, 0x99, 0xd9, 0x99, 0x9d, 0x85, 0x63, 0x2e, 0xd2, 0x9c, 0x79, 0xf2, 0x9b, - 0x4d, 0x3d, 0xb1, 0xc8, 0x18, 0x77, 0xb3, 0x3c, 0x15, 0x29, 0x36, 0x44, 0x48, 0x93, 0x94, 0x9f, - 0x1c, 0x05, 0x69, 0x90, 0x4a, 0x97, 0x57, 0x5a, 0x15, 0x3d, 0xa9, 0x03, 0x67, 0x74, 0xca, 0x66, - 0xbb, 0x81, 0xce, 0x27, 0x04, 0x8d, 0xe7, 0x61, 0x91, 0xdc, 0xe2, 0x53, 0xd0, 0x4b, 0x60, 0xa1, - 0x2e, 0xea, 0xfd, 0x7f, 0x7e, 0xdf, 0xad, 0x32, 0xba, 0x12, 0xba, 0xc3, 0xc4, 0x4f, 0x6f, 0xa2, - 0x24, 0x20, 0x52, 0x83, 0x31, 0xe8, 0x37, 0x54, 0x50, 0x4b, 0xed, 0xa2, 0xde, 0x01, 0x91, 0x36, - 0xb6, 0x40, 0x0f, 0x29, 0x0f, 0x2d, 0xad, 0x8b, 0x7a, 0xfa, 0x40, 0x5f, 0xae, 0x3a, 0x88, 0x48, - 0x8f, 0xe3, 0x40, 0x6b, 0x1b, 0x8f, 0x9b, 0xa0, 0xbd, 0x1b, 0x11, 0x53, 0xc1, 0x87, 0xd0, 0x7e, - 0xf9, 0x6a, 0x3c, 0x19, 0xbd, 0x20, 0xfd, 0x0b, 0x13, 0x39, 0xdf, 0x10, 0x18, 0x63, 0x96, 0x47, - 0x8c, 0x63, 0x1f, 0x0c, 0x79, 0x52, 0x6e, 0xa1, 0xae, 0xd6, 0xfb, 0xef, 0xfc, 0x70, 0x7b, 0x94, - 0xd7, 0xa5, 0x77, 0xf0, 0x6c, 0xb9, 0xea, 0x28, 0x3f, 0x56, 0x9d, 0xa7, 0x41, 0x24, 0xc2, 0x62, - 0xea, 0xfa, 0x69, 0xec, 0x55, 0x82, 0xb3, 0x28, 0xad, 0x2d, 0x2f, 0xbb, 0x0d, 0xbc, 0x9d, 0xa6, - 0xdd, 0x6b, 0x19, 0x4d, 0xea, 0xd4, 0xd8, 0x03, 0xc3, 0x2f, 0x3b, 0xe3, 0x96, 0x2a, 0x8b, 0xdc, - 0xdb, 0x16, 0xe9, 0x07, 0x41, 0x2e, 0x7b, 0x96, 0x2d, 0x28, 0xa4, 0x96, 0x39, 0x5f, 0x55, 0x68, - 0xff, 0x61, 0xf8, 0x18, 0x5a, 0x71, 0x94, 0xbc, 0x17, 0x51, 0x5c, 0x0d, 0x4c, 0x23, 0xcd, 0x38, - 0x4a, 0x26, 0x51, 0xcc, 0x24, 0xa2, 0xf3, 0x0a, 0xa9, 0x35, 0xa2, 0x73, 0x89, 0x3a, 0xa0, 0xe5, - 0xf4, 0xa3, 0x9c, 0xd0, 0x3f, 0x6d, 0xc9, 0x8c, 0xa4, 0x24, 0xf8, 0x21, 0x34, 0xfc, 0xb4, 0x48, - 0x84, 0xa5, 0xef, 0x93, 0x54, 0xac, 0xcc, 0xc2, 0x8b, 0xd8, 0x6a, 0xec, 0xcd, 0xc2, 0x8b, 0xb8, - 0x14, 0xc4, 0x51, 0x62, 0x19, 0x7b, 0x05, 0x71, 0x94, 0x48, 0x01, 0x9d, 0x5b, 0xcd, 0xfd, 0x02, - 0x3a, 0xc7, 0x8f, 0xa1, 0x29, 0x6b, 0xb1, 0xdc, 0x6a, 0xed, 0x13, 0x6d, 0xa9, 0xf3, 0x05, 0xc1, - 0x81, 0x1c, 0xec, 0x05, 0x15, 0x7e, 0xc8, 0x72, 0x7c, 0xb6, 0xb3, 0x45, 0xc7, 0x3b, 0x57, 0x57, - 0x6b, 0xdc, 0xc9, 0x22, 0x63, 0x7f, 0x17, 0x29, 0xa1, 0xf5, 0xa0, 0xda, 0x44, 0xda, 0xf8, 0x08, - 0x1a, 0x1f, 0xe8, 0xac, 0x60, 0x72, 0x4e, 0x6d, 0x52, 0xfd, 0x38, 0x3d, 0xd0, 0xcb, 0x38, 0x6c, - 0x80, 0x3a, 0xbc, 0x32, 0x95, 0x72, 0x91, 0x2e, 0x87, 0x57, 0x26, 0x2a, 0x1d, 0x64, 0x68, 0xaa, - 0xd2, 0x41, 0x86, 0xa6, 0x76, 0xea, 0xc2, 0x83, 0x37, 0x34, 0x17, 0x11, 0x9d, 0x11, 0xc6, 0xb3, - 0x34, 0xe1, 0x6c, 0x2c, 0x72, 0x2a, 0x58, 0xb0, 0xc0, 0x2d, 0xd0, 0xdf, 0xf6, 0xc9, 0xa5, 0xa9, - 0xe0, 0x36, 0x34, 0xfa, 0x83, 0x11, 0x99, 0x98, 0x68, 0xf0, 0x68, 0xf9, 0xcb, 0x56, 0x96, 0x6b, - 0x1b, 0xdd, 0xad, 0x6d, 0xf4, 0x73, 0x6d, 0xa3, 0xcf, 0x1b, 0x5b, 0xb9, 0xdb, 0xd8, 0xca, 0xf7, - 0x8d, 0xad, 0x5c, 0x37, 0xeb, 0xe7, 0x36, 0x35, 0xe4, 0x83, 0x79, 0xf2, 0x3b, 0x00, 0x00, 0xff, - 0xff, 0x44, 0x9d, 0x95, 0xa2, 0x86, 0x03, 0x00, 0x00, + // 565 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x53, 0x4d, 0x6f, 0xd3, 0x40, + 0x10, 0xf5, 0xda, 0x8e, 0x93, 0x0c, 0x2d, 0x98, 0xa5, 0x02, 0xb7, 0x07, 0x27, 0x32, 0x42, 0x44, + 0x95, 0x6a, 0x4b, 0x05, 0x89, 0x0b, 0x97, 0x04, 0x85, 0x0f, 0xa9, 0x6d, 0xe8, 0x26, 0x12, 0xa8, + 0x97, 0x6a, 0xe3, 0xae, 0x6c, 0xab, 0xf1, 0x87, 0xec, 0x35, 0x24, 0xff, 0x02, 0xc4, 0x8d, 0x03, + 0xbf, 0x27, 0xc7, 0x1e, 0x11, 0x87, 0x08, 0x92, 0x3f, 0x82, 0xbc, 0x76, 0x28, 0x91, 0x72, 0xb1, + 0xc6, 0xef, 0xbd, 0x99, 0xd9, 0x79, 0x3b, 0x0b, 0xfb, 0x19, 0x8f, 0x53, 0xe6, 0x88, 0x6f, 0x32, + 0x76, 0xf8, 0x2c, 0x61, 0x99, 0x9d, 0xa4, 0x31, 0x8f, 0xb1, 0xc6, 0x7d, 0x1a, 0xc5, 0xd9, 0xc1, + 0x9e, 0x17, 0x7b, 0xb1, 0x80, 0x9c, 0x22, 0x2a, 0xd9, 0x83, 0x2a, 0x71, 0x42, 0xc7, 0x6c, 0xb2, + 0x99, 0x68, 0x7d, 0x47, 0x50, 0x7b, 0xe5, 0xe7, 0xd1, 0x35, 0x3e, 0x04, 0xb5, 0x20, 0x0c, 0xd4, + 0x46, 0x9d, 0xbb, 0xc7, 0x0f, 0xed, 0xb2, 0xa2, 0x2d, 0x48, 0xbb, 0x1f, 0xb9, 0xf1, 0x55, 0x10, + 0x79, 0x44, 0x68, 0x30, 0x06, 0xf5, 0x8a, 0x72, 0x6a, 0xc8, 0x6d, 0xd4, 0xd9, 0x21, 0x22, 0xc6, + 0x06, 0xa8, 0x3e, 0xcd, 0x7c, 0x43, 0x69, 0xa3, 0x8e, 0xda, 0x53, 0xe7, 0x8b, 0x16, 0x22, 0x02, + 0xb1, 0x5e, 0x40, 0x63, 0x9d, 0x8f, 0xeb, 0xa0, 0x7c, 0x1c, 0x10, 0x5d, 0xc2, 0xbb, 0xd0, 0x7c, + 0xfb, 0x6e, 0x38, 0x1a, 0xbc, 0x21, 0xdd, 0x53, 0x1d, 0xe1, 0x07, 0x70, 0xef, 0xf5, 0xc9, 0xa0, + 0x3b, 0xba, 0xbc, 0x05, 0x65, 0xeb, 0x07, 0x02, 0x6d, 0xc8, 0xd2, 0x80, 0x65, 0xd8, 0x05, 0x4d, + 0x1c, 0x3f, 0x33, 0x50, 0x5b, 0xe9, 0xdc, 0x39, 0xde, 0x5d, 0x9f, 0xef, 0xa4, 0x40, 0x7b, 0x2f, + 0xe7, 0x8b, 0x96, 0xf4, 0x6b, 0xd1, 0x7a, 0xee, 0x05, 0xdc, 0xcf, 0xc7, 0xb6, 0x1b, 0x87, 0x4e, + 0x29, 0x38, 0x0a, 0xe2, 0x2a, 0x72, 0x92, 0x6b, 0xcf, 0xd9, 0x70, 0xc2, 0xbe, 0x10, 0xd9, 0xa4, + 0x2a, 0x8d, 0x1d, 0xd0, 0xdc, 0x62, 0xdc, 0xcc, 0x90, 0x45, 0x93, 0xfb, 0xeb, 0x26, 0x5d, 0xcf, + 0x4b, 0x85, 0x11, 0x62, 0x2e, 0x89, 0x54, 0x32, 0xeb, 0x9b, 0x0c, 0xcd, 0x7f, 0x1c, 0xde, 0x87, + 0x46, 0x18, 0x44, 0x97, 0x3c, 0x08, 0x4b, 0x17, 0x15, 0x52, 0x0f, 0x83, 0x68, 0x14, 0x84, 0x4c, + 0x50, 0x74, 0x5a, 0x52, 0x72, 0x45, 0xd1, 0xa9, 0xa0, 0x5a, 0xa0, 0xa4, 0xf4, 0xb3, 0xb0, 0xed, + 0xbf, 0xb1, 0x44, 0x45, 0x52, 0x30, 0xf8, 0x31, 0xd4, 0xdc, 0x38, 0x8f, 0xb8, 0xa1, 0x6e, 0x93, + 0x94, 0x5c, 0x51, 0x25, 0xcb, 0x43, 0xa3, 0xb6, 0xb5, 0x4a, 0x96, 0x87, 0x85, 0x20, 0x0c, 0x22, + 0x43, 0xdb, 0x2a, 0x08, 0x83, 0x48, 0x08, 0xe8, 0xd4, 0xa8, 0x6f, 0x17, 0xd0, 0x29, 0x7e, 0x0a, + 0x75, 0xd1, 0x8b, 0xa5, 0x46, 0x63, 0x9b, 0x68, 0xcd, 0x5a, 0x5f, 0x11, 0xec, 0x08, 0x63, 0x4f, + 0x29, 0x77, 0x7d, 0x96, 0xe2, 0xa3, 0x8d, 0xd5, 0xda, 0xdf, 0xb8, 0xba, 0x4a, 0x63, 0x8f, 0x66, + 0x09, 0xbb, 0xdd, 0xae, 0x88, 0x56, 0x46, 0x35, 0x89, 0x88, 0xf1, 0x1e, 0xd4, 0x3e, 0xd1, 0x49, + 0xce, 0x84, 0x4f, 0x4d, 0x52, 0xfe, 0x58, 0x1d, 0x50, 0x8b, 0x3c, 0xac, 0x81, 0xdc, 0x3f, 0xd7, + 0xa5, 0x62, 0xbb, 0xce, 0xfa, 0xe7, 0x3a, 0x2a, 0x00, 0xd2, 0xd7, 0x65, 0x01, 0x90, 0xbe, 0xae, + 0x1c, 0xda, 0xf0, 0xe8, 0x3d, 0x4d, 0x79, 0x40, 0x27, 0x84, 0x65, 0x49, 0x1c, 0x65, 0x6c, 0xc8, + 0x53, 0xca, 0x99, 0x37, 0xc3, 0x0d, 0x50, 0x3f, 0x74, 0xc9, 0x99, 0x2e, 0xe1, 0x26, 0xd4, 0xba, + 0xbd, 0x01, 0x19, 0xe9, 0xa8, 0xf7, 0x64, 0xfe, 0xc7, 0x94, 0xe6, 0x4b, 0x13, 0xdd, 0x2c, 0x4d, + 0xf4, 0x7b, 0x69, 0xa2, 0x2f, 0x2b, 0x53, 0xba, 0x59, 0x99, 0xd2, 0xcf, 0x95, 0x29, 0x5d, 0xd4, + 0xab, 0x37, 0x38, 0xd6, 0xc4, 0x2b, 0x7a, 0xf6, 0x37, 0x00, 0x00, 0xff, 0xff, 0x9e, 0x6d, 0x25, + 0xf3, 0x9b, 0x03, 0x00, 0x00, } func (m *Chunk) Marshal() (dAtA []byte, err error) { diff --git a/pkg/store/storepb/types.proto b/pkg/store/storepb/types.proto index 67c93fa52e..840d3c5188 100644 --- a/pkg/store/storepb/types.proto +++ b/pkg/store/storepb/types.proto @@ -24,6 +24,7 @@ message Chunk { enum Encoding { XOR = 0; HISTOGRAM = 1; + FLOAT_HISTOGRAM = 2; } Encoding type = 1; bytes data = 2; diff --git a/test/e2e/native_histograms_test.go b/test/e2e/native_histograms_test.go index 799974a5be..48d1f25639 100644 --- a/test/e2e/native_histograms_test.go +++ b/test/e2e/native_histograms_test.go @@ -19,6 +19,7 @@ import ( "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/storage/remote" "github.com/prometheus/prometheus/tsdb/tsdbutil" + "github.com/thanos-io/thanos/pkg/promclient" "github.com/thanos-io/thanos/pkg/queryfrontend" "github.com/thanos-io/thanos/pkg/receive" @@ -50,19 +51,19 @@ func TestQueryNativeHistograms(t *testing.T) { histograms := tsdbutil.GenerateTestHistograms(4) now := time.Now() - _, err = writeHistograms(ctx, now, histograms, rawRemoteWriteURL1) + _, err = writeHistograms(ctx, now, testHistogramMetricName, histograms, nil, rawRemoteWriteURL1) testutil.Ok(t, err) - _, err = writeHistograms(ctx, now, histograms, rawRemoteWriteURL2) + _, err = writeHistograms(ctx, now, testHistogramMetricName, histograms, nil, rawRemoteWriteURL2) testutil.Ok(t, err) ts := func() time.Time { return now } // Make sure we can query histogram from both Prometheus instances. - queryAndAssert(t, ctx, prom1.Endpoint("http"), func() string { return testHistogramMetricName }, ts, promclient.QueryOptions{}, expectedHistogramModelVector(histograms[len(histograms)-1], nil)) - queryAndAssert(t, ctx, prom2.Endpoint("http"), func() string { return testHistogramMetricName }, ts, promclient.QueryOptions{}, expectedHistogramModelVector(histograms[len(histograms)-1], nil)) + queryAndAssert(t, ctx, prom1.Endpoint("http"), func() string { return testHistogramMetricName }, ts, promclient.QueryOptions{}, expectedHistogramModelVector(testHistogramMetricName, histograms[len(histograms)-1], nil, nil)) + queryAndAssert(t, ctx, prom2.Endpoint("http"), func() string { return testHistogramMetricName }, ts, promclient.QueryOptions{}, expectedHistogramModelVector(testHistogramMetricName, histograms[len(histograms)-1], nil, nil)) t.Run("query deduplicated histogram", func(t *testing.T) { - queryAndAssert(t, ctx, querier.Endpoint("http"), func() string { return testHistogramMetricName }, ts, promclient.QueryOptions{Deduplicate: true}, expectedHistogramModelVector(histograms[len(histograms)-1], map[string]string{ + queryAndAssert(t, ctx, querier.Endpoint("http"), func() string { return testHistogramMetricName }, ts, promclient.QueryOptions{Deduplicate: true}, expectedHistogramModelVector(testHistogramMetricName, histograms[len(histograms)-1], nil, map[string]string{ "prometheus": "prom-ha", })) }) @@ -123,14 +124,22 @@ func TestWriteNativeHistograms(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) t.Cleanup(cancel) + timeNow := time.Now() + histograms := tsdbutil.GenerateTestHistograms(1) - now := time.Now() - _, err = writeHistograms(ctx, now, histograms, rawRemoteWriteURL) + _, err = writeHistograms(ctx, timeNow, testHistogramMetricName, histograms, nil, rawRemoteWriteURL) testutil.Ok(t, err) - ts := func() time.Time { return now } + testFloatHistogramMetricName := testHistogramMetricName + "_float" + floatHistograms := tsdbutil.GenerateTestFloatHistograms(1) + _, err = writeHistograms(ctx, timeNow, testFloatHistogramMetricName, nil, floatHistograms, rawRemoteWriteURL) + testutil.Ok(t, err) - queryAndAssert(t, ctx, querier.Endpoint("http"), func() string { return testHistogramMetricName }, ts, promclient.QueryOptions{Deduplicate: true}, expectedHistogramModelVector(histograms[0], map[string]string{ + queryAndAssert(t, ctx, querier.Endpoint("http"), func() string { return testHistogramMetricName }, time.Now, promclient.QueryOptions{Deduplicate: true}, expectedHistogramModelVector(testHistogramMetricName, histograms[0], nil, map[string]string{ + "tenant_id": "default-tenant", + })) + + queryAndAssert(t, ctx, querier.Endpoint("http"), func() string { return testFloatHistogramMetricName }, time.Now, promclient.QueryOptions{Deduplicate: true}, expectedHistogramModelVector(testFloatHistogramMetricName, nil, floatHistograms[0], map[string]string{ "tenant_id": "default-tenant", })) } @@ -166,9 +175,9 @@ func TestQueryFrontendNativeHistograms(t *testing.T) { histograms := tsdbutil.GenerateTestHistograms(4) now := time.Now() - _, err = writeHistograms(ctx, now, histograms, rawRemoteWriteURL1) + _, err = writeHistograms(ctx, now, testHistogramMetricName, histograms, nil, rawRemoteWriteURL1) testutil.Ok(t, err) - startTime, err := writeHistograms(ctx, now, histograms, rawRemoteWriteURL2) + startTime, err := writeHistograms(ctx, now, testHistogramMetricName, histograms, nil, rawRemoteWriteURL2) testutil.Ok(t, err) // Ensure we can get the result from Querier first so that it @@ -193,7 +202,7 @@ func TestQueryFrontendNativeHistograms(t *testing.T) { ts := func() time.Time { return now } t.Run("query frontend works for instant query", func(t *testing.T) { - queryAndAssert(t, ctx, queryFrontend.Endpoint("http"), func() string { return testHistogramMetricName }, ts, promclient.QueryOptions{Deduplicate: true}, expectedHistogramModelVector(histograms[len(histograms)-1], map[string]string{ + queryAndAssert(t, ctx, queryFrontend.Endpoint("http"), func() string { return testHistogramMetricName }, ts, promclient.QueryOptions{Deduplicate: true}, expectedHistogramModelVector(testHistogramMetricName, histograms[len(histograms)-1], nil, map[string]string{ "prometheus": "prom-ha", })) @@ -211,7 +220,7 @@ func TestQueryFrontendNativeHistograms(t *testing.T) { }) t.Run("query range query, all but last histogram", func(t *testing.T) { - expectedRes := expectedHistogramModelMatrix(histograms[:len(histograms)-1], startTime, map[string]string{ + expectedRes := expectedHistogramModelMatrix(testHistogramMetricName, histograms[:len(histograms)-1], nil, startTime, map[string]string{ "prometheus": "prom-ha", }) @@ -247,7 +256,7 @@ func TestQueryFrontendNativeHistograms(t *testing.T) { }) t.Run("query range, all histograms", func(t *testing.T) { - expectedRes := expectedHistogramModelMatrix(histograms, startTime, map[string]string{ + expectedRes := expectedHistogramModelMatrix(testHistogramMetricName, histograms, nil, startTime, map[string]string{ "prometheus": "prom-ha", }) @@ -280,7 +289,7 @@ func TestQueryFrontendNativeHistograms(t *testing.T) { }) t.Run("query range, all histograms again", func(t *testing.T) { - expectedRes := expectedHistogramModelMatrix(histograms, startTime, map[string]string{ + expectedRes := expectedHistogramModelMatrix(testHistogramMetricName, histograms, nil, startTime, map[string]string{ "prometheus": "prom-ha", }) @@ -314,10 +323,15 @@ func TestQueryFrontendNativeHistograms(t *testing.T) { }) } -func writeHistograms(ctx context.Context, now time.Time, histograms []*histogram.Histogram, rawRemoteWriteURL string) (time.Time, error) { +func writeHistograms(ctx context.Context, now time.Time, name string, histograms []*histogram.Histogram, floatHistograms []*histogram.FloatHistogram, rawRemoteWriteURL string) (time.Time, error) { startTime := now.Add(time.Duration(len(histograms)-1) * -30 * time.Second).Truncate(30 * time.Second) prompbHistograms := make([]prompb.Histogram, 0, len(histograms)) + for i, fh := range floatHistograms { + ts := startTime.Add(time.Duration(i) * 30 * time.Second).UnixMilli() + prompbHistograms = append(prompbHistograms, remote.FloatHistogramToHistogramProto(ts, fh)) + } + for i, h := range histograms { ts := startTime.Add(time.Duration(i) * 30 * time.Second).UnixMilli() prompbHistograms = append(prompbHistograms, remote.HistogramToHistogramProto(ts, h)) @@ -325,7 +339,7 @@ func writeHistograms(ctx context.Context, now time.Time, histograms []*histogram timeSeriespb := prompb.TimeSeries{ Labels: []prompb.Label{ - {Name: "__name__", Value: testHistogramMetricName}, + {Name: "__name__", Value: name}, {Name: "foo", Value: "bar"}, }, Histograms: prompbHistograms, @@ -336,45 +350,60 @@ func writeHistograms(ctx context.Context, now time.Time, histograms []*histogram }) } -func expectedHistogramModelVector(histogram *histogram.Histogram, externalLabels map[string]string) model.Vector { +func expectedHistogramModelVector(metricName string, histogram *histogram.Histogram, floatHistogram *histogram.FloatHistogram, externalLabels map[string]string) model.Vector { metrics := model.Metric{ - "__name__": testHistogramMetricName, + "__name__": model.LabelValue(metricName), "foo": "bar", } for labelKey, labelValue := range externalLabels { metrics[model.LabelName(labelKey)] = model.LabelValue(labelValue) } + var sampleHistogram *model.SampleHistogram + + if histogram != nil { + sampleHistogram = histogramToSampleHistogram(histogram) + } else { + sampleHistogram = floatHistogramToSampleHistogram(floatHistogram) + } + return model.Vector{ &model.Sample{ Metric: metrics, - Histogram: histogramToSampleHistogram(histogram), + Histogram: sampleHistogram, }, } } -func expectedHistogramModelMatrix(histograms []*histogram.Histogram, startTime time.Time, externalLabels map[string]string) model.Matrix { +func expectedHistogramModelMatrix(metricName string, histograms []*histogram.Histogram, floatHistograms []*histogram.FloatHistogram, startTime time.Time, externalLabels map[string]string) model.Matrix { metrics := model.Metric{ - "__name__": testHistogramMetricName, + "__name__": model.LabelValue(metricName), "foo": "bar", } for labelKey, labelValue := range externalLabels { metrics[model.LabelName(labelKey)] = model.LabelValue(labelValue) } - shp := make([]model.SampleHistogramPair, 0, len(histograms)) + sampleHistogramPair := make([]model.SampleHistogramPair, 0, len(histograms)) for i, h := range histograms { - shp = append(shp, model.SampleHistogramPair{ + sampleHistogramPair = append(sampleHistogramPair, model.SampleHistogramPair{ Timestamp: model.Time(startTime.Add(time.Duration(i) * 30 * time.Second).UnixMilli()), Histogram: histogramToSampleHistogram(h), }) } + for i, fh := range floatHistograms { + sampleHistogramPair = append(sampleHistogramPair, model.SampleHistogramPair{ + Timestamp: model.Time(startTime.Add(time.Duration(i) * 30 * time.Second).UnixMilli()), + Histogram: floatHistogramToSampleHistogram(fh), + }) + } + return model.Matrix{ &model.SampleStream{ Metric: metrics, - Histograms: shp, + Histograms: sampleHistogramPair, }, } } @@ -401,7 +430,29 @@ func histogramToSampleHistogram(h *histogram.Histogram) *model.SampleHistogram { } } -func bucketToSampleHistogramBucket(bucket histogram.Bucket[uint64]) *model.HistogramBucket { +func floatHistogramToSampleHistogram(fh *histogram.FloatHistogram) *model.SampleHistogram { + var buckets []*model.HistogramBucket + + it := fh.NegativeBucketIterator() + for it.Next() { + buckets = append([]*model.HistogramBucket{bucketToSampleHistogramBucket(it.At())}, buckets...) + } + + buckets = append(buckets, bucketToSampleHistogramBucket(fh.ZeroBucket())) + + it = fh.PositiveBucketIterator() + for it.Next() { + buckets = append(buckets, bucketToSampleHistogramBucket(it.At())) + } + + return &model.SampleHistogram{ + Count: model.FloatString(fh.Count), + Sum: model.FloatString(fh.Sum), + Buckets: buckets, + } +} + +func bucketToSampleHistogramBucket[BC histogram.BucketCount](bucket histogram.Bucket[BC]) *model.HistogramBucket { return &model.HistogramBucket{ Lower: model.FloatString(bucket.Lower), Upper: model.FloatString(bucket.Upper), @@ -410,7 +461,7 @@ func bucketToSampleHistogramBucket(bucket histogram.Bucket[uint64]) *model.Histo } } -func boundaries(bucket histogram.Bucket[uint64]) int32 { +func boundaries[BC histogram.BucketCount](bucket histogram.Bucket[BC]) int32 { switch { case bucket.UpperInclusive && !bucket.LowerInclusive: return 0 From f94b6edc27e05bed903415a39f0411bca23c66b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Mon, 10 Jul 2023 19:15:27 +0300 Subject: [PATCH 2/7] go.mod: bump e2e framework version (#6516) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bump e2e framework version and add a note about E2E_DOCKER_CPUS. Signed-off-by: Giedrius Statkevičius --- Makefile | 3 +++ go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 986503f1e0..3569214c65 100644 --- a/Makefile +++ b/Makefile @@ -326,6 +326,9 @@ test-e2e: docker $(GOTESPLIT) @echo ">> running /test/e2e tests." # NOTE(bwplotka): # * If you see errors on CI (timeouts), but not locally, try to add -parallel 1 (Wiard note: to the GOTEST_OPTS arg) to limit to single CPU to reproduce small 1CPU machine. + # NOTE(GiedriusS): + # * If you want to limit CPU time available in e2e tests then pass E2E_DOCKER_CPUS environment variable. For example, E2E_DOCKER_CPUS=0.05 limits CPU time available + # to spawned Docker containers to 0.05 cores. @$(GOTESPLIT) -total ${GH_PARALLEL} -index ${GH_INDEX} ./test/e2e/... -- ${GOTEST_OPTS} .PHONY: test-e2e-local diff --git a/go.mod b/go.mod index f03d501f3e..49a0df2444 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/chromedp/chromedp v0.8.2 github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.0 - github.com/efficientgo/e2e v0.14.1-0.20230530075226-84bb33e062c4 + github.com/efficientgo/e2e v0.14.1-0.20230710114240-c316eb95ae5b github.com/efficientgo/tools/extkingpin v0.0.0-20220817170617-6c25e3b627dd github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb github.com/fatih/structtag v1.2.0 diff --git a/go.sum b/go.sum index 30b1525b6e..ad4edf82e4 100644 --- a/go.sum +++ b/go.sum @@ -243,8 +243,8 @@ github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/efficientgo/core v1.0.0-rc.2 h1:7j62qHLnrZqO3V3UA0AqOGd5d5aXV3AX6m/NZBHp78I= github.com/efficientgo/core v1.0.0-rc.2/go.mod h1:FfGdkzWarkuzOlY04VY+bGfb1lWrjaL6x/GLcQ4vJps= -github.com/efficientgo/e2e v0.14.1-0.20230530075226-84bb33e062c4 h1:8w29+abukpj2UWN19wZ4xmOswdrYBjfqmbot9oppp3U= -github.com/efficientgo/e2e v0.14.1-0.20230530075226-84bb33e062c4/go.mod h1:plsKU0YHE9uX+7utvr7SiDtVBSHJyEfHRO4UnUgDmts= +github.com/efficientgo/e2e v0.14.1-0.20230710114240-c316eb95ae5b h1:8VX23BNufsa4KCqnnEonvI3yrou2Pjp8JLcbdVn0Fs8= +github.com/efficientgo/e2e v0.14.1-0.20230710114240-c316eb95ae5b/go.mod h1:plsKU0YHE9uX+7utvr7SiDtVBSHJyEfHRO4UnUgDmts= github.com/efficientgo/tools/extkingpin v0.0.0-20220817170617-6c25e3b627dd h1:VaYzzXeUbC5fVheskcKVNOyJMEYD+HgrJNzIAg/mRIM= github.com/efficientgo/tools/extkingpin v0.0.0-20220817170617-6c25e3b627dd/go.mod h1:ZV0utlglOczUWv3ih2AbqPSoLoFzdplUYxwV62eZi6Q= github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= From 1dcbd7af909069d0845ee7d811d08a662bd0821e Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Mon, 10 Jul 2023 19:00:08 +0200 Subject: [PATCH 3/7] Add changelog for #6474 (#6518) Signed-off-by: Filip Petkovski --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 133784cd8a..c6f012aebe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#6405](https://github.com/thanos-io/thanos/pull/6405) Index Cache: Change postings cache key to include the encoding format used so that older Thanos versions would not try to decode it during the deployment of a new version. - [#6432](https://github.com/thanos-io/thanos/pull/6432) Receive: Remove duplicated `gopkg.in/fsnotify.v1` dependency - [#6479](https://github.com/thanos-io/thanos/pull/6479) Store: *breaking :warning:* Rename `thanos_bucket_store_cached_series_fetch_duration_seconds` to `thanos_bucket_store_series_fetch_duration_seconds` and `thanos_bucket_store_cached_postings_fetch_duration_seconds` to `thanos_bucket_store_postings_fetch_duration_seconds`. +- [#6474](https://github.com/thanos-io/thanos/pull/6474) Store/Compact: Reduce a large amount of `Exists` API calls against object storage when synchronizing meta files in favour of a recursive `Iter` call. ### Removed From 4dd2667b670f3a21ee8d1a7041c435f061def534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Tue, 11 Jul 2023 12:45:36 +0300 Subject: [PATCH 4/7] cortex/redisclient: use rueidis client (#6520) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cortex/redisclient: use rueidis client Use the same rueidis client in query-frontend. Solves https://github.com/thanos-io/thanos/issues/6094. Signed-off-by: Giedrius Statkevičius * cortex/cache: gofumpt + fix errors.Errorf Signed-off-by: Giedrius Statkevičius * cacheutil/docs: clean up more old stuff Signed-off-by: Giedrius Statkevičius --------- Signed-off-by: Giedrius Statkevičius --- CHANGELOG.md | 1 + docs/components/query-frontend.md | 4 - docs/components/store.md | 4 - go.mod | 7 +- go.sum | 19 ++- internal/cortex/chunk/cache/cache.go | 12 +- .../cortex/chunk/cache/redis_cache_test.go | 14 +- internal/cortex/chunk/cache/redis_client.go | 121 +++++++++--------- .../cortex/chunk/cache/redis_client_test.go | 33 +++-- pkg/cacheutil/redis_client.go | 19 --- pkg/queryfrontend/config.go | 15 +-- 11 files changed, 127 insertions(+), 122 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6f012aebe..ba7292b23a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#6010](https://github.com/thanos-io/thanos/pull/6010) *: Upgrade Prometheus to v0.42.0. - [#5999](https://github.com/thanos-io/thanos/pull/5999) *: Upgrade Alertmanager dependency to v0.25.0. +- [#6520](https://github.com/thanos-io/thanos/pull/6520): Switch query-frontend to use [Rueidis](https://github.com/redis/rueidis) client. Deleted `idle_timeout`, `max_conn_age`, `pool_size`, `min_idle_conns` fields as they are not used anymore. - [#5887](https://github.com/thanos-io/thanos/pull/5887) Tracing: Make sure rate limiting sampler is the default, as was the case in version pre-0.29.0. - [#5997](https://github.com/thanos-io/thanos/pull/5997) Rule: switch to miekgdns DNS resolver as the default one. - [#6126](https://github.com/thanos-io/thanos/pull/6126) Build with Go 1.20 diff --git a/docs/components/query-frontend.md b/docs/components/query-frontend.md index 8ae6aee8e7..8e21c21830 100644 --- a/docs/components/query-frontend.md +++ b/docs/components/query-frontend.md @@ -117,10 +117,6 @@ config: dial_timeout: 5s read_timeout: 3s write_timeout: 3s - pool_size: 100 - min_idle_conns: 10 - idle_timeout: 5m0s - max_conn_age: 0s max_get_multi_concurrency: 100 get_multi_batch_size: 100 max_set_multi_concurrency: 100 diff --git a/docs/components/store.md b/docs/components/store.md index a8b03a224a..229c036775 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -342,10 +342,6 @@ config: dial_timeout: 5s read_timeout: 3s write_timeout: 3s - pool_size: 100 - min_idle_conns: 10 - idle_timeout: 5m0s - max_conn_age: 0s max_get_multi_concurrency: 100 get_multi_batch_size: 100 max_set_multi_concurrency: 100 diff --git a/go.mod b/go.mod index 49a0df2444..80114896e0 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,6 @@ require ( github.com/fsnotify/fsnotify v1.6.0 github.com/go-kit/log v0.2.1 github.com/go-openapi/strfmt v0.21.7 - github.com/go-redis/redis/v8 v8.11.5 github.com/gogo/protobuf v1.3.2 github.com/gogo/status v1.1.1 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da @@ -124,7 +123,10 @@ require ( golang.org/x/exp v0.0.0-20230321023759-10a507213a29 ) -require go4.org/unsafe/assume-no-moving-gc v0.0.0-20230209150437-ee73d164e760 // indirect +require ( + github.com/onsi/ginkgo v1.16.5 // indirect + go4.org/unsafe/assume-no-moving-gc v0.0.0-20230209150437-ee73d164e760 // indirect +) require ( cloud.google.com/go/compute/metadata v0.2.3 // indirect @@ -154,7 +156,6 @@ require ( github.com/clbanning/mxj v1.8.4 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/dennwc/varint v1.0.0 // indirect - github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dnaeon/go-vcr v1.2.0 // indirect github.com/edsrzf/mmap-go v1.1.0 // indirect github.com/elastic/go-sysinfo v1.8.1 // indirect diff --git a/go.sum b/go.sum index ad4edf82e4..1a2568d345 100644 --- a/go.sum +++ b/go.sum @@ -223,8 +223,6 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE= github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/digitalocean/godo v1.99.0 h1:gUHO7n9bDaZFWvbzOum4bXE0/09ZuYA9yA8idQHX57E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= @@ -274,6 +272,7 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -338,11 +337,10 @@ github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/validate v0.22.1 h1:G+c2ub6q47kfX1sOBLwIQwzBVt8qmOAARyo/9Fqs9NU= github.com/go-openapi/validate v0.22.1/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= -github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= -github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= @@ -709,7 +707,9 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/ncw/swift v1.0.53 h1:luHjjTNtekIEvHg5KdAFIBaH7bWfNkefwFnpDffSIks= github.com/ncw/swift v1.0.53/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= @@ -721,9 +721,13 @@ github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= @@ -880,6 +884,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -1108,6 +1113,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -1197,9 +1203,12 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1226,6 +1235,7 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1346,6 +1356,7 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= diff --git a/internal/cortex/chunk/cache/cache.go b/internal/cortex/chunk/cache/cache.go index 691a7415f4..26324c2982 100644 --- a/internal/cortex/chunk/cache/cache.go +++ b/internal/cortex/chunk/cache/cache.go @@ -5,10 +5,11 @@ package cache import ( "context" - "errors" "flag" "time" + "github.com/pkg/errors" + "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" ) @@ -46,7 +47,7 @@ type Config struct { } // RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet -func (cfg *Config) RegisterFlagsWithPrefix(prefix string, description string, f *flag.FlagSet) { +func (cfg *Config) RegisterFlagsWithPrefix(prefix, description string, f *flag.FlagSet) { cfg.Background.RegisterFlagsWithPrefix(prefix, description, f) cfg.Memcache.RegisterFlagsWithPrefix(prefix, description, f) cfg.MemcacheClient.RegisterFlagsWithPrefix(prefix, description, f) @@ -58,6 +59,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, description string, f cfg.Prefix = prefix } + func (cfg *Config) Validate() error { return cfg.Fifocache.Validate() } @@ -101,7 +103,11 @@ func New(cfg Config, reg prometheus.Registerer, logger log.Logger) (Cache, error cfg.Redis.Expiration = cfg.DefaultValidity } cacheName := cfg.Prefix + "redis" - cache := NewRedisCache(cacheName, NewRedisClient(&cfg.Redis), reg, logger) + redisClient, err := NewRedisClient(&cfg.Redis) + if err != nil { + return nil, errors.Errorf("creating redis client: %v", err) + } + cache := NewRedisCache(cacheName, redisClient, reg, logger) caches = append(caches, NewBackground(cacheName, cfg.Background, Instrument(cacheName, cache, reg), reg)) } diff --git a/internal/cortex/chunk/cache/redis_cache_test.go b/internal/cortex/chunk/cache/redis_cache_test.go index 25220b19f6..3ba57b8509 100644 --- a/internal/cortex/chunk/cache/redis_cache_test.go +++ b/internal/cortex/chunk/cache/redis_cache_test.go @@ -10,7 +10,7 @@ import ( "github.com/alicebob/miniredis/v2" "github.com/go-kit/log" - "github.com/go-redis/redis/v8" + "github.com/redis/rueidis" "github.com/stretchr/testify/require" ) @@ -57,14 +57,18 @@ func mockRedisCache() (*RedisCache, error) { redisServer, err := miniredis.Run() if err != nil { return nil, err - + } + cl, err := rueidis.NewClient(rueidis.ClientOption{ + InitAddress: []string{redisServer.Addr()}, + DisableCache: true, + }) + if err != nil { + return nil, err } redisClient := &RedisClient{ expiration: time.Minute, timeout: 100 * time.Millisecond, - rdb: redis.NewUniversalClient(&redis.UniversalOptions{ - Addrs: []string{redisServer.Addr()}, - }), + rdb: cl, } return NewRedisCache("mock", redisClient, nil, log.NewNopLogger()), nil } diff --git a/internal/cortex/chunk/cache/redis_client.go b/internal/cortex/chunk/cache/redis_client.go index 2b8cafcbcb..ac63ec3396 100644 --- a/internal/cortex/chunk/cache/redis_client.go +++ b/internal/cortex/chunk/cache/redis_client.go @@ -7,12 +7,13 @@ import ( "context" "crypto/tls" "flag" - "fmt" + "net" "strings" "time" "unsafe" - "github.com/go-redis/redis/v8" + "github.com/pkg/errors" + "github.com/redis/rueidis" "github.com/thanos-io/thanos/internal/cortex/util/flagext" ) @@ -24,12 +25,9 @@ type RedisConfig struct { Timeout time.Duration `yaml:"timeout"` Expiration time.Duration `yaml:"expiration"` DB int `yaml:"db"` - PoolSize int `yaml:"pool_size"` Password flagext.Secret `yaml:"password"` EnableTLS bool `yaml:"tls_enabled"` InsecureSkipVerify bool `yaml:"tls_insecure_skip_verify"` - IdleTimeout time.Duration `yaml:"idle_timeout"` - MaxConnAge time.Duration `yaml:"max_connection_age"` } // RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet @@ -39,39 +37,47 @@ func (cfg *RedisConfig) RegisterFlagsWithPrefix(prefix, description string, f *f f.DurationVar(&cfg.Timeout, prefix+"redis.timeout", 500*time.Millisecond, description+"Maximum time to wait before giving up on redis requests.") f.DurationVar(&cfg.Expiration, prefix+"redis.expiration", 0, description+"How long keys stay in the redis.") f.IntVar(&cfg.DB, prefix+"redis.db", 0, description+"Database index.") - f.IntVar(&cfg.PoolSize, prefix+"redis.pool-size", 0, description+"Maximum number of connections in the pool.") f.Var(&cfg.Password, prefix+"redis.password", description+"Password to use when connecting to redis.") f.BoolVar(&cfg.EnableTLS, prefix+"redis.tls-enabled", false, description+"Enable connecting to redis with TLS.") f.BoolVar(&cfg.InsecureSkipVerify, prefix+"redis.tls-insecure-skip-verify", false, description+"Skip validating server certificate.") - f.DurationVar(&cfg.IdleTimeout, prefix+"redis.idle-timeout", 0, description+"Close connections after remaining idle for this duration. If the value is zero, then idle connections are not closed.") - f.DurationVar(&cfg.MaxConnAge, prefix+"redis.max-connection-age", 0, description+"Close connections older than this duration. If the value is zero, then the pool does not close connections based on age.") } type RedisClient struct { expiration time.Duration timeout time.Duration - rdb redis.UniversalClient + rdb rueidis.Client } // NewRedisClient creates Redis client -func NewRedisClient(cfg *RedisConfig) *RedisClient { - opt := &redis.UniversalOptions{ - Addrs: strings.Split(cfg.Endpoint, ","), - MasterName: cfg.MasterName, - Password: cfg.Password.Value, - DB: cfg.DB, - PoolSize: cfg.PoolSize, - IdleTimeout: cfg.IdleTimeout, - MaxConnAge: cfg.MaxConnAge, +func NewRedisClient(cfg *RedisConfig) (*RedisClient, error) { + clientOpts := rueidis.ClientOption{ + InitAddress: strings.Split(cfg.Endpoint, ","), + ShuffleInit: true, + Password: cfg.Password.Value, + SelectDB: cfg.DB, + Dialer: net.Dialer{Timeout: cfg.Timeout}, + ConnWriteTimeout: cfg.Timeout, + DisableCache: true, } if cfg.EnableTLS { - opt.TLSConfig = &tls.Config{InsecureSkipVerify: cfg.InsecureSkipVerify} + clientOpts.TLSConfig = &tls.Config{InsecureSkipVerify: cfg.InsecureSkipVerify} } + if cfg.MasterName != "" { + clientOpts.Sentinel = rueidis.SentinelOption{ + MasterSet: cfg.MasterName, + } + } + + client, err := rueidis.NewClient(clientOpts) + if err != nil { + return nil, err + } + return &RedisClient{ expiration: cfg.Expiration, timeout: cfg.Timeout, - rdb: redis.NewUniversalClient(opt), - } + rdb: client, + }, nil } func (c *RedisClient) Ping(ctx context.Context) error { @@ -81,12 +87,13 @@ func (c *RedisClient) Ping(ctx context.Context) error { defer cancel() } - pong, err := c.rdb.Ping(ctx).Result() + resp := c.rdb.Do(ctx, c.rdb.B().Ping().Build()) + pingResp, err := resp.ToString() if err != nil { - return err + return errors.New("converting PING response to string") } - if pong != "PONG" { - return fmt.Errorf("redis: Unexpected PING response %q", pong) + if pingResp != "PONG" { + return errors.Errorf("redis: Unexpected PING response %q", pingResp) } return nil } @@ -99,15 +106,19 @@ func (c *RedisClient) MSet(ctx context.Context, keys []string, values [][]byte) } if len(keys) != len(values) { - return fmt.Errorf("MSet the length of keys and values not equal, len(keys)=%d, len(values)=%d", len(keys), len(values)) + return errors.Errorf("MSet the length of keys and values not equal, len(keys)=%d, len(values)=%d", len(keys), len(values)) } - pipe := c.rdb.TxPipeline() + cmds := make(rueidis.Commands, 0, len(keys)) for i := range keys { - pipe.Set(ctx, keys[i], values[i], c.expiration) + cmds = append(cmds, c.rdb.B().Set().Key(keys[i]).Value(rueidis.BinaryString(values[i])).Ex(c.expiration).Build()) } - _, err := pipe.Exec(ctx) - return err + for _, resp := range c.rdb.DoMulti(ctx, cmds...) { + if err := resp.Error(); err != nil { + return err + } + } + return nil } func (c *RedisClient) MGet(ctx context.Context, keys []string) ([][]byte, error) { @@ -117,47 +128,37 @@ func (c *RedisClient) MGet(ctx context.Context, keys []string) ([][]byte, error) defer cancel() } - ret := make([][]byte, len(keys)) - - // redis.UniversalClient can take redis.Client and redis.ClusterClient. - // if redis.Client is set, then Single node or sentinel configuration. mget is always supported. - // if redis.ClusterClient is set, then Redis Cluster configuration. mget may not be supported. - _, isCluster := c.rdb.(*redis.ClusterClient) - - if isCluster { - for i, key := range keys { - cmd := c.rdb.Get(ctx, key) - err := cmd.Err() - if err == redis.Nil { - // if key not found, response nil - continue - } else if err != nil { - return nil, err - } - ret[i] = StringToBytes(cmd.Val()) + ret := make([][]byte, 0, len(keys)) + + mgetRet, err := rueidis.MGet(c.rdb, ctx, keys) + if err != nil { + return nil, err + } + for _, k := range keys { + m, ok := mgetRet[k] + if !ok { + return nil, errors.Errorf("not found key %s in results", k) } - } else { - cmd := c.rdb.MGet(ctx, keys...) - if err := cmd.Err(); err != nil { - return nil, err + if m.IsNil() { + ret = append(ret, nil) + continue } - - for i, val := range cmd.Val() { - if val != nil { - ret[i] = StringToBytes(val.(string)) - } + r, err := m.ToString() + if err != nil { + return nil, errors.Errorf("failed to convert %s resp to string", k) } + ret = append(ret, stringToBytes(r)) } return ret, nil } func (c *RedisClient) Close() error { - return c.rdb.Close() + c.rdb.Close() + return nil } -// StringToBytes converts string to byte slice. (copied from vendor/github.com/go-redis/redis/v8/internal/util/unsafe.go) -func StringToBytes(s string) []byte { +func stringToBytes(s string) []byte { return *(*[]byte)(unsafe.Pointer( &struct { string diff --git a/internal/cortex/chunk/cache/redis_client_test.go b/internal/cortex/chunk/cache/redis_client_test.go index 413c8ea76d..369a676d5e 100644 --- a/internal/cortex/chunk/cache/redis_client_test.go +++ b/internal/cortex/chunk/cache/redis_client_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/alicebob/miniredis/v2" - "github.com/go-redis/redis/v8" + "github.com/redis/rueidis" "github.com/stretchr/testify/require" ) @@ -22,6 +22,9 @@ func TestRedisClient(t *testing.T) { require.Nil(t, err) defer cluster.Close() + require.NoError(t, single.Ping(context.Background())) + require.NoError(t, cluster.Ping(context.Background())) + ctx := context.Background() tests := []struct { @@ -58,7 +61,7 @@ func TestRedisClient(t *testing.T) { // get missing keys values, err = tt.client.MGet(ctx, miss) - require.Nil(t, err) + require.NoError(t, err) require.Len(t, values, len(miss)) for _, value := range values { require.Nil(t, value) @@ -72,12 +75,18 @@ func mockRedisClientSingle() (*RedisClient, error) { if err != nil { return nil, err } + + cl, err := rueidis.NewClient(rueidis.ClientOption{ + InitAddress: []string{redisServer.Addr()}, + DisableCache: true, + }) + if err != nil { + return nil, err + } return &RedisClient{ expiration: time.Minute, timeout: 100 * time.Millisecond, - rdb: redis.NewClient(&redis.Options{ - Addr: redisServer.Addr(), - }), + rdb: cl, }, nil } @@ -90,14 +99,16 @@ func mockRedisClientCluster() (*RedisClient, error) { if err != nil { return nil, err } + cl, err := rueidis.NewClient(rueidis.ClientOption{ + InitAddress: []string{redisServer1.Addr(), redisServer2.Addr()}, + DisableCache: true, + }) + if err != nil { + return nil, err + } return &RedisClient{ expiration: time.Minute, timeout: 100 * time.Millisecond, - rdb: redis.NewClusterClient(&redis.ClusterOptions{ - Addrs: []string{ - redisServer1.Addr(), - redisServer2.Addr(), - }, - }), + rdb: cl, }, nil } diff --git a/pkg/cacheutil/redis_client.go b/pkg/cacheutil/redis_client.go index a7ab81fadf..6a701b14f8 100644 --- a/pkg/cacheutil/redis_client.go +++ b/pkg/cacheutil/redis_client.go @@ -31,9 +31,6 @@ var ( DialTimeout: time.Second * 5, ReadTimeout: time.Second * 3, WriteTimeout: time.Second * 3, - PoolSize: 100, - MinIdleConns: 10, - IdleTimeout: time.Minute * 5, MaxGetMultiConcurrency: 100, GetMultiBatchSize: 100, MaxSetMultiConcurrency: 100, @@ -84,22 +81,6 @@ type RedisClientConfig struct { // WriteTimeout specifies the client write timeout. WriteTimeout time.Duration `yaml:"write_timeout"` - // Maximum number of socket connections. - PoolSize int `yaml:"pool_size"` - - // MinIdleConns specifies the minimum number of idle connections which is useful when establishing - // new connection is slow. - MinIdleConns int `yaml:"min_idle_conns"` - - // Amount of time after which client closes idle connections. - // Should be less than server's timeout. - // -1 disables idle timeout check. - IdleTimeout time.Duration `yaml:"idle_timeout"` - - // Connection age at which client retires (closes) the connection. - // Default 0 is to not close aged connections. - MaxConnAge time.Duration `yaml:"max_conn_age"` - // MaxGetMultiConcurrency specifies the maximum number of concurrent GetMulti() operations. // If set to 0, concurrency is unlimited. MaxGetMultiConcurrency int `yaml:"max_get_multi_concurrency"` diff --git a/pkg/queryfrontend/config.go b/pkg/queryfrontend/config.go index 59d6cb62a6..a56551995c 100644 --- a/pkg/queryfrontend/config.go +++ b/pkg/queryfrontend/config.go @@ -160,15 +160,12 @@ func NewCacheConfig(logger log.Logger, confContentYaml []byte) (*cortexcache.Con } return &cortexcache.Config{ Redis: cortexcache.RedisConfig{ - Endpoint: config.Redis.Addr, - Timeout: config.Redis.ReadTimeout, - MasterName: config.Redis.MasterName, - Expiration: config.Expiration, - DB: config.Redis.DB, - PoolSize: config.Redis.PoolSize, - Password: flagext.Secret{Value: config.Redis.Password}, - IdleTimeout: config.Redis.IdleTimeout, - MaxConnAge: config.Redis.MaxConnAge, + Endpoint: config.Redis.Addr, + Timeout: config.Redis.ReadTimeout, + MasterName: config.Redis.MasterName, + Expiration: config.Expiration, + DB: config.Redis.DB, + Password: flagext.Secret{Value: config.Redis.Password}, }, Background: cortexcache.BackgroundConfig{ WriteBackBuffer: config.Redis.MaxSetMultiConcurrency * config.Redis.SetMultiBatchSize, From df3a5f8087268d069a13a062a77d7f650eb6760a Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Tue, 11 Jul 2023 09:01:12 -0700 Subject: [PATCH 5/7] index header: Remove memWriter from fileWriter (#6509) * index header: remove memWriter from fileWriter Signed-off-by: Ben Ye * update changelog Signed-off-by: Ben Ye * refactor Signed-off-by: Ben Ye * fix test Signed-off-by: Ben Ye * update comment Signed-off-by: Ben Ye --------- Signed-off-by: Ben Ye --- CHANGELOG.md | 1 + pkg/block/indexheader/binary_reader.go | 72 ++++++++++++-------------- 2 files changed, 33 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba7292b23a..045f045f0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#6325](https://github.com/thanos-io/thanos/pull/6325) Store: return gRPC resource exhausted error for byte limiter. - [#6399](https://github.com/thanos-io/thanos/pull/6399) *: Fix double-counting bug in http_request_duration metric - [#6428](https://github.com/thanos-io/thanos/pull/6428) Report gRPC connnection errors in the logs. +- [#6509](https://github.com/thanos-io/thanos/pull/6509) Store Gateway: Remove `memWriter` from `fileWriter` to reduce memory usage when sync index headers. ### Changed - [#6049](https://github.com/thanos-io/thanos/pull/6049) Compact: *breaking :warning:* Replace group with resolution in compact metrics to avoid cardinality explosion on compact metrics for large numbers of groups. diff --git a/pkg/block/indexheader/binary_reader.go b/pkg/block/indexheader/binary_reader.go index 0c7d062c9c..1befe63a7f 100644 --- a/pkg/block/indexheader/binary_reader.go +++ b/pkg/block/indexheader/binary_reader.go @@ -250,13 +250,7 @@ type binaryWriter struct { } func newBinaryWriter(id ulid.ULID, cacheFilename string, buf []byte) (w *binaryWriter, err error) { - var memoryWriter *MemoryWriter - memoryWriter, err = NewMemoryWriter(id, len(buf)) - if err != nil { - return nil, err - } - var binWriter PosWriter = memoryWriter - + var binWriter PosWriter if cacheFilename != "" { dir := filepath.Dir(cacheFilename) @@ -277,9 +271,8 @@ func newBinaryWriter(id ulid.ULID, cacheFilename string, buf []byte) (w *binaryW return nil, errors.Wrap(err, "remove any existing index at path") } - // We use file writer for buffers not larger than reused one. var fileWriter *FileWriter - fileWriter, err = NewFileWriter(cacheFilename, memoryWriter) + fileWriter, err = NewFileWriter(cacheFilename, len(buf)) if err != nil { return nil, err } @@ -287,6 +280,8 @@ func newBinaryWriter(id ulid.ULID, cacheFilename string, buf []byte) (w *binaryW return nil, errors.Wrap(err, "sync dir") } binWriter = fileWriter + } else { + binWriter = NewMemoryWriter(id, len(buf)) } w = &binaryWriter{ @@ -304,10 +299,14 @@ func newBinaryWriter(id ulid.ULID, cacheFilename string, buf []byte) (w *binaryW return w, w.writer.Write(w.buf.Get()) } +type PosWriterWithBuffer interface { + PosWriter + Buffer() []byte +} + type PosWriter interface { Pos() uint64 Write(bufs ...[]byte) error - Buffer() []byte Flush() error Sync() error Close() error @@ -315,18 +314,16 @@ type PosWriter interface { type MemoryWriter struct { id ulid.ULID - buf bytes.Buffer + buf *bytes.Buffer pos uint64 } -// TODO(bwplotka): Added size to method, upstream this. -func NewMemoryWriter(id ulid.ULID, size int) (*MemoryWriter, error) { - var buf bytes.Buffer +func NewMemoryWriter(id ulid.ULID, size int) *MemoryWriter { return &MemoryWriter{ id: id, - buf: buf, + buf: bytes.NewBuffer(make([]byte, 0, size)), pos: 0, - }, nil + } } func (mw *MemoryWriter) Pos() uint64 { @@ -369,58 +366,52 @@ func (mw *MemoryWriter) Close() error { type FileWriter struct { f *os.File - memWriter *MemoryWriter fileWriter *bufio.Writer name string + pos uint64 } // TODO(bwplotka): Added size to method, upstream this. -func NewFileWriter(name string, memWriter *MemoryWriter) (*FileWriter, error) { +func NewFileWriter(name string, size int) (*FileWriter, error) { f, err := os.OpenFile(filepath.Clean(name), os.O_CREATE|os.O_RDWR, 0600) if err != nil { return nil, err } return &FileWriter{ f: f, - memWriter: memWriter, - fileWriter: bufio.NewWriterSize(f, memWriter.buf.Len()), + fileWriter: bufio.NewWriterSize(f, size), name: name, + pos: 0, }, nil } func (fw *FileWriter) Pos() uint64 { - return fw.memWriter.Pos() + return fw.pos } func (fw *FileWriter) Write(bufs ...[]byte) error { - if err := fw.memWriter.Write(bufs...); err != nil { - return err - } for _, b := range bufs { - _, err := fw.fileWriter.Write(b) + n, err := fw.fileWriter.Write(b) + fw.pos += uint64(n) if err != nil { return err } + // For now the index file must not grow beyond 64GiB. Some of the fixed-sized + // offset references in v1 are only 4 bytes large. + // Once we move to compressed/varint representations in those areas, this limitation + // can be lifted. + if fw.pos > 16*math.MaxUint32 { + return errors.Errorf("%q exceeding max size of 64GiB", fw.name) + } } return nil } -func (fw *FileWriter) Buffer() []byte { - return fw.memWriter.Buffer() -} - func (fw *FileWriter) Flush() error { - if err := fw.memWriter.Flush(); err != nil { - return err - } - return fw.fileWriter.Flush() } func (fw *FileWriter) Close() error { - if err := fw.memWriter.Close(); err != nil { - return err - } if err := fw.Flush(); err != nil { return err } @@ -431,9 +422,6 @@ func (fw *FileWriter) Close() error { } func (fw *FileWriter) Sync() error { - if err := fw.memWriter.Sync(); err != nil { - return err - } return fw.f.Sync() } @@ -476,7 +464,11 @@ func (w *binaryWriter) Write(p []byte) (int, error) { } func (w *binaryWriter) Buffer() []byte { - return w.writer.Buffer() + pwb, ok := w.writer.(PosWriterWithBuffer) + if ok { + return pwb.Buffer() + } + return nil } func (w *binaryWriter) Close() error { From 9e7bd8eb7de09860c05ffbf586353c9998347e9b Mon Sep 17 00:00:00 2001 From: Douglas Camata <159076+douglascamata@users.noreply.github.com> Date: Tue, 11 Jul 2023 22:35:06 +0200 Subject: [PATCH 6/7] Wrap object store Prometheus registry (#6152) * Wrap object store Prometheus registery In preparation for the work being done at https://github.com/thanos-io/objstore/pull/26. Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> * Update objstore to the latest version This version removes the `thanos_` prefix from metrics, which is the reason for wrapping the objstore's metrics registry in the first place. Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> * Fix modules Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> * Rerun CI Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> * Wrap metrics registerer for objstore bucket Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> * Remove prefix from Thanos Store metrics Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> * Fix goimports Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> * Rerun CI Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> * Rerun CI Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> * Put back upgraded objstore dep Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> * Rerun CI Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> * Rerun CI Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> * Move to more recent ref of thanos/objstore Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> * Ignore OBS in objstore tests Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> * Fix linting error on main test after objstore upgrade Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> * Skip OCS objstore test in circle ci Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> * Fix echo in makefile Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> * Fix typo Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> --------- Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> --- .circleci/config.yml | 2 +- Makefile | 6 +++--- cmd/thanos/compact.go | 2 +- cmd/thanos/downsample.go | 2 +- cmd/thanos/main_test.go | 4 ++++ cmd/thanos/receive.go | 2 +- cmd/thanos/rule.go | 2 +- cmd/thanos/sidecar.go | 2 +- cmd/thanos/store.go | 2 +- cmd/thanos/tools_bucket.go | 16 ++++++++-------- go.mod | 3 ++- go.sum | 6 ++++-- pkg/block/block_test.go | 4 +++- pkg/shipper/shipper_e2e_test.go | 4 +++- 14 files changed, 34 insertions(+), 23 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 97cdc78e28..50c82994f0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,7 +36,7 @@ jobs: - run: name: "Run unit tests." environment: - THANOS_TEST_OBJSTORE_SKIP: GCS,S3,AZURE,COS,ALIYUNOSS,BOS,OCI + THANOS_TEST_OBJSTORE_SKIP: GCS,S3,AZURE,COS,ALIYUNOSS,BOS,OCI,OBS # Variables for Swift testing. OS_AUTH_URL: http://127.0.0.1:5000/v2.0 OS_PASSWORD: s3cr3t diff --git a/Makefile b/Makefile index 3569214c65..4c3e2dba78 100644 --- a/Makefile +++ b/Makefile @@ -307,12 +307,12 @@ test: export THANOS_TEST_PROMETHEUS_PATHS= $(PROMETHEUS) test: export THANOS_TEST_ALERTMANAGER_PATH= $(ALERTMANAGER) test: check-git install-tool-deps @echo ">> install thanos GOOPTS=${GOOPTS}" - @echo ">> running unit tests (without /test/e2e). Do export THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI if you want to skip e2e tests against all real store buckets. Current value: ${THANOS_TEST_OBJSTORE_SKIP}" + @echo ">> running unit tests (without /test/e2e). Do export THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI,OBS if you want to skip e2e tests against all real store buckets. Current value: ${THANOS_TEST_OBJSTORE_SKIP}" @go test -timeout 15m $(shell go list ./... | grep -v /vendor/ | grep -v /test/e2e); .PHONY: test-local test-local: ## Runs test excluding tests for ALL object storage integrations. -test-local: export THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI +test-local: export THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI,OBS test-local: $(MAKE) test @@ -333,7 +333,7 @@ test-e2e: docker $(GOTESPLIT) .PHONY: test-e2e-local test-e2e-local: ## Runs all thanos e2e tests locally. -test-e2e-local: export THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI +test-e2e-local: export THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI,OBS test-e2e-local: $(MAKE) test-e2e diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 1368b56989..2649236d6a 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -202,7 +202,7 @@ func runCompact( return err } - bkt, err := client.NewBucket(logger, confContentYaml, reg, component.String()) + bkt, err := client.NewBucket(logger, confContentYaml, extprom.WrapRegistererWithPrefix("thanos_", reg), component.String()) if err != nil { return err } diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index 366eb16145..b89d4d6ee2 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -80,7 +80,7 @@ func RunDownsample( return err } - bkt, err := client.NewBucket(logger, confContentYaml, reg, component.Downsample.String()) + bkt, err := client.NewBucket(logger, confContentYaml, extprom.WrapRegistererWithPrefix("thanos_", reg), component.Downsample.String()) if err != nil { return err } diff --git a/cmd/thanos/main_test.go b/cmd/thanos/main_test.go index deeb085640..2b67c1af70 100644 --- a/cmd/thanos/main_test.go +++ b/cmd/thanos/main_test.go @@ -31,6 +31,10 @@ type erroringBucket struct { bkt objstore.InstrumentedBucket } +func (b *erroringBucket) IsCustomerManagedKeyError(err error) bool { + return b.bkt.IsCustomerManagedKeyError(err) +} + func (b *erroringBucket) Close() error { return b.bkt.Close() } diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index ee9dbb5b16..cf0ca6da77 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -175,7 +175,7 @@ func runReceive( } // The background shipper continuously scans the data directory and uploads // new blocks to object storage service. - bkt, err = client.NewBucket(logger, confContentYaml, reg, comp.String()) + bkt, err = client.NewBucket(logger, confContentYaml, extprom.WrapRegistererWithPrefix("thanos_", reg), comp.String()) if err != nil { return err } diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 6a57a55296..bc1f049de6 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -716,7 +716,7 @@ func runRule( if len(confContentYaml) > 0 { // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, confContentYaml, reg, component.Rule.String()) + bkt, err := client.NewBucket(logger, confContentYaml, extprom.WrapRegistererWithPrefix("thanos_", reg), component.Rule.String()) if err != nil { return err } diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index ddd24104ed..4f8208ccde 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -308,7 +308,7 @@ func runSidecar( if uploads { // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, confContentYaml, reg, component.Sidecar.String()) + bkt, err := client.NewBucket(logger, confContentYaml, extprom.WrapRegistererWithPrefix("thanos_", reg), component.Sidecar.String()) if err != nil { return err } diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 4ac27ddeb3..d189586ac5 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -283,7 +283,7 @@ func runStore( return err } - bkt, err := client.NewBucket(logger, confContentYaml, reg, conf.component.String()) + bkt, err := client.NewBucket(logger, confContentYaml, extprom.WrapRegistererWithPrefix("thanos_", reg), conf.component.String()) if err != nil { return errors.Wrap(err, "create bucket client") } diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index b9682406df..71549c5e6f 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -299,7 +299,7 @@ func registerBucketVerify(app extkingpin.AppClause, objStoreConfig *extflag.Path return err } - bkt, err := client.NewBucket(logger, confContentYaml, reg, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, extprom.WrapRegistererWithPrefix("thanos_", reg), component.Bucket.String()) if err != nil { return err } @@ -380,7 +380,7 @@ func registerBucketLs(app extkingpin.AppClause, objStoreConfig *extflag.PathOrCo return err } - bkt, err := client.NewBucket(logger, confContentYaml, reg, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, extprom.WrapRegistererWithPrefix("thanos_", reg), component.Bucket.String()) if err != nil { return err } @@ -486,7 +486,7 @@ func registerBucketInspect(app extkingpin.AppClause, objStoreConfig *extflag.Pat return err } - bkt, err := client.NewBucket(logger, confContentYaml, reg, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, extprom.WrapRegistererWithPrefix("thanos_", reg), component.Bucket.String()) if err != nil { return err } @@ -594,7 +594,7 @@ func registerBucketWeb(app extkingpin.AppClause, objStoreConfig *extflag.PathOrC return err } - bkt, err := client.NewBucket(logger, confContentYaml, reg, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, extprom.WrapRegistererWithPrefix("thanos_", reg), component.Bucket.String()) if err != nil { return errors.Wrap(err, "bucket client") } @@ -779,7 +779,7 @@ func registerBucketCleanup(app extkingpin.AppClause, objStoreConfig *extflag.Pat return err } - bkt, err := client.NewBucket(logger, confContentYaml, reg, component.Cleanup.String()) + bkt, err := client.NewBucket(logger, confContentYaml, extprom.WrapRegistererWithPrefix("thanos_", reg), component.Cleanup.String()) if err != nil { return err } @@ -1034,7 +1034,7 @@ func registerBucketMarkBlock(app extkingpin.AppClause, objStoreConfig *extflag.P return err } - bkt, err := client.NewBucket(logger, confContentYaml, reg, component.Mark.String()) + bkt, err := client.NewBucket(logger, confContentYaml, extprom.WrapRegistererWithPrefix("thanos_", reg), component.Mark.String()) if err != nil { return err } @@ -1113,7 +1113,7 @@ func registerBucketRewrite(app extkingpin.AppClause, objStoreConfig *extflag.Pat return err } - bkt, err := client.NewBucket(logger, confContentYaml, reg, component.Rewrite.String()) + bkt, err := client.NewBucket(logger, confContentYaml, extprom.WrapRegistererWithPrefix("thanos_", reg), component.Rewrite.String()) if err != nil { return err } @@ -1320,7 +1320,7 @@ func registerBucketRetention(app extkingpin.AppClause, objStoreConfig *extflag.P return err } - bkt, err := client.NewBucket(logger, confContentYaml, reg, component.Retention.String()) + bkt, err := client.NewBucket(logger, confContentYaml, extprom.WrapRegistererWithPrefix("thanos_", reg), component.Retention.String()) if err != nil { return err } diff --git a/go.mod b/go.mod index 80114896e0..b99785ba8a 100644 --- a/go.mod +++ b/go.mod @@ -65,7 +65,7 @@ require ( github.com/prometheus/prometheus v0.44.1-0.20230524110021-37e5249e33e4 github.com/sony/gobreaker v0.5.0 github.com/stretchr/testify v1.8.2 - github.com/thanos-io/objstore v0.0.0-20230201072718-11ffbc490204 + github.com/thanos-io/objstore v0.0.0-20230710163637-47c0118da0ca github.com/thanos-io/promql-engine v0.0.0-20230524050847-9a1a33217cf2 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible // indirect @@ -124,6 +124,7 @@ require ( ) require ( + github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible // indirect github.com/onsi/ginkgo v1.16.5 // indirect go4.org/unsafe/assume-no-moving-gc v0.0.0-20230209150437-ee73d164e760 // indirect ) diff --git a/go.sum b/go.sum index 1a2568d345..5cf06bd83a 100644 --- a/go.sum +++ b/go.sum @@ -565,6 +565,8 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= github.com/hetznercloud/hcloud-go v1.45.1 h1:nl0OOklFfQT5J6AaNIOhl5Ruh3fhmGmhvZEqHbibVuk= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible h1:tKTaPHNVwikS3I1rdyf1INNvgJXWSf/+TzqsiGbrgnQ= +github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -898,8 +900,8 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4 github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1Zsv7OAU9iQhZwigp50Yl38W10g/vd5NC8Rdk1Jzng= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM= -github.com/thanos-io/objstore v0.0.0-20230201072718-11ffbc490204 h1:W4w5Iph7j32Sf1QFWLJDCqvO0WgZS0jHGID+qnq3wV0= -github.com/thanos-io/objstore v0.0.0-20230201072718-11ffbc490204/go.mod h1:STSgpY8M6EKF2G/raUFdbIMf2U9GgYlEjAEHJxjvpAo= +github.com/thanos-io/objstore v0.0.0-20230710163637-47c0118da0ca h1:JRF7i58HovirZQVJGwCClQsMK6CCmK2fvialXjeoSpI= +github.com/thanos-io/objstore v0.0.0-20230710163637-47c0118da0ca/go.mod h1:5V7lzXuaxwt6XFQoA/zJrhdnQrxq1+r0bwQ1iYOq3gM= github.com/thanos-io/promql-engine v0.0.0-20230524050847-9a1a33217cf2 h1:bz8GzwMTYV0cRxdgeESmqrRRXE2yH/NiyMD+zVVtWJo= github.com/thanos-io/promql-engine v0.0.0-20230524050847-9a1a33217cf2/go.mod h1:eIgPaXWgOhNAv6CPPrgu09r0AtT7byBTZy+7WkX0D18= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU= diff --git a/pkg/block/block_test.go b/pkg/block/block_test.go index 432129e3b1..2a671f8950 100644 --- a/pkg/block/block_test.go +++ b/pkg/block/block_test.go @@ -15,6 +15,8 @@ import ( "testing" "time" + "github.com/thanos-io/thanos/pkg/extprom" + "github.com/go-kit/log" "github.com/oklog/ulid" "github.com/pkg/errors" @@ -449,7 +451,7 @@ func TestHashDownload(t *testing.T) { bkt := objstore.NewInMemBucket() r := prometheus.NewRegistry() - instrumentedBkt := objstore.BucketWithMetrics("test", bkt, r) + instrumentedBkt := objstore.BucketWithMetrics("test", bkt, extprom.WrapRegistererWithPrefix("thanos_", r)) b1, err := e2eutil.CreateBlockWithTombstone(ctx, tmpDir, []labels.Labels{ {{Name: "a", Value: "1"}}, diff --git a/pkg/shipper/shipper_e2e_test.go b/pkg/shipper/shipper_e2e_test.go index 95846e3b07..985ee4329f 100644 --- a/pkg/shipper/shipper_e2e_test.go +++ b/pkg/shipper/shipper_e2e_test.go @@ -16,6 +16,8 @@ import ( "testing" "time" + "github.com/thanos-io/thanos/pkg/extprom" + "github.com/go-kit/log" "github.com/oklog/ulid" "github.com/prometheus/client_golang/prometheus" @@ -37,7 +39,7 @@ func TestShipper_SyncBlocks_e2e(t *testing.T) { objtesting.ForeachStore(t, func(t *testing.T, bkt objstore.Bucket) { // TODO(GiedriusS): consider switching to BucketWithMetrics() everywhere? metrics := prometheus.NewRegistry() - metricsBucket := objstore.BucketWithMetrics("test", bkt, metrics) + metricsBucket := objstore.BucketWithMetrics("test", bkt, extprom.WrapRegistererWithPrefix("thanos_", metrics)) dir := t.TempDir() From 4218109434022f059a1e250dcaf961f545c9dd10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Wed, 12 Jul 2023 12:11:50 +0300 Subject: [PATCH 7/7] e2e/store: try to fix Series() limit test again (#6522) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I finally managed to reproduce this failure locally with https://github.com/efficientgo/e2e/commit/c316eb95ae5b0fa65b0be110f8cfc3c48e8810bb. The added t.Logf() showed that is the problem is that with a lower bytes limit, it might hit the series or chunks part first. I have bumped the bytes limit. I calculated the new bytes limit by checking how much bytes are allocated before sending the last chunk. I have also noticed that one block is created without a delay. Update it so that it would be like the others. Include objstore@main update with https://github.com/thanos-io/objstore/pull/62/files so that Iter() would always return an error on a timeout. Signed-off-by: Giedrius Statkevičius --- .circleci/config.yml | 2 +- docs/storage.md | 4 ++-- pkg/replicate/replicator.go | 4 ++-- test/e2e/store_gateway_test.go | 10 +++++++--- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 50c82994f0..7098932b84 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,7 @@ jobs: test: executor: golang-test environment: - GO111MODULE: 'on' + GO111MODULE: "on" steps: - git-shallow-clone/checkout - go/mod-download-cached diff --git a/docs/storage.md b/docs/storage.md index f283e5b5e0..24634ad3c7 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -209,7 +209,7 @@ Example working AWS IAM policy for user: To test the policy, set env vars for S3 access for *empty, not used* bucket as well as: ``` -THANOS_TEST_OBJSTORE_SKIP=GCS,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI +THANOS_TEST_OBJSTORE_SKIP=GCS,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI,OBS THANOS_ALLOW_EXISTING_BUCKET_USE=true ``` @@ -243,7 +243,7 @@ We need access to CreateBucket and DeleteBucket and access to all buckets: } ``` -With this policy you should be able to run set `THANOS_TEST_OBJSTORE_SKIP=GCS,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI` and unset `S3_BUCKET` and run all tests using `make test`. +With this policy you should be able to run set `THANOS_TEST_OBJSTORE_SKIP=GCS,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI,OBS` and unset `S3_BUCKET` and run all tests using `make test`. Details about AWS policies: https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html diff --git a/pkg/replicate/replicator.go b/pkg/replicate/replicator.go index 1c42c14c8a..151c667197 100644 --- a/pkg/replicate/replicator.go +++ b/pkg/replicate/replicator.go @@ -117,7 +117,7 @@ func RunReplicate( fromBkt, err := client.NewBucket( logger, fromConfContentYaml, - prometheus.WrapRegistererWith(prometheus.Labels{"replicate": "from"}, reg), + prometheus.WrapRegistererWithPrefix("thanos_", prometheus.WrapRegistererWith(prometheus.Labels{"replicate": "from"}, reg)), component.Replicate.String(), ) if err != nil { @@ -136,7 +136,7 @@ func RunReplicate( toBkt, err := client.NewBucket( logger, toConfContentYaml, - prometheus.WrapRegistererWith(prometheus.Labels{"replicate": "to"}, reg), + prometheus.WrapRegistererWithPrefix("thanos_", prometheus.WrapRegistererWith(prometheus.Labels{"replicate": "to"}, reg)), component.Replicate.String(), ) if err != nil { diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index b5039ab408..4a888b1ff1 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -825,7 +825,7 @@ config: }, string(cacheCfg), "", - []string{"--store.grpc.downloaded-bytes-limit=196627B"}, + []string{"--store.grpc.downloaded-bytes-limit=310176B"}, ) testutil.Ok(t, e2e.StartAndWaitReady(store1, store2, store3)) @@ -842,6 +842,7 @@ config: extLset := labels.FromStrings("ext1", "value1", "replica", "1") extLset2 := labels.FromStrings("ext1", "value1", "replica", "2") extLset3 := labels.FromStrings("ext1", "value2", "replica", "3") + extLset4 := labels.FromStrings("ext1", "value2", "replica", "4") ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) t.Cleanup(cancel) @@ -853,7 +854,7 @@ config: testutil.Ok(t, err) id3, err := e2eutil.CreateBlockWithBlockDelay(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), 30*time.Minute, extLset3, 0, metadata.NoneFunc) testutil.Ok(t, err) - id4, err := e2eutil.CreateBlock(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), extLset, 0, metadata.NoneFunc) + id4, err := e2eutil.CreateBlockWithBlockDelay(ctx, dir, series, 10, timestamp.FromTime(now), timestamp.FromTime(now.Add(2*time.Hour)), 30*time.Minute, extLset4, 0, metadata.NoneFunc) testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, @@ -897,8 +898,11 @@ config: testutil.Ok(t, runutil.RetryWithLog(log.NewLogfmtLogger(os.Stdout), 5*time.Second, ctx.Done(), func() error { if _, _, _, err := promclient.NewDefaultClient().QueryInstant(ctx, urlParse(t, "http://"+q3.Endpoint("http")), testQuery, now, opts); err != nil { + if err != nil { + t.Logf("got error: %s", err) + } e := err.Error() - if strings.Contains(e, "load chunks") && strings.Contains(e, "exceeded bytes limit while fetching chunks: limit 196627 violated") { + if strings.Contains(e, "load chunks") && strings.Contains(e, "exceeded bytes limit while fetching chunks: limit 310176 violated") { return nil } return err