From 99f0ee50e54f8d1e1ca8180fea47bc039d9645c5 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Wed, 4 Nov 2020 17:35:13 +0100 Subject: [PATCH] compact: Added index size limiting planner detecting output index size over 64GB. Fixes: https://github.com/thanos-io/thanos/issues/1424 Signed-off-by: Bartlomiej Plotka --- pkg/block/block.go | 32 +++++ pkg/block/block_test.go | 56 +++++++++ pkg/block/metadata/markers.go | 2 + pkg/compact/planner.go | 91 +++++++++++++- pkg/compact/planner_test.go | 224 +++++++++++++++++++++++++++++++++- 5 files changed, 403 insertions(+), 2 deletions(-) diff --git a/pkg/block/block.go b/pkg/block/block.go index d02ba26a7ae..164f8f4ba76 100644 --- a/pkg/block/block.go +++ b/pkg/block/block.go @@ -284,3 +284,35 @@ func gatherFileStats(blockDir string) (res []metadata.File, _ error) { // TODO(bwplotka): Add optional files like tombstones? return res, err } + +// MarkForNoCompact creates a file which marks block to be not compacted. +func MarkForNoCompact(ctx context.Context, logger log.Logger, bkt objstore.Bucket, id ulid.ULID, reason metadata.NoCompactReason, noCompactDetails string, markedForNoCompact prometheus.Counter) error { + m := path.Join(id.String(), metadata.NoCompactMarkFilename) + noCompactMarkExists, err := bkt.Exists(ctx, m) + if err != nil { + return errors.Wrapf(err, "check exists %s in bucket", m) + } + if noCompactMarkExists { + level.Warn(logger).Log("msg", "requested to mark for no compaction, but file already exists; this should not happen; investigate", "err", errors.Errorf("file %s already exists in bucket", m)) + return nil + } + + noCompactMark, err := json.Marshal(metadata.NoCompactMark{ + ID: id, + Version: metadata.NoCompactMarkVersion1, + + Time: time.Now().Unix(), + Reason: reason, + Details: noCompactDetails, + }) + if err != nil { + return errors.Wrap(err, "json encode no compact mark") + } + + if err := bkt.Upload(ctx, m, bytes.NewBuffer(noCompactMark)); err != nil { + return errors.Wrapf(err, "upload file %s to bucket", m) + } + markedForNoCompact.Inc() + level.Info(logger).Log("msg", "block has been marked for no compaction", "block", id) + return nil +} diff --git a/pkg/block/block_test.go b/pkg/block/block_test.go index fb584bc7584..15598eaa18f 100644 --- a/pkg/block/block_test.go +++ b/pkg/block/block_test.go @@ -311,3 +311,59 @@ func TestMarkForDeletion(t *testing.T) { }) } } + +func TestMarkForNoCompact(t *testing.T) { + defer testutil.TolerantVerifyLeak(t) + ctx := context.Background() + + tmpDir, err := ioutil.TempDir("", "test-block-mark-for-no-compact") + testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(tmpDir)) }() + + for _, tcase := range []struct { + name string + preUpload func(t testing.TB, id ulid.ULID, bkt objstore.Bucket) + + blocksMarked int + }{ + { + name: "block marked", + preUpload: func(t testing.TB, id ulid.ULID, bkt objstore.Bucket) {}, + blocksMarked: 1, + }, + { + name: "block with no-compact mark already, expected log and no metric increment", + preUpload: func(t testing.TB, id ulid.ULID, bkt objstore.Bucket) { + m, err := json.Marshal(metadata.NoCompactMark{ + ID: id, + Time: time.Now().Unix(), + Version: metadata.NoCompactMarkVersion1, + }) + testutil.Ok(t, err) + testutil.Ok(t, bkt.Upload(ctx, path.Join(id.String(), metadata.NoCompactMarkFilename), bytes.NewReader(m))) + }, + blocksMarked: 0, + }, + } { + t.Run(tcase.name, func(t *testing.T) { + bkt := objstore.NewInMemBucket() + id, err := e2eutil.CreateBlock(ctx, tmpDir, []labels.Labels{ + {{Name: "a", Value: "1"}}, + {{Name: "a", Value: "2"}}, + {{Name: "a", Value: "3"}}, + {{Name: "a", Value: "4"}}, + {{Name: "b", Value: "1"}}, + }, 100, 0, 1000, labels.Labels{{Name: "ext1", Value: "val1"}}, 124) + testutil.Ok(t, err) + + tcase.preUpload(t, id, bkt) + + testutil.Ok(t, Upload(ctx, log.NewNopLogger(), bkt, path.Join(tmpDir, id.String()))) + + c := promauto.With(nil).NewCounter(prometheus.CounterOpts{}) + err = MarkForNoCompact(ctx, log.NewNopLogger(), bkt, id, metadata.ManualNoCompactReason, "", c) + testutil.Ok(t, err) + testutil.Equals(t, float64(tcase.blocksMarked), promtest.ToFloat64(c)) + }) + } +} diff --git a/pkg/block/metadata/markers.go b/pkg/block/metadata/markers.go index 075311192a7..494544be360 100644 --- a/pkg/block/metadata/markers.go +++ b/pkg/block/metadata/markers.go @@ -74,6 +74,8 @@ type NoCompactMark struct { // Version of the file. Version int `json:"version"` + // Time is a unix timestamp of when the block was marked for no compact. + Time int64 `json:"time"` Reason NoCompactReason `json:"reason"` // Details is a human readable string giving details of reason. Details string `json:"details"` diff --git a/pkg/compact/planner.go b/pkg/compact/planner.go index 12c541c2007..5bcd72209f0 100644 --- a/pkg/compact/planner.go +++ b/pkg/compact/planner.go @@ -5,10 +5,17 @@ package compact import ( "context" + "fmt" + "math" + "path/filepath" "github.com/go-kit/kit/log" "github.com/oklog/ulid" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/thanos-io/thanos/pkg/objstore" ) type tsdbBasedPlanner struct { @@ -36,7 +43,10 @@ func NewPlanner(logger log.Logger, ranges []int64, noCompBlocks *GatherNoCompact // TODO(bwplotka): Consider smarter algorithm, this prefers smaller iterative compactions vs big single one: https://github.com/thanos-io/thanos/issues/3405 func (p *tsdbBasedPlanner) Plan(_ context.Context, metasByMinTime []*metadata.Meta) ([]*metadata.Meta, error) { - noCompactMarked := p.noCompBlocksFunc() + return p.plan(p.noCompBlocksFunc(), metasByMinTime) +} + +func (p *tsdbBasedPlanner) plan(noCompactMarked map[ulid.ULID]*metadata.NoCompactMark, metasByMinTime []*metadata.Meta) ([]*metadata.Meta, error) { notExcludedMetasByMinTime := make([]*metadata.Meta, 0, len(metasByMinTime)) for _, meta := range metasByMinTime { if _, excluded := noCompactMarked[meta.ULID]; excluded { @@ -203,3 +213,82 @@ func splitByRange(metasByMinTime []*metadata.Meta, tr int64) [][]*metadata.Meta return splitDirs } + +type largeTotalIndexSizeFilter struct { + *tsdbBasedPlanner + + bkt objstore.Bucket + markedForNoCompact prometheus.Counter + totalMaxIndexSizeBytes int64 +} + +var _ Planner = &largeTotalIndexSizeFilter{} + +// WithLargeTotalIndexSizeFilter wraps Planner with largeTotalIndexSizeFilter that checks the given plans and estimates total index size. +// When found, it marks block for no compaction by placing no-compact.json and updating cache. +// NOTE: The estimation is very rough as it assumes extreme cases of indexes sharing no bytes, thus summing all source index sizes. +// Adjust limit accordingly reducing to some % of actual limit you want to give. +// TODO(bwplotka): This is short term fix for https://github.com/thanos-io/thanos/issues/1424, replace with vertical block sharding https://github.com/thanos-io/thanos/pull/3390. +func WithLargeTotalIndexSizeFilter(with *tsdbBasedPlanner, bkt objstore.Bucket, totalMaxIndexSizeBytes int64, markedForNoCompact prometheus.Counter) *largeTotalIndexSizeFilter { + return &largeTotalIndexSizeFilter{tsdbBasedPlanner: with, bkt: bkt, totalMaxIndexSizeBytes: totalMaxIndexSizeBytes, markedForNoCompact: markedForNoCompact} +} + +func (t *largeTotalIndexSizeFilter) Plan(ctx context.Context, metasByMinTime []*metadata.Meta) ([]*metadata.Meta, error) { + noCompactMarked := t.noCompBlocksFunc() + copiedNoCompactMarked := make(map[ulid.ULID]*metadata.NoCompactMark, len(noCompactMarked)) + for k, v := range noCompactMarked { + copiedNoCompactMarked[k] = v + } + +PlanLoop: + for { + plan, err := t.plan(copiedNoCompactMarked, metasByMinTime) + if err != nil { + return nil, err + } + var totalIndexBytes, maxIndexSize int64 = 0, math.MinInt64 + var biggestIndex int + for i, p := range plan { + indexSize := int64(-1) + for _, f := range p.Thanos.Files { + if f.RelPath == block.IndexFilename { + indexSize = f.SizeBytes + } + } + if indexSize <= 0 { + // Get size from bkt instead. + attr, err := t.bkt.Attributes(ctx, filepath.Join(p.ULID.String(), block.IndexFilename)) + if err != nil { + return nil, errors.Wrapf(err, "get attr of %v", filepath.Join(p.ULID.String(), block.IndexFilename)) + } + indexSize = attr.Size + } + + if maxIndexSize < indexSize { + maxIndexSize = indexSize + biggestIndex = i + } + totalIndexBytes += indexSize + if totalIndexBytes >= t.totalMaxIndexSizeBytes { + // Marking blocks for no compact to limit size. + // TODO(bwplotka): Make sure to reset cache once this is done: https://github.com/thanos-io/thanos/issues/3408 + if err := block.MarkForNoCompact( + ctx, + t.logger, + t.bkt, + plan[biggestIndex].ULID, + metadata.IndexSizeExceedingNoCompactReason, + fmt.Sprintf("largeTotalIndexSizeFilter: Total compacted block's index size could exceed: %v with this block. See https://github.com/thanos-io/thanos/issues/1424", t.totalMaxIndexSizeBytes), + t.markedForNoCompact, + ); err != nil { + return nil, errors.Wrapf(err, "mark %v for no compaction", plan[biggestIndex].ULID.String()) + } + // Make sure wrapped planner exclude this block. + copiedNoCompactMarked[plan[biggestIndex].ULID] = &metadata.NoCompactMark{ID: plan[biggestIndex].ULID, Version: metadata.NoCompactMarkVersion1} + continue PlanLoop + } + } + // Planned blocks should not exceed limit. + return plan, nil + } +} diff --git a/pkg/compact/planner_test.go b/pkg/compact/planner_test.go index be8d0729095..9821d42b9ac 100644 --- a/pkg/compact/planner_test.go +++ b/pkg/compact/planner_test.go @@ -4,6 +4,7 @@ package compact import ( + "bytes" "context" "io/ioutil" "os" @@ -14,8 +15,13 @@ import ( "github.com/go-kit/kit/log" "github.com/oklog/ulid" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + promtest "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/prometheus/tsdb" + "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/testutil" ) @@ -438,7 +444,7 @@ func TestRangeWithFailedCompactionWontGetSelected(t *testing.T) { } } -func TestTSDBBasedPlanners_PlanWithNoCompactMarkers(t *testing.T) { +func TestTSDBBasedPlanner_PlanWithNoCompactMarks(t *testing.T) { ranges := []int64{ 20, 60, @@ -629,3 +635,219 @@ func TestTSDBBasedPlanners_PlanWithNoCompactMarkers(t *testing.T) { }) } } + +func TestLargeTotalIndexSizeFilter_Plan(t *testing.T) { + ranges := []int64{ + 20, + 60, + 180, + 540, + 1620, + } + + bkt := objstore.NewInMemBucket() + g := &GatherNoCompactionMarkFilter{} + + marked := promauto.With(nil).NewCounter(prometheus.CounterOpts{}) + planner := WithLargeTotalIndexSizeFilter(NewPlanner(log.NewNopLogger(), ranges, g), bkt, 100, marked) + var lastMarkValue float64 + for _, c := range []struct { + name string + metas []*metadata.Meta + + expected []*metadata.Meta + expectedMarks float64 + }{ + { + name: "Outside range and excluded", + metas: []*metadata.Meta{ + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 100}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(1, nil), MinTime: 0, MaxTime: 20}}, + }, + expectedMarks: 0, + }, + { + name: "Blocks to fill the entire parent, but with first one too large.", + metas: []*metadata.Meta{ + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 41}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(1, nil), MinTime: 0, MaxTime: 20}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(2, nil), MinTime: 20, MaxTime: 40}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(3, nil), MinTime: 40, MaxTime: 60}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(4, nil), MinTime: 60, MaxTime: 80}}, + }, + expectedMarks: 1, + expected: []*metadata.Meta{ + {BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(2, nil), MinTime: 20, MaxTime: 40}}, + {BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(3, nil), MinTime: 40, MaxTime: 60}}, + }, + }, + { + name: "Blocks to fill the entire parent, but with second one too large.", + metas: []*metadata.Meta{ + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(1, nil), MinTime: 0, MaxTime: 20}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 41}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(2, nil), MinTime: 20, MaxTime: 40}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(3, nil), MinTime: 40, MaxTime: 60}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 20}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(4, nil), MinTime: 60, MaxTime: 80}}, + }, + expectedMarks: 1, + }, + { + name: "Blocks to fill the entire parent, but with last size exceeded (should not matter and not even marked).", + metas: []*metadata.Meta{ + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(1, nil), MinTime: 0, MaxTime: 20}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(2, nil), MinTime: 20, MaxTime: 40}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(3, nil), MinTime: 40, MaxTime: 60}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 90}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(4, nil), MinTime: 60, MaxTime: 80}}, + }, + expected: []*metadata.Meta{ + {BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(1, nil), MinTime: 0, MaxTime: 20}}, + {BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(2, nil), MinTime: 20, MaxTime: 40}}, + {BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(3, nil), MinTime: 40, MaxTime: 60}}, + }, + }, + { + name: "Blocks to fill the entire parent, but with pre-last one and first too large.", + metas: []*metadata.Meta{ + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 90}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(1, nil), MinTime: 0, MaxTime: 20}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(2, nil), MinTime: 20, MaxTime: 40}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(3, nil), MinTime: 40, MaxTime: 50}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 90}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(4, nil), MinTime: 50, MaxTime: 60}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 90}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(5, nil), MinTime: 60, MaxTime: 80}}, + }, + expected: []*metadata.Meta{ + {BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(2, nil), MinTime: 20, MaxTime: 40}}, + {BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(3, nil), MinTime: 40, MaxTime: 50}}, + }, + expectedMarks: 2, + }, + { + name: `Block for the next parent range appeared, and we have a gap with size 20 between second and third block. + Second block is excluded.`, + metas: []*metadata.Meta{ + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(1, nil), MinTime: 0, MaxTime: 20}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 90}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(2, nil), MinTime: 20, MaxTime: 40}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(4, nil), MinTime: 60, MaxTime: 80}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(5, nil), MinTime: 80, MaxTime: 100}}, + }, + expectedMarks: 1, + }, + { + name: "We have 20, 60, 20, 60, 240 range blocks. We could compact 20 + 60 + 60, but sixth 6th is excluded", + metas: []*metadata.Meta{ + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(2, nil), MinTime: 20, MaxTime: 40}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(4, nil), MinTime: 60, MaxTime: 120}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(5, nil), MinTime: 960, MaxTime: 980}}, // Fresh one. + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 90}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(6, nil), MinTime: 120, MaxTime: 180}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(7, nil), MinTime: 720, MaxTime: 960}}, + }, + expected: []*metadata.Meta{ + {BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(2, nil), MinTime: 20, MaxTime: 40}}, + {BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(4, nil), MinTime: 60, MaxTime: 120}}, + }, + expectedMarks: 1, + }, + // |--------------| + // |----------------| + // |--------------| + { + name: "Overlapping blocks 1, but total is too large", + metas: []*metadata.Meta{ + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 90}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(1, nil), MinTime: 0, MaxTime: 20}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(2, nil), MinTime: 19, MaxTime: 40}}, + {Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: block.IndexFilename, SizeBytes: 30}}}, + BlockMeta: tsdb.BlockMeta{Version: 1, ULID: ulid.MustNew(3, nil), MinTime: 40, MaxTime: 60}}, + }, + expectedMarks: 1, + }, + } { + if !t.Run(c.name, func(t *testing.T) { + t.Run("from meta", func(t *testing.T) { + obj := bkt.Objects() + for o := range obj { + delete(obj, o) + } + + metasByMinTime := make([]*metadata.Meta, len(c.metas)) + for i := range metasByMinTime { + orig := c.metas[i] + m := &metadata.Meta{} + *m = *orig + metasByMinTime[i] = m + } + sort.Slice(metasByMinTime, func(i, j int) bool { + return metasByMinTime[i].MinTime < metasByMinTime[j].MinTime + }) + + plan, err := planner.Plan(context.Background(), metasByMinTime) + testutil.Ok(t, err) + + for _, m := range plan { + // For less boilerplate. + m.Thanos = metadata.Thanos{} + } + testutil.Equals(t, c.expected, plan) + testutil.Equals(t, c.expectedMarks, promtest.ToFloat64(marked)-lastMarkValue) + lastMarkValue = promtest.ToFloat64(marked) + }) + t.Run("from bkt", func(t *testing.T) { + obj := bkt.Objects() + for o := range obj { + delete(obj, o) + } + + metasByMinTime := make([]*metadata.Meta, len(c.metas)) + for i := range metasByMinTime { + orig := c.metas[i] + m := &metadata.Meta{} + *m = *orig + metasByMinTime[i] = m + } + sort.Slice(metasByMinTime, func(i, j int) bool { + return metasByMinTime[i].MinTime < metasByMinTime[j].MinTime + }) + + for _, m := range metasByMinTime { + testutil.Ok(t, bkt.Upload(nil, filepath.Join(m.ULID.String(), block.IndexFilename), bytes.NewReader(make([]byte, m.Thanos.Files[0].SizeBytes)))) + m.Thanos = metadata.Thanos{} + } + + plan, err := planner.Plan(context.Background(), metasByMinTime) + testutil.Ok(t, err) + testutil.Equals(t, c.expected, plan) + testutil.Equals(t, c.expectedMarks, promtest.ToFloat64(marked)-lastMarkValue) + + lastMarkValue = promtest.ToFloat64(marked) + }) + + }) { + return + } + } +}