diff --git a/block_test.go b/block_test.go index 68cad4b8..bddddbe5 100644 --- a/block_test.go +++ b/block_test.go @@ -45,7 +45,7 @@ func TestSetCompactionFailed(t *testing.T) { testutil.Ok(t, err) defer os.RemoveAll(tmpdir) - blockDir := createBlock(t, tmpdir, genSeries(1, 1, 0, 0)) + blockDir := createBlock(t, tmpdir, genSeries(1, 1, nil, 0, 0)) b, err := OpenBlock(nil, blockDir, nil) testutil.Ok(t, err) testutil.Equals(t, false, b.meta.Compaction.Failed) @@ -96,12 +96,30 @@ func createBlock(tb testing.TB, dir string, series []Series) string { } // genSeries generates series with a given number of labels and values. -func genSeries(totalSeries, labelCount int, mint, maxt int64) []Series { - if totalSeries == 0 || labelCount == 0 { +// If `prefilledLabels` is not empty, then the number of series and the labels +// for them will be taken from `prefilledLabels`, and `totalSeries` `labelCount` +// will be ignored. +func genSeries(totalSeries, labelCount int, prefilledLabels []map[string]string, mint, maxt int64) []Series { + if (totalSeries == 0 || labelCount == 0) && len(prefilledLabels) == 0 { return nil } - series := make([]Series, totalSeries) + if len(prefilledLabels) > 0 { + series := make([]Series, 0, len(prefilledLabels)) + for _, lbls := range prefilledLabels { + if len(lbls) == 0 { + continue + } + samples := make([]tsdbutil.Sample, 0, maxt-mint+1) + for t := mint; t <= maxt; t++ { + samples = append(samples, sample{t: t, v: rand.Float64()}) + } + series = append(series, newSeries(lbls, samples)) + } + return series + } + + series := make([]Series, totalSeries) for i := 0; i < totalSeries; i++ { lbls := make(map[string]string, labelCount) for len(lbls) < labelCount { @@ -113,7 +131,6 @@ func genSeries(totalSeries, labelCount int, mint, maxt int64) []Series { } series[i] = newSeries(lbls, samples) } - return series } diff --git a/compact_test.go b/compact_test.go index 24ef0cba..d49ce13c 100644 --- a/compact_test.go +++ b/compact_test.go @@ -771,7 +771,7 @@ func BenchmarkCompaction(b *testing.B) { blockDirs := make([]string, 0, len(c.ranges)) var blocks []*Block for _, r := range c.ranges { - block, err := OpenBlock(nil, createBlock(b, dir, genSeries(nSeries, 10, r[0], r[1])), nil) + block, err := OpenBlock(nil, createBlock(b, dir, genSeries(nSeries, 10, nil, r[0], r[1])), nil) testutil.Ok(b, err) blocks = append(blocks, block) defer block.Close() diff --git a/db_test.go b/db_test.go index 6797be98..d4a2fba9 100644 --- a/db_test.go +++ b/db_test.go @@ -91,7 +91,7 @@ func TestDB_reloadOrder(t *testing.T) { {MinTime: 100, MaxTime: 110}, } for _, m := range metas { - createBlock(t, db.Dir(), genSeries(1, 1, m.MinTime, m.MaxTime)) + createBlock(t, db.Dir(), genSeries(1, 1, nil, m.MinTime, m.MaxTime)) } testutil.Ok(t, db.reload()) @@ -833,7 +833,7 @@ func TestTombstoneCleanFail(t *testing.T) { // totalBlocks should be >=2 so we have enough blocks to trigger compaction failure. totalBlocks := 2 for i := 0; i < totalBlocks; i++ { - blockDir := createBlock(t, db.Dir(), genSeries(1, 1, 0, 0)) + blockDir := createBlock(t, db.Dir(), genSeries(1, 1, nil, 0, 0)) block, err := OpenBlock(nil, blockDir, nil) testutil.Ok(t, err) // Add some some fake tombstones to trigger the compaction. @@ -877,7 +877,7 @@ func (c *mockCompactorFailing) Write(dest string, b BlockReader, mint, maxt int6 return ulid.ULID{}, fmt.Errorf("the compactor already did the maximum allowed blocks so it is time to fail") } - block, err := OpenBlock(nil, createBlock(c.t, dest, genSeries(1, 1, 0, 0)), nil) + block, err := OpenBlock(nil, createBlock(c.t, dest, genSeries(1, 1, nil, 0, 0)), nil) testutil.Ok(c.t, err) testutil.Ok(c.t, block.Close()) // Close block as we won't be using anywhere. c.blocks = append(c.blocks, block) @@ -915,7 +915,7 @@ func TestTimeRetention(t *testing.T) { } for _, m := range blocks { - createBlock(t, db.Dir(), genSeries(10, 10, m.MinTime, m.MaxTime)) + createBlock(t, db.Dir(), genSeries(10, 10, nil, m.MinTime, m.MaxTime)) } testutil.Ok(t, db.reload()) // Reload the db to register the new blocks. @@ -949,7 +949,7 @@ func TestSizeRetention(t *testing.T) { } for _, m := range blocks { - createBlock(t, db.Dir(), genSeries(100, 10, m.MinTime, m.MaxTime)) + createBlock(t, db.Dir(), genSeries(100, 10, nil, m.MinTime, m.MaxTime)) } // Test that registered size matches the actual disk size. @@ -1316,7 +1316,7 @@ func TestInitializeHeadTimestamp(t *testing.T) { testutil.Ok(t, err) defer os.RemoveAll(dir) - createBlock(t, dir, genSeries(1, 1, 1000, 2000)) + createBlock(t, dir, genSeries(1, 1, nil, 1000, 2000)) db, err := Open(dir, nil, nil, nil) testutil.Ok(t, err) @@ -1329,7 +1329,7 @@ func TestInitializeHeadTimestamp(t *testing.T) { testutil.Ok(t, err) defer os.RemoveAll(dir) - createBlock(t, dir, genSeries(1, 1, 1000, 6000)) + createBlock(t, dir, genSeries(1, 1, nil, 1000, 6000)) testutil.Ok(t, os.MkdirAll(path.Join(dir, "wal"), 0777)) w, err := wal.New(nil, nil, path.Join(dir, "wal")) @@ -1447,7 +1447,7 @@ func TestNoEmptyBlocks(t *testing.T) { {MinTime: currentTime + 100, MaxTime: currentTime + 100 + db.opts.BlockRanges[0]}, } for _, m := range blocks { - createBlock(t, db.Dir(), genSeries(2, 2, m.MinTime, m.MaxTime)) + createBlock(t, db.Dir(), genSeries(2, 2, nil, m.MinTime, m.MaxTime)) } oldBlocks := db.Blocks() @@ -1924,7 +1924,7 @@ func TestBlockRanges(t *testing.T) { // Test that the compactor doesn't create overlapping blocks // when a non standard block already exists. firstBlockMaxT := int64(3) - createBlock(t, dir, genSeries(1, 1, 0, firstBlockMaxT)) + createBlock(t, dir, genSeries(1, 1, nil, 0, firstBlockMaxT)) db, err := Open(dir, logger, nil, DefaultOptions) if err != nil { t.Fatalf("Opening test storage failed: %s", err) @@ -1974,7 +1974,7 @@ func TestBlockRanges(t *testing.T) { testutil.Ok(t, db.Close()) thirdBlockMaxt := secondBlockMaxt + 2 - createBlock(t, dir, genSeries(1, 1, secondBlockMaxt+1, thirdBlockMaxt)) + createBlock(t, dir, genSeries(1, 1, nil, secondBlockMaxt+1, thirdBlockMaxt)) db, err = Open(dir, logger, nil, DefaultOptions) if err != nil { diff --git a/querier_test.go b/querier_test.go index c3e26023..94f362b3 100644 --- a/querier_test.go +++ b/querier_test.go @@ -1255,7 +1255,7 @@ func BenchmarkPersistedQueries(b *testing.B) { testutil.Ok(b, err) defer os.RemoveAll(dir) - block, err := OpenBlock(nil, createBlock(b, dir, genSeries(nSeries, 10, 1, int64(nSamples))), nil) + block, err := OpenBlock(nil, createBlock(b, dir, genSeries(nSeries, 10, nil, 1, int64(nSamples))), nil) testutil.Ok(b, err) defer block.Close() @@ -1547,7 +1547,7 @@ func BenchmarkQueryIterator(b *testing.B) { numBlocks: 20, numSeries: 1000, numSamplesPerSeriesPerBlock: 20000, - overlapPercentages: []int{0, 10, 30, 50}, + overlapPercentages: []int{0, 10, 30}, }, } @@ -1563,13 +1563,25 @@ func BenchmarkQueryIterator(b *testing.B) { testutil.Ok(b, os.RemoveAll(dir)) }() - var blocks []*Block - overlapDelta := int64(overlapPercentage * c.numSamplesPerSeriesPerBlock / 100) + var ( + blocks []*Block + overlapDelta = int64(overlapPercentage * c.numSamplesPerSeriesPerBlock / 100) + prefilledLabels []map[string]string + generatedSeries []Series + ) for i := int64(0); i < int64(c.numBlocks); i++ { offset := i * overlapDelta mint := i*int64(c.numSamplesPerSeriesPerBlock) - offset maxt := mint + int64(c.numSamplesPerSeriesPerBlock) - 1 - block, err := OpenBlock(nil, createBlock(b, dir, genSeries(c.numSeries, 10, mint, maxt)), nil) + if len(prefilledLabels) == 0 { + generatedSeries = genSeries(c.numSeries, 10, nil, mint, maxt) + for _, s := range generatedSeries { + prefilledLabels = append(prefilledLabels, s.Labels().Map()) + } + } else { + generatedSeries = genSeries(c.numSeries, 10, prefilledLabels, mint, maxt) + } + block, err := OpenBlock(nil, createBlock(b, dir, generatedSeries), nil) testutil.Ok(b, err) blocks = append(blocks, block) defer block.Close() @@ -1637,13 +1649,25 @@ func BenchmarkQuerySeek(b *testing.B) { testutil.Ok(b, os.RemoveAll(dir)) }() - var blocks []*Block - overlapDelta := int64(overlapPercentage * c.numSamplesPerSeriesPerBlock / 100) + var ( + blocks []*Block + overlapDelta = int64(overlapPercentage * c.numSamplesPerSeriesPerBlock / 100) + prefilledLabels []map[string]string + generatedSeries []Series + ) for i := int64(0); i < int64(c.numBlocks); i++ { offset := i * overlapDelta mint := i*int64(c.numSamplesPerSeriesPerBlock) - offset maxt := mint + int64(c.numSamplesPerSeriesPerBlock) - 1 - block, err := OpenBlock(nil, createBlock(b, dir, genSeries(c.numSeries, 10, mint, maxt)), nil) + if len(prefilledLabels) == 0 { + generatedSeries = genSeries(c.numSeries, 10, nil, mint, maxt) + for _, s := range generatedSeries { + prefilledLabels = append(prefilledLabels, s.Labels().Map()) + } + } else { + generatedSeries = genSeries(c.numSeries, 10, prefilledLabels, mint, maxt) + } + block, err := OpenBlock(nil, createBlock(b, dir, generatedSeries), nil) testutil.Ok(b, err) blocks = append(blocks, block) defer block.Close()