Skip to content
This repository has been archived by the owner on Aug 13, 2019. It is now read-only.

Commit

Permalink
genSeries takes optional labels. Updated BenchmarkQueryIterator and B…
Browse files Browse the repository at this point in the history 
…enchmarkQuerySeek.

Signed-off-by: Ganesh Vernekar <[email protected]>
  • Loading branch information
@codesome
codesome committed Feb 10, 2019
1 parent 117cef8 commit 5f8d911
Show file tree
Hide file tree
Showing 4 changed files with 65 additions and 24 deletions.
27 changes: 22 additions & 5 deletions block_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func TestSetCompactionFailed(t *testing.T) {
testutil.Ok(t, err)
defer os.RemoveAll(tmpdir)

blockDir := createBlock(t, tmpdir, genSeries(1, 1, 0, 0))
blockDir := createBlock(t, tmpdir, genSeries(1, 1, nil, 0, 0))
b, err := OpenBlock(nil, blockDir, nil)
testutil.Ok(t, err)
testutil.Equals(t, false, b.meta.Compaction.Failed)
Expand Down Expand Up @@ -96,12 +96,30 @@ func createBlock(tb testing.TB, dir string, series []Series) string {
}

// genSeries generates series with a given number of labels and values.
func genSeries(totalSeries, labelCount int, mint, maxt int64) []Series {
if totalSeries == 0 || labelCount == 0 {
// If `prefilledLabels` is not empty, then the number of series and the labels
// for them will be taken from `prefilledLabels`, and `totalSeries` `labelCount`
// will be ignored.
func genSeries(totalSeries, labelCount int, prefilledLabels []map[string]string, mint, maxt int64) []Series {
if (totalSeries == 0 || labelCount == 0) && len(prefilledLabels) == 0 {
return nil
}
series := make([]Series, totalSeries)

if len(prefilledLabels) > 0 {
series := make([]Series, 0, len(prefilledLabels))
for _, lbls := range prefilledLabels {
if len(lbls) == 0 {
continue
}
samples := make([]tsdbutil.Sample, 0, maxt-mint+1)
for t := mint; t <= maxt; t++ {
samples = append(samples, sample{t: t, v: rand.Float64()})
}
series = append(series, newSeries(lbls, samples))
}
return series
}

series := make([]Series, totalSeries)
for i := 0; i < totalSeries; i++ {
lbls := make(map[string]string, labelCount)
for len(lbls) < labelCount {
Expand All @@ -113,7 +131,6 @@ func genSeries(totalSeries, labelCount int, mint, maxt int64) []Series {
}
series[i] = newSeries(lbls, samples)
}

return series
}

Expand Down
2 changes: 1 addition & 1 deletion compact_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -771,7 +771,7 @@ func BenchmarkCompaction(b *testing.B) {
blockDirs := make([]string, 0, len(c.ranges))
var blocks []*Block
for _, r := range c.ranges {
block, err := OpenBlock(nil, createBlock(b, dir, genSeries(nSeries, 10, r[0], r[1])), nil)
block, err := OpenBlock(nil, createBlock(b, dir, genSeries(nSeries, 10, nil, r[0], r[1])), nil)
testutil.Ok(b, err)
blocks = append(blocks, block)
defer block.Close()
Expand Down
20 changes: 10 additions & 10 deletions db_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ func TestDB_reloadOrder(t *testing.T) {
{MinTime: 100, MaxTime: 110},
}
for _, m := range metas {
createBlock(t, db.Dir(), genSeries(1, 1, m.MinTime, m.MaxTime))
createBlock(t, db.Dir(), genSeries(1, 1, nil, m.MinTime, m.MaxTime))
}

testutil.Ok(t, db.reload())
Expand Down Expand Up @@ -833,7 +833,7 @@ func TestTombstoneCleanFail(t *testing.T) {
// totalBlocks should be >=2 so we have enough blocks to trigger compaction failure.
totalBlocks := 2
for i := 0; i < totalBlocks; i++ {
blockDir := createBlock(t, db.Dir(), genSeries(1, 1, 0, 0))
blockDir := createBlock(t, db.Dir(), genSeries(1, 1, nil, 0, 0))
block, err := OpenBlock(nil, blockDir, nil)
testutil.Ok(t, err)
// Add some some fake tombstones to trigger the compaction.
Expand Down Expand Up @@ -877,7 +877,7 @@ func (c *mockCompactorFailing) Write(dest string, b BlockReader, mint, maxt int6
return ulid.ULID{}, fmt.Errorf("the compactor already did the maximum allowed blocks so it is time to fail")
}

block, err := OpenBlock(nil, createBlock(c.t, dest, genSeries(1, 1, 0, 0)), nil)
block, err := OpenBlock(nil, createBlock(c.t, dest, genSeries(1, 1, nil, 0, 0)), nil)
testutil.Ok(c.t, err)
testutil.Ok(c.t, block.Close()) // Close block as we won't be using anywhere.
c.blocks = append(c.blocks, block)
Expand Down Expand Up @@ -915,7 +915,7 @@ func TestTimeRetention(t *testing.T) {
}

for _, m := range blocks {
createBlock(t, db.Dir(), genSeries(10, 10, m.MinTime, m.MaxTime))
createBlock(t, db.Dir(), genSeries(10, 10, nil, m.MinTime, m.MaxTime))
}

testutil.Ok(t, db.reload()) // Reload the db to register the new blocks.
Expand Down Expand Up @@ -949,7 +949,7 @@ func TestSizeRetention(t *testing.T) {
}

for _, m := range blocks {
createBlock(t, db.Dir(), genSeries(100, 10, m.MinTime, m.MaxTime))
createBlock(t, db.Dir(), genSeries(100, 10, nil, m.MinTime, m.MaxTime))
}

// Test that registered size matches the actual disk size.
Expand Down Expand Up @@ -1316,7 +1316,7 @@ func TestInitializeHeadTimestamp(t *testing.T) {
testutil.Ok(t, err)
defer os.RemoveAll(dir)

createBlock(t, dir, genSeries(1, 1, 1000, 2000))
createBlock(t, dir, genSeries(1, 1, nil, 1000, 2000))

db, err := Open(dir, nil, nil, nil)
testutil.Ok(t, err)
Expand All @@ -1329,7 +1329,7 @@ func TestInitializeHeadTimestamp(t *testing.T) {
testutil.Ok(t, err)
defer os.RemoveAll(dir)

createBlock(t, dir, genSeries(1, 1, 1000, 6000))
createBlock(t, dir, genSeries(1, 1, nil, 1000, 6000))

testutil.Ok(t, os.MkdirAll(path.Join(dir, "wal"), 0777))
w, err := wal.New(nil, nil, path.Join(dir, "wal"))
Expand Down Expand Up @@ -1447,7 +1447,7 @@ func TestNoEmptyBlocks(t *testing.T) {
{MinTime: currentTime + 100, MaxTime: currentTime + 100 + db.opts.BlockRanges[0]},
}
for _, m := range blocks {
createBlock(t, db.Dir(), genSeries(2, 2, m.MinTime, m.MaxTime))
createBlock(t, db.Dir(), genSeries(2, 2, nil, m.MinTime, m.MaxTime))
}

oldBlocks := db.Blocks()
Expand Down Expand Up @@ -1924,7 +1924,7 @@ func TestBlockRanges(t *testing.T) {
// Test that the compactor doesn't create overlapping blocks
// when a non standard block already exists.
firstBlockMaxT := int64(3)
createBlock(t, dir, genSeries(1, 1, 0, firstBlockMaxT))
createBlock(t, dir, genSeries(1, 1, nil, 0, firstBlockMaxT))
db, err := Open(dir, logger, nil, DefaultOptions)
if err != nil {
t.Fatalf("Opening test storage failed: %s", err)
Expand Down Expand Up @@ -1974,7 +1974,7 @@ func TestBlockRanges(t *testing.T) {
testutil.Ok(t, db.Close())

thirdBlockMaxt := secondBlockMaxt + 2
createBlock(t, dir, genSeries(1, 1, secondBlockMaxt+1, thirdBlockMaxt))
createBlock(t, dir, genSeries(1, 1, nil, secondBlockMaxt+1, thirdBlockMaxt))

db, err = Open(dir, logger, nil, DefaultOptions)
if err != nil {
Expand Down
40 changes: 32 additions & 8 deletions querier_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1255,7 +1255,7 @@ func BenchmarkPersistedQueries(b *testing.B) {
testutil.Ok(b, err)
defer os.RemoveAll(dir)

block, err := OpenBlock(nil, createBlock(b, dir, genSeries(nSeries, 10, 1, int64(nSamples))), nil)
block, err := OpenBlock(nil, createBlock(b, dir, genSeries(nSeries, 10, nil, 1, int64(nSamples))), nil)
testutil.Ok(b, err)
defer block.Close()

Expand Down Expand Up @@ -1547,7 +1547,7 @@ func BenchmarkQueryIterator(b *testing.B) {
numBlocks: 20,
numSeries: 1000,
numSamplesPerSeriesPerBlock: 20000,
overlapPercentages: []int{0, 10, 30, 50},
overlapPercentages: []int{0, 10, 30},
},
}

Expand All @@ -1563,13 +1563,25 @@ func BenchmarkQueryIterator(b *testing.B) {
testutil.Ok(b, os.RemoveAll(dir))
}()

var blocks []*Block
overlapDelta := int64(overlapPercentage * c.numSamplesPerSeriesPerBlock / 100)
var (
blocks []*Block
overlapDelta = int64(overlapPercentage * c.numSamplesPerSeriesPerBlock / 100)
prefilledLabels []map[string]string
generatedSeries []Series
)
for i := int64(0); i < int64(c.numBlocks); i++ {
offset := i * overlapDelta
mint := i*int64(c.numSamplesPerSeriesPerBlock) - offset
maxt := mint + int64(c.numSamplesPerSeriesPerBlock) - 1
block, err := OpenBlock(nil, createBlock(b, dir, genSeries(c.numSeries, 10, mint, maxt)), nil)
if len(prefilledLabels) == 0 {
generatedSeries = genSeries(c.numSeries, 10, nil, mint, maxt)
for _, s := range generatedSeries {
prefilledLabels = append(prefilledLabels, s.Labels().Map())
}
} else {
generatedSeries = genSeries(c.numSeries, 10, prefilledLabels, mint, maxt)
}
block, err := OpenBlock(nil, createBlock(b, dir, generatedSeries), nil)
testutil.Ok(b, err)
blocks = append(blocks, block)
defer block.Close()
Expand Down Expand Up @@ -1637,13 +1649,25 @@ func BenchmarkQuerySeek(b *testing.B) {
testutil.Ok(b, os.RemoveAll(dir))
}()

var blocks []*Block
overlapDelta := int64(overlapPercentage * c.numSamplesPerSeriesPerBlock / 100)
var (
blocks []*Block
overlapDelta = int64(overlapPercentage * c.numSamplesPerSeriesPerBlock / 100)
prefilledLabels []map[string]string
generatedSeries []Series
)
for i := int64(0); i < int64(c.numBlocks); i++ {
offset := i * overlapDelta
mint := i*int64(c.numSamplesPerSeriesPerBlock) - offset
maxt := mint + int64(c.numSamplesPerSeriesPerBlock) - 1
block, err := OpenBlock(nil, createBlock(b, dir, genSeries(c.numSeries, 10, mint, maxt)), nil)
if len(prefilledLabels) == 0 {
generatedSeries = genSeries(c.numSeries, 10, nil, mint, maxt)
for _, s := range generatedSeries {
prefilledLabels = append(prefilledLabels, s.Labels().Map())
}
} else {
generatedSeries = genSeries(c.numSeries, 10, prefilledLabels, mint, maxt)
}
block, err := OpenBlock(nil, createBlock(b, dir, generatedSeries), nil)
testutil.Ok(b, err)
blocks = append(blocks, block)
defer block.Close()
Expand Down

0 comments on commit 5f8d911

Please sign in to comment.