-
Notifications
You must be signed in to change notification settings - Fork 2.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Signed-off-by: Krasi Georgiev <[email protected]>
- Loading branch information
1 parent
bb1610f
commit 3646e13
Showing
6 changed files
with
173 additions
and
57 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,150 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "io/ioutil" | ||
| "os" | ||
| "path" | ||
|
|
||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/go-kit/kit/log" | ||
| "github.com/oklog/ulid" | ||
| "github.com/prometheus/client_golang/prometheus" | ||
| promtest "github.com/prometheus/client_golang/prometheus/testutil" | ||
| dto "github.com/prometheus/client_model/go" | ||
| "github.com/prometheus/prometheus/tsdb" | ||
| "github.com/prometheus/prometheus/tsdb/labels" | ||
| "github.com/thanos-io/thanos/pkg/block" | ||
| "github.com/thanos-io/thanos/pkg/compact" | ||
| "github.com/thanos-io/thanos/pkg/compact/downsample" | ||
| "github.com/thanos-io/thanos/pkg/objstore" | ||
| "github.com/thanos-io/thanos/pkg/objstore/inmem" | ||
| "github.com/thanos-io/thanos/pkg/testutil" | ||
| ) | ||
|
|
||
| func TestCleanupCompactCacheFolder(t *testing.T) { | ||
| ctx, logger, dir, _, bkt, actReg := bootstrap(t) | ||
| defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() | ||
|
|
||
| sy, err := compact.NewSyncer(logger, actReg, bkt, 0*time.Second, 1, false) | ||
| testutil.Ok(t, err) | ||
|
|
||
| expReg := prometheus.NewRegistry() | ||
| syncExp := prometheus.NewCounter(prometheus.CounterOpts{ | ||
| Name: compact.MetricSyncMetaName, | ||
| Help: compact.MetricSyncMetaHelp, | ||
| }) | ||
| expReg.MustRegister(syncExp) | ||
|
|
||
| GatherAndCompare(t, expReg, actReg, compact.MetricSyncMetaName) | ||
|
|
||
| comp, err := tsdb.NewLeveledCompactor(ctx, nil, logger, []int64{1}, nil) | ||
| testutil.Ok(t, err) | ||
|
|
||
| bComp, err := compact.NewBucketCompactor(logger, sy, comp, dir, bkt, 1) | ||
| testutil.Ok(t, err) | ||
|
|
||
| // Even with with a single uploaded block the bucker compactor needs to | ||
| // downloads the meta file to plan the compaction groups. | ||
| testutil.Ok(t, bComp.Compact(ctx)) | ||
|
|
||
| syncExp.Inc() | ||
|
|
||
| GatherAndCompare(t, expReg, actReg, compact.MetricSyncMetaName) | ||
|
|
||
| _, err = os.Stat(dir) | ||
| testutil.Assert(t, os.IsNotExist(err), "index cache dir shouldn't not exist at the end of execution") | ||
|
|
||
| } | ||
|
|
||
| func TestCleanupIndexCacheFolder(t *testing.T) { | ||
| ctx, logger, dir, _, bkt, actReg := bootstrap(t) | ||
| defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() | ||
|
|
||
| expReg := prometheus.NewRegistry() | ||
| genIndexExp := prometheus.NewCounter(prometheus.CounterOpts{ | ||
| Name: MetricIndexGenerateName, | ||
| Help: MetricIndexGenerateHelp, | ||
| }) | ||
| expReg.MustRegister(genIndexExp) | ||
|
|
||
| GatherAndCompare(t, expReg, actReg, compact.MetricSyncMetaName) | ||
|
|
||
| genMissingIndexCacheFiles(ctx, logger, actReg, bkt, dir) | ||
|
|
||
| genIndexExp.Inc() | ||
| GatherAndCompare(t, expReg, actReg, compact.MetricSyncMetaName) | ||
|
|
||
| _, err := os.Stat(dir) | ||
| testutil.Assert(t, os.IsNotExist(err), "index cache dir shouldn't not exist at the end of execution") | ||
| } | ||
|
|
||
| func TestCleanupDownsampleCacheFolder(t *testing.T) { | ||
| ctx, logger, dir, blckID, bkt, reg := bootstrap(t) | ||
| defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() | ||
|
|
||
| meta, err := block.DownloadMeta(ctx, logger, bkt, blckID) | ||
| testutil.Ok(t, err) | ||
|
|
||
| metrics := newDownsampleMetrics(reg) | ||
| testutil.Equals(t, 0.0, promtest.ToFloat64(metrics.downsamples.WithLabelValues(compact.GroupKey(meta)))) | ||
| testutil.Ok(t, downsampleBucket(ctx, logger, metrics, bkt, dir)) | ||
| testutil.Equals(t, 1.0, promtest.ToFloat64(metrics.downsamples.WithLabelValues(compact.GroupKey(meta)))) | ||
|
|
||
| _, err = os.Stat(dir) | ||
| testutil.Assert(t, os.IsNotExist(err), "index cache dir shouldn't not exist at the end of execution") | ||
| } | ||
|
|
||
| func bootstrap(t *testing.T) (context.Context, log.Logger, string, ulid.ULID, objstore.Bucket, *prometheus.Registry) { | ||
| logger := log.NewLogfmtLogger(os.Stderr) | ||
| dir, err := ioutil.TempDir("", "test-compact-cleanup") | ||
| testutil.Ok(t, err) | ||
|
|
||
| ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) | ||
| defer cancel() | ||
|
|
||
| bkt := inmem.NewBucket() | ||
| var blckID ulid.ULID | ||
|
|
||
| // Create and upload a single block to the bucker. | ||
| // The compaction will download the meta block of this block | ||
| // to plan the compaction groups. | ||
| { | ||
| blckID, err = testutil.CreateBlock( | ||
| ctx, | ||
| dir, | ||
| []labels.Labels{ | ||
| {{Name: "a", Value: "1"}}, | ||
| }, | ||
| 1, 0, downsample.DownsampleRange0+1, // Pass the minimum DownsampleRange0 check. | ||
| labels.Labels{{Name: "e1", Value: "1"}}, | ||
| downsample.ResLevel0) | ||
| testutil.Ok(t, err) | ||
| testutil.Ok(t, block.Upload(ctx, logger, bkt, path.Join(dir, blckID.String()))) | ||
| } | ||
|
|
||
| return ctx, logger, dir, blckID, bkt, prometheus.NewRegistry() | ||
| } | ||
|
|
||
| func GatherAndCompare(t *testing.T, g1 prometheus.Gatherer, g2 prometheus.Gatherer, filter string) { | ||
| g1m, err := g1.Gather() | ||
| testutil.Ok(t, err) | ||
| g2m, err := g2.Gather() | ||
| testutil.Ok(t, err) | ||
|
|
||
| var m1 *dto.MetricFamily | ||
| for _, m := range g1m { | ||
| if *m.Name == filter { | ||
| m1 = m | ||
| } | ||
| } | ||
| var m2 *dto.MetricFamily | ||
| for _, m := range g2m { | ||
| if *m.Name == filter { | ||
| m2 = m | ||
| } | ||
| } | ||
| testutil.Equals(t, m1.String(), m2.String()) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters