From fe98ddd861df5319ea6380bd25efca79291c365c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Tue, 16 Nov 2021 16:48:55 +0100 Subject: [PATCH 1/9] Let store-gateways ignore blocks that are too young. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- docs/blocks-storage/querier.md | 6 ++++ docs/blocks-storage/store-gateway.md | 6 ++++ docs/configuration/config-file-reference.md | 6 ++++ pkg/storage/tsdb/config.go | 2 ++ pkg/storegateway/bucket_stores.go | 3 ++ pkg/storegateway/metadata_fetcher_filters.go | 26 +++++++++++++++ .../metadata_fetcher_filters_test.go | 32 +++++++++++++++++++ 7 files changed, 81 insertions(+) diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index 9a3ffdbc04a..1125da22e51 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -731,6 +731,12 @@ blocks_storage: # CLI flag: -blocks-storage.bucket-store.bucket-index.max-stale-period [max_stale_period: | default = 1h] + # Blocks with minimum time within this duration are ignored, and not loaded + # by store-gateway. Useful when used together with + # -querier.query-store-after to prevent loading young blocks. + # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within + [ignore_blocks_within: | default = 0s] + # Max size - in bytes - of a chunks pool, used to reduce memory allocations. # The pool is shared across all tenants. 0 to disable the limit. # CLI flag: -blocks-storage.bucket-store.max-chunk-pool-bytes diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md index 1d7580d6e55..829c86d7cf6 100644 --- a/docs/blocks-storage/store-gateway.md +++ b/docs/blocks-storage/store-gateway.md @@ -784,6 +784,12 @@ blocks_storage: # CLI flag: -blocks-storage.bucket-store.bucket-index.max-stale-period [max_stale_period: | default = 1h] + # Blocks with minimum time within this duration are ignored, and not loaded + # by store-gateway. Useful when used together with + # -querier.query-store-after to prevent loading young blocks. + # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within + [ignore_blocks_within: | default = 0s] + # Max size - in bytes - of a chunks pool, used to reduce memory allocations. # The pool is shared across all tenants. 0 to disable the limit. # CLI flag: -blocks-storage.bucket-store.max-chunk-pool-bytes diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 35352684bf1..d9664fa7406 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -4985,6 +4985,12 @@ bucket_store: # CLI flag: -blocks-storage.bucket-store.bucket-index.max-stale-period [max_stale_period: | default = 1h] + # Blocks with minimum time within this duration are ignored, and not loaded by + # store-gateway. Useful when used together with -querier.query-store-after to + # prevent loading young blocks. + # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within + [ignore_blocks_within: | default = 0s] + # Max size - in bytes - of a chunks pool, used to reduce memory allocations. # The pool is shared across all tenants. 0 to disable the limit. # CLI flag: -blocks-storage.bucket-store.max-chunk-pool-bytes diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index ba580e27626..f127429aaed 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -259,6 +259,7 @@ type BucketStoreConfig struct { MetadataCache MetadataCacheConfig `yaml:"metadata_cache"` IgnoreDeletionMarksDelay time.Duration `yaml:"ignore_deletion_mark_delay"` BucketIndex BucketIndexConfig `yaml:"bucket_index"` + IgnoreBlocksWithin time.Duration `yaml:"ignore_blocks_within"` // Chunk pool. MaxChunkPoolBytes uint64 `yaml:"max_chunk_pool_bytes"` @@ -305,6 +306,7 @@ func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.IgnoreDeletionMarksDelay, "blocks-storage.bucket-store.ignore-deletion-marks-delay", time.Hour*6, "Duration after which the blocks marked for deletion will be filtered out while fetching blocks. "+ "The idea of ignore-deletion-marks-delay is to ignore blocks that are marked for deletion with some delay. This ensures store can still serve blocks that are meant to be deleted but do not have a replacement yet. "+ "Default is 6h, half of the default value for -compactor.deletion-delay.") + f.DurationVar(&cfg.IgnoreBlocksWithin, "blocks-storage.bucket-store.ignore-blocks-within", 0, "Blocks with minimum time within this duration are ignored, and not loaded by store-gateway. Useful when used together with -querier.query-store-after to prevent loading young blocks.") f.IntVar(&cfg.PostingOffsetsInMemSampling, "blocks-storage.bucket-store.posting-offsets-in-mem-sampling", DefaultPostingOffsetInMemorySampling, "Controls what is the ratio of postings offsets that the store will hold in memory.") f.BoolVar(&cfg.IndexHeaderLazyLoadingEnabled, "blocks-storage.bucket-store.index-header-lazy-loading-enabled", false, "If enabled, store-gateway will lazy load an index-header only once required by a query.") f.DurationVar(&cfg.IndexHeaderLazyLoadingIdleTimeout, "blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout", 20*time.Minute, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") diff --git a/pkg/storegateway/bucket_stores.go b/pkg/storegateway/bucket_stores.go index 23d46bf276b..7af4a19cfba 100644 --- a/pkg/storegateway/bucket_stores.go +++ b/pkg/storegateway/bucket_stores.go @@ -451,6 +451,9 @@ func (u *BucketStores) getOrCreateStore(userID string) (*BucketStore, error) { // but if the store-gateway removes redundant blocks before the querier discovers them, the // consistency check on the querier will fail. }...) + if u.cfg.BucketStore.IgnoreBlocksWithin > 0 { + filters = append(filters, NewTimeMetaFilter(u.cfg.BucketStore.IgnoreBlocksWithin)) + } modifiers := []block.MetadataModifier{ // Remove Mimir external labels so that they're not injected when querying blocks. diff --git a/pkg/storegateway/metadata_fetcher_filters.go b/pkg/storegateway/metadata_fetcher_filters.go index 7a103ebc888..e6d6ec9dde4 100644 --- a/pkg/storegateway/metadata_fetcher_filters.go +++ b/pkg/storegateway/metadata_fetcher_filters.go @@ -11,6 +11,7 @@ import ( "github.com/go-kit/log" "github.com/oklog/ulid" + "github.com/prometheus/prometheus/pkg/timestamp" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/extprom" @@ -81,3 +82,28 @@ func (f *IgnoreDeletionMarkFilter) FilterWithBucketIndex(_ context.Context, meta return nil } + +const timeExcludedMeta = "time-excluded" // Matches block.timeExcludedMeta value. + +// TimeMetaFilter filters out blocks that contain the most recent data. +type TimeMetaFilter struct { + limit time.Duration +} + +func NewTimeMetaFilter(limit time.Duration) *TimeMetaFilter { + return &TimeMetaFilter{limit: limit} +} + +func (f *TimeMetaFilter) Filter(_ context.Context, metas map[ulid.ULID]*metadata.Meta, synced *extprom.TxGaugeVec) error { + limitTime := timestamp.FromTime(time.Now().Add(-f.limit)) + + for id, m := range metas { + if m.MinTime < limitTime { + continue + } + + synced.WithLabelValues(timeExcludedMeta).Inc() + delete(metas, id) + } + return nil +} diff --git a/pkg/storegateway/metadata_fetcher_filters_test.go b/pkg/storegateway/metadata_fetcher_filters_test.go index 9cc2a4619f5..37d2d127bdd 100644 --- a/pkg/storegateway/metadata_fetcher_filters_test.go +++ b/pkg/storegateway/metadata_fetcher_filters_test.go @@ -17,6 +17,8 @@ import ( "github.com/oklog/ulid" "github.com/prometheus/client_golang/prometheus" promtest "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/prometheus/prometheus/pkg/timestamp" + "github.com/prometheus/prometheus/tsdb" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/thanos-io/thanos/pkg/block" @@ -110,3 +112,33 @@ func testIgnoreDeletionMarkFilter(t *testing.T, bucketIndexEnabled bool) { assert.Equal(t, expectedMetas, inputMetas) assert.Equal(t, expectedDeletionMarks, f.DeletionMarkBlocks()) } + +func TestTimeMetaFilter(t *testing.T) { + now := time.Now() + limit := 10 * time.Minute + limitTime := now.Add(-limit) + + ulid1 := ulid.MustNew(1, nil) + ulid2 := ulid.MustNew(2, nil) + ulid3 := ulid.MustNew(3, nil) + ulid4 := ulid.MustNew(4, nil) + + inputMetas := map[ulid.ULID]*metadata.Meta{ + ulid1: {BlockMeta: tsdb.BlockMeta{MinTime: 100}}, // Very old, keep it + ulid2: {BlockMeta: tsdb.BlockMeta{MinTime: timestamp.FromTime(now)}}, // Fresh block, remove. + ulid3: {BlockMeta: tsdb.BlockMeta{MinTime: timestamp.FromTime(limitTime.Add(time.Minute))}}, // Inside limit time, remove. + ulid4: {BlockMeta: tsdb.BlockMeta{MinTime: timestamp.FromTime(limitTime.Add(-time.Minute))}}, // Before limit time, keep. + } + + expectedMetas := map[ulid.ULID]*metadata.Meta{} + expectedMetas[ulid1] = inputMetas[ulid1] + expectedMetas[ulid4] = inputMetas[ulid4] + + synced := extprom.NewTxGaugeVec(nil, prometheus.GaugeOpts{Name: "synced"}, []string{"state"}) + f := NewTimeMetaFilter(limit) + + require.NoError(t, f.Filter(context.Background(), inputMetas, synced)) + + assert.Equal(t, expectedMetas, inputMetas) + assert.Equal(t, 2.0, promtest.ToFloat64(synced.WithLabelValues(timeExcludedMeta))) +} From 732d9b9ab43a293ba4a30089d941162366555285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Tue, 16 Nov 2021 16:52:47 +0100 Subject: [PATCH 2/9] CHANGELOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 029d0cdc822..5cf3fec94ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -117,6 +117,7 @@ * [ENHANCEMENT] Querier&Ruler: reduce cpu usage, latency and peak memory consumption. #459 #463 * [ENHANCEMENT] Overrides Exporter: Add `max_fetched_chunks_per_query` limit to the default and per-tenant limits exported as metrics. #471 * [ENHANCEMENT] Compactor (blocks cleaner): Delete blocks marked for deletion faster. #490 +* [ENHANCEMENT] Store-gateway: store-gateway can now ignore blocks with minimum time within `-blocks-storage.bucket-store.ignore-blocks-within` duration. Useful when used together with `-querier.query-store-after`. #502 * [BUGFIX] Frontend: Fixes @ modifier functions (start/end) when splitting queries by time. #206 * [BUGFIX] Fixes a panic in the query-tee when comparing result. #207 * [BUGFIX] Upgrade Prometheus. TSDB now waits for pending readers before truncating Head block, fixing the `chunk not found` error and preventing wrong query results. #16 From b359ef86e795b63e870e64721e3a1fb5dc211ed3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Wed, 17 Nov 2021 15:15:02 +0100 Subject: [PATCH 3/9] Renamed TimeMetaFilter to minTimeMetaFilter, added it to TestBucketIndexMetadataFetcher_Fetch test. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/storegateway/bucket_index_metadata_fetcher_test.go | 8 ++++++-- pkg/storegateway/bucket_stores.go | 2 +- pkg/storegateway/metadata_fetcher_filters.go | 10 +++++----- pkg/storegateway/metadata_fetcher_filters_test.go | 2 +- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pkg/storegateway/bucket_index_metadata_fetcher_test.go b/pkg/storegateway/bucket_index_metadata_fetcher_test.go index 1944c175016..8626ef0a893 100644 --- a/pkg/storegateway/bucket_index_metadata_fetcher_test.go +++ b/pkg/storegateway/bucket_index_metadata_fetcher_test.go @@ -18,6 +18,7 @@ import ( "github.com/oklog/ulid" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/prometheus/prometheus/pkg/timestamp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -43,12 +44,14 @@ func TestBucketIndexMetadataFetcher_Fetch(t *testing.T) { block1 := &bucketindex.Block{ID: ulid.MustNew(1, nil)} block2 := &bucketindex.Block{ID: ulid.MustNew(2, nil)} block3 := &bucketindex.Block{ID: ulid.MustNew(3, nil)} + block4 := &bucketindex.Block{ID: ulid.MustNew(4, nil), MinTime: timestamp.FromTime(now.Add(-30 * time.Minute))} // Has most-recent data, to be ignored by minTimeMetaFilter. + mark1 := &bucketindex.BlockDeletionMark{ID: block1.ID, DeletionTime: now.Add(-time.Hour).Unix()} // Below the ignore delay threshold. mark2 := &bucketindex.BlockDeletionMark{ID: block2.ID, DeletionTime: now.Add(-3 * time.Hour).Unix()} // Above the ignore delay threshold. require.NoError(t, bucketindex.WriteIndex(ctx, bkt, userID, nil, &bucketindex.Index{ Version: bucketindex.IndexVersion1, - Blocks: bucketindex.Blocks{block1, block2, block3}, + Blocks: bucketindex.Blocks{block1, block2, block3, block4}, BlockDeletionMarks: bucketindex.BlockDeletionMarks{mark1, mark2}, UpdatedAt: now.Unix(), })) @@ -56,6 +59,7 @@ func TestBucketIndexMetadataFetcher_Fetch(t *testing.T) { // Create a metadata fetcher with filters. filters := []block.MetadataFilter{ NewIgnoreDeletionMarkFilter(logger, bucket.NewUserBucketClient(userID, bkt, nil), 2*time.Hour, 1), + NewMinTimeMetaFilter(1 * time.Hour), } fetcher := NewBucketIndexMetadataFetcher(userID, bkt, NewNoShardingStrategy(), nil, logger, reg, filters, nil) @@ -89,7 +93,7 @@ func TestBucketIndexMetadataFetcher_Fetch(t *testing.T) { blocks_meta_synced{state="marked-for-no-compact"} 0 blocks_meta_synced{state="no-bucket-index"} 0 blocks_meta_synced{state="no-meta-json"} 0 - blocks_meta_synced{state="time-excluded"} 0 + blocks_meta_synced{state="time-excluded"} 1 blocks_meta_synced{state="too-fresh"} 0 # HELP blocks_meta_syncs_total Total blocks metadata synchronization attempts diff --git a/pkg/storegateway/bucket_stores.go b/pkg/storegateway/bucket_stores.go index 7af4a19cfba..7be585c8ec3 100644 --- a/pkg/storegateway/bucket_stores.go +++ b/pkg/storegateway/bucket_stores.go @@ -452,7 +452,7 @@ func (u *BucketStores) getOrCreateStore(userID string) (*BucketStore, error) { // consistency check on the querier will fail. }...) if u.cfg.BucketStore.IgnoreBlocksWithin > 0 { - filters = append(filters, NewTimeMetaFilter(u.cfg.BucketStore.IgnoreBlocksWithin)) + filters = append(filters, NewMinTimeMetaFilter(u.cfg.BucketStore.IgnoreBlocksWithin)) } modifiers := []block.MetadataModifier{ diff --git a/pkg/storegateway/metadata_fetcher_filters.go b/pkg/storegateway/metadata_fetcher_filters.go index e6d6ec9dde4..e2fe9d016a7 100644 --- a/pkg/storegateway/metadata_fetcher_filters.go +++ b/pkg/storegateway/metadata_fetcher_filters.go @@ -85,16 +85,16 @@ func (f *IgnoreDeletionMarkFilter) FilterWithBucketIndex(_ context.Context, meta const timeExcludedMeta = "time-excluded" // Matches block.timeExcludedMeta value. -// TimeMetaFilter filters out blocks that contain the most recent data. -type TimeMetaFilter struct { +// minTimeMetaFilter filters out blocks that contain the most recent data (based on block MinTime). +type minTimeMetaFilter struct { limit time.Duration } -func NewTimeMetaFilter(limit time.Duration) *TimeMetaFilter { - return &TimeMetaFilter{limit: limit} +func NewMinTimeMetaFilter(limit time.Duration) *minTimeMetaFilter { + return &minTimeMetaFilter{limit: limit} } -func (f *TimeMetaFilter) Filter(_ context.Context, metas map[ulid.ULID]*metadata.Meta, synced *extprom.TxGaugeVec) error { +func (f *minTimeMetaFilter) Filter(_ context.Context, metas map[ulid.ULID]*metadata.Meta, synced *extprom.TxGaugeVec) error { limitTime := timestamp.FromTime(time.Now().Add(-f.limit)) for id, m := range metas { diff --git a/pkg/storegateway/metadata_fetcher_filters_test.go b/pkg/storegateway/metadata_fetcher_filters_test.go index 37d2d127bdd..217dca4b39a 100644 --- a/pkg/storegateway/metadata_fetcher_filters_test.go +++ b/pkg/storegateway/metadata_fetcher_filters_test.go @@ -135,7 +135,7 @@ func TestTimeMetaFilter(t *testing.T) { expectedMetas[ulid4] = inputMetas[ulid4] synced := extprom.NewTxGaugeVec(nil, prometheus.GaugeOpts{Name: "synced"}, []string{"state"}) - f := NewTimeMetaFilter(limit) + f := NewMinTimeMetaFilter(limit) require.NoError(t, f.Filter(context.Background(), inputMetas, synced)) From 950b576984d9c7378f91994691230d0afbe79a27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Wed, 17 Nov 2021 15:20:51 +0100 Subject: [PATCH 4/9] Unexport NewMinTimeMetaFilter. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/storegateway/bucket_index_metadata_fetcher_test.go | 2 +- pkg/storegateway/bucket_stores.go | 2 +- pkg/storegateway/metadata_fetcher_filters.go | 2 +- pkg/storegateway/metadata_fetcher_filters_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/storegateway/bucket_index_metadata_fetcher_test.go b/pkg/storegateway/bucket_index_metadata_fetcher_test.go index 8626ef0a893..6329f9b01ae 100644 --- a/pkg/storegateway/bucket_index_metadata_fetcher_test.go +++ b/pkg/storegateway/bucket_index_metadata_fetcher_test.go @@ -59,7 +59,7 @@ func TestBucketIndexMetadataFetcher_Fetch(t *testing.T) { // Create a metadata fetcher with filters. filters := []block.MetadataFilter{ NewIgnoreDeletionMarkFilter(logger, bucket.NewUserBucketClient(userID, bkt, nil), 2*time.Hour, 1), - NewMinTimeMetaFilter(1 * time.Hour), + newMinTimeMetaFilter(1 * time.Hour), } fetcher := NewBucketIndexMetadataFetcher(userID, bkt, NewNoShardingStrategy(), nil, logger, reg, filters, nil) diff --git a/pkg/storegateway/bucket_stores.go b/pkg/storegateway/bucket_stores.go index 7be585c8ec3..50f4d44e342 100644 --- a/pkg/storegateway/bucket_stores.go +++ b/pkg/storegateway/bucket_stores.go @@ -452,7 +452,7 @@ func (u *BucketStores) getOrCreateStore(userID string) (*BucketStore, error) { // consistency check on the querier will fail. }...) if u.cfg.BucketStore.IgnoreBlocksWithin > 0 { - filters = append(filters, NewMinTimeMetaFilter(u.cfg.BucketStore.IgnoreBlocksWithin)) + filters = append(filters, newMinTimeMetaFilter(u.cfg.BucketStore.IgnoreBlocksWithin)) } modifiers := []block.MetadataModifier{ diff --git a/pkg/storegateway/metadata_fetcher_filters.go b/pkg/storegateway/metadata_fetcher_filters.go index e2fe9d016a7..d3e0d1191d9 100644 --- a/pkg/storegateway/metadata_fetcher_filters.go +++ b/pkg/storegateway/metadata_fetcher_filters.go @@ -90,7 +90,7 @@ type minTimeMetaFilter struct { limit time.Duration } -func NewMinTimeMetaFilter(limit time.Duration) *minTimeMetaFilter { +func newMinTimeMetaFilter(limit time.Duration) *minTimeMetaFilter { return &minTimeMetaFilter{limit: limit} } diff --git a/pkg/storegateway/metadata_fetcher_filters_test.go b/pkg/storegateway/metadata_fetcher_filters_test.go index 217dca4b39a..2a439f25add 100644 --- a/pkg/storegateway/metadata_fetcher_filters_test.go +++ b/pkg/storegateway/metadata_fetcher_filters_test.go @@ -135,7 +135,7 @@ func TestTimeMetaFilter(t *testing.T) { expectedMetas[ulid4] = inputMetas[ulid4] synced := extprom.NewTxGaugeVec(nil, prometheus.GaugeOpts{Name: "synced"}, []string{"state"}) - f := NewMinTimeMetaFilter(limit) + f := newMinTimeMetaFilter(limit) require.NoError(t, f.Filter(context.Background(), inputMetas, synced)) From 8bad8ca7daae1ad82d0ecce2754631636cc92996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Thu, 18 Nov 2021 10:46:35 +0100 Subject: [PATCH 5/9] Enhanced field description. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- docs/blocks-storage/querier.md | 4 +++- docs/blocks-storage/store-gateway.md | 4 +++- docs/configuration/config-file-reference.md | 3 ++- pkg/storage/tsdb/config.go | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index 1125da22e51..2c29afdc07b 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -733,7 +733,9 @@ blocks_storage: # Blocks with minimum time within this duration are ignored, and not loaded # by store-gateway. Useful when used together with - # -querier.query-store-after to prevent loading young blocks. + # -querier.query-store-after to prevent loading young blocks, because there + # is usually many of them (depending on number of ingesters) and they are + # not yet compacted. # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within [ignore_blocks_within: | default = 0s] diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md index 829c86d7cf6..2f9a703080e 100644 --- a/docs/blocks-storage/store-gateway.md +++ b/docs/blocks-storage/store-gateway.md @@ -786,7 +786,9 @@ blocks_storage: # Blocks with minimum time within this duration are ignored, and not loaded # by store-gateway. Useful when used together with - # -querier.query-store-after to prevent loading young blocks. + # -querier.query-store-after to prevent loading young blocks, because there + # is usually many of them (depending on number of ingesters) and they are + # not yet compacted. # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within [ignore_blocks_within: | default = 0s] diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index d9664fa7406..2a034288de3 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -4987,7 +4987,8 @@ bucket_store: # Blocks with minimum time within this duration are ignored, and not loaded by # store-gateway. Useful when used together with -querier.query-store-after to - # prevent loading young blocks. + # prevent loading young blocks, because there is usually many of them + # (depending on number of ingesters) and they are not yet compacted. # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within [ignore_blocks_within: | default = 0s] diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index f127429aaed..df6ed5f72c3 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -306,7 +306,7 @@ func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.IgnoreDeletionMarksDelay, "blocks-storage.bucket-store.ignore-deletion-marks-delay", time.Hour*6, "Duration after which the blocks marked for deletion will be filtered out while fetching blocks. "+ "The idea of ignore-deletion-marks-delay is to ignore blocks that are marked for deletion with some delay. This ensures store can still serve blocks that are meant to be deleted but do not have a replacement yet. "+ "Default is 6h, half of the default value for -compactor.deletion-delay.") - f.DurationVar(&cfg.IgnoreBlocksWithin, "blocks-storage.bucket-store.ignore-blocks-within", 0, "Blocks with minimum time within this duration are ignored, and not loaded by store-gateway. Useful when used together with -querier.query-store-after to prevent loading young blocks.") + f.DurationVar(&cfg.IgnoreBlocksWithin, "blocks-storage.bucket-store.ignore-blocks-within", 0, "Blocks with minimum time within this duration are ignored, and not loaded by store-gateway. Useful when used together with -querier.query-store-after to prevent loading young blocks, because there is usually many of them (depending on number of ingesters) and they are not yet compacted.") f.IntVar(&cfg.PostingOffsetsInMemSampling, "blocks-storage.bucket-store.posting-offsets-in-mem-sampling", DefaultPostingOffsetInMemorySampling, "Controls what is the ratio of postings offsets that the store will hold in memory.") f.BoolVar(&cfg.IndexHeaderLazyLoadingEnabled, "blocks-storage.bucket-store.index-header-lazy-loading-enabled", false, "If enabled, store-gateway will lazy load an index-header only once required by a query.") f.DurationVar(&cfg.IndexHeaderLazyLoadingIdleTimeout, "blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout", 20*time.Minute, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") From 019fe8fc7a50dcd560868ec7bfc3c0d11616677d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Thu, 18 Nov 2021 12:27:49 +0100 Subject: [PATCH 6/9] Fix wording as suggested by Arve. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- docs/blocks-storage/querier.md | 2 +- docs/blocks-storage/store-gateway.md | 2 +- docs/configuration/config-file-reference.md | 2 +- pkg/storage/tsdb/config.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index 2c29afdc07b..1e8314811ca 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -734,7 +734,7 @@ blocks_storage: # Blocks with minimum time within this duration are ignored, and not loaded # by store-gateway. Useful when used together with # -querier.query-store-after to prevent loading young blocks, because there - # is usually many of them (depending on number of ingesters) and they are + # are usually many of them (depending on number of ingesters) and they are # not yet compacted. # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within [ignore_blocks_within: | default = 0s] diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md index 2f9a703080e..c1b2da4566e 100644 --- a/docs/blocks-storage/store-gateway.md +++ b/docs/blocks-storage/store-gateway.md @@ -787,7 +787,7 @@ blocks_storage: # Blocks with minimum time within this duration are ignored, and not loaded # by store-gateway. Useful when used together with # -querier.query-store-after to prevent loading young blocks, because there - # is usually many of them (depending on number of ingesters) and they are + # are usually many of them (depending on number of ingesters) and they are # not yet compacted. # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within [ignore_blocks_within: | default = 0s] diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 2a034288de3..0c9fa3c20aa 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -4987,7 +4987,7 @@ bucket_store: # Blocks with minimum time within this duration are ignored, and not loaded by # store-gateway. Useful when used together with -querier.query-store-after to - # prevent loading young blocks, because there is usually many of them + # prevent loading young blocks, because there are usually many of them # (depending on number of ingesters) and they are not yet compacted. # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within [ignore_blocks_within: | default = 0s] diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index df6ed5f72c3..ae6af39eaf9 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -306,7 +306,7 @@ func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.IgnoreDeletionMarksDelay, "blocks-storage.bucket-store.ignore-deletion-marks-delay", time.Hour*6, "Duration after which the blocks marked for deletion will be filtered out while fetching blocks. "+ "The idea of ignore-deletion-marks-delay is to ignore blocks that are marked for deletion with some delay. This ensures store can still serve blocks that are meant to be deleted but do not have a replacement yet. "+ "Default is 6h, half of the default value for -compactor.deletion-delay.") - f.DurationVar(&cfg.IgnoreBlocksWithin, "blocks-storage.bucket-store.ignore-blocks-within", 0, "Blocks with minimum time within this duration are ignored, and not loaded by store-gateway. Useful when used together with -querier.query-store-after to prevent loading young blocks, because there is usually many of them (depending on number of ingesters) and they are not yet compacted.") + f.DurationVar(&cfg.IgnoreBlocksWithin, "blocks-storage.bucket-store.ignore-blocks-within", 0, "Blocks with minimum time within this duration are ignored, and not loaded by store-gateway. Useful when used together with -querier.query-store-after to prevent loading young blocks, because there are usually many of them (depending on number of ingesters) and they are not yet compacted.") f.IntVar(&cfg.PostingOffsetsInMemSampling, "blocks-storage.bucket-store.posting-offsets-in-mem-sampling", DefaultPostingOffsetInMemorySampling, "Controls what is the ratio of postings offsets that the store will hold in memory.") f.BoolVar(&cfg.IndexHeaderLazyLoadingEnabled, "blocks-storage.bucket-store.index-header-lazy-loading-enabled", false, "If enabled, store-gateway will lazy load an index-header only once required by a query.") f.DurationVar(&cfg.IndexHeaderLazyLoadingIdleTimeout, "blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout", 20*time.Minute, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") From 93e3b08689aebd390d93dc3573c871a44fd56919 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Thu, 18 Nov 2021 14:18:27 +0100 Subject: [PATCH 7/9] Address review feedback. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/storage/tsdb/config.go | 2 +- pkg/storegateway/bucket_stores.go | 7 +++---- pkg/storegateway/metadata_fetcher_filters.go | 4 ++++ pkg/storegateway/metadata_fetcher_filters_test.go | 8 +++++++- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index ae6af39eaf9..b8f78610a2c 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -306,7 +306,7 @@ func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.IgnoreDeletionMarksDelay, "blocks-storage.bucket-store.ignore-deletion-marks-delay", time.Hour*6, "Duration after which the blocks marked for deletion will be filtered out while fetching blocks. "+ "The idea of ignore-deletion-marks-delay is to ignore blocks that are marked for deletion with some delay. This ensures store can still serve blocks that are meant to be deleted but do not have a replacement yet. "+ "Default is 6h, half of the default value for -compactor.deletion-delay.") - f.DurationVar(&cfg.IgnoreBlocksWithin, "blocks-storage.bucket-store.ignore-blocks-within", 0, "Blocks with minimum time within this duration are ignored, and not loaded by store-gateway. Useful when used together with -querier.query-store-after to prevent loading young blocks, because there are usually many of them (depending on number of ingesters) and they are not yet compacted.") + f.DurationVar(&cfg.IgnoreBlocksWithin, "blocks-storage.bucket-store.ignore-blocks-within", 0, "Blocks with minimum time within this duration are ignored, and not loaded by store-gateway. Useful when used together with -querier.query-store-after to prevent loading young blocks, because there are usually many of them (depending on number of ingesters) and they are not yet compacted. Negative values or 0 disable the filter.") f.IntVar(&cfg.PostingOffsetsInMemSampling, "blocks-storage.bucket-store.posting-offsets-in-mem-sampling", DefaultPostingOffsetInMemorySampling, "Controls what is the ratio of postings offsets that the store will hold in memory.") f.BoolVar(&cfg.IndexHeaderLazyLoadingEnabled, "blocks-storage.bucket-store.index-header-lazy-loading-enabled", false, "If enabled, store-gateway will lazy load an index-header only once required by a query.") f.DurationVar(&cfg.IndexHeaderLazyLoadingIdleTimeout, "blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout", 20*time.Minute, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") diff --git a/pkg/storegateway/bucket_stores.go b/pkg/storegateway/bucket_stores.go index 50f4d44e342..aa5a5eb8b40 100644 --- a/pkg/storegateway/bucket_stores.go +++ b/pkg/storegateway/bucket_stores.go @@ -442,17 +442,16 @@ func (u *BucketStores) getOrCreateStore(userID string) (*BucketStore, error) { fetcherReg := prometheus.NewRegistry() // The sharding strategy filter MUST be before the ones we create here (order matters). - filters := append([]block.MetadataFilter{NewShardingMetadataFilterAdapter(userID, u.shardingStrategy)}, []block.MetadataFilter{ + filters := []block.MetadataFilter{ + NewShardingMetadataFilterAdapter(userID, u.shardingStrategy), block.NewConsistencyDelayMetaFilter(userLogger, u.cfg.BucketStore.ConsistencyDelay, fetcherReg), + newMinTimeMetaFilter(u.cfg.BucketStore.IgnoreBlocksWithin), // Use our own custom implementation. NewIgnoreDeletionMarkFilter(userLogger, userBkt, u.cfg.BucketStore.IgnoreDeletionMarksDelay, u.cfg.BucketStore.MetaSyncConcurrency), // The duplicate filter has been intentionally omitted because it could cause troubles with // the consistency check done on the querier. The duplicate filter removes redundant blocks // but if the store-gateway removes redundant blocks before the querier discovers them, the // consistency check on the querier will fail. - }...) - if u.cfg.BucketStore.IgnoreBlocksWithin > 0 { - filters = append(filters, newMinTimeMetaFilter(u.cfg.BucketStore.IgnoreBlocksWithin)) } modifiers := []block.MetadataModifier{ diff --git a/pkg/storegateway/metadata_fetcher_filters.go b/pkg/storegateway/metadata_fetcher_filters.go index d3e0d1191d9..187aa8ac382 100644 --- a/pkg/storegateway/metadata_fetcher_filters.go +++ b/pkg/storegateway/metadata_fetcher_filters.go @@ -95,6 +95,10 @@ func newMinTimeMetaFilter(limit time.Duration) *minTimeMetaFilter { } func (f *minTimeMetaFilter) Filter(_ context.Context, metas map[ulid.ULID]*metadata.Meta, synced *extprom.TxGaugeVec) error { + if f.limit <= 0 { + return nil + } + limitTime := timestamp.FromTime(time.Now().Add(-f.limit)) for id, m := range metas { diff --git a/pkg/storegateway/metadata_fetcher_filters_test.go b/pkg/storegateway/metadata_fetcher_filters_test.go index 2a439f25add..64fd4af7e4a 100644 --- a/pkg/storegateway/metadata_fetcher_filters_test.go +++ b/pkg/storegateway/metadata_fetcher_filters_test.go @@ -135,8 +135,14 @@ func TestTimeMetaFilter(t *testing.T) { expectedMetas[ulid4] = inputMetas[ulid4] synced := extprom.NewTxGaugeVec(nil, prometheus.GaugeOpts{Name: "synced"}, []string{"state"}) - f := newMinTimeMetaFilter(limit) + // Test negative limit. + f := newMinTimeMetaFilter(-10 * time.Minute) + require.NoError(t, f.Filter(context.Background(), inputMetas, synced)) + assert.Equal(t, inputMetas, inputMetas) + assert.Equal(t, 0.0, promtest.ToFloat64(synced.WithLabelValues(timeExcludedMeta))) + + f = newMinTimeMetaFilter(limit) require.NoError(t, f.Filter(context.Background(), inputMetas, synced)) assert.Equal(t, expectedMetas, inputMetas) From 43735fa5d6cf1c3697af7f8c8b1e87dd435bd3b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Thu, 18 Nov 2021 14:41:52 +0100 Subject: [PATCH 8/9] Address review feedback. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- docs/blocks-storage/querier.md | 2 +- docs/blocks-storage/store-gateway.md | 2 +- docs/configuration/config-file-reference.md | 3 ++- pkg/storegateway/bucket_index_metadata_fetcher.go | 2 +- pkg/storegateway/metadata_fetcher_filters.go | 4 ++-- pkg/storegateway/metadata_fetcher_filters_test.go | 4 ++-- 6 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index 1e8314811ca..06d8ed33d4b 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -735,7 +735,7 @@ blocks_storage: # by store-gateway. Useful when used together with # -querier.query-store-after to prevent loading young blocks, because there # are usually many of them (depending on number of ingesters) and they are - # not yet compacted. + # not yet compacted. Negative values or 0 disable the filter. # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within [ignore_blocks_within: | default = 0s] diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md index c1b2da4566e..abbedacdaa3 100644 --- a/docs/blocks-storage/store-gateway.md +++ b/docs/blocks-storage/store-gateway.md @@ -788,7 +788,7 @@ blocks_storage: # by store-gateway. Useful when used together with # -querier.query-store-after to prevent loading young blocks, because there # are usually many of them (depending on number of ingesters) and they are - # not yet compacted. + # not yet compacted. Negative values or 0 disable the filter. # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within [ignore_blocks_within: | default = 0s] diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 0c9fa3c20aa..37ce735c388 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -4988,7 +4988,8 @@ bucket_store: # Blocks with minimum time within this duration are ignored, and not loaded by # store-gateway. Useful when used together with -querier.query-store-after to # prevent loading young blocks, because there are usually many of them - # (depending on number of ingesters) and they are not yet compacted. + # (depending on number of ingesters) and they are not yet compacted. Negative + # values or 0 disable the filter. # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within [ignore_blocks_within: | default = 0s] diff --git a/pkg/storegateway/bucket_index_metadata_fetcher.go b/pkg/storegateway/bucket_index_metadata_fetcher.go index 63947c51727..5b57b5d1897 100644 --- a/pkg/storegateway/bucket_index_metadata_fetcher.go +++ b/pkg/storegateway/bucket_index_metadata_fetcher.go @@ -57,7 +57,7 @@ func NewBucketIndexMetadataFetcher( logger: logger, filters: filters, modifiers: modifiers, - metrics: block.NewFetcherMetrics(reg, [][]string{{corruptedBucketIndex}, {noBucketIndex}}, nil), + metrics: block.NewFetcherMetrics(reg, [][]string{{corruptedBucketIndex}, {noBucketIndex}, {minTimeExcludedMeta}}, nil), } } diff --git a/pkg/storegateway/metadata_fetcher_filters.go b/pkg/storegateway/metadata_fetcher_filters.go index 187aa8ac382..9a1e16837c8 100644 --- a/pkg/storegateway/metadata_fetcher_filters.go +++ b/pkg/storegateway/metadata_fetcher_filters.go @@ -83,7 +83,7 @@ func (f *IgnoreDeletionMarkFilter) FilterWithBucketIndex(_ context.Context, meta return nil } -const timeExcludedMeta = "time-excluded" // Matches block.timeExcludedMeta value. +const minTimeExcludedMeta = "min-time-excluded" // minTimeMetaFilter filters out blocks that contain the most recent data (based on block MinTime). type minTimeMetaFilter struct { @@ -106,7 +106,7 @@ func (f *minTimeMetaFilter) Filter(_ context.Context, metas map[ulid.ULID]*metad continue } - synced.WithLabelValues(timeExcludedMeta).Inc() + synced.WithLabelValues(minTimeExcludedMeta).Inc() delete(metas, id) } return nil diff --git a/pkg/storegateway/metadata_fetcher_filters_test.go b/pkg/storegateway/metadata_fetcher_filters_test.go index 64fd4af7e4a..e54bb49e0e3 100644 --- a/pkg/storegateway/metadata_fetcher_filters_test.go +++ b/pkg/storegateway/metadata_fetcher_filters_test.go @@ -140,11 +140,11 @@ func TestTimeMetaFilter(t *testing.T) { f := newMinTimeMetaFilter(-10 * time.Minute) require.NoError(t, f.Filter(context.Background(), inputMetas, synced)) assert.Equal(t, inputMetas, inputMetas) - assert.Equal(t, 0.0, promtest.ToFloat64(synced.WithLabelValues(timeExcludedMeta))) + assert.Equal(t, 0.0, promtest.ToFloat64(synced.WithLabelValues(minTimeExcludedMeta))) f = newMinTimeMetaFilter(limit) require.NoError(t, f.Filter(context.Background(), inputMetas, synced)) assert.Equal(t, expectedMetas, inputMetas) - assert.Equal(t, 2.0, promtest.ToFloat64(synced.WithLabelValues(timeExcludedMeta))) + assert.Equal(t, 2.0, promtest.ToFloat64(synced.WithLabelValues(minTimeExcludedMeta))) } From e55890ff70cda2b0de43c5324bac8c0990d83820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Thu, 18 Nov 2021 14:49:12 +0100 Subject: [PATCH 9/9] Fix tests after changing label value. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/storegateway/bucket_index_metadata_fetcher_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/storegateway/bucket_index_metadata_fetcher_test.go b/pkg/storegateway/bucket_index_metadata_fetcher_test.go index 6329f9b01ae..7562ddc6e91 100644 --- a/pkg/storegateway/bucket_index_metadata_fetcher_test.go +++ b/pkg/storegateway/bucket_index_metadata_fetcher_test.go @@ -93,7 +93,8 @@ func TestBucketIndexMetadataFetcher_Fetch(t *testing.T) { blocks_meta_synced{state="marked-for-no-compact"} 0 blocks_meta_synced{state="no-bucket-index"} 0 blocks_meta_synced{state="no-meta-json"} 0 - blocks_meta_synced{state="time-excluded"} 1 + blocks_meta_synced{state="time-excluded"} 0 + blocks_meta_synced{state="min-time-excluded"} 1 blocks_meta_synced{state="too-fresh"} 0 # HELP blocks_meta_syncs_total Total blocks metadata synchronization attempts @@ -145,6 +146,7 @@ func TestBucketIndexMetadataFetcher_Fetch_NoBucketIndex(t *testing.T) { blocks_meta_synced{state="no-bucket-index"} 1 blocks_meta_synced{state="no-meta-json"} 0 blocks_meta_synced{state="time-excluded"} 0 + blocks_meta_synced{state="min-time-excluded"} 0 blocks_meta_synced{state="too-fresh"} 0 # HELP blocks_meta_syncs_total Total blocks metadata synchronization attempts @@ -199,6 +201,7 @@ func TestBucketIndexMetadataFetcher_Fetch_CorruptedBucketIndex(t *testing.T) { blocks_meta_synced{state="no-bucket-index"} 0 blocks_meta_synced{state="no-meta-json"} 0 blocks_meta_synced{state="time-excluded"} 0 + blocks_meta_synced{state="min-time-excluded"} 0 blocks_meta_synced{state="too-fresh"} 0 # HELP blocks_meta_syncs_total Total blocks metadata synchronization attempts @@ -245,6 +248,7 @@ func TestBucketIndexMetadataFetcher_Fetch_ShouldResetGaugeMetrics(t *testing.T) blocks_meta_synced{state="no-bucket-index"} 0 blocks_meta_synced{state="no-meta-json"} 0 blocks_meta_synced{state="time-excluded"} 0 + blocks_meta_synced{state="min-time-excluded"} 0 blocks_meta_synced{state="too-fresh"} 0 `), "blocks_meta_synced")) @@ -269,6 +273,7 @@ func TestBucketIndexMetadataFetcher_Fetch_ShouldResetGaugeMetrics(t *testing.T) blocks_meta_synced{state="no-bucket-index"} 1 blocks_meta_synced{state="no-meta-json"} 0 blocks_meta_synced{state="time-excluded"} 0 + blocks_meta_synced{state="min-time-excluded"} 0 blocks_meta_synced{state="too-fresh"} 0 `), "blocks_meta_synced")) @@ -301,6 +306,7 @@ func TestBucketIndexMetadataFetcher_Fetch_ShouldResetGaugeMetrics(t *testing.T) blocks_meta_synced{state="no-bucket-index"} 0 blocks_meta_synced{state="no-meta-json"} 0 blocks_meta_synced{state="time-excluded"} 0 + blocks_meta_synced{state="min-time-excluded"} 0 blocks_meta_synced{state="too-fresh"} 0 `), "blocks_meta_synced")) @@ -327,6 +333,7 @@ func TestBucketIndexMetadataFetcher_Fetch_ShouldResetGaugeMetrics(t *testing.T) blocks_meta_synced{state="no-bucket-index"} 0 blocks_meta_synced{state="no-meta-json"} 0 blocks_meta_synced{state="time-excluded"} 0 + blocks_meta_synced{state="min-time-excluded"} 0 blocks_meta_synced{state="too-fresh"} 0 `), "blocks_meta_synced")) }