-
Notifications
You must be signed in to change notification settings - Fork 2.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
compact: add schedule-delete and delete-delay
Signed-off-by: khyatisoneji <[email protected]>
- Loading branch information
1 parent
0a417f0
commit 19cc734
Showing
11 changed files
with
229 additions
and
20 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| // Copyright (c) The Thanos Authors. | ||
| // Licensed under the Apache License 2.0. | ||
|
|
||
| package metadata | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "io/ioutil" | ||
| "path" | ||
|
|
||
| "github.com/go-kit/kit/log" | ||
| "github.com/oklog/ulid" | ||
| "github.com/pkg/errors" | ||
| "github.com/thanos-io/thanos/pkg/objstore" | ||
| "github.com/thanos-io/thanos/pkg/runutil" | ||
| ) | ||
|
|
||
| const ( | ||
| // DeletionMarkFilename is the known json filename to store details about when block is marked for deletion. | ||
| DeletionMarkFilename = "deletion-mark.json" | ||
|
|
||
| // DeletionMarkVersion1 is a enumeration of deletion-mark versions supported by Thanos. | ||
| DeletionMarkVersion1 = iota + 1 | ||
| ) | ||
|
|
||
| // ErrorDeletionMarkNotFound is the error when deletion-mark.json file is not found. | ||
| var ErrorDeletionMarkNotFound = errors.New("deletion-mark.json not found") | ||
|
|
||
| // DeletionMark stores block id and when block was marked for deletion. | ||
| type DeletionMark struct { | ||
| // ID of the tsdb block. | ||
| ID ulid.ULID `json:"id"` | ||
|
|
||
| // DeletionTime is a unix timestamp of when the block was marked to be deleted. | ||
| DeletionTime int64 `json:"deletion_time"` | ||
|
|
||
| // Version of the file. | ||
| Version int `json:"version"` | ||
| } | ||
|
|
||
| // ReadDeletionMark reads the given compactor meta from <dir>/deletion-mark.json in bucket. | ||
| func ReadDeletionMark(ctx context.Context, bkt objstore.Bucket, logger log.Logger, dir string) (*DeletionMark, error) { | ||
| deletionMarkFile := path.Join(dir, DeletionMarkFilename) | ||
|
|
||
| r, err := bkt.Get(ctx, deletionMarkFile) | ||
| if bkt.IsObjNotFoundErr(err) { | ||
| return nil, ErrorDeletionMarkNotFound | ||
| } | ||
| if err != nil { | ||
| return nil, errors.Wrapf(err, "get file: %v", deletionMarkFile) | ||
| } | ||
|
|
||
| defer runutil.CloseWithLogOnErr(logger, r, "close bkt deletion-mark reader") | ||
|
|
||
| metaContent, err := ioutil.ReadAll(r) | ||
| if err != nil { | ||
| return nil, errors.Wrapf(err, "read file: %v", deletionMarkFile) | ||
| } | ||
|
|
||
| deletionMark := DeletionMark{} | ||
| if err := json.Unmarshal(metaContent, &deletionMark); err != nil { | ||
| return nil, errors.Wrap(err, "unmarshal compactor meta") | ||
| } | ||
|
|
||
| if deletionMark.Version != DeletionMarkVersion1 { | ||
| return nil, errors.Errorf("unexpected deletion-mark file version %d", deletionMark.Version) | ||
| } | ||
|
|
||
| return &deletionMark, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| // Copyright (c) The Thanos Authors. | ||
| // Licensed under the Apache License 2.0. | ||
|
|
||
| package compact | ||
|
|
||
| import ( | ||
| "context" | ||
| "time" | ||
|
|
||
| "github.com/go-kit/kit/log" | ||
| "github.com/go-kit/kit/log/level" | ||
| "github.com/pkg/errors" | ||
| "github.com/thanos-io/thanos/pkg/block" | ||
| "github.com/thanos-io/thanos/pkg/block/metadata" | ||
| "github.com/thanos-io/thanos/pkg/objstore" | ||
| ) | ||
|
|
||
| // BlocksCleaner is a struct that deletes blocks from bucket | ||
| // which are marked for deletion. | ||
| type BlocksCleaner struct { | ||
| logger log.Logger | ||
| deleteDelay time.Duration | ||
| bkt objstore.Bucket | ||
| } | ||
|
|
||
| // NewBlocksCleaner creates a new BlocksCleaner. | ||
| func NewBlocksCleaner(logger log.Logger, bkt objstore.Bucket, deleteDelay time.Duration) *BlocksCleaner { | ||
| return &BlocksCleaner{ | ||
| logger: logger, | ||
| deleteDelay: deleteDelay, | ||
| bkt: bkt, | ||
| } | ||
| } | ||
|
|
||
| // DeleteMarkedBlocks reads compactor-meta.json file in block to check when the block was marked for deletion and | ||
| // deletes the block deleteDelay duration after block is marked for deletion. | ||
| func (s *BlocksCleaner) DeleteMarkedBlocks(ctx context.Context) error { | ||
| return s.bkt.Iter(ctx, "", func(name string) error { | ||
| id, ok := block.IsBlockDir(name) | ||
| if !ok { | ||
| return nil | ||
| } | ||
|
|
||
| deletionMark, err := metadata.ReadDeletionMark(ctx, s.bkt, s.logger, id.String()) | ||
| if err != nil { | ||
| if errors.Cause(err) != metadata.ErrorDeletionMarkNotFound { | ||
| return errors.Wrap(err, "read file: %s") | ||
| } | ||
| } | ||
|
|
||
| if time.Since(time.Unix(deletionMark.DeletionTime, 0)) > s.deleteDelay { | ||
| if err := block.Delete(ctx, s.logger, s.bkt, deletionMark.ID); err != nil { | ||
| return errors.Wrap(err, "delete block") | ||
| } | ||
| level.Info(s.logger).Log("msg", "blocks cleaner: deleting block", "id", id) | ||
| } | ||
|
|
||
| return nil | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.