From 1c7ad574eb6fa428175e0a8a2182d36e705e97bc Mon Sep 17 00:00:00 2001 From: Xiang Dai <764524258@qq.com> Date: Wed, 12 Feb 2020 10:02:21 +0800 Subject: [PATCH] feedback Signed-off-by: Xiang Dai <764524258@qq.com> --- cmd/thanos/bucket.go | 10 +++++----- docs/components/bucket.md | 12 ++++++++---- go.mod | 1 - go.sum | 3 --- pkg/{replicater => replicator}/replicater.go | 2 +- pkg/{replicater => replicator}/scheme.go | 10 +++++++++- pkg/{replicater => replicator}/scheme_test.go | 4 ++-- 7 files changed, 25 insertions(+), 17 deletions(-) rename pkg/{replicater => replicator}/replicater.go (99%) rename pkg/{replicater => replicator}/scheme.go (98%) rename pkg/{replicater => replicator}/scheme_test.go (99%) diff --git a/cmd/thanos/bucket.go b/cmd/thanos/bucket.go index 8a7daf57d7b..f5fefbbe37a 100644 --- a/cmd/thanos/bucket.go +++ b/cmd/thanos/bucket.go @@ -35,7 +35,7 @@ import ( "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/prober" - "github.com/thanos-io/thanos/pkg/replicater" + "github.com/thanos-io/thanos/pkg/replicator" "github.com/thanos-io/thanos/pkg/runutil" httpserver "github.com/thanos-io/thanos/pkg/server/http" "github.com/thanos-io/thanos/pkg/ui" @@ -380,21 +380,21 @@ func registerBucketWeb(m map[string]setupFunc, root *kingpin.CmdClause, name str } func registerBucketReplicate(m map[string]setupFunc, root *kingpin.CmdClause, name string, objStoreConfig *extflag.PathOrContent) { - cmd := root.Command("replicate", "Replicate data from one object storage to another") + cmd := root.Command("replicate", "Replicate data from one object storage to another. NOTE: Currently it works only with Thanos blocks (meta.json has to have Thanos metadata).") httpMetricsBindAddr, _ := regHTTPFlags(cmd) toObjStoreConfig := regCommonObjStoreFlags(cmd, "-to", false, "The object storage which replicate data to.") resolution := cmd.Flag("resolution", "Only blocks with this resolution will be replicated.").Default(strconv.FormatInt(downsample.ResLevel0, 10)).Int64() compaction := cmd.Flag("compaction", "Only blocks with this compaction level will be replicated.").Default("1").Int() - matcherStrs := cmd.Flag("matcher", "Only blocks whose labels match this matcher will be replicated.").PlaceHolder("key=\"value\"").Strings() + matcherStrs := cmd.Flag("matcher", "Only blocks whose external labels exactly match this matcher will be replicated.").PlaceHolder("key=\"value\"").Strings() singleRun := cmd.Flag("single-run", "Run replication only one time, then exit.").Default("false").Bool() m[name+" replicate"] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { - matchers, err := replicater.ParseFlagMatchers(*matcherStrs) + matchers, err := replicator.ParseFlagMatchers(*matcherStrs) if err != nil { return errors.Wrap(err, "parse block label matchers") } - return replicater.RunReplicate( + return replicator.RunReplicate( g, logger, reg, diff --git a/docs/components/bucket.md b/docs/components/bucket.md index c1ca1ea9f13..3dc3c7aaf91 100644 --- a/docs/components/bucket.md +++ b/docs/components/bucket.md @@ -75,7 +75,8 @@ Subcommands: Web interface for remote storage bucket bucket replicate [] - Replicate data from one object storage to another + Replicate data from one object storage to another. NOTE: Currently it works + only with Thanos blocks (meta.json has to have Thanos metadata). ``` @@ -323,6 +324,8 @@ Flags: `bucket replicate` is used to replicate buckets from one object storage to another. +NOTE: Currently it works only with Thanos blocks (meta.json has to have Thanos metadata). + Example: ``` $ thanos bucket replicate --objstore.config-file="..." --objstore-to.config="..." @@ -332,7 +335,8 @@ $ thanos bucket replicate --objstore.config-file="..." --objstore-to.config="... ```txt usage: thanos bucket replicate [] -Replicate data from one object storage to another +Replicate data from one object storage to another. NOTE: Currently it works only +with Thanos blocks (meta.json has to have Thanos metadata). Flags: -h, --help Show context-sensitive help (also try @@ -380,8 +384,8 @@ Flags: replicated. --compaction=1 Only blocks with this compaction level will be replicated. - --matcher=key="value" ... Only blocks whose labels match this matcher - will be replicated. + --matcher=key="value" ... Only blocks whose external labels exactly match + this matcher will be replicated. --single-run Run replication only one time, then exit. ``` diff --git a/go.mod b/go.mod index 0ed5f7a9ff4..beb3c75d701 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,6 @@ require ( github.com/prometheus/common v0.7.0 github.com/prometheus/procfs v0.0.6 // indirect github.com/prometheus/prometheus v1.8.2-0.20200110114423-1e64d757f711 // master ~ v2.15.2 - github.com/prometheus/tsdb v0.10.0 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da // indirect github.com/satori/go.uuid v1.2.0 // indirect github.com/smartystreets/assertions v1.0.1 // indirect diff --git a/go.sum b/go.sum index ef05edbcd40..a4ddc0f20a9 100644 --- a/go.sum +++ b/go.sum @@ -118,7 +118,6 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -504,8 +503,6 @@ github.com/prometheus/procfs v0.0.6/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/prometheus v0.0.0-20180315085919-58e2a31db8de/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= github.com/prometheus/prometheus v1.8.2-0.20200110114423-1e64d757f711 h1:uEq+8hKI4kfycPLSKNw844YYkdMNpC2eZpov73AvlFk= github.com/prometheus/prometheus v1.8.2-0.20200110114423-1e64d757f711/go.mod h1:7U90zPoLkWjEIQcy/rweQla82OCTUzxVHE51G3OhJbI= -github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic= -github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= diff --git a/pkg/replicater/replicater.go b/pkg/replicator/replicater.go similarity index 99% rename from pkg/replicater/replicater.go rename to pkg/replicator/replicater.go index ea9de508a9c..186cd6017ba 100644 --- a/pkg/replicater/replicater.go +++ b/pkg/replicator/replicater.go @@ -1,7 +1,7 @@ // Copyright (c) The Thanos Authors. // Licensed under the Apache License 2.0. -package replicater +package replicator import ( "context" diff --git a/pkg/replicater/scheme.go b/pkg/replicator/scheme.go similarity index 98% rename from pkg/replicater/scheme.go rename to pkg/replicator/scheme.go index 560d86bc085..5ee6ed43999 100644 --- a/pkg/replicater/scheme.go +++ b/pkg/replicator/scheme.go @@ -1,7 +1,7 @@ // Copyright (c) The Thanos Authors. // Licensed under the Apache License 2.0. -package replicater +package replicator import ( "bytes" @@ -51,6 +51,11 @@ func NewBlockFilter( // Filter return true if block is non-compacted and matches selector. func (bf *BlockFilter) Filter(b *metadata.Meta) bool { + if len(b.Thanos.Labels) == 0 { + level.Error(bf.logger).Log("msg", "filtering block", "reason", "labels should not be empty") + return false + } + blockLabels := labels.FromMap(b.Thanos.Labels) labelMatch := bf.labelSelector.Matches(blockLabels) @@ -174,6 +179,7 @@ func (rs *replicationScheme) execute(ctx context.Context) error { level.Debug(rs.logger).Log("msg", "scanning blocks available blocks for replication") + // TODO: Use block.MetaFetcher with filters instead. if err := rs.fromBkt.Iter(ctx, "", func(name string) error { rs.metrics.originIterations.Inc() @@ -354,10 +360,12 @@ func loadMeta(ctx context.Context, bucket objstore.BucketReader, id ulid.ULID) ( r, err := bucket.Get(ctx, src) if bucket.IsObjNotFoundErr(err) { + r.Close() return nil, true, fmt.Errorf("get meta file: %w", err) } if err != nil { + r.Close() return nil, false, fmt.Errorf("get meta file: %w", err) } diff --git a/pkg/replicater/scheme_test.go b/pkg/replicator/scheme_test.go similarity index 99% rename from pkg/replicater/scheme_test.go rename to pkg/replicator/scheme_test.go index a282325eefe..fc5a9a95a8a 100644 --- a/pkg/replicater/scheme_test.go +++ b/pkg/replicator/scheme_test.go @@ -1,7 +1,7 @@ // Copyright (c) The Thanos Authors. // Licensed under the Apache License 2.0. -package replicater +package replicator import ( "bytes" @@ -19,11 +19,11 @@ import ( "github.com/oklog/ulid" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb" - "github.com/prometheus/tsdb/testutil" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/compact" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/inmem" + "github.com/thanos-io/thanos/pkg/testutil" ) func testLogger(testName string) log.Logger {