From bbb31144d73d31dfccc73597f6b3fc40f9da03a5 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Fri, 21 Jul 2023 17:49:27 +0100 Subject: [PATCH 01/38] [FEAT] Add uploaded TSDB bytes metric Signed-off-by: rita.canavarro --- objstore.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/objstore.go b/objstore.go index 84eafce3..9d704783 100644 --- a/objstore.go +++ b/objstore.go @@ -443,6 +443,12 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu Help: "Second timestamp of the last successful upload to the bucket.", ConstLabels: prometheus.Labels{"bucket": name}, }), + + opsUploadedBytes: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ + Name: "objstore_bucket_operation_uploaded_bytes_total", + Help: "Total number of bytes uploaded from TSDB block, per operation.", + ConstLabels: prometheus.Labels{"bucket": name}, + }, []string{"operation"}), } for _, op := range []string{ OpIter, @@ -457,6 +463,7 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu bkt.opsFailures.WithLabelValues(op) bkt.opsDuration.WithLabelValues(op) bkt.opsFetchedBytes.WithLabelValues(op) + bkt.opsUploadedBytes.WithLabelValues(op) } // fetched bytes only relevant for get and getrange for _, op := range []string{ @@ -478,6 +485,7 @@ type metricBucket struct { opsFetchedBytes *prometheus.CounterVec opsTransferredBytes *prometheus.HistogramVec + opsUploadedBytes *prometheus.CounterVec opsDuration *prometheus.HistogramVec lastSuccessfulUploadTime prometheus.Gauge } From 8f0a6ab88312ed85a6da36ea33e3d60ab0cccc7b Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Fri, 21 Jul 2023 17:54:41 +0100 Subject: [PATCH 02/38] [FIX] Tests Signed-off-by: rita.canavarro --- objstore.go | 7 ++++++- objstore_test.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/objstore.go b/objstore.go index 9d704783..67fd0d16 100644 --- a/objstore.go +++ b/objstore.go @@ -497,6 +497,7 @@ func (b *metricBucket) WithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket { opsFailures: b.opsFailures, opsFetchedBytes: b.opsFetchedBytes, opsTransferredBytes: b.opsTransferredBytes, + opsUploadedBytes: b.opsUploadedBytes, isOpFailureExpected: fn, opsDuration: b.opsDuration, lastSuccessfulUploadTime: b.lastSuccessfulUploadTime, @@ -555,6 +556,7 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err b.isOpFailureExpected, b.opsFetchedBytes, b.opsTransferredBytes, + b.opsUploadedBytes, ), nil } @@ -577,6 +579,7 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in b.isOpFailureExpected, b.opsFetchedBytes, b.opsTransferredBytes, + b.opsUploadedBytes, ), nil } @@ -659,9 +662,10 @@ type timingReadCloser struct { isFailureExpected IsOpFailureExpectedFunc fetchedBytes *prometheus.CounterVec transferredBytes *prometheus.HistogramVec + uploadedBytes *prometheus.CounterVec } -func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) *timingReadCloser { +func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, uploadedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) *timingReadCloser { // Initialize the metrics with 0. dur.WithLabelValues(op) failed.WithLabelValues(op) @@ -678,6 +682,7 @@ func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramV fetchedBytes: fetchedBytes, transferredBytes: transferredBytes, readBytes: 0, + uploadedBytes: uploadedBytes, } } diff --git a/objstore_test.go b/objstore_test.go index 4b105281..cc5ef021 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -194,7 +194,7 @@ func TestTimingTracingReader(t *testing.T) { tr := NopCloserWithSize(r) tr = newTimingReadCloser(tr, "", m.opsDuration, m.opsFailures, func(err error) bool { return false - }, m.opsFetchedBytes, m.opsTransferredBytes) + }, m.opsFetchedBytes, m.opsTransferredBytes, m.opsUploadedBytes) size, err := TryToGetSize(tr) From 0b3153e1964d91cea5b2b321f7a948bf88b38869 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Fri, 21 Jul 2023 17:54:53 +0100 Subject: [PATCH 03/38] [FEAT] Changelog Signed-off-by: rita.canavarro --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32d0f742..f4fd871d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#61](https://github.com/thanos-io/objstore/pull/61) Add OpenTelemetry TracingBucket. > This also changes the behaviour of `client.NewBucket`. Now it returns, uninstrumented and untraced bucket. You can combine `objstore.WrapWithMetrics` and `tracing/{opentelemetry,opentracing}.WrapWithTraces` to have old behavior. +- [#66](https://github.com/thanos-io/objstore/pull/66) Add for the number of total uploaded blocks. - [#64](https://github.com/thanos-io/objstore/pull/64) OCI: OKE Workload Identity support. - [#73](https://github.com/thanos-io/objstore/pull/73) Аdded file path to erros from DownloadFile - [#51](https://github.com/thanos-io/objstore/pull/51) Azure: Support using connection string authentication. From 54d6fb552844dbf23a3253eccc994d4aeb4180e9 Mon Sep 17 00:00:00 2001 From: Rita Canavarro <98762287+ritaCanavarro@users.noreply.github.com> Date: Tue, 25 Jul 2023 09:15:55 +0100 Subject: [PATCH 04/38] Update CHANGELOG.md Co-authored-by: Bartlomiej Plotka Signed-off-by: Rita Canavarro <98762287+ritaCanavarro@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4fd871d..7231f68a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#61](https://github.com/thanos-io/objstore/pull/61) Add OpenTelemetry TracingBucket. > This also changes the behaviour of `client.NewBucket`. Now it returns, uninstrumented and untraced bucket. You can combine `objstore.WrapWithMetrics` and `tracing/{opentelemetry,opentracing}.WrapWithTraces` to have old behavior. -- [#66](https://github.com/thanos-io/objstore/pull/66) Add for the number of total uploaded blocks. +- [#66](https://github.com/thanos-io/objstore/pull/66) Add `objstore_bucket_operation_written_bytes_total` that counts the number of total bytes per written to the bucket operation. - [#64](https://github.com/thanos-io/objstore/pull/64) OCI: OKE Workload Identity support. - [#73](https://github.com/thanos-io/objstore/pull/73) Аdded file path to erros from DownloadFile - [#51](https://github.com/thanos-io/objstore/pull/51) Azure: Support using connection string authentication. From a3d95a12296462317d60801b3fe8042690932c4e Mon Sep 17 00:00:00 2001 From: Rita Canavarro <98762287+ritaCanavarro@users.noreply.github.com> Date: Tue, 25 Jul 2023 09:16:37 +0100 Subject: [PATCH 05/38] Update objstore.go Co-authored-by: Bartlomiej Plotka Signed-off-by: Rita Canavarro <98762287+ritaCanavarro@users.noreply.github.com> --- objstore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/objstore.go b/objstore.go index 67fd0d16..84bce17d 100644 --- a/objstore.go +++ b/objstore.go @@ -445,7 +445,7 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu }), opsUploadedBytes: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ - Name: "objstore_bucket_operation_uploaded_bytes_total", + Name: "objstore_bucket_operation_written_bytes_total", Help: "Total number of bytes uploaded from TSDB block, per operation.", ConstLabels: prometheus.Labels{"bucket": name}, }, []string{"operation"}), From 64d79d1dedf45b3cfdf8dfaf5106f9272e708e25 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 25 Jul 2023 14:05:53 +0100 Subject: [PATCH 06/38] [FEAT] Increment metric Signed-off-by: rita.canavarro --- inmem.go | 9 +++---- objstore.go | 31 +++++++++++++------------ objstore_test.go | 22 +++++++++++------- prefixed_bucket.go | 2 +- prefixed_bucket_test.go | 9 ++++--- providers/bos/bos.go | 20 ++++++++-------- providers/cos/cos.go | 18 +++++++------- providers/filesystem/filesystem.go | 15 ++++++------ providers/filesystem/filesystem_test.go | 8 ++++--- providers/gcs/gcs.go | 9 +++---- providers/obs/obs.go | 16 ++++++------- providers/oci/oci.go | 6 ++--- providers/oss/oss.go | 16 ++++++------- providers/s3/s3.go | 8 +++---- providers/s3/s3_e2e_test.go | 3 ++- providers/swift/swift.go | 15 ++++++------ testing.go | 29 +++++++++++++++-------- tracing/opentelemetry/opentelemetry.go | 2 +- tracing/opentracing/opentracing.go | 4 ++-- 19 files changed, 134 insertions(+), 108 deletions(-) diff --git a/inmem.go b/inmem.go index 3f6f35e9..45d13ef3 100644 --- a/inmem.go +++ b/inmem.go @@ -175,19 +175,20 @@ func (b *InMemBucket) Attributes(_ context.Context, name string) (ObjectAttribut } // Upload writes the file specified in src to into the memory. -func (b *InMemBucket) Upload(_ context.Context, name string, r io.Reader) error { +func (b *InMemBucket) Upload(_ context.Context, name string, r io.Reader) (int64, error) { b.mtx.Lock() defer b.mtx.Unlock() body, err := io.ReadAll(r) if err != nil { - return err + return 0, err } b.objects[name] = body + size := int64(len(body)) b.attrs[name] = ObjectAttributes{ - Size: int64(len(body)), + Size: size, LastModified: time.Now(), } - return nil + return size, nil } // Delete removes all data prefixed with the dir. diff --git a/objstore.go b/objstore.go index 84bce17d..c9ec22d2 100644 --- a/objstore.go +++ b/objstore.go @@ -42,7 +42,7 @@ type Bucket interface { // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. - Upload(ctx context.Context, name string, r io.Reader) error + Upload(ctx context.Context, name string, r io.Reader) (int64, error) // Delete removes the object with the given name. // If object does not exist in the moment of deletion, Delete should throw error. @@ -285,7 +285,7 @@ func UploadFile(ctx context.Context, logger log.Logger, bkt Bucket, src, dst str } defer logerrcapture.Do(logger, r.Close, "close file %s", src) - if err := bkt.Upload(ctx, dst, r); err != nil { + if _, err := bkt.Upload(ctx, dst, r); err != nil { return errors.Wrapf(err, "upload file %s as %s", src, dst) } level.Debug(logger).Log("msg", "uploaded file", "from", src, "dst", dst, "bucket", bkt.Name()) @@ -444,7 +444,7 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu ConstLabels: prometheus.Labels{"bucket": name}, }), - opsUploadedBytes: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ + opsWrittenBytes: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ Name: "objstore_bucket_operation_written_bytes_total", Help: "Total number of bytes uploaded from TSDB block, per operation.", ConstLabels: prometheus.Labels{"bucket": name}, @@ -463,7 +463,7 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu bkt.opsFailures.WithLabelValues(op) bkt.opsDuration.WithLabelValues(op) bkt.opsFetchedBytes.WithLabelValues(op) - bkt.opsUploadedBytes.WithLabelValues(op) + bkt.opsWrittenBytes.WithLabelValues(op) } // fetched bytes only relevant for get and getrange for _, op := range []string{ @@ -485,7 +485,7 @@ type metricBucket struct { opsFetchedBytes *prometheus.CounterVec opsTransferredBytes *prometheus.HistogramVec - opsUploadedBytes *prometheus.CounterVec + opsWrittenBytes *prometheus.CounterVec opsDuration *prometheus.HistogramVec lastSuccessfulUploadTime prometheus.Gauge } @@ -497,7 +497,7 @@ func (b *metricBucket) WithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket { opsFailures: b.opsFailures, opsFetchedBytes: b.opsFetchedBytes, opsTransferredBytes: b.opsTransferredBytes, - opsUploadedBytes: b.opsUploadedBytes, + opsWrittenBytes: b.opsWrittenBytes, isOpFailureExpected: fn, opsDuration: b.opsDuration, lastSuccessfulUploadTime: b.lastSuccessfulUploadTime, @@ -556,7 +556,7 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err b.isOpFailureExpected, b.opsFetchedBytes, b.opsTransferredBytes, - b.opsUploadedBytes, + b.opsWrittenBytes, ), nil } @@ -579,7 +579,7 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in b.isOpFailureExpected, b.opsFetchedBytes, b.opsTransferredBytes, - b.opsUploadedBytes, + b.opsWrittenBytes, ), nil } @@ -599,20 +599,21 @@ func (b *metricBucket) Exists(ctx context.Context, name string) (bool, error) { return ok, nil } -func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) error { +func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { const op = OpUpload b.ops.WithLabelValues(op).Inc() start := time.Now() - if err := b.bkt.Upload(ctx, name, r); err != nil { + writtenBytes, err := b.bkt.Upload(ctx, name, r) + if err != nil { if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.opsFailures.WithLabelValues(op).Inc() } - return err + return 0, err } b.lastSuccessfulUploadTime.SetToCurrentTime() b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) - return nil + return writtenBytes, nil } func (b *metricBucket) Delete(ctx context.Context, name string) error { @@ -662,10 +663,10 @@ type timingReadCloser struct { isFailureExpected IsOpFailureExpectedFunc fetchedBytes *prometheus.CounterVec transferredBytes *prometheus.HistogramVec - uploadedBytes *prometheus.CounterVec + writtenBytes *prometheus.CounterVec } -func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, uploadedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) *timingReadCloser { +func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, writtenBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) *timingReadCloser { // Initialize the metrics with 0. dur.WithLabelValues(op) failed.WithLabelValues(op) @@ -682,7 +683,7 @@ func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramV fetchedBytes: fetchedBytes, transferredBytes: transferredBytes, readBytes: 0, - uploadedBytes: uploadedBytes, + writtenBytes: writtenBytes, } } diff --git a/objstore_test.go b/objstore_test.go index cc5ef021..26b86ab4 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -85,9 +85,12 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), r, "") tempDir := t.TempDir() - testutil.Ok(t, m.Upload(context.Background(), "dir/obj1", bytes.NewReader([]byte("1")))) - testutil.Ok(t, m.Upload(context.Background(), "dir/obj2", bytes.NewReader([]byte("2")))) - testutil.Ok(t, m.Upload(context.Background(), "dir/obj3", bytes.NewReader(bytes.Repeat([]byte("3"), 1024*1024)))) + _, err := m.Upload(context.Background(), "dir/obj1", bytes.NewReader([]byte("1"))) + testutil.Ok(t, err) + _, err = m.Upload(context.Background(), "dir/obj2", bytes.NewReader([]byte("2"))) + testutil.Ok(t, err) + _, err = m.Upload(context.Background(), "dir/obj3", bytes.NewReader(bytes.Repeat([]byte("3"), 1024*1024))) + testutil.Ok(t, err) testutil.Ok(t, promtest.GatherAndCompare(r, strings.NewReader(` # HELP objstore_bucket_operations_total Total number of all attempted operations against a bucket. @@ -194,7 +197,7 @@ func TestTimingTracingReader(t *testing.T) { tr := NopCloserWithSize(r) tr = newTimingReadCloser(tr, "", m.opsDuration, m.opsFailures, func(err error) bool { return false - }, m.opsFetchedBytes, m.opsTransferredBytes, m.opsUploadedBytes) + }, m.opsFetchedBytes, m.opsTransferredBytes, m.opsWrittenBytes) size, err := TryToGetSize(tr) @@ -221,13 +224,16 @@ func TestDownloadDir_CleanUp(t *testing.T) { } tempDir := t.TempDir() - testutil.Ok(t, b.Upload(context.Background(), "dir/obj1", bytes.NewReader([]byte("1")))) - testutil.Ok(t, b.Upload(context.Background(), "dir/obj2", bytes.NewReader([]byte("2")))) - testutil.Ok(t, b.Upload(context.Background(), "dir/obj3", bytes.NewReader([]byte("3")))) + _, err := b.Upload(context.Background(), "dir/obj1", bytes.NewReader([]byte("1"))) + testutil.Ok(t, err) + _, err = b.Upload(context.Background(), "dir/obj2", bytes.NewReader([]byte("2"))) + testutil.Ok(t, err) + _, err = b.Upload(context.Background(), "dir/obj3", bytes.NewReader([]byte("3"))) + testutil.Ok(t, err) // We exapect the third Get to fail testutil.NotOk(t, DownloadDir(context.Background(), log.NewNopLogger(), b, "dir/", "dir/", tempDir)) - _, err := os.Stat(tempDir) + _, err = os.Stat(tempDir) testutil.Assert(t, os.IsNotExist(err)) } diff --git a/prefixed_bucket.go b/prefixed_bucket.go index f2b71434..cbc5b491 100644 --- a/prefixed_bucket.go +++ b/prefixed_bucket.go @@ -86,7 +86,7 @@ func (p PrefixedBucket) Attributes(ctx context.Context, name string) (ObjectAttr // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. -func (p *PrefixedBucket) Upload(ctx context.Context, name string, r io.Reader) error { +func (p *PrefixedBucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { return p.bkt.Upload(ctx, conditionalPrefix(p.prefix, name), r) } diff --git a/prefixed_bucket_test.go b/prefixed_bucket_test.go index f93c8580..e7f2d9bc 100644 --- a/prefixed_bucket_test.go +++ b/prefixed_bucket_test.go @@ -29,7 +29,8 @@ func TestPrefixedBucket_Acceptance(t *testing.T) { } func UsesPrefixTest(t *testing.T, bkt Bucket, prefix string) { - testutil.Ok(t, bkt.Upload(context.Background(), strings.Trim(prefix, "/")+"/file1.jpg", strings.NewReader("test-data1"))) + _, err := bkt.Upload(context.Background(), strings.Trim(prefix, "/")+"/file1.jpg", strings.NewReader("test-data1")) + testutil.Ok(t, err) pBkt := NewPrefixedBucket(bkt, prefix) rc1, err := pBkt.Get(context.Background(), "file1.jpg") @@ -41,7 +42,8 @@ func UsesPrefixTest(t *testing.T, bkt Bucket, prefix string) { testutil.Ok(t, err) testutil.Equals(t, "test-data1", string(content)) - testutil.Ok(t, pBkt.Upload(context.Background(), "file2.jpg", strings.NewReader("test-data2"))) + _, err = pBkt.Upload(context.Background(), "file2.jpg", strings.NewReader("test-data2")) + testutil.Ok(t, err) rc2, err := bkt.Get(context.Background(), strings.Trim(prefix, "/")+"/file2.jpg") testutil.Ok(t, err) defer func() { testutil.Ok(t, rc2.Close()) }() @@ -69,7 +71,8 @@ func UsesPrefixTest(t *testing.T, bkt Bucket, prefix string) { testutil.Ok(t, err) testutil.Assert(t, attrs.Size == 10, "expected size to be equal to 10") - testutil.Ok(t, bkt.Upload(context.Background(), strings.Trim(prefix, "/")+"/dir/file1.jpg", strings.NewReader("test-data1"))) + _, err = bkt.Upload(context.Background(), strings.Trim(prefix, "/")+"/dir/file1.jpg", strings.NewReader("test-data1")) + testutil.Ok(t, err) seen := []string{} testutil.Ok(t, pBkt.Iter(context.Background(), "", func(fn string) error { seen = append(seen, fn) diff --git a/providers/bos/bos.go b/providers/bos/bos.go index 72e1b1e0..f5aefd8a 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -110,29 +110,29 @@ func (b *Bucket) Delete(_ context.Context, name string) error { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { +func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) (int64, error) { size, err := objstore.TryToGetSize(r) if err != nil { - return errors.Wrapf(err, "getting size of %s", name) + return 0, errors.Wrapf(err, "getting size of %s", name) } partNums, lastSlice := int(math.Floor(float64(size)/partSize)), size%partSize if partNums == 0 { body, err := bce.NewBodyFromSizedReader(r, lastSlice) if err != nil { - return errors.Wrapf(err, "failed to create SizedReader for %s", name) + return 0, errors.Wrapf(err, "failed to create SizedReader for %s", name) } if _, err := b.client.PutObject(b.name, name, body, nil); err != nil { - return errors.Wrapf(err, "failed to upload %s", name) + return 0, errors.Wrapf(err, "failed to upload %s", name) } - return nil + return 0, nil } result, err := b.client.BasicInitiateMultipartUpload(b.name, name) if err != nil { - return errors.Wrapf(err, "failed to initiate MultipartUpload for %s", name) + return 0, errors.Wrapf(err, "failed to initiate MultipartUpload for %s", name) } uploadEveryPart := func(partSize int64, part int, uploadId string) (string, error) { @@ -156,7 +156,7 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { for part := 1; part <= partNums; part++ { etag, err := uploadEveryPart(partSize, part, result.UploadId) if err != nil { - return errors.Wrapf(err, "failed to upload part %d for %s", part, name) + return 0, errors.Wrapf(err, "failed to upload part %d for %s", part, name) } parts = append(parts, api.UploadInfoType{PartNumber: part, ETag: etag}) } @@ -164,15 +164,15 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { if lastSlice != 0 { etag, err := uploadEveryPart(lastSlice, partNums+1, result.UploadId) if err != nil { - return errors.Wrapf(err, "failed to upload the last part for %s", name) + return 0, errors.Wrapf(err, "failed to upload the last part for %s", name) } parts = append(parts, api.UploadInfoType{PartNumber: partNums + 1, ETag: etag}) } if _, err := b.client.CompleteMultipartUploadFromStruct(b.name, name, result.UploadId, &api.CompleteMultipartUploadArgs{Parts: parts}); err != nil { - return errors.Wrapf(err, "failed to set %s upload completed", name) + return 0, errors.Wrapf(err, "failed to set %s upload completed", name) } - return nil + return size, nil } // Iter calls f for each entry in the given directory (not recursive). The argument to f is the full diff --git a/providers/cos/cos.go b/providers/cos/cos.go index e518cae2..1b99f640 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -197,24 +197,24 @@ func (r fixedLengthReader) Size() int64 { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { size, err := objstore.TryToGetSize(r) if err != nil { - return errors.Wrapf(err, "getting size of %s", name) + return 0, errors.Wrapf(err, "getting size of %s", name) } // partSize 128MB. const partSize = 1024 * 1024 * 128 partNums, lastSlice := int(math.Floor(float64(size)/partSize)), size%partSize if partNums == 0 { if _, err := b.client.Object.Put(ctx, name, r, nil); err != nil { - return errors.Wrapf(err, "Put object: %s", name) + return 0, errors.Wrapf(err, "Put object: %s", name) } - return nil + return 0, nil } // 1. init. result, _, err := b.client.Object.InitiateMultipartUpload(ctx, name, nil) if err != nil { - return errors.Wrapf(err, "InitiateMultipartUpload %s", name) + return 0, errors.Wrapf(err, "InitiateMultipartUpload %s", name) } uploadEveryPart := func(partSize int64, part int, uploadID string) (string, error) { r := newFixedLengthReader(r, partSize) @@ -235,7 +235,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { for part := 1; part <= partNums; part++ { etag, err := uploadEveryPart(partSize, part, result.UploadID) if err != nil { - return errors.Wrapf(err, "uploadPart %d, %s", part, name) + return 0, errors.Wrapf(err, "uploadPart %d, %s", part, name) } optcom.Parts = append(optcom.Parts, cos.Object{ PartNumber: part, ETag: etag}, @@ -246,7 +246,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { part := partNums + 1 etag, err := uploadEveryPart(lastSlice, part, result.UploadID) if err != nil { - return errors.Wrapf(err, "uploadPart %d, %s", part, name) + return 0, errors.Wrapf(err, "uploadPart %d, %s", part, name) } optcom.Parts = append(optcom.Parts, cos.Object{ PartNumber: part, ETag: etag}, @@ -254,9 +254,9 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { } // 4. complete. if _, _, err := b.client.Object.CompleteMultipartUpload(ctx, name, result.UploadID, optcom); err != nil { - return errors.Wrapf(err, "CompleteMultipartUpload %s", name) + return 0, errors.Wrapf(err, "CompleteMultipartUpload %s", name) } - return nil + return size, nil } // Delete removes the object with the given name. diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index 21c70485..026f2368 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -191,26 +191,27 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { } // Upload writes the file specified in src to into the memory. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (err error) { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (writtenBytes int64, err error) { if ctx.Err() != nil { - return ctx.Err() + return 0, ctx.Err() } file := filepath.Join(b.rootDir, name) if err := os.MkdirAll(filepath.Dir(file), os.ModePerm); err != nil { - return err + return 0, err } f, err := os.Create(file) if err != nil { - return err + return 0, err } defer errcapture.Do(&err, f.Close, "close") - if _, err := io.Copy(f, r); err != nil { - return errors.Wrapf(err, "copy to %s", file) + written, err := io.Copy(f, r) + if err != nil { + return 0, errors.Wrapf(err, "copy to %s", file) } - return nil + return written, nil } func isDirEmpty(name string) (ok bool, err error) { diff --git a/providers/filesystem/filesystem_test.go b/providers/filesystem/filesystem_test.go index c3621fe0..d8a0a91d 100644 --- a/providers/filesystem/filesystem_test.go +++ b/providers/filesystem/filesystem_test.go @@ -23,8 +23,10 @@ func TestDelete_EmptyDirDeletionRaceCondition(t *testing.T) { testutil.Ok(t, err) // Upload 2 objects in a subfolder. - testutil.Ok(t, b.Upload(ctx, "subfolder/first", strings.NewReader("first"))) - testutil.Ok(t, b.Upload(ctx, "subfolder/second", strings.NewReader("second"))) + _, err = b.Upload(ctx, "subfolder/first", strings.NewReader("first")) + testutil.Ok(t, err) + _, err = b.Upload(ctx, "subfolder/second", strings.NewReader("second")) + testutil.Ok(t, err) // Prepare goroutines to concurrently delete the 2 objects (each one deletes a different object) start := make(chan struct{}) @@ -116,7 +118,7 @@ func TestUpload_CancelledContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - err = b.Upload(ctx, "some-file", bytes.NewReader([]byte("file content"))) + _, err = b.Upload(ctx, "some-file", bytes.NewReader([]byte("file content"))) testutil.NotOk(t, err) testutil.Equals(t, context.Canceled, err) } diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 5ea45c7e..be17b6d1 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -171,13 +171,14 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { } // Upload writes the file specified in src to remote GCS location specified as target. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { w := b.bkt.Object(name).NewWriter(ctx) - if _, err := io.Copy(w, r); err != nil { - return err + written, err := io.Copy(w, r) + if err != nil { + return 0, err } - return w.Close() + return written, w.Close() } // Delete removes the object with the given name. diff --git a/providers/obs/obs.go b/providers/obs/obs.go index 35a04c80..1d95f7b2 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -128,26 +128,26 @@ func (b *Bucket) Delete(ctx context.Context, name string) error { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { size, err := objstore.TryToGetSize(r) if err != nil { - return errors.Wrapf(err, "failed to get size apriori to upload %s", name) + return 0, errors.Wrapf(err, "failed to get size apriori to upload %s", name) } if size <= 0 { - return errors.New("object size must be provided") + return 0, errors.New("object size must be provided") } if size <= MinMultipartUploadSize { err = b.putObjectSingle(name, r) if err != nil { - return err + return 0, err } } else { var initOutput *obs.InitiateMultipartUploadOutput initOutput, err = b.initiateMultipartUpload(name) if err != nil { - return err + return 0, err } uploadId := initOutput.UploadId @@ -165,7 +165,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { }() parts, err := b.multipartUpload(size, name, uploadId, r) if err != nil { - return err + return 0, err } _, err = b.client.CompleteMultipartUpload(&obs.CompleteMultipartUploadInput{ @@ -175,10 +175,10 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { Parts: parts, }) if err != nil { - return errors.Wrap(err, "failed to complete multipart upload") + return 0, errors.Wrap(err, "failed to complete multipart upload") } } - return nil + return size, nil } func (b *Bucket) putObjectSingle(key string, body io.Reader) error { diff --git a/providers/oci/oci.go b/providers/oci/oci.go index 2db35461..f57f5fe4 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -168,7 +168,7 @@ func (b *Bucket) GetRange(ctx context.Context, name string, offset, length int64 // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (err error) { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (written int64, err error) { req := transfer.UploadStreamRequest{ UploadRequest: transfer.UploadRequest{ NamespaceName: common.String(b.namespace), @@ -185,9 +185,9 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (err erro } uploadManager := transfer.NewUploadManager() - _, err = uploadManager.UploadStream(ctx, req) + response, err := uploadManager.UploadStream(ctx, req) - return err + return response.PutObjectResponse.RawResponse.ContentLength, err } // Exists checks if the given object exists in the bucket. diff --git a/providers/oss/oss.go b/providers/oss/oss.go index e7e3a648..b259f576 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -68,11 +68,11 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { +func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) (int64, error) { // TODO(https://github.com/thanos-io/thanos/issues/678): Remove guessing length when minio provider will support multipart upload without this. size, err := objstore.TryToGetSize(r) if err != nil { - return errors.Wrapf(err, "failed to get size apriori to upload %s", name) + return 0, errors.Wrapf(err, "failed to get size apriori to upload %s", name) } chunksnum, lastslice := int(math.Floor(float64(size)/PartSize)), size%PartSize @@ -81,13 +81,13 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { switch chunksnum { case 0: if err := b.bucket.PutObject(name, ncloser); err != nil { - return errors.Wrap(err, "failed to upload oss object") + return 0, errors.Wrap(err, "failed to upload oss object") } default: { init, err := b.bucket.InitiateMultipartUpload(name) if err != nil { - return errors.Wrap(err, "failed to initiate multi-part upload") + return 0, errors.Wrap(err, "failed to initiate multi-part upload") } chunk := 0 uploadEveryPart := func(everypartsize int64, cnk int) (alioss.UploadPart, error) { @@ -105,23 +105,23 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { for ; chunk < chunksnum; chunk++ { part, err := uploadEveryPart(PartSize, chunk+1) if err != nil { - return errors.Wrap(err, "failed to upload every part") + return 0, errors.Wrap(err, "failed to upload every part") } parts = append(parts, part) } if lastslice != 0 { part, err := uploadEveryPart(lastslice, chunksnum+1) if err != nil { - return errors.Wrap(err, "failed to upload the last chunk") + return 0, errors.Wrap(err, "failed to upload the last chunk") } parts = append(parts, part) } if _, err := b.bucket.CompleteMultipartUpload(init, parts); err != nil { - return errors.Wrap(err, "failed to set multi-part upload completive") + return 0, errors.Wrap(err, "failed to set multi-part upload completive") } } } - return nil + return size, nil } // Delete removes the object with the given name. diff --git a/providers/s3/s3.go b/providers/s3/s3.go index f92d3973..3ee29d24 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -475,10 +475,10 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { sse, err := b.getServerSideEncryption(ctx) if err != nil { - return err + return 0, err } // TODO(https://github.com/thanos-io/thanos/issues/678): Remove guessing length when minio provider will support multipart upload without this. @@ -509,10 +509,10 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { NumThreads: 4, }, ); err != nil { - return errors.Wrap(err, "upload s3 object") + return 0, errors.Wrap(err, "upload s3 object") } - return nil + return size, nil } // Attributes returns information about the specified object. diff --git a/providers/s3/s3_e2e_test.go b/providers/s3/s3_e2e_test.go index 4b75a014..2bec75d9 100644 --- a/providers/s3/s3_e2e_test.go +++ b/providers/s3/s3_e2e_test.go @@ -50,6 +50,7 @@ func BenchmarkUpload(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - testutil.Ok(b, bkt.Upload(ctx, "test", strings.NewReader(str))) + _, err = bkt.Upload(ctx, "test", strings.NewReader(str)) + testutil.Ok(b, err) } } diff --git a/providers/swift/swift.go b/providers/swift/swift.go index 9bfa3cf8..f45376ea 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -296,7 +296,7 @@ func (c *Container) IsAccessDeniedErr(err error) bool { } // Upload writes the contents of the reader as an object into the container. -func (c *Container) Upload(_ context.Context, name string, r io.Reader) (err error) { +func (c *Container) Upload(_ context.Context, name string, r io.Reader) (written int64, err error) { size, err := objstore.TryToGetSize(r) if err != nil { level.Warn(c.logger).Log("msg", "could not guess file size, using large object to avoid issues if the file is larger than limit", "name", name, "err", err) @@ -314,23 +314,24 @@ func (c *Container) Upload(_ context.Context, name string, r io.Reader) (err err } if c.useDynamicLargeObjects { if file, err = c.connection.DynamicLargeObjectCreateFile(&opts); err != nil { - return errors.Wrap(err, "create DLO file") + return 0, errors.Wrap(err, "create DLO file") } } else { if file, err = c.connection.StaticLargeObjectCreateFile(&opts); err != nil { - return errors.Wrap(err, "create SLO file") + return 0, errors.Wrap(err, "create SLO file") } } } else { if file, err = c.connection.ObjectCreate(c.name, name, true, "", "", swift.Headers{}); err != nil { - return errors.Wrap(err, "create file") + return 0, errors.Wrap(err, "create file") } } defer errcapture.Do(&err, file.Close, "upload object close") - if _, err := io.Copy(file, r); err != nil { - return errors.Wrap(err, "uploading object") + written, err = io.Copy(file, r) + if err != nil { + return 0, errors.Wrap(err, "uploading object") } - return nil + return written, nil } // Delete removes the object with the given name. diff --git a/testing.go b/testing.go index b8e3744c..b8e9bd35 100644 --- a/testing.go +++ b/testing.go @@ -100,7 +100,8 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { testutil.Assert(t, bkt.IsObjNotFoundErr(err), "expected not found error but got %s", err) // Upload first object. - testutil.Ok(t, bkt.Upload(ctx, "id1/obj_1.some", strings.NewReader("@test-data@"))) + _, err = bkt.Upload(ctx, "id1/obj_1.some", strings.NewReader("@test-data@")) + testutil.Ok(t, err) // Double check we can immediately read it. rc1, err := bkt.Get(ctx, "id1/obj_1.some") @@ -150,14 +151,21 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { testutil.Assert(t, ok, "expected exits") // Upload other objects. - testutil.Ok(t, bkt.Upload(ctx, "id1/obj_2.some", strings.NewReader("@test-data2@"))) + _, err = bkt.Upload(ctx, "id1/obj_2.some", strings.NewReader("@test-data2@")) + testutil.Ok(t, err) // Upload should be idempotent. - testutil.Ok(t, bkt.Upload(ctx, "id1/obj_2.some", strings.NewReader("@test-data2@"))) - testutil.Ok(t, bkt.Upload(ctx, "id1/obj_3.some", strings.NewReader("@test-data3@"))) - testutil.Ok(t, bkt.Upload(ctx, "id1/sub/subobj_1.some", strings.NewReader("@test-data4@"))) - testutil.Ok(t, bkt.Upload(ctx, "id1/sub/subobj_2.some", strings.NewReader("@test-data5@"))) - testutil.Ok(t, bkt.Upload(ctx, "id2/obj_4.some", strings.NewReader("@test-data6@"))) - testutil.Ok(t, bkt.Upload(ctx, "obj_5.some", strings.NewReader("@test-data7@"))) + _, err = bkt.Upload(ctx, "id1/obj_2.some", strings.NewReader("@test-data2@")) + testutil.Ok(t, err) + _, err = bkt.Upload(ctx, "id1/obj_3.some", strings.NewReader("@test-data3@")) + testutil.Ok(t, err) + _, err = bkt.Upload(ctx, "id1/sub/subobj_1.some", strings.NewReader("@test-data4@")) + testutil.Ok(t, err) + _, err = bkt.Upload(ctx, "id1/sub/subobj_2.some", strings.NewReader("@test-data5@")) + testutil.Ok(t, err) + _, err = bkt.Upload(ctx, "id2/obj_4.some", strings.NewReader("@test-data6@")) + testutil.Ok(t, err) + _, err = bkt.Upload(ctx, "obj_5.some", strings.NewReader("@test-data7@")) + testutil.Ok(t, err) // Can we iter over items from top dir? var seen []string @@ -247,7 +255,8 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { sort.Strings(seen) testutil.Equals(t, expected, seen) - testutil.Ok(t, bkt.Upload(ctx, "obj_6.som", bytes.NewReader(make([]byte, 1024*1024*200)))) + _, err = bkt.Upload(ctx, "obj_6.som", bytes.NewReader(make([]byte, 1024*1024*200))) + testutil.Ok(t, err) testutil.Ok(t, bkt.Delete(ctx, "obj_6.som")) } @@ -285,7 +294,7 @@ func (d *delayingBucket) Exists(ctx context.Context, name string) (bool, error) return d.bkt.Exists(ctx, name) } -func (d *delayingBucket) Upload(ctx context.Context, name string, r io.Reader) error { +func (d *delayingBucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { time.Sleep(d.delay) return d.bkt.Upload(ctx, name, r) } diff --git a/tracing/opentelemetry/opentelemetry.go b/tracing/opentelemetry/opentelemetry.go index f65b0f0e..1bedb348 100644 --- a/tracing/opentelemetry/opentelemetry.go +++ b/tracing/opentelemetry/opentelemetry.go @@ -90,7 +90,7 @@ func (t TracingBucket) Attributes(ctx context.Context, name string) (_ objstore. return t.bkt.Attributes(ctx, name) } -func (t TracingBucket) Upload(ctx context.Context, name string, r io.Reader) (err error) { +func (t TracingBucket) Upload(ctx context.Context, name string, r io.Reader) (writtenBytes int64, err error) { ctx, span := t.tracer.Start(ctx, "bucket_upload") defer span.End() span.SetAttributes(attribute.String("name", name)) diff --git a/tracing/opentracing/opentracing.go b/tracing/opentracing/opentracing.go index 8b99e304..543fbfdd 100644 --- a/tracing/opentracing/opentracing.go +++ b/tracing/opentracing/opentracing.go @@ -96,10 +96,10 @@ func (t TracingBucket) Attributes(ctx context.Context, name string) (attrs objst return } -func (t TracingBucket) Upload(ctx context.Context, name string, r io.Reader) (err error) { +func (t TracingBucket) Upload(ctx context.Context, name string, r io.Reader) (writtenBytes int64, err error) { doWithSpan(ctx, "bucket_upload", func(spanCtx context.Context, span opentracing.Span) { span.LogKV("name", name) - err = t.bkt.Upload(spanCtx, name, r) + writtenBytes, err = t.bkt.Upload(spanCtx, name, r) }) return } From 312ee41de61c7e3f2b54eeb7b35b63aee7c96be5 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 25 Jul 2023 14:24:02 +0100 Subject: [PATCH 07/38] [FIX] Azure upload logic Signed-off-by: rita.canavarro --- providers/azure/azure.go | 12 +++++++++--- providers/s3/s3.go | 7 ++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/providers/azure/azure.go b/providers/azure/azure.go index a72f3052..42e0e7f6 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -307,17 +307,23 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { level.Debug(b.logger).Log("msg", "uploading blob", "blob", name) + size, err := objstore.TryToGetSize(r) + if err != nil { + return 0, errors.Wrapf(err, "failed to get size apriori to upload %s", name) + } + blobClient := b.containerClient.NewBlockBlobClient(name) opts := &blockblob.UploadStreamOptions{ BlockSize: 3 * 1024 * 1024, Concurrency: 4, } + if _, err := blobClient.UploadStream(ctx, r, opts); err != nil { - return errors.Wrapf(err, "cannot upload Azure blob, address: %s", name) + return 0, errors.Wrapf(err, "cannot upload Azure blob, address: %s", name) } - return nil + return size, nil } // Delete removes the object with the given name. diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 3ee29d24..86b1feaf 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -492,7 +492,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e if size < int64(partSize) { partSize = 0 } - if _, err := b.client.PutObject( + resp, err := b.client.PutObject( ctx, b.name, name, @@ -508,11 +508,12 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e // TODO(bwplotka): Consider adjusting this number to GOMAXPROCS or to expose this in config if it becomes bottleneck. NumThreads: 4, }, - ); err != nil { + ) + if err != nil { return 0, errors.Wrap(err, "upload s3 object") } - return size, nil + return resp.Size, nil } // Attributes returns information about the specified object. From ea0fd63a1f69338265b1f82c4ff2c2c2d115ba3d Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Fri, 28 Jul 2023 09:55:55 +0100 Subject: [PATCH 08/38] [FIX] Function declaration Signed-off-by: rita.canavarro --- objstore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/objstore.go b/objstore.go index c9ec22d2..7849ea8a 100644 --- a/objstore.go +++ b/objstore.go @@ -666,7 +666,7 @@ type timingReadCloser struct { writtenBytes *prometheus.CounterVec } -func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, writtenBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) *timingReadCloser { +func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec, writtenBytes *prometheus.CounterVec) *timingReadCloser { // Initialize the metrics with 0. dur.WithLabelValues(op) failed.WithLabelValues(op) From bc60f0b3c497632d4f6c3bd2aa5b926f39fe504e Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Fri, 28 Jul 2023 10:34:25 +0100 Subject: [PATCH 09/38] [FEAT] Update README Signed-off-by: rita.canavarro --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2a155271..f0e71bd9 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ type Bucket interface { // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. - Upload(ctx context.Context, name string, r io.Reader) error + Upload(ctx context.Context, name string, r io.Reader) (int64, error) // Delete removes the object with the given name. // If object does not exist in the moment of deletion, Delete should throw error. From 3f65cd73cffdec56bb6f8980d20ce68ef1a0f790 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 31 Jul 2023 16:33:33 +0100 Subject: [PATCH 10/38] [FEAT] Clean up Signed-off-by: rita.canavarro --- README.md | 2 +- go.mod | 3 ++- go.sum | 12 +++++++++++ inmem.go | 6 +++--- objstore.go | 12 +++++------ objstore_test.go | 18 ++++++---------- prefixed_bucket.go | 2 +- prefixed_bucket_test.go | 9 +++----- providers/azure/azure.go | 10 ++++----- providers/bos/bos.go | 20 +++++++++--------- providers/cos/cos.go | 18 ++++++++-------- providers/filesystem/filesystem.go | 14 ++++++------- providers/gcs/gcs.go | 9 ++++---- providers/obs/obs.go | 16 +++++++------- providers/oci/oci.go | 6 +++--- providers/oss/oss.go | 16 +++++++------- providers/s3/s3.go | 10 ++++----- providers/s3/s3_e2e_test.go | 3 +-- providers/swift/swift.go | 14 ++++++------- testing.go | 29 +++++++++----------------- tracing/opentelemetry/opentelemetry.go | 2 +- tracing/opentracing/opentracing.go | 4 ++-- 22 files changed, 114 insertions(+), 121 deletions(-) diff --git a/README.md b/README.md index f0e71bd9..2a155271 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ type Bucket interface { // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. - Upload(ctx context.Context, name string, r io.Reader) (int64, error) + Upload(ctx context.Context, name string, r io.Reader) error // Delete removes the object with the given name. // If object does not exist in the moment of deletion, Delete should throw error. diff --git a/go.mod b/go.mod index dee14caa..81d39061 100644 --- a/go.mod +++ b/go.mod @@ -62,6 +62,7 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/google/s2a-go v0.1.4 // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect github.com/googleapis/gax-go/v2 v2.7.0 // indirect @@ -99,7 +100,7 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 github.com/kr/text v0.2.0 // indirect diff --git a/go.sum b/go.sum index 860aca72..a6266e2c 100644 --- a/go.sum +++ b/go.sum @@ -44,10 +44,13 @@ cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 h1:Oj853U9kG+RLTCQXpjvOnrv0WaZHxgmZz1TlLywgOPY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= @@ -142,6 +145,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -206,6 +210,8 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -270,6 +276,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -365,6 +372,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -427,6 +436,7 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= @@ -493,6 +503,7 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= @@ -506,6 +517,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/inmem.go b/inmem.go index 45d13ef3..84e27780 100644 --- a/inmem.go +++ b/inmem.go @@ -175,12 +175,12 @@ func (b *InMemBucket) Attributes(_ context.Context, name string) (ObjectAttribut } // Upload writes the file specified in src to into the memory. -func (b *InMemBucket) Upload(_ context.Context, name string, r io.Reader) (int64, error) { +func (b *InMemBucket) Upload(_ context.Context, name string, r io.Reader) error { b.mtx.Lock() defer b.mtx.Unlock() body, err := io.ReadAll(r) if err != nil { - return 0, err + return err } b.objects[name] = body size := int64(len(body)) @@ -188,7 +188,7 @@ func (b *InMemBucket) Upload(_ context.Context, name string, r io.Reader) (int64 Size: size, LastModified: time.Now(), } - return size, nil + return nil } // Delete removes all data prefixed with the dir. diff --git a/objstore.go b/objstore.go index 7849ea8a..14ba8c47 100644 --- a/objstore.go +++ b/objstore.go @@ -42,7 +42,7 @@ type Bucket interface { // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. - Upload(ctx context.Context, name string, r io.Reader) (int64, error) + Upload(ctx context.Context, name string, r io.Reader) error // Delete removes the object with the given name. // If object does not exist in the moment of deletion, Delete should throw error. @@ -285,7 +285,7 @@ func UploadFile(ctx context.Context, logger log.Logger, bkt Bucket, src, dst str } defer logerrcapture.Do(logger, r.Close, "close file %s", src) - if _, err := bkt.Upload(ctx, dst, r); err != nil { + if err := bkt.Upload(ctx, dst, r); err != nil { return errors.Wrapf(err, "upload file %s as %s", src, dst) } level.Debug(logger).Log("msg", "uploaded file", "from", src, "dst", dst, "bucket", bkt.Name()) @@ -599,21 +599,21 @@ func (b *metricBucket) Exists(ctx context.Context, name string) (bool, error) { return ok, nil } -func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { +func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) error { const op = OpUpload b.ops.WithLabelValues(op).Inc() start := time.Now() - writtenBytes, err := b.bkt.Upload(ctx, name, r) + err := b.bkt.Upload(ctx, name, r) if err != nil { if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.opsFailures.WithLabelValues(op).Inc() } - return 0, err + return err } b.lastSuccessfulUploadTime.SetToCurrentTime() b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) - return writtenBytes, nil + return nil } func (b *metricBucket) Delete(ctx context.Context, name string) error { diff --git a/objstore_test.go b/objstore_test.go index 26b86ab4..370d53fc 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -85,12 +85,9 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), r, "") tempDir := t.TempDir() - _, err := m.Upload(context.Background(), "dir/obj1", bytes.NewReader([]byte("1"))) - testutil.Ok(t, err) - _, err = m.Upload(context.Background(), "dir/obj2", bytes.NewReader([]byte("2"))) - testutil.Ok(t, err) - _, err = m.Upload(context.Background(), "dir/obj3", bytes.NewReader(bytes.Repeat([]byte("3"), 1024*1024))) - testutil.Ok(t, err) + testutil.Ok(t, m.Upload(context.Background(), "dir/obj1", bytes.NewReader([]byte("1")))) + testutil.Ok(t, m.Upload(context.Background(), "dir/obj2", bytes.NewReader([]byte("2")))) + testutil.Ok(t, m.Upload(context.Background(), "dir/obj3", bytes.NewReader(bytes.Repeat([]byte("3"), 1024*1024)))) testutil.Ok(t, promtest.GatherAndCompare(r, strings.NewReader(` # HELP objstore_bucket_operations_total Total number of all attempted operations against a bucket. @@ -224,12 +221,9 @@ func TestDownloadDir_CleanUp(t *testing.T) { } tempDir := t.TempDir() - _, err := b.Upload(context.Background(), "dir/obj1", bytes.NewReader([]byte("1"))) - testutil.Ok(t, err) - _, err = b.Upload(context.Background(), "dir/obj2", bytes.NewReader([]byte("2"))) - testutil.Ok(t, err) - _, err = b.Upload(context.Background(), "dir/obj3", bytes.NewReader([]byte("3"))) - testutil.Ok(t, err) + testutil.Ok(t, b.Upload(context.Background(), "dir/obj1", bytes.NewReader([]byte("1")))) + testutil.Ok(t, b.Upload(context.Background(), "dir/obj2", bytes.NewReader([]byte("2")))) + testutil.Ok(t, b.Upload(context.Background(), "dir/obj3", bytes.NewReader([]byte("3")))) // We exapect the third Get to fail testutil.NotOk(t, DownloadDir(context.Background(), log.NewNopLogger(), b, "dir/", "dir/", tempDir)) diff --git a/prefixed_bucket.go b/prefixed_bucket.go index cbc5b491..f2b71434 100644 --- a/prefixed_bucket.go +++ b/prefixed_bucket.go @@ -86,7 +86,7 @@ func (p PrefixedBucket) Attributes(ctx context.Context, name string) (ObjectAttr // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. -func (p *PrefixedBucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { +func (p *PrefixedBucket) Upload(ctx context.Context, name string, r io.Reader) error { return p.bkt.Upload(ctx, conditionalPrefix(p.prefix, name), r) } diff --git a/prefixed_bucket_test.go b/prefixed_bucket_test.go index e7f2d9bc..f93c8580 100644 --- a/prefixed_bucket_test.go +++ b/prefixed_bucket_test.go @@ -29,8 +29,7 @@ func TestPrefixedBucket_Acceptance(t *testing.T) { } func UsesPrefixTest(t *testing.T, bkt Bucket, prefix string) { - _, err := bkt.Upload(context.Background(), strings.Trim(prefix, "/")+"/file1.jpg", strings.NewReader("test-data1")) - testutil.Ok(t, err) + testutil.Ok(t, bkt.Upload(context.Background(), strings.Trim(prefix, "/")+"/file1.jpg", strings.NewReader("test-data1"))) pBkt := NewPrefixedBucket(bkt, prefix) rc1, err := pBkt.Get(context.Background(), "file1.jpg") @@ -42,8 +41,7 @@ func UsesPrefixTest(t *testing.T, bkt Bucket, prefix string) { testutil.Ok(t, err) testutil.Equals(t, "test-data1", string(content)) - _, err = pBkt.Upload(context.Background(), "file2.jpg", strings.NewReader("test-data2")) - testutil.Ok(t, err) + testutil.Ok(t, pBkt.Upload(context.Background(), "file2.jpg", strings.NewReader("test-data2"))) rc2, err := bkt.Get(context.Background(), strings.Trim(prefix, "/")+"/file2.jpg") testutil.Ok(t, err) defer func() { testutil.Ok(t, rc2.Close()) }() @@ -71,8 +69,7 @@ func UsesPrefixTest(t *testing.T, bkt Bucket, prefix string) { testutil.Ok(t, err) testutil.Assert(t, attrs.Size == 10, "expected size to be equal to 10") - _, err = bkt.Upload(context.Background(), strings.Trim(prefix, "/")+"/dir/file1.jpg", strings.NewReader("test-data1")) - testutil.Ok(t, err) + testutil.Ok(t, bkt.Upload(context.Background(), strings.Trim(prefix, "/")+"/dir/file1.jpg", strings.NewReader("test-data1"))) seen := []string{} testutil.Ok(t, pBkt.Iter(context.Background(), "", func(fn string) error { seen = append(seen, fn) diff --git a/providers/azure/azure.go b/providers/azure/azure.go index 42e0e7f6..bbddac64 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -307,11 +307,11 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { level.Debug(b.logger).Log("msg", "uploading blob", "blob", name) - size, err := objstore.TryToGetSize(r) + _, err := objstore.TryToGetSize(r) if err != nil { - return 0, errors.Wrapf(err, "failed to get size apriori to upload %s", name) + return errors.Wrapf(err, "failed to get size apriori to upload %s", name) } blobClient := b.containerClient.NewBlockBlobClient(name) @@ -321,9 +321,9 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e } if _, err := blobClient.UploadStream(ctx, r, opts); err != nil { - return 0, errors.Wrapf(err, "cannot upload Azure blob, address: %s", name) + return errors.Wrapf(err, "cannot upload Azure blob, address: %s", name) } - return size, nil + return nil } // Delete removes the object with the given name. diff --git a/providers/bos/bos.go b/providers/bos/bos.go index f5aefd8a..72e1b1e0 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -110,29 +110,29 @@ func (b *Bucket) Delete(_ context.Context, name string) error { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) (int64, error) { +func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { size, err := objstore.TryToGetSize(r) if err != nil { - return 0, errors.Wrapf(err, "getting size of %s", name) + return errors.Wrapf(err, "getting size of %s", name) } partNums, lastSlice := int(math.Floor(float64(size)/partSize)), size%partSize if partNums == 0 { body, err := bce.NewBodyFromSizedReader(r, lastSlice) if err != nil { - return 0, errors.Wrapf(err, "failed to create SizedReader for %s", name) + return errors.Wrapf(err, "failed to create SizedReader for %s", name) } if _, err := b.client.PutObject(b.name, name, body, nil); err != nil { - return 0, errors.Wrapf(err, "failed to upload %s", name) + return errors.Wrapf(err, "failed to upload %s", name) } - return 0, nil + return nil } result, err := b.client.BasicInitiateMultipartUpload(b.name, name) if err != nil { - return 0, errors.Wrapf(err, "failed to initiate MultipartUpload for %s", name) + return errors.Wrapf(err, "failed to initiate MultipartUpload for %s", name) } uploadEveryPart := func(partSize int64, part int, uploadId string) (string, error) { @@ -156,7 +156,7 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) (int64, err for part := 1; part <= partNums; part++ { etag, err := uploadEveryPart(partSize, part, result.UploadId) if err != nil { - return 0, errors.Wrapf(err, "failed to upload part %d for %s", part, name) + return errors.Wrapf(err, "failed to upload part %d for %s", part, name) } parts = append(parts, api.UploadInfoType{PartNumber: part, ETag: etag}) } @@ -164,15 +164,15 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) (int64, err if lastSlice != 0 { etag, err := uploadEveryPart(lastSlice, partNums+1, result.UploadId) if err != nil { - return 0, errors.Wrapf(err, "failed to upload the last part for %s", name) + return errors.Wrapf(err, "failed to upload the last part for %s", name) } parts = append(parts, api.UploadInfoType{PartNumber: partNums + 1, ETag: etag}) } if _, err := b.client.CompleteMultipartUploadFromStruct(b.name, name, result.UploadId, &api.CompleteMultipartUploadArgs{Parts: parts}); err != nil { - return 0, errors.Wrapf(err, "failed to set %s upload completed", name) + return errors.Wrapf(err, "failed to set %s upload completed", name) } - return size, nil + return nil } // Iter calls f for each entry in the given directory (not recursive). The argument to f is the full diff --git a/providers/cos/cos.go b/providers/cos/cos.go index 1b99f640..e518cae2 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -197,24 +197,24 @@ func (r fixedLengthReader) Size() int64 { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { size, err := objstore.TryToGetSize(r) if err != nil { - return 0, errors.Wrapf(err, "getting size of %s", name) + return errors.Wrapf(err, "getting size of %s", name) } // partSize 128MB. const partSize = 1024 * 1024 * 128 partNums, lastSlice := int(math.Floor(float64(size)/partSize)), size%partSize if partNums == 0 { if _, err := b.client.Object.Put(ctx, name, r, nil); err != nil { - return 0, errors.Wrapf(err, "Put object: %s", name) + return errors.Wrapf(err, "Put object: %s", name) } - return 0, nil + return nil } // 1. init. result, _, err := b.client.Object.InitiateMultipartUpload(ctx, name, nil) if err != nil { - return 0, errors.Wrapf(err, "InitiateMultipartUpload %s", name) + return errors.Wrapf(err, "InitiateMultipartUpload %s", name) } uploadEveryPart := func(partSize int64, part int, uploadID string) (string, error) { r := newFixedLengthReader(r, partSize) @@ -235,7 +235,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e for part := 1; part <= partNums; part++ { etag, err := uploadEveryPart(partSize, part, result.UploadID) if err != nil { - return 0, errors.Wrapf(err, "uploadPart %d, %s", part, name) + return errors.Wrapf(err, "uploadPart %d, %s", part, name) } optcom.Parts = append(optcom.Parts, cos.Object{ PartNumber: part, ETag: etag}, @@ -246,7 +246,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e part := partNums + 1 etag, err := uploadEveryPart(lastSlice, part, result.UploadID) if err != nil { - return 0, errors.Wrapf(err, "uploadPart %d, %s", part, name) + return errors.Wrapf(err, "uploadPart %d, %s", part, name) } optcom.Parts = append(optcom.Parts, cos.Object{ PartNumber: part, ETag: etag}, @@ -254,9 +254,9 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e } // 4. complete. if _, _, err := b.client.Object.CompleteMultipartUpload(ctx, name, result.UploadID, optcom); err != nil { - return 0, errors.Wrapf(err, "CompleteMultipartUpload %s", name) + return errors.Wrapf(err, "CompleteMultipartUpload %s", name) } - return size, nil + return nil } // Delete removes the object with the given name. diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index 026f2368..5d5085e9 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -191,27 +191,27 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { } // Upload writes the file specified in src to into the memory. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (writtenBytes int64, err error) { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (err error) { if ctx.Err() != nil { - return 0, ctx.Err() + return ctx.Err() } file := filepath.Join(b.rootDir, name) if err := os.MkdirAll(filepath.Dir(file), os.ModePerm); err != nil { - return 0, err + return err } f, err := os.Create(file) if err != nil { - return 0, err + return err } defer errcapture.Do(&err, f.Close, "close") - written, err := io.Copy(f, r) + _, err = io.Copy(f, r) if err != nil { - return 0, errors.Wrapf(err, "copy to %s", file) + return errors.Wrapf(err, "copy to %s", file) } - return written, nil + return nil } func isDirEmpty(name string) (ok bool, err error) { diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index be17b6d1..df902ed4 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -171,14 +171,13 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { } // Upload writes the file specified in src to remote GCS location specified as target. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { w := b.bkt.Object(name).NewWriter(ctx) - - written, err := io.Copy(w, r) + _, err := io.Copy(w, r) if err != nil { - return 0, err + return err } - return written, w.Close() + return w.Close() } // Delete removes the object with the given name. diff --git a/providers/obs/obs.go b/providers/obs/obs.go index 1d95f7b2..35a04c80 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -128,26 +128,26 @@ func (b *Bucket) Delete(ctx context.Context, name string) error { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { size, err := objstore.TryToGetSize(r) if err != nil { - return 0, errors.Wrapf(err, "failed to get size apriori to upload %s", name) + return errors.Wrapf(err, "failed to get size apriori to upload %s", name) } if size <= 0 { - return 0, errors.New("object size must be provided") + return errors.New("object size must be provided") } if size <= MinMultipartUploadSize { err = b.putObjectSingle(name, r) if err != nil { - return 0, err + return err } } else { var initOutput *obs.InitiateMultipartUploadOutput initOutput, err = b.initiateMultipartUpload(name) if err != nil { - return 0, err + return err } uploadId := initOutput.UploadId @@ -165,7 +165,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e }() parts, err := b.multipartUpload(size, name, uploadId, r) if err != nil { - return 0, err + return err } _, err = b.client.CompleteMultipartUpload(&obs.CompleteMultipartUploadInput{ @@ -175,10 +175,10 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e Parts: parts, }) if err != nil { - return 0, errors.Wrap(err, "failed to complete multipart upload") + return errors.Wrap(err, "failed to complete multipart upload") } } - return size, nil + return nil } func (b *Bucket) putObjectSingle(key string, body io.Reader) error { diff --git a/providers/oci/oci.go b/providers/oci/oci.go index f57f5fe4..2db35461 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -168,7 +168,7 @@ func (b *Bucket) GetRange(ctx context.Context, name string, offset, length int64 // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (written int64, err error) { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (err error) { req := transfer.UploadStreamRequest{ UploadRequest: transfer.UploadRequest{ NamespaceName: common.String(b.namespace), @@ -185,9 +185,9 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (written } uploadManager := transfer.NewUploadManager() - response, err := uploadManager.UploadStream(ctx, req) + _, err = uploadManager.UploadStream(ctx, req) - return response.PutObjectResponse.RawResponse.ContentLength, err + return err } // Exists checks if the given object exists in the bucket. diff --git a/providers/oss/oss.go b/providers/oss/oss.go index b259f576..e7e3a648 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -68,11 +68,11 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) (int64, error) { +func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { // TODO(https://github.com/thanos-io/thanos/issues/678): Remove guessing length when minio provider will support multipart upload without this. size, err := objstore.TryToGetSize(r) if err != nil { - return 0, errors.Wrapf(err, "failed to get size apriori to upload %s", name) + return errors.Wrapf(err, "failed to get size apriori to upload %s", name) } chunksnum, lastslice := int(math.Floor(float64(size)/PartSize)), size%PartSize @@ -81,13 +81,13 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) (int64, err switch chunksnum { case 0: if err := b.bucket.PutObject(name, ncloser); err != nil { - return 0, errors.Wrap(err, "failed to upload oss object") + return errors.Wrap(err, "failed to upload oss object") } default: { init, err := b.bucket.InitiateMultipartUpload(name) if err != nil { - return 0, errors.Wrap(err, "failed to initiate multi-part upload") + return errors.Wrap(err, "failed to initiate multi-part upload") } chunk := 0 uploadEveryPart := func(everypartsize int64, cnk int) (alioss.UploadPart, error) { @@ -105,23 +105,23 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) (int64, err for ; chunk < chunksnum; chunk++ { part, err := uploadEveryPart(PartSize, chunk+1) if err != nil { - return 0, errors.Wrap(err, "failed to upload every part") + return errors.Wrap(err, "failed to upload every part") } parts = append(parts, part) } if lastslice != 0 { part, err := uploadEveryPart(lastslice, chunksnum+1) if err != nil { - return 0, errors.Wrap(err, "failed to upload the last chunk") + return errors.Wrap(err, "failed to upload the last chunk") } parts = append(parts, part) } if _, err := b.bucket.CompleteMultipartUpload(init, parts); err != nil { - return 0, errors.Wrap(err, "failed to set multi-part upload completive") + return errors.Wrap(err, "failed to set multi-part upload completive") } } } - return size, nil + return nil } // Delete removes the object with the given name. diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 86b1feaf..0e5e24e1 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -475,10 +475,10 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { } // Upload the contents of the reader as an object into the bucket. -func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { sse, err := b.getServerSideEncryption(ctx) if err != nil { - return 0, err + return err } // TODO(https://github.com/thanos-io/thanos/issues/678): Remove guessing length when minio provider will support multipart upload without this. @@ -492,7 +492,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e if size < int64(partSize) { partSize = 0 } - resp, err := b.client.PutObject( + _, err = b.client.PutObject( ctx, b.name, name, @@ -510,10 +510,10 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (int64, e }, ) if err != nil { - return 0, errors.Wrap(err, "upload s3 object") + return errors.Wrap(err, "upload s3 object") } - return resp.Size, nil + return nil } // Attributes returns information about the specified object. diff --git a/providers/s3/s3_e2e_test.go b/providers/s3/s3_e2e_test.go index 2bec75d9..2bb3a851 100644 --- a/providers/s3/s3_e2e_test.go +++ b/providers/s3/s3_e2e_test.go @@ -50,7 +50,6 @@ func BenchmarkUpload(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - _, err = bkt.Upload(ctx, "test", strings.NewReader(str)) - testutil.Ok(b, err) + testutil.Ok(bkt.Upload(ctx, "test", strings.NewReader(str)), err) } } diff --git a/providers/swift/swift.go b/providers/swift/swift.go index f45376ea..b5bbd903 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -296,7 +296,7 @@ func (c *Container) IsAccessDeniedErr(err error) bool { } // Upload writes the contents of the reader as an object into the container. -func (c *Container) Upload(_ context.Context, name string, r io.Reader) (written int64, err error) { +func (c *Container) Upload(_ context.Context, name string, r io.Reader) (err error) { size, err := objstore.TryToGetSize(r) if err != nil { level.Warn(c.logger).Log("msg", "could not guess file size, using large object to avoid issues if the file is larger than limit", "name", name, "err", err) @@ -314,24 +314,24 @@ func (c *Container) Upload(_ context.Context, name string, r io.Reader) (written } if c.useDynamicLargeObjects { if file, err = c.connection.DynamicLargeObjectCreateFile(&opts); err != nil { - return 0, errors.Wrap(err, "create DLO file") + return errors.Wrap(err, "create DLO file") } } else { if file, err = c.connection.StaticLargeObjectCreateFile(&opts); err != nil { - return 0, errors.Wrap(err, "create SLO file") + return errors.Wrap(err, "create SLO file") } } } else { if file, err = c.connection.ObjectCreate(c.name, name, true, "", "", swift.Headers{}); err != nil { - return 0, errors.Wrap(err, "create file") + return errors.Wrap(err, "create file") } } defer errcapture.Do(&err, file.Close, "upload object close") - written, err = io.Copy(file, r) + _, err = io.Copy(file, r) if err != nil { - return 0, errors.Wrap(err, "uploading object") + return errors.Wrap(err, "uploading object") } - return written, nil + return nil } // Delete removes the object with the given name. diff --git a/testing.go b/testing.go index b8e9bd35..b8e3744c 100644 --- a/testing.go +++ b/testing.go @@ -100,8 +100,7 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { testutil.Assert(t, bkt.IsObjNotFoundErr(err), "expected not found error but got %s", err) // Upload first object. - _, err = bkt.Upload(ctx, "id1/obj_1.some", strings.NewReader("@test-data@")) - testutil.Ok(t, err) + testutil.Ok(t, bkt.Upload(ctx, "id1/obj_1.some", strings.NewReader("@test-data@"))) // Double check we can immediately read it. rc1, err := bkt.Get(ctx, "id1/obj_1.some") @@ -151,21 +150,14 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { testutil.Assert(t, ok, "expected exits") // Upload other objects. - _, err = bkt.Upload(ctx, "id1/obj_2.some", strings.NewReader("@test-data2@")) - testutil.Ok(t, err) + testutil.Ok(t, bkt.Upload(ctx, "id1/obj_2.some", strings.NewReader("@test-data2@"))) // Upload should be idempotent. - _, err = bkt.Upload(ctx, "id1/obj_2.some", strings.NewReader("@test-data2@")) - testutil.Ok(t, err) - _, err = bkt.Upload(ctx, "id1/obj_3.some", strings.NewReader("@test-data3@")) - testutil.Ok(t, err) - _, err = bkt.Upload(ctx, "id1/sub/subobj_1.some", strings.NewReader("@test-data4@")) - testutil.Ok(t, err) - _, err = bkt.Upload(ctx, "id1/sub/subobj_2.some", strings.NewReader("@test-data5@")) - testutil.Ok(t, err) - _, err = bkt.Upload(ctx, "id2/obj_4.some", strings.NewReader("@test-data6@")) - testutil.Ok(t, err) - _, err = bkt.Upload(ctx, "obj_5.some", strings.NewReader("@test-data7@")) - testutil.Ok(t, err) + testutil.Ok(t, bkt.Upload(ctx, "id1/obj_2.some", strings.NewReader("@test-data2@"))) + testutil.Ok(t, bkt.Upload(ctx, "id1/obj_3.some", strings.NewReader("@test-data3@"))) + testutil.Ok(t, bkt.Upload(ctx, "id1/sub/subobj_1.some", strings.NewReader("@test-data4@"))) + testutil.Ok(t, bkt.Upload(ctx, "id1/sub/subobj_2.some", strings.NewReader("@test-data5@"))) + testutil.Ok(t, bkt.Upload(ctx, "id2/obj_4.some", strings.NewReader("@test-data6@"))) + testutil.Ok(t, bkt.Upload(ctx, "obj_5.some", strings.NewReader("@test-data7@"))) // Can we iter over items from top dir? var seen []string @@ -255,8 +247,7 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { sort.Strings(seen) testutil.Equals(t, expected, seen) - _, err = bkt.Upload(ctx, "obj_6.som", bytes.NewReader(make([]byte, 1024*1024*200))) - testutil.Ok(t, err) + testutil.Ok(t, bkt.Upload(ctx, "obj_6.som", bytes.NewReader(make([]byte, 1024*1024*200)))) testutil.Ok(t, bkt.Delete(ctx, "obj_6.som")) } @@ -294,7 +285,7 @@ func (d *delayingBucket) Exists(ctx context.Context, name string) (bool, error) return d.bkt.Exists(ctx, name) } -func (d *delayingBucket) Upload(ctx context.Context, name string, r io.Reader) (int64, error) { +func (d *delayingBucket) Upload(ctx context.Context, name string, r io.Reader) error { time.Sleep(d.delay) return d.bkt.Upload(ctx, name, r) } diff --git a/tracing/opentelemetry/opentelemetry.go b/tracing/opentelemetry/opentelemetry.go index 1bedb348..f65b0f0e 100644 --- a/tracing/opentelemetry/opentelemetry.go +++ b/tracing/opentelemetry/opentelemetry.go @@ -90,7 +90,7 @@ func (t TracingBucket) Attributes(ctx context.Context, name string) (_ objstore. return t.bkt.Attributes(ctx, name) } -func (t TracingBucket) Upload(ctx context.Context, name string, r io.Reader) (writtenBytes int64, err error) { +func (t TracingBucket) Upload(ctx context.Context, name string, r io.Reader) (err error) { ctx, span := t.tracer.Start(ctx, "bucket_upload") defer span.End() span.SetAttributes(attribute.String("name", name)) diff --git a/tracing/opentracing/opentracing.go b/tracing/opentracing/opentracing.go index 543fbfdd..8b99e304 100644 --- a/tracing/opentracing/opentracing.go +++ b/tracing/opentracing/opentracing.go @@ -96,10 +96,10 @@ func (t TracingBucket) Attributes(ctx context.Context, name string) (attrs objst return } -func (t TracingBucket) Upload(ctx context.Context, name string, r io.Reader) (writtenBytes int64, err error) { +func (t TracingBucket) Upload(ctx context.Context, name string, r io.Reader) (err error) { doWithSpan(ctx, "bucket_upload", func(spanCtx context.Context, span opentracing.Span) { span.LogKV("name", name) - writtenBytes, err = t.bkt.Upload(spanCtx, name, r) + err = t.bkt.Upload(spanCtx, name, r) }) return } From 361e530e86bfe85f16a73557eec580ab0f4cab17 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 31 Jul 2023 16:36:30 +0100 Subject: [PATCH 11/38] [FEAT] Clean up Signed-off-by: rita.canavarro --- go.mod | 10 +++++----- go.sum | 12 ++++++++++++ objstore_test.go | 2 +- providers/s3/s3_e2e_test.go | 2 +- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 81d39061..ff862035 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,8 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/oracle/oci-go-sdk/v65 v65.41.1 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.12.2 - github.com/prometheus/common v0.36.0 + github.com/prometheus/client_golang v1.16.0 + github.com/prometheus/common v0.44.0 github.com/tencentyun/cos-go-sdk-v5 v0.7.40 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 @@ -71,7 +71,7 @@ require ( github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect @@ -80,8 +80,8 @@ require ( github.com/mozillazg/go-httpheader v0.2.1 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sony/gobreaker v0.5.0 // indirect diff --git a/go.sum b/go.sum index a6266e2c..b453b431 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,7 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfR github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 h1:Oj853U9kG+RLTCQXpjvOnrv0WaZHxgmZz1TlLywgOPY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= @@ -261,6 +262,8 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.61 h1:87c+x8J3jxQ5VUGimV9oHdpjsAvy3fhneEBKuoKEVUI= @@ -303,17 +306,26 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.36.0 h1:78hJTing+BLYLjhXE+Z2BubeEymH5Lr0/Mt8FKkxxYo= github.com/prometheus/common v0.36.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= diff --git a/objstore_test.go b/objstore_test.go index 370d53fc..934dbb50 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -227,7 +227,7 @@ func TestDownloadDir_CleanUp(t *testing.T) { // We exapect the third Get to fail testutil.NotOk(t, DownloadDir(context.Background(), log.NewNopLogger(), b, "dir/", "dir/", tempDir)) - _, err = os.Stat(tempDir) + _, err := os.Stat(tempDir) testutil.Assert(t, os.IsNotExist(err)) } diff --git a/providers/s3/s3_e2e_test.go b/providers/s3/s3_e2e_test.go index 2bb3a851..4b75a014 100644 --- a/providers/s3/s3_e2e_test.go +++ b/providers/s3/s3_e2e_test.go @@ -50,6 +50,6 @@ func BenchmarkUpload(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - testutil.Ok(bkt.Upload(ctx, "test", strings.NewReader(str)), err) + testutil.Ok(b, bkt.Upload(ctx, "test", strings.NewReader(str))) } } From 8b22c6fd29c6c756c3994e33fa469e1ea0364f2f Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 31 Jul 2023 16:42:16 +0100 Subject: [PATCH 12/38] [FEAT] Clean up Signed-off-by: rita.canavarro --- inmem.go | 3 +-- providers/azure/azure.go | 5 ----- providers/filesystem/filesystem.go | 3 +-- providers/filesystem/filesystem_test.go | 9 +++------ providers/gcs/gcs.go | 3 +-- providers/s3/s3.go | 5 ++--- providers/swift/swift.go | 3 +-- 7 files changed, 9 insertions(+), 22 deletions(-) diff --git a/inmem.go b/inmem.go index 84e27780..3f6f35e9 100644 --- a/inmem.go +++ b/inmem.go @@ -183,9 +183,8 @@ func (b *InMemBucket) Upload(_ context.Context, name string, r io.Reader) error return err } b.objects[name] = body - size := int64(len(body)) b.attrs[name] = ObjectAttributes{ - Size: size, + Size: int64(len(body)), LastModified: time.Now(), } return nil diff --git a/providers/azure/azure.go b/providers/azure/azure.go index bbddac64..f28ab025 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -309,11 +309,6 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { // Upload the contents of the reader as an object into the bucket. func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { level.Debug(b.logger).Log("msg", "uploading blob", "blob", name) - _, err := objstore.TryToGetSize(r) - if err != nil { - return errors.Wrapf(err, "failed to get size apriori to upload %s", name) - } - blobClient := b.containerClient.NewBlockBlobClient(name) opts := &blockblob.UploadStreamOptions{ BlockSize: 3 * 1024 * 1024, diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index 5d5085e9..cb0a094e 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -207,8 +207,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (err erro } defer errcapture.Do(&err, f.Close, "close") - _, err = io.Copy(f, r) - if err != nil { + if _, err = io.Copy(f, r); err != nil { return errors.Wrapf(err, "copy to %s", file) } return nil diff --git a/providers/filesystem/filesystem_test.go b/providers/filesystem/filesystem_test.go index d8a0a91d..fb9a7d3d 100644 --- a/providers/filesystem/filesystem_test.go +++ b/providers/filesystem/filesystem_test.go @@ -23,10 +23,8 @@ func TestDelete_EmptyDirDeletionRaceCondition(t *testing.T) { testutil.Ok(t, err) // Upload 2 objects in a subfolder. - _, err = b.Upload(ctx, "subfolder/first", strings.NewReader("first")) - testutil.Ok(t, err) - _, err = b.Upload(ctx, "subfolder/second", strings.NewReader("second")) - testutil.Ok(t, err) + testutil.Ok(t, b.Upload(ctx, "subfolder/first", strings.NewReader("first"))) + testutil.Ok(t, b.Upload(ctx, "subfolder/second", strings.NewReader("second"))) // Prepare goroutines to concurrently delete the 2 objects (each one deletes a different object) start := make(chan struct{}) @@ -118,8 +116,7 @@ func TestUpload_CancelledContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err = b.Upload(ctx, "some-file", bytes.NewReader([]byte("file content"))) - testutil.NotOk(t, err) + testutil.NotOk(t, b.Upload(ctx, "some-file", bytes.NewReader([]byte("file content")))) testutil.Equals(t, context.Canceled, err) } diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index df902ed4..bee75fa3 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -173,8 +173,7 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { // Upload writes the file specified in src to remote GCS location specified as target. func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { w := b.bkt.Object(name).NewWriter(ctx) - _, err := io.Copy(w, r) - if err != nil { + if _, err := io.Copy(w, r); err != nil { return err } return w.Close() diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 0e5e24e1..f92d3973 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -492,7 +492,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { if size < int64(partSize) { partSize = 0 } - _, err = b.client.PutObject( + if _, err := b.client.PutObject( ctx, b.name, name, @@ -508,8 +508,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { // TODO(bwplotka): Consider adjusting this number to GOMAXPROCS or to expose this in config if it becomes bottleneck. NumThreads: 4, }, - ) - if err != nil { + ); err != nil { return errors.Wrap(err, "upload s3 object") } diff --git a/providers/swift/swift.go b/providers/swift/swift.go index b5bbd903..ad7a90fa 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -327,8 +327,7 @@ func (c *Container) Upload(_ context.Context, name string, r io.Reader) (err err } } defer errcapture.Do(&err, file.Close, "upload object close") - _, err = io.Copy(file, r) - if err != nil { + if _, err = io.Copy(file, r); err != nil { return errors.Wrap(err, "uploading object") } return nil From 18e198e8eb88be015e2b53a3ff10ab76ec3cb224 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 31 Jul 2023 17:43:32 +0100 Subject: [PATCH 13/38] [FIX] Filesystem test Signed-off-by: rita.canavarro --- providers/filesystem/filesystem_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/providers/filesystem/filesystem_test.go b/providers/filesystem/filesystem_test.go index fb9a7d3d..c3621fe0 100644 --- a/providers/filesystem/filesystem_test.go +++ b/providers/filesystem/filesystem_test.go @@ -116,7 +116,8 @@ func TestUpload_CancelledContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - testutil.NotOk(t, b.Upload(ctx, "some-file", bytes.NewReader([]byte("file content")))) + err = b.Upload(ctx, "some-file", bytes.NewReader([]byte("file content"))) + testutil.NotOk(t, err) testutil.Equals(t, context.Canceled, err) } From 66868e8746f23c13119890b9a7b7ee42970a01f2 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 1 Aug 2023 09:10:44 +0100 Subject: [PATCH 14/38] [FIX] Nit pick Signed-off-by: rita.canavarro --- providers/filesystem/filesystem.go | 2 +- providers/gcs/gcs.go | 1 + providers/swift/swift.go | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index cb0a094e..21c70485 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -207,7 +207,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) (err erro } defer errcapture.Do(&err, f.Close, "close") - if _, err = io.Copy(f, r); err != nil { + if _, err := io.Copy(f, r); err != nil { return errors.Wrapf(err, "copy to %s", file) } return nil diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index bee75fa3..5ea45c7e 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -173,6 +173,7 @@ func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { // Upload writes the file specified in src to remote GCS location specified as target. func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { w := b.bkt.Object(name).NewWriter(ctx) + if _, err := io.Copy(w, r); err != nil { return err } diff --git a/providers/swift/swift.go b/providers/swift/swift.go index ad7a90fa..9bfa3cf8 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -327,7 +327,7 @@ func (c *Container) Upload(_ context.Context, name string, r io.Reader) (err err } } defer errcapture.Do(&err, file.Close, "upload object close") - if _, err = io.Copy(file, r); err != nil { + if _, err := io.Copy(file, r); err != nil { return errors.Wrap(err, "uploading object") } return nil From 200a5149a2c7817334252b804f0be73e152de6b6 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 1 Aug 2023 09:56:43 +0100 Subject: [PATCH 15/38] [FEAT] Upload written bytes histogram Signed-off-by: rita.canavarro --- objstore.go | 15 ++++++--------- objstore_test.go | 2 +- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/objstore.go b/objstore.go index 14ba8c47..aae04e6e 100644 --- a/objstore.go +++ b/objstore.go @@ -444,10 +444,11 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu ConstLabels: prometheus.Labels{"bucket": name}, }), - opsWrittenBytes: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ - Name: "objstore_bucket_operation_written_bytes_total", - Help: "Total number of bytes uploaded from TSDB block, per operation.", + opsWrittenBytes: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ + Name: "objstore_bucket_operation_written_bytes", + Help: "Number of bytes uploaded from TSDB block, per operation.", ConstLabels: prometheus.Labels{"bucket": name}, + Buckets: prometheus.ExponentialBuckets(50<<100, 2, 10), }, []string{"operation"}), } for _, op := range []string{ @@ -485,7 +486,7 @@ type metricBucket struct { opsFetchedBytes *prometheus.CounterVec opsTransferredBytes *prometheus.HistogramVec - opsWrittenBytes *prometheus.CounterVec + opsWrittenBytes *prometheus.HistogramVec opsDuration *prometheus.HistogramVec lastSuccessfulUploadTime prometheus.Gauge } @@ -556,7 +557,6 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err b.isOpFailureExpected, b.opsFetchedBytes, b.opsTransferredBytes, - b.opsWrittenBytes, ), nil } @@ -579,7 +579,6 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in b.isOpFailureExpected, b.opsFetchedBytes, b.opsTransferredBytes, - b.opsWrittenBytes, ), nil } @@ -663,10 +662,9 @@ type timingReadCloser struct { isFailureExpected IsOpFailureExpectedFunc fetchedBytes *prometheus.CounterVec transferredBytes *prometheus.HistogramVec - writtenBytes *prometheus.CounterVec } -func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec, writtenBytes *prometheus.CounterVec) *timingReadCloser { +func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) *timingReadCloser { // Initialize the metrics with 0. dur.WithLabelValues(op) failed.WithLabelValues(op) @@ -683,7 +681,6 @@ func newTimingReadCloser(rc io.ReadCloser, op string, dur *prometheus.HistogramV fetchedBytes: fetchedBytes, transferredBytes: transferredBytes, readBytes: 0, - writtenBytes: writtenBytes, } } diff --git a/objstore_test.go b/objstore_test.go index 934dbb50..4b105281 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -194,7 +194,7 @@ func TestTimingTracingReader(t *testing.T) { tr := NopCloserWithSize(r) tr = newTimingReadCloser(tr, "", m.opsDuration, m.opsFailures, func(err error) bool { return false - }, m.opsFetchedBytes, m.opsTransferredBytes, m.opsWrittenBytes) + }, m.opsFetchedBytes, m.opsTransferredBytes) size, err := TryToGetSize(tr) From 7fb13ca24614f997f7d41ed775a7477f21cccee1 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 1 Aug 2023 11:40:25 +0100 Subject: [PATCH 16/38] [FEAT] Remove written bytes histogram Signed-off-by: rita.canavarro --- objstore.go | 1 - 1 file changed, 1 deletion(-) diff --git a/objstore.go b/objstore.go index aae04e6e..7eadcc39 100644 --- a/objstore.go +++ b/objstore.go @@ -464,7 +464,6 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu bkt.opsFailures.WithLabelValues(op) bkt.opsDuration.WithLabelValues(op) bkt.opsFetchedBytes.WithLabelValues(op) - bkt.opsWrittenBytes.WithLabelValues(op) } // fetched bytes only relevant for get and getrange for _, op := range []string{ From 0548f4f14ab54eb4117f6f248443c8ecf5572952 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 7 Aug 2023 10:33:18 +0100 Subject: [PATCH 17/38] [FEAT] Read transferred bytes per upload Signed-off-by: rita.canavarro --- objstore.go | 21 +++++++++++++++++++-- objstore_test.go | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/objstore.go b/objstore.go index 7eadcc39..19e40ea2 100644 --- a/objstore.go +++ b/objstore.go @@ -465,11 +465,11 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu bkt.opsDuration.WithLabelValues(op) bkt.opsFetchedBytes.WithLabelValues(op) } - // fetched bytes only relevant for get and getrange + // fetched bytes only relevant for get, getrange and upload for _, op := range []string{ OpGet, OpGetRange, - // TODO: Add uploads + OpUpload, } { bkt.opsTransferredBytes.WithLabelValues(op) } @@ -611,6 +611,23 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err } b.lastSuccessfulUploadTime.SetToCurrentTime() b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) + + rc := nopCloserWithObjectSize{ + Reader: r, + } + + trc := newTimingReadCloser( + rc, + op, + b.opsDuration, + b.opsFailures, + b.isOpFailureExpected, + b.opsFetchedBytes, + b.opsTransferredBytes, + ) + + trc.Close() + return nil } diff --git a/objstore_test.go b/objstore_test.go index 4b105281..6c893ab0 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -170,6 +170,25 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="+Inf"} 0 objstore_bucket_operation_transferred_bytes_sum{bucket="",operation="get_range"} 0 objstore_bucket_operation_transferred_bytes_count{bucket="",operation="get_range"} 0 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="32768"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="65536"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="131072"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="262144"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="524288"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="1.048576e+06"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="2.097152e+06"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="4.194304e+06"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="8.388608e+06"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="1.6777216e+07"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="3.3554432e+07"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="6.7108864e+07"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="1.34217728e+08"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="2.68435456e+08"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="5.36870912e+08"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="1.073741824e+09"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="+Inf"} 3 + objstore_bucket_operation_transferred_bytes_sum{bucket="",operation="upload"} 0 + objstore_bucket_operation_transferred_bytes_count{bucket="",operation="upload"} 3 `), `objstore_bucket_operation_transferred_bytes`)) testutil.Ok(t, UploadDir(context.Background(), log.NewNopLogger(), m, tempDir, "/dir-copy", WithUploadConcurrency(10))) From 8f5c7b5e75327206f996860d8773ce0b0146a3a4 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 7 Aug 2023 10:50:19 +0100 Subject: [PATCH 18/38] [FIX] Conflicts after rebase Signed-off-by: rita.canavarro --- go.mod | 10 +++++----- go.sum | 35 +++++++++++++++++++++++++++++++++++ objstore.go | 7 ------- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index ff862035..5bc96bf2 100644 --- a/go.mod +++ b/go.mod @@ -24,8 +24,8 @@ require ( go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/atomic v1.9.0 - golang.org/x/oauth2 v0.4.0 - golang.org/x/sync v0.1.0 + golang.org/x/oauth2 v0.8.0 + golang.org/x/sync v0.3.0 google.golang.org/api v0.103.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 @@ -36,7 +36,7 @@ require ( cloud.google.com/go/compute v1.15.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v0.8.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect @@ -59,7 +59,7 @@ require ( github.com/gofrs/flock v0.8.1 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.4 // indirect @@ -94,7 +94,7 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect google.golang.org/grpc v1.53.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/protobuf v1.30.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index b453b431..5465ea17 100644 --- a/go.sum +++ b/go.sum @@ -69,6 +69,7 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAu github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible h1:9gWa46nstkJ9miBReJcN8Gq34cBFbzSpQZVVT9N09TM= github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aws/aws-sdk-go-v2 v1.16.0 h1:cBAYjiiexRAg9v2z9vb6IdxAa7ef4KCtjW7w7e3GxGo= github.com/aws/aws-sdk-go-v2 v1.16.0/go.mod h1:lJYcuZZEHWNIb6ugJjbQY1fykdoobWbOS7kJYb4APoI= github.com/aws/aws-sdk-go-v2/config v1.15.1 h1:hTIZFepYESYyowQUBo47lu69WSxsYqGUILY9Nu8+7pY= @@ -111,6 +112,11 @@ github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -125,9 +131,12 @@ github.com/efficientgo/e2e v0.13.1-0.20220922081603-45de9fc588a8/go.mod h1:Hi+sz github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -179,6 +188,8 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -203,6 +214,7 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -223,6 +235,7 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible h1:tKTaPHNVwikS3I1rdyf1INNvgJXWSf/+TzqsiGbrgnQ= @@ -332,6 +345,9 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -352,6 +368,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -365,6 +382,7 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7y github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -376,6 +394,7 @@ go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -418,6 +437,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -447,10 +467,12 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -462,6 +484,7 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -471,8 +494,11 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -514,6 +540,7 @@ golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -577,6 +604,7 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -632,6 +660,7 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= @@ -652,7 +681,10 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -669,6 +701,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -681,6 +715,7 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/objstore.go b/objstore.go index 19e40ea2..cbe49ecf 100644 --- a/objstore.go +++ b/objstore.go @@ -443,13 +443,6 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu Help: "Second timestamp of the last successful upload to the bucket.", ConstLabels: prometheus.Labels{"bucket": name}, }), - - opsWrittenBytes: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ - Name: "objstore_bucket_operation_written_bytes", - Help: "Number of bytes uploaded from TSDB block, per operation.", - ConstLabels: prometheus.Labels{"bucket": name}, - Buckets: prometheus.ExponentialBuckets(50<<100, 2, 10), - }, []string{"operation"}), } for _, op := range []string{ OpIter, From 72636c357fb92c99c71387df897ab332e17b3b80 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 7 Aug 2023 10:59:41 +0100 Subject: [PATCH 19/38] [FIX] Dependencies Signed-off-by: rita.canavarro --- go.mod | 30 +++++++++++++++--------------- go.sum | 20 ++++++++++++++++++++ 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 5bc96bf2..fcbc4bb8 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/thanos-io/objstore go 1.18 require ( - cloud.google.com/go/storage v1.27.0 + cloud.google.com/go/storage v1.29.0 github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible github.com/aws/aws-sdk-go-v2 v1.16.0 github.com/aws/aws-sdk-go-v2/config v1.15.1 @@ -24,18 +24,18 @@ require ( go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/atomic v1.9.0 - golang.org/x/oauth2 v0.8.0 + golang.org/x/oauth2 v0.11.0 golang.org/x/sync v0.3.0 - google.golang.org/api v0.103.0 + google.golang.org/api v0.126.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 ) require ( - cloud.google.com/go v0.107.0 // indirect - cloud.google.com/go/compute v1.15.1 // indirect + cloud.google.com/go v0.110.2 // indirect + cloud.google.com/go/compute v1.20.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.8.0 // indirect + cloud.google.com/go/iam v0.13.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect @@ -64,8 +64,8 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect - github.com/googleapis/gax-go/v2 v2.7.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/googleapis/gax-go/v2 v2.11.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.7 // indirect @@ -86,15 +86,15 @@ require ( github.com/sirupsen/logrus v1.9.3 // indirect github.com/sony/gobreaker v0.5.0 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/net v0.12.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect - google.golang.org/grpc v1.53.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/grpc v1.55.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) @@ -105,5 +105,5 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect - golang.org/x/crypto v0.11.0 // indirect + golang.org/x/crypto v0.12.0 // indirect ) diff --git a/go.sum b/go.sum index 5465ea17..a30a4d66 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,8 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.107.0 h1:qkj22L7bgkl6vIeZDlOY2po43Mx/TIa2Wsa7VR+PEww= cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= +cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA= +cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -23,12 +25,15 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE= cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg= +cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -41,6 +46,7 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= @@ -231,10 +237,12 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -407,6 +415,7 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -475,6 +484,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -485,6 +496,8 @@ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -547,6 +560,7 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -559,6 +573,7 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -629,6 +644,7 @@ google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSr google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -669,6 +685,7 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -687,6 +704,7 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -703,6 +721,8 @@ google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From ae27ef19f98d0aac57363dc475fd74d718828f0e Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 7 Aug 2023 11:02:08 +0100 Subject: [PATCH 20/38] [FIX] Dependencies Signed-off-by: rita.canavarro --- go.mod | 2 ++ go.sum | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/go.mod b/go.mod index fcbc4bb8..9d8e4993 100644 --- a/go.mod +++ b/go.mod @@ -93,6 +93,8 @@ require ( golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect google.golang.org/grpc v1.55.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/go.sum b/go.sum index a30a4d66..c1a657db 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,7 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -46,6 +47,7 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= @@ -237,11 +239,13 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -415,6 +419,7 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -560,6 +565,7 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -573,6 +579,7 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -644,6 +651,7 @@ google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSr google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o= google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -685,7 +693,12 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao= google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -704,6 +717,7 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 510968f7723310f4bdf1d5ed2f648f74fca577c9 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 7 Aug 2023 11:04:19 +0100 Subject: [PATCH 21/38] [FIX] Remove opsWrittenBytes Signed-off-by: rita.canavarro --- objstore.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/objstore.go b/objstore.go index cbe49ecf..fa4e8ab8 100644 --- a/objstore.go +++ b/objstore.go @@ -478,7 +478,6 @@ type metricBucket struct { opsFetchedBytes *prometheus.CounterVec opsTransferredBytes *prometheus.HistogramVec - opsWrittenBytes *prometheus.HistogramVec opsDuration *prometheus.HistogramVec lastSuccessfulUploadTime prometheus.Gauge } @@ -490,7 +489,6 @@ func (b *metricBucket) WithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket { opsFailures: b.opsFailures, opsFetchedBytes: b.opsFetchedBytes, opsTransferredBytes: b.opsTransferredBytes, - opsWrittenBytes: b.opsWrittenBytes, isOpFailureExpected: fn, opsDuration: b.opsDuration, lastSuccessfulUploadTime: b.lastSuccessfulUploadTime, From 4fe2701f2bae382db3bd3e31d61f4e00462d92e2 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Mon, 7 Aug 2023 15:04:18 +0100 Subject: [PATCH 22/38] [FEAT] Clean up Signed-off-by: rita.canavarro --- objstore.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/objstore.go b/objstore.go index fa4e8ab8..e130215e 100644 --- a/objstore.go +++ b/objstore.go @@ -603,12 +603,10 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err b.lastSuccessfulUploadTime.SetToCurrentTime() b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) - rc := nopCloserWithObjectSize{ - Reader: r, - } - trc := newTimingReadCloser( - rc, + nopCloserWithObjectSize{ + Reader: r, + }, op, b.opsDuration, b.opsFailures, From 62c969d6b3758566f9e5ebe6472d0d8900065e69 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 8 Aug 2023 12:54:56 +0100 Subject: [PATCH 23/38] [FIX] Wrapping of reader Signed-off-by: rita.canavarro --- objstore.go | 19 +++++++++---------- objstore_test.go | 14 +++++++------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/objstore.go b/objstore.go index e130215e..54d4eccd 100644 --- a/objstore.go +++ b/objstore.go @@ -593,16 +593,6 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err b.ops.WithLabelValues(op).Inc() start := time.Now() - err := b.bkt.Upload(ctx, name, r) - if err != nil { - if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { - b.opsFailures.WithLabelValues(op).Inc() - } - return err - } - b.lastSuccessfulUploadTime.SetToCurrentTime() - b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) - trc := newTimingReadCloser( nopCloserWithObjectSize{ Reader: r, @@ -614,6 +604,15 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err b.opsFetchedBytes, b.opsTransferredBytes, ) + err := b.bkt.Upload(ctx, name, trc) + if err != nil { + if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { + b.opsFailures.WithLabelValues(op).Inc() + } + return err + } + b.lastSuccessfulUploadTime.SetToCurrentTime() + b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) trc.Close() diff --git a/objstore_test.go b/objstore_test.go index 6c893ab0..aa73b2bd 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -126,7 +126,7 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { objstore_bucket_operation_fetched_bytes_total{bucket="",operation="get"} 1.048578e+06 objstore_bucket_operation_fetched_bytes_total{bucket="",operation="get_range"} 0 objstore_bucket_operation_fetched_bytes_total{bucket="",operation="iter"} 0 - objstore_bucket_operation_fetched_bytes_total{bucket="",operation="upload"} 0 + objstore_bucket_operation_fetched_bytes_total{bucket="",operation="upload"} 1.048578e+06 `), `objstore_bucket_operation_fetched_bytes_total`)) testutil.Ok(t, promtest.GatherAndCompare(r, strings.NewReader(` @@ -170,11 +170,11 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="+Inf"} 0 objstore_bucket_operation_transferred_bytes_sum{bucket="",operation="get_range"} 0 objstore_bucket_operation_transferred_bytes_count{bucket="",operation="get_range"} 0 - objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="32768"} 3 - objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="65536"} 3 - objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="131072"} 3 - objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="262144"} 3 - objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="524288"} 3 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="32768"} 2 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="65536"} 2 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="131072"} 2 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="262144"} 2 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="524288"} 2 objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="1.048576e+06"} 3 objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="2.097152e+06"} 3 objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="4.194304e+06"} 3 @@ -187,7 +187,7 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="5.36870912e+08"} 3 objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="1.073741824e+09"} 3 objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="+Inf"} 3 - objstore_bucket_operation_transferred_bytes_sum{bucket="",operation="upload"} 0 + objstore_bucket_operation_transferred_bytes_sum{bucket="",operation="upload"} 1.048578e+06 objstore_bucket_operation_transferred_bytes_count{bucket="",operation="upload"} 3 `), `objstore_bucket_operation_transferred_bytes`)) From db8d21c070dec982ed72c22c7e07599b6ccadaa6 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 8 Aug 2023 14:12:24 +0100 Subject: [PATCH 24/38] [FEAT] Minor updates Signed-off-by: rita.canavarro --- objstore.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/objstore.go b/objstore.go index 54d4eccd..049ceddc 100644 --- a/objstore.go +++ b/objstore.go @@ -594,9 +594,7 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err start := time.Now() trc := newTimingReadCloser( - nopCloserWithObjectSize{ - Reader: r, - }, + io.NopCloser(r), op, b.opsDuration, b.opsFailures, @@ -609,6 +607,8 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.opsFailures.WithLabelValues(op).Inc() } + + trc.Close() return err } b.lastSuccessfulUploadTime.SetToCurrentTime() From 3554ffd1569c54730b646e6edad88aef22ebaa6a Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 8 Aug 2023 14:22:55 +0100 Subject: [PATCH 25/38] [REVERT] GO mod Signed-off-by: rita.canavarro --- go.mod | 51 ++++++++++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 9d8e4993..dee14caa 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/thanos-io/objstore go 1.18 require ( - cloud.google.com/go/storage v1.29.0 + cloud.google.com/go/storage v1.27.0 github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible github.com/aws/aws-sdk-go-v2 v1.16.0 github.com/aws/aws-sdk-go-v2/config v1.15.1 @@ -18,25 +18,25 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/oracle/oci-go-sdk/v65 v65.41.1 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.16.0 - github.com/prometheus/common v0.44.0 + github.com/prometheus/client_golang v1.12.2 + github.com/prometheus/common v0.36.0 github.com/tencentyun/cos-go-sdk-v5 v0.7.40 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/atomic v1.9.0 - golang.org/x/oauth2 v0.11.0 - golang.org/x/sync v0.3.0 - google.golang.org/api v0.126.0 + golang.org/x/oauth2 v0.4.0 + golang.org/x/sync v0.1.0 + google.golang.org/api v0.103.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 ) require ( - cloud.google.com/go v0.110.2 // indirect - cloud.google.com/go/compute v1.20.1 // indirect + cloud.google.com/go v0.107.0 // indirect + cloud.google.com/go/compute v1.15.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.13.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect + cloud.google.com/go/iam v0.8.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect @@ -59,19 +59,18 @@ require ( github.com/gofrs/flock v0.8.1 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/google/s2a-go v0.1.4 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.11.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect @@ -80,32 +79,30 @@ require ( github.com/mozillazg/go-httpheader v0.2.1 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/procfs v0.7.3 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sony/gobreaker v0.5.0 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/net v0.14.0 // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect + golang.org/x/net v0.12.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect - google.golang.org/grpc v1.55.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect + google.golang.org/grpc v1.53.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect - golang.org/x/crypto v0.12.0 // indirect + golang.org/x/crypto v0.11.0 // indirect ) From 7e4dd6ac0bbf73c423d7e44a485076d03ee33f4f Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 8 Aug 2023 16:13:13 +0100 Subject: [PATCH 26/38] [FEAT] Changelog Signed-off-by: rita.canavarro --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7231f68a..b1803c3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,8 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#61](https://github.com/thanos-io/objstore/pull/61) Add OpenTelemetry TracingBucket. > This also changes the behaviour of `client.NewBucket`. Now it returns, uninstrumented and untraced bucket. You can combine `objstore.WrapWithMetrics` and `tracing/{opentelemetry,opentracing}.WrapWithTraces` to have old behavior. -- [#66](https://github.com/thanos-io/objstore/pull/66) Add `objstore_bucket_operation_written_bytes_total` that counts the number of total bytes per written to the bucket operation. +- [#69](https://github.com/thanos-io/objstore/pull/69) Add `objstore_bucket_operation_transferred_bytes` that counts the number of total bytes read from the bucket operation Get/GetRange. +- [#66](https://github.com/thanos-io/objstore/pull/66) Made `objstore_bucket_operation_transferred_bytes` also count the number of total bytes per written to the bucket operation Upload. - [#64](https://github.com/thanos-io/objstore/pull/64) OCI: OKE Workload Identity support. - [#73](https://github.com/thanos-io/objstore/pull/73) Аdded file path to erros from DownloadFile - [#51](https://github.com/thanos-io/objstore/pull/51) Azure: Support using connection string authentication. From 96b3e99a690e5337b6146a51d1383b1fd5da2b48 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Tue, 8 Aug 2023 17:51:00 +0100 Subject: [PATCH 27/38] [FEAT] Improvements Signed-off-by: rita.canavarro --- CHANGELOG.md | 2 +- go.sum | 93 ------------------------------------------------ objstore.go | 8 +++-- objstore_test.go | 2 +- 4 files changed, 8 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1803c3a..cf10fe59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re > This also changes the behaviour of `client.NewBucket`. Now it returns, uninstrumented and untraced bucket. You can combine `objstore.WrapWithMetrics` and `tracing/{opentelemetry,opentracing}.WrapWithTraces` to have old behavior. - [#69](https://github.com/thanos-io/objstore/pull/69) Add `objstore_bucket_operation_transferred_bytes` that counts the number of total bytes read from the bucket operation Get/GetRange. -- [#66](https://github.com/thanos-io/objstore/pull/66) Made `objstore_bucket_operation_transferred_bytes` also count the number of total bytes per written to the bucket operation Upload. +- [#69](https://github.com/thanos-io/objstore/pull/69) [#66](https://github.com/thanos-io/objstore/pull/66) Add `objstore_bucket_operation_transferred_bytes` that counts the number of total bytes read from the bucket operation Get/GetRange and also counts the number of total bytes per written to the bucket operation Upload. - [#64](https://github.com/thanos-io/objstore/pull/64) OCI: OKE Workload Identity support. - [#73](https://github.com/thanos-io/objstore/pull/73) Аdded file path to erros from DownloadFile - [#51](https://github.com/thanos-io/objstore/pull/51) Azure: Support using connection string authentication. diff --git a/go.sum b/go.sum index c1a657db..860aca72 100644 --- a/go.sum +++ b/go.sum @@ -15,8 +15,6 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.107.0 h1:qkj22L7bgkl6vIeZDlOY2po43Mx/TIa2Wsa7VR+PEww= cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= -cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA= -cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -25,16 +23,12 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE= cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= -cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg= -cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= -cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= -cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -47,19 +41,13 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= -cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= -cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0 h1:sVW/AFBTGyJxDaMYlq0ct3jUXTtj12tQ6zE2GZUgVQw= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.2.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 h1:Oj853U9kG+RLTCQXpjvOnrv0WaZHxgmZz1TlLywgOPY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= @@ -77,7 +65,6 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAu github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible h1:9gWa46nstkJ9miBReJcN8Gq34cBFbzSpQZVVT9N09TM= github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aws/aws-sdk-go-v2 v1.16.0 h1:cBAYjiiexRAg9v2z9vb6IdxAa7ef4KCtjW7w7e3GxGo= github.com/aws/aws-sdk-go-v2 v1.16.0/go.mod h1:lJYcuZZEHWNIb6ugJjbQY1fykdoobWbOS7kJYb4APoI= github.com/aws/aws-sdk-go-v2/config v1.15.1 h1:hTIZFepYESYyowQUBo47lu69WSxsYqGUILY9Nu8+7pY= @@ -120,11 +107,6 @@ github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -139,12 +121,9 @@ github.com/efficientgo/e2e v0.13.1-0.20220922081603-45de9fc588a8/go.mod h1:Hi+sz github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -163,7 +142,6 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -196,8 +174,6 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -222,7 +198,6 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -231,23 +206,16 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= -github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= -github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= -github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= -github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= -github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible h1:tKTaPHNVwikS3I1rdyf1INNvgJXWSf/+TzqsiGbrgnQ= @@ -287,8 +255,6 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.61 h1:87c+x8J3jxQ5VUGimV9oHdpjsAvy3fhneEBKuoKEVUI= @@ -304,7 +270,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -331,35 +296,23 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.36.0 h1:78hJTing+BLYLjhXE+Z2BubeEymH5Lr0/Mt8FKkxxYo= github.com/prometheus/common v0.36.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= -github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -380,7 +333,6 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -394,7 +346,6 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7y github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -406,7 +357,6 @@ go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -415,12 +365,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -451,7 +397,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -481,16 +426,11 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -500,9 +440,6 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= -golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= -golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= -golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -512,11 +449,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -558,15 +492,11 @@ golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -576,11 +506,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -626,7 +553,6 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -651,8 +577,6 @@ google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSr google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= -google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o= -google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -684,7 +608,6 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= @@ -693,12 +616,6 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= -google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao= -google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= -google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= -google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -711,14 +628,9 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= -google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -733,10 +645,6 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -749,7 +657,6 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/objstore.go b/objstore.go index 049ceddc..24e5d8a1 100644 --- a/objstore.go +++ b/objstore.go @@ -458,6 +458,7 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu bkt.opsDuration.WithLabelValues(op) bkt.opsFetchedBytes.WithLabelValues(op) } + // fetched bytes only relevant for get, getrange and upload for _, op := range []string{ OpGet, @@ -599,7 +600,7 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err b.opsDuration, b.opsFailures, b.isOpFailureExpected, - b.opsFetchedBytes, + nil, b.opsTransferredBytes, ) err := b.bkt.Upload(ctx, name, trc) @@ -707,7 +708,10 @@ func (rc *timingReadCloser) Close() error { func (rc *timingReadCloser) Read(b []byte) (n int, err error) { n, err = rc.ReadCloser.Read(b) - rc.fetchedBytes.WithLabelValues(rc.op).Add(float64(n)) + if rc.fetchedBytes != nil { + rc.fetchedBytes.WithLabelValues(rc.op).Add(float64(n)) + } + rc.readBytes += int64(n) // Report metric just once. if !rc.alreadyGotErr && err != nil && err != io.EOF { diff --git a/objstore_test.go b/objstore_test.go index aa73b2bd..4b737c16 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -126,7 +126,7 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { objstore_bucket_operation_fetched_bytes_total{bucket="",operation="get"} 1.048578e+06 objstore_bucket_operation_fetched_bytes_total{bucket="",operation="get_range"} 0 objstore_bucket_operation_fetched_bytes_total{bucket="",operation="iter"} 0 - objstore_bucket_operation_fetched_bytes_total{bucket="",operation="upload"} 1.048578e+06 + objstore_bucket_operation_fetched_bytes_total{bucket="",operation="upload"} 0 `), `objstore_bucket_operation_fetched_bytes_total`)) testutil.Ok(t, promtest.GatherAndCompare(r, strings.NewReader(` From 4253402e19b907624de33ba46c55adb831c8d60c Mon Sep 17 00:00:00 2001 From: Rita Canavarro <98762287+ritaCanavarro@users.noreply.github.com> Date: Wed, 9 Aug 2023 09:07:36 +0100 Subject: [PATCH 28/38] [FEAT] Clean whitespace Co-authored-by: Christian Simon Signed-off-by: Rita Canavarro <98762287+ritaCanavarro@users.noreply.github.com> --- objstore_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/objstore_test.go b/objstore_test.go index 4b737c16..5feb089b 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -170,7 +170,7 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="get_range",le="+Inf"} 0 objstore_bucket_operation_transferred_bytes_sum{bucket="",operation="get_range"} 0 objstore_bucket_operation_transferred_bytes_count{bucket="",operation="get_range"} 0 - objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="32768"} 2 + objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="32768"} 2 objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="65536"} 2 objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="131072"} 2 objstore_bucket_operation_transferred_bytes_bucket{bucket="",operation="upload",le="262144"} 2 From 1e55b009c378eaa669aac1233bc0402b9dec0fa4 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Wed, 9 Aug 2023 09:10:10 +0100 Subject: [PATCH 29/38] [FEAT] Clean up Signed-off-by: rita.canavarro --- CHANGELOG.md | 1 - objstore.go | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf10fe59..42b77fa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,6 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#61](https://github.com/thanos-io/objstore/pull/61) Add OpenTelemetry TracingBucket. > This also changes the behaviour of `client.NewBucket`. Now it returns, uninstrumented and untraced bucket. You can combine `objstore.WrapWithMetrics` and `tracing/{opentelemetry,opentracing}.WrapWithTraces` to have old behavior. -- [#69](https://github.com/thanos-io/objstore/pull/69) Add `objstore_bucket_operation_transferred_bytes` that counts the number of total bytes read from the bucket operation Get/GetRange. - [#69](https://github.com/thanos-io/objstore/pull/69) [#66](https://github.com/thanos-io/objstore/pull/66) Add `objstore_bucket_operation_transferred_bytes` that counts the number of total bytes read from the bucket operation Get/GetRange and also counts the number of total bytes per written to the bucket operation Upload. - [#64](https://github.com/thanos-io/objstore/pull/64) OCI: OKE Workload Identity support. - [#73](https://github.com/thanos-io/objstore/pull/73) Аdded file path to erros from DownloadFile diff --git a/objstore.go b/objstore.go index 24e5d8a1..f772d51a 100644 --- a/objstore.go +++ b/objstore.go @@ -593,7 +593,6 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err const op = OpUpload b.ops.WithLabelValues(op).Inc() - start := time.Now() trc := newTimingReadCloser( io.NopCloser(r), op, @@ -603,6 +602,7 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err nil, b.opsTransferredBytes, ) + defer trc.Close() err := b.bkt.Upload(ctx, name, trc) if err != nil { if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { @@ -613,9 +613,6 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err return err } b.lastSuccessfulUploadTime.SetToCurrentTime() - b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) - - trc.Close() return nil } From 35b9ac6b3c062c675e4a43a6224f3f532a424162 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Wed, 9 Aug 2023 09:17:34 +0100 Subject: [PATCH 30/38] [FEAT] Clean up Signed-off-by: rita.canavarro --- providers/azure/azure.go | 1 - 1 file changed, 1 deletion(-) diff --git a/providers/azure/azure.go b/providers/azure/azure.go index f28ab025..a72f3052 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -314,7 +314,6 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { BlockSize: 3 * 1024 * 1024, Concurrency: 4, } - if _, err := blobClient.UploadStream(ctx, r, opts); err != nil { return errors.Wrapf(err, "cannot upload Azure blob, address: %s", name) } From d5dfb062a64e58f35a23f1f5ccebb54ed832fbdf Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Wed, 16 Aug 2023 13:07:27 +0100 Subject: [PATCH 31/38] [FEAT] Clean up Signed-off-by: rita.canavarro --- objstore.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/objstore.go b/objstore.go index f772d51a..5bcc55f1 100644 --- a/objstore.go +++ b/objstore.go @@ -608,8 +608,6 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.opsFailures.WithLabelValues(op).Inc() } - - trc.Close() return err } b.lastSuccessfulUploadTime.SetToCurrentTime() From 47a0da286d6afa0e4208bf2f9880c0abfc5b8559 Mon Sep 17 00:00:00 2001 From: "rita.canavarro" Date: Thu, 7 Sep 2023 15:29:33 +0100 Subject: [PATCH 32/38] [FEAT] Update changelog Signed-off-by: rita.canavarro --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42b77fa8..4b9387df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#61](https://github.com/thanos-io/objstore/pull/61) Add OpenTelemetry TracingBucket. > This also changes the behaviour of `client.NewBucket`. Now it returns, uninstrumented and untraced bucket. You can combine `objstore.WrapWithMetrics` and `tracing/{opentelemetry,opentracing}.WrapWithTraces` to have old behavior. -- [#69](https://github.com/thanos-io/objstore/pull/69) [#66](https://github.com/thanos-io/objstore/pull/66) Add `objstore_bucket_operation_transferred_bytes` that counts the number of total bytes read from the bucket operation Get/GetRange and also counts the number of total bytes per written to the bucket operation Upload. +- [#69](https://github.com/thanos-io/objstore/pull/69) [#66](https://github.com/thanos-io/objstore/pull/66) Add `objstore_bucket_operation_transferred_bytes` that counts the number of total bytes read from the bucket operation Get/GetRange and also counts the number of total bytes written to the bucket operation Upload. - [#64](https://github.com/thanos-io/objstore/pull/64) OCI: OKE Workload Identity support. - [#73](https://github.com/thanos-io/objstore/pull/73) Аdded file path to erros from DownloadFile - [#51](https://github.com/thanos-io/objstore/pull/51) Azure: Support using connection string authentication. From 25673b61e4d9514f3f5d244fc755b5ff224a488f Mon Sep 17 00:00:00 2001 From: Andy Asp Date: Tue, 12 Sep 2023 13:38:16 -0400 Subject: [PATCH 33/38] GCS: use Query.SetAttrSelection when listing Signed-off-by: Andy Asp --- providers/gcs/gcs.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 5ea45c7e..ad305d6e 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -108,10 +108,16 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt delimiter = "" } - it := b.bkt.Objects(ctx, &storage.Query{ + query := &storage.Query{ Prefix: dir, Delimiter: delimiter, - }) + } + err := query.SetAttrSelection([]string{"Name"}) + if err != nil { + return err + } + + it := b.bkt.Objects(ctx, query) for { select { case <-ctx.Done(): From 02d54dc163e2e0bddcc35b52c5e69bb1e33cbd85 Mon Sep 17 00:00:00 2001 From: Andy Asp Date: Tue, 12 Sep 2023 16:09:45 -0400 Subject: [PATCH 34/38] Add changelog entry Signed-off-by: Andy Asp --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b9387df..72f8f180 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#64](https://github.com/thanos-io/objstore/pull/64) OCI: OKE Workload Identity support. - [#73](https://github.com/thanos-io/objstore/pull/73) Аdded file path to erros from DownloadFile - [#51](https://github.com/thanos-io/objstore/pull/51) Azure: Support using connection string authentication. +- [#76](https://github.com/thanos-io/objstore/pull/76) GCS: Query for object names only in `Iter` to possibly improve performance when listing objects. ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. From 0cc6ab5601c744400c2540f52b397897334804ac Mon Sep 17 00:00:00 2001 From: Andy Asp Date: Wed, 20 Sep 2023 12:42:20 -0400 Subject: [PATCH 35/38] Fix metricBucket losing object size in Upload Signed-off-by: Andy Asp --- objstore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/objstore.go b/objstore.go index 5bcc55f1..20723704 100644 --- a/objstore.go +++ b/objstore.go @@ -594,7 +594,7 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err b.ops.WithLabelValues(op).Inc() trc := newTimingReadCloser( - io.NopCloser(r), + NopCloserWithSize(r), op, b.opsDuration, b.opsFailures, From 6175521d7140b1b76cbd7ae53436f09dccb43b4c Mon Sep 17 00:00:00 2001 From: Andy Asp Date: Wed, 20 Sep 2023 13:42:53 -0400 Subject: [PATCH 36/38] Add changelog entry Signed-off-by: Andy Asp --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72f8f180..e26b1548 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#33](https://github.com/thanos-io/objstore/pull/33) Tracing: Add `ContextWithTracer()` to inject the tracer into the context. - [#34](https://github.com/thanos-io/objstore/pull/34) Fix ignored options when creating shared credential Azure client. - [#62](https://github.com/thanos-io/objstore/pull/62) S3: Fix ignored context cancellation in `Iter` method. +- [#77](https://github.com/thanos-io/objstore/pull/77) Fix buckets wrapped with metrics from being unable to determine object sizes in `Upload`. ### Added - [#15](https://github.com/thanos-io/objstore/pull/15) Add Oracle Cloud Infrastructure Object Storage Bucket support. From ff39b162a6d3bbfc5a9f46097f7d6e7a224fe2fb Mon Sep 17 00:00:00 2001 From: Andy Asp Date: Wed, 20 Sep 2023 12:31:49 -0400 Subject: [PATCH 37/38] S3: clone putUserMetadata map to prevent concurrent modification Signed-off-by: Andy Asp --- providers/s3/s3.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/providers/s3/s3.go b/providers/s3/s3.go index f92d3973..83e3a2de 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -492,6 +492,13 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { if size < int64(partSize) { partSize = 0 } + + // Cloning map since minio may modify it + userMetadata := make(map[string]string, len(b.putUserMetadata)) + for k, v := range b.putUserMetadata { + userMetadata[k] = v + } + if _, err := b.client.PutObject( ctx, b.name, @@ -501,7 +508,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { minio.PutObjectOptions{ PartSize: partSize, ServerSideEncryption: sse, - UserMetadata: b.putUserMetadata, + UserMetadata: userMetadata, StorageClass: b.storageClass, // 4 is what minio-go have as the default. To be certain we do micro benchmark before any changes we // ensure we pin this number to four. From a557e75bcb9ea67a663b6329571b5a1d0794969c Mon Sep 17 00:00:00 2001 From: Andy Asp Date: Wed, 20 Sep 2023 14:16:35 -0400 Subject: [PATCH 38/38] Add changelog entry Signed-off-by: Andy Asp --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e26b1548..0dc65033 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#34](https://github.com/thanos-io/objstore/pull/34) Fix ignored options when creating shared credential Azure client. - [#62](https://github.com/thanos-io/objstore/pull/62) S3: Fix ignored context cancellation in `Iter` method. - [#77](https://github.com/thanos-io/objstore/pull/77) Fix buckets wrapped with metrics from being unable to determine object sizes in `Upload`. +- [#78](https://github.com/thanos-io/objstore/pull/78) S3: Fix possible concurrent modification of the PutUserMetadata map. ### Added - [#15](https://github.com/thanos-io/objstore/pull/15) Add Oracle Cloud Infrastructure Object Storage Bucket support.