chore(storage): add retry metric headers

storage/acl.go

@@ -121,12 +121,12 @@ func (a *ACLHandle) List(ctx context.Context) (rules []ACLRule, err error) {
 func (a *ACLHandle) bucketDefaultList(ctx context.Context) ([]ACLRule, error) {
 	var acls *raw.ObjectAccessControls
 	var err error
+	req := a.c.raw.DefaultObjectAccessControls.List(a.bucket)
+	a.configureCall(ctx, req)
 	err = run(ctx, func() error {
-		req := a.c.raw.DefaultObjectAccessControls.List(a.bucket)
-		a.configureCall(ctx, req)
 		acls, err = req.Do()
 		return err
-	}, a.retry, true)
+	}, a.retry, true, setRetryHeaderHTTP(req))
 	if err != nil {
 		return nil, err
 	}
@@ -139,18 +139,18 @@ func (a *ACLHandle) bucketDefaultDelete(ctx context.Context, entity ACLEntity) e
 
 	return run(ctx, func() error {
 		return req.Do()
-	}, a.retry, false)
+	}, a.retry, false, setRetryHeaderHTTP(req))
 }
 
 func (a *ACLHandle) bucketList(ctx context.Context) ([]ACLRule, error) {
 	var acls *raw.BucketAccessControls
 	var err error
+	req := a.c.raw.BucketAccessControls.List(a.bucket)
+	a.configureCall(ctx, req)
 	err = run(ctx, func() error {
-		req := a.c.raw.BucketAccessControls.List(a.bucket)
-		a.configureCall(ctx, req)
 		acls, err = req.Do()
 		return err
-	}, a.retry, true)
+	}, a.retry, true, setRetryHeaderHTTP(req))
 	if err != nil {
 		return nil, err
 	}
@@ -168,26 +168,26 @@ func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRol
 	return run(ctx, func() error {
 		_, err := req.Do()
 		return err
-	}, a.retry, false)
+	}, a.retry, false, setRetryHeaderHTTP(req))
 }
 
 func (a *ACLHandle) bucketDelete(ctx context.Context, entity ACLEntity) error {
 	req := a.c.raw.BucketAccessControls.Delete(a.bucket, string(entity))
 	a.configureCall(ctx, req)
 	return run(ctx, func() error {
 		return req.Do()
-	}, a.retry, false)
+	}, a.retry, false, setRetryHeaderHTTP(req))
 }
 
 func (a *ACLHandle) objectList(ctx context.Context) ([]ACLRule, error) {
 	var acls *raw.ObjectAccessControls
 	var err error
+	req := a.c.raw.ObjectAccessControls.List(a.bucket, a.object)
+	a.configureCall(ctx, req)
 	err = run(ctx, func() error {
-		req := a.c.raw.ObjectAccessControls.List(a.bucket, a.object)
-		a.configureCall(ctx, req)
 		acls, err = req.Do()
 		return err
-	}, a.retry, true)
+	}, a.retry, true, setRetryHeaderHTTP(req))
 	if err != nil {
 		return nil, err
 	}
@@ -215,15 +215,15 @@ func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRol
 	return run(ctx, func() error {
 		_, err := req.Do()
 		return err
-	}, a.retry, false)
+	}, a.retry, false, setRetryHeaderHTTP(req))
 }
 
 func (a *ACLHandle) objectDelete(ctx context.Context, entity ACLEntity) error {
 	req := a.c.raw.ObjectAccessControls.Delete(a.bucket, a.object, string(entity))
 	a.configureCall(ctx, req)
 	return run(ctx, func() error {
 		return req.Do()
-	}, a.retry, false)
+	}, a.retry, false, setRetryHeaderHTTP(req))
 }
 
 func (a *ACLHandle) configureCall(ctx context.Context, call interface{ Header() http.Header }) {

storage/bucket.go

@@ -103,7 +103,7 @@ func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *Buck
 	if attrs != nil && attrs.PredefinedDefaultObjectACL != "" {
 		req.PredefinedDefaultObjectAcl(attrs.PredefinedDefaultObjectACL)
 	}
-	return run(ctx, func() error { _, err := req.Context(ctx).Do(); return err }, b.retry, true)
+	return run(ctx, func() error { _, err := req.Context(ctx).Do(); return err }, b.retry, true, setRetryHeaderHTTP(req))
 }
 
 // Delete deletes the Bucket.
@@ -116,7 +116,7 @@ func (b *BucketHandle) Delete(ctx context.Context) (err error) {
 		return err
 	}
 
-	return run(ctx, func() error { return req.Context(ctx).Do() }, b.retry, true)
+	return run(ctx, func() error { return req.Context(ctx).Do() }, b.retry, true, setRetryHeaderHTTP(req))
 }
 
 func (b *BucketHandle) newDeleteCall() (*raw.BucketsDeleteCall, error) {
@@ -184,7 +184,7 @@ func (b *BucketHandle) Attrs(ctx context.Context) (attrs *BucketAttrs, err error
 	err = run(ctx, func() error {
 		resp, err = req.Context(ctx).Do()
 		return err
-	}, b.retry, true)
+	}, b.retry, true, setRetryHeaderHTTP(req))
 	var e *googleapi.Error
 	if ok := errors.As(err, &e); ok && e.Code == http.StatusNotFound {
 		return nil, ErrBucketNotExist
@@ -232,7 +232,7 @@ func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) (
 		return err
 	}
 
-	if err := run(ctx, call, b.retry, isIdempotent); err != nil {
+	if err := run(ctx, call, b.retry, isIdempotent, setRetryHeaderHTTP(req)); err != nil {
 		return nil, err
 	}
 	return newBucket(rawBucket)
@@ -1341,7 +1341,7 @@ func (b *BucketHandle) LockRetentionPolicy(ctx context.Context) error {
 	return run(ctx, func() error {
 		_, err := req.Context(ctx).Do()
 		return err
-	}, b.retry, true)
+	}, b.retry, true, setRetryHeaderHTTP(req))
 }
 
 // applyBucketConds modifies the provided call using the conditions in conds.
@@ -1999,7 +1999,7 @@ func (it *ObjectIterator) fetch(pageSize int, pageToken string) (string, error)
 	err = run(it.ctx, func() error {
 		resp, err = req.Context(it.ctx).Do()
 		return err
-	}, it.bucket.retry, true)
+	}, it.bucket.retry, true, setRetryHeaderHTTP(req))
 	if err != nil {
 		var e *googleapi.Error
 		if ok := errors.As(err, &e); ok && e.Code == http.StatusNotFound {
@@ -2086,7 +2086,7 @@ func (it *BucketIterator) fetch(pageSize int, pageToken string) (token string, e
 	err = run(it.ctx, func() error {
 		resp, err = req.Context(it.ctx).Do()
 		return err
-	}, it.client.retry, true)
+	}, it.client.retry, true, setRetryHeaderHTTP(req))
 	if err != nil {
 		return "", err
 	}

storage/copy.go

@@ -142,7 +142,7 @@ func (c *Copier) callRewrite(ctx context.Context, rawObj *raw.Object) (*raw.Rewr
 	retryCall := func() error { res, err = call.Do(); return err }
 	isIdempotent := c.dst.conds != nil && (c.dst.conds.GenerationMatch != 0 || c.dst.conds.DoesNotExist)
 
-	if err := run(ctx, retryCall, c.dst.retry, isIdempotent); err != nil {
+	if err := run(ctx, retryCall, c.dst.retry, isIdempotent, setRetryHeaderHTTP(call)); err != nil {
 		return nil, err
 	}
 	c.RewriteToken = res.RewriteToken
@@ -237,7 +237,7 @@ func (c *Composer) Run(ctx context.Context) (attrs *ObjectAttrs, err error) {
 	retryCall := func() error { obj, err = call.Do(); return err }
 	isIdempotent := c.dst.conds != nil && (c.dst.conds.GenerationMatch != 0 || c.dst.conds.DoesNotExist)
 
-	if err := run(ctx, retryCall, c.dst.retry, isIdempotent); err != nil {
+	if err := run(ctx, retryCall, c.dst.retry, isIdempotent, setRetryHeaderHTTP(call)); err != nil {
 		return nil, err
 	}
 	return newObject(obj), nil

storage/go.mod

@@ -8,6 +8,7 @@ require (
 	cloud.google.com/go/iam v0.3.0
 	github.com/golang/protobuf v1.5.2
 	github.com/google/go-cmp v0.5.7
+	github.com/google/uuid v1.1.2
 	github.com/googleapis/gax-go/v2 v2.3.0
 	github.com/googleapis/go-type-adapters v1.0.0
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5

storage/go.sum

@@ -157,6 +157,7 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=

storage/grpc_client.go

@@ -121,7 +121,7 @@ func (c *grpcStorageClient) GetServiceAccount(ctx context.Context, project strin
 		var err error
 		resp, err = c.raw.GetServiceAccount(ctx, req, s.gax...)
 		return err
-	}, s.retry, s.idempotent)
+	}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))
 	if err != nil {
 		return "", err
 	}
@@ -153,7 +153,7 @@ func (c *grpcStorageClient) CreateBucket(ctx context.Context, project string, at
 		battrs = newBucketFromProto(res)
 
 		return err
-	}, s.retry, s.idempotent)
+	}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))
 
 	return battrs, err
 }
@@ -186,7 +186,7 @@ func (c *grpcStorageClient) ListBuckets(ctx context.Context, project string, opt
 		err = run(it.ctx, func() error {
 			buckets, next, err = gitr.InternalFetch(pageSize, pageToken)
 			return err
-		}, s.retry, s.idempotent)
+		}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))
 		if err != nil {
 			return "", err
 		}
@@ -224,7 +224,7 @@ func (c *grpcStorageClient) DeleteBucket(ctx context.Context, bucket string, con
 
 	return run(ctx, func() error {
 		return c.raw.DeleteBucket(ctx, req, s.gax...)
-	}, s.retry, s.idempotent)
+	}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))
 }
 
 func (c *grpcStorageClient) GetBucket(ctx context.Context, bucket string, conds *BucketConditions, opts ...storageOption) (*BucketAttrs, error) {
@@ -248,7 +248,7 @@ func (c *grpcStorageClient) GetBucket(ctx context.Context, bucket string, conds
 		battrs = newBucketFromProto(res)
 
 		return err
-	}, s.retry, s.idempotent)
+	}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))
 
 	if s, ok := status.FromError(err); ok && s.Code() == codes.NotFound {
 		return nil, ErrBucketNotExist
@@ -338,7 +338,7 @@ func (c *grpcStorageClient) UpdateBucket(ctx context.Context, bucket string, uat
 		res, err := c.raw.UpdateBucket(ctx, req, s.gax...)
 		battrs = newBucketFromProto(res)
 		return err
-	}, s.retry, s.idempotent)
+	}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))
 
 	return battrs, err
 }
@@ -359,7 +359,7 @@ func (c *grpcStorageClient) LockBucketRetentionPolicy(ctx context.Context, bucke
 	return run(ctx, func() error {
 		_, err := c.raw.LockBucketRetentionPolicy(ctx, req, s.gax...)
 		return err
-	}, s.retry, s.idempotent)
+	}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))
 
 }
 func (c *grpcStorageClient) ListObjects(ctx context.Context, bucket string, q *Query, opts ...storageOption) *ObjectIterator {
@@ -389,7 +389,7 @@ func (c *grpcStorageClient) ListObjects(ctx context.Context, bucket string, q *Q
 		err = run(it.ctx, func() error {
 			objects, token, err = gitr.InternalFetch(pageSize, pageToken)
 			return err
-		}, s.retry, s.idempotent)
+		}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))
 		if err != nil {
 			if st, ok := status.FromError(err); ok && st.Code() == codes.NotFound {
 				err = ErrBucketNotExist
@@ -552,7 +552,7 @@ func (c *grpcStorageClient) GetIamPolicy(ctx context.Context, resource string, v
 		var err error
 		rp, err = c.raw.GetIamPolicy(ctx, req, s.gax...)
 		return err
-	}, s.retry, s.idempotent)
+	}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))
 
 	return rp, err
 }
@@ -569,7 +569,7 @@ func (c *grpcStorageClient) SetIamPolicy(ctx context.Context, resource string, p
 	return run(ctx, func() error {
 		_, err := c.raw.SetIamPolicy(ctx, req, s.gax...)
 		return err
-	}, s.retry, s.idempotent)
+	}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))
 }
 
 func (c *grpcStorageClient) TestIamPermissions(ctx context.Context, resource string, permissions []string, opts ...storageOption) ([]string, error) {
@@ -584,7 +584,7 @@ func (c *grpcStorageClient) TestIamPermissions(ctx context.Context, resource str
 		var err error
 		res, err = c.raw.TestIamPermissions(ctx, req, s.gax...)
 		return err
-	}, s.retry, s.idempotent)
+	}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))
 	if err != nil {
 		return nil, err
 	}

storage/hmac.go

@@ -130,7 +130,7 @@ func (hkh *HMACKeyHandle) Get(ctx context.Context, opts ...HMACKeyOption) (*HMAC
 	err = run(ctx, func() error {
 		metadata, err = call.Context(ctx).Do()
 		return err
-	}, hkh.retry, true)
+	}, hkh.retry, true, setRetryHeaderHTTP(call))
 	if err != nil {
 		return nil, err
 	}
@@ -159,7 +159,7 @@ func (hkh *HMACKeyHandle) Delete(ctx context.Context, opts ...HMACKeyOption) err
 
 	return run(ctx, func() error {
 		return delCall.Context(ctx).Do()
-	}, hkh.retry, true)
+	}, hkh.retry, true, setRetryHeaderHTTP(delCall))
 }
 
 func pbHmacKeyToHMACKey(pb *raw.HmacKey, updatedTimeCanBeNil bool) (*HMACKey, error) {
@@ -221,7 +221,7 @@ func (c *Client) CreateHMACKey(ctx context.Context, projectID, serviceAccountEma
 		h, err := call.Context(ctx).Do()
 		hkPb = h
 		return err
-	}, c.retry, false); err != nil {
+	}, c.retry, false, setRetryHeaderHTTP(call)); err != nil {
 		return nil, err
 	}
 
@@ -267,7 +267,7 @@ func (h *HMACKeyHandle) Update(ctx context.Context, au HMACKeyAttrsToUpdate, opt
 	err = run(ctx, func() error {
 		metadata, err = call.Context(ctx).Do()
 		return err
-	}, h.retry, isIdempotent)
+	}, h.retry, isIdempotent, setRetryHeaderHTTP(call))
 
 	if err != nil {
 		return nil, err
@@ -373,7 +373,7 @@ func (it *HMACKeysIterator) fetch(pageSize int, pageToken string) (token string,
 	err = run(it.ctx, func() error {
 		resp, err = call.Context(ctx).Do()
 		return err
-	}, it.retry, true)
+	}, it.retry, true, setRetryHeaderHTTP(call))
 	if err != nil {
 		return "", err
 	}