Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore(storage): add retry metric headers #6013

Merged
merged 10 commits into from
May 18, 2022
Merged

chore(storage): add retry metric headers #6013

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
2c5f31d
chore(storage): add retry metric headers [WIP]
tritone May 6, 2022
91e7eb2
finish implementation, add tests
tritone May 9, 2022
22e10d3
don't clobber existing header
tritone May 9, 2022
492a89b
Merge branch 'main' into retry-metric
tritone May 9, 2022
0f4f920
add TODO
tritone May 9, 2022
9144cc9
update to use setter func
tritone May 16, 2022
cca4a0e
Merge branch 'main' into retry-metric
tritone May 16, 2022
ddcc4c7
update added calls from merge
tritone May 16, 2022
4b83569
make test check that ID is consistent
tritone May 18, 2022
05dbf21
Merge branch 'main' into retry-metric
tritone May 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions storage/acl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ func (a *ACLHandle) bucketDefaultList(ctx context.Context) ([]ACLRule, error) {
err = run(ctx, func() error {
acls, err = req.Do()
return err
}, a.retry, true, req)
}, a.retry, true, setRetryHeaderHTTP(req))
if err != nil {
return nil, err
}
Expand All @@ -139,7 +139,7 @@ func (a *ACLHandle) bucketDefaultDelete(ctx context.Context, entity ACLEntity) e

return run(ctx, func() error {
return req.Do()
}, a.retry, false, req)
}, a.retry, false, setRetryHeaderHTTP(req))
}

func (a *ACLHandle) bucketList(ctx context.Context) ([]ACLRule, error) {
Expand All @@ -150,7 +150,7 @@ func (a *ACLHandle) bucketList(ctx context.Context) ([]ACLRule, error) {
err = run(ctx, func() error {
acls, err = req.Do()
return err
}, a.retry, true, req)
}, a.retry, true, setRetryHeaderHTTP(req))
if err != nil {
return nil, err
}
Expand All @@ -168,15 +168,15 @@ func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRol
return run(ctx, func() error {
_, err := req.Do()
return err
}, a.retry, false, req)
}, a.retry, false, setRetryHeaderHTTP(req))
}

func (a *ACLHandle) bucketDelete(ctx context.Context, entity ACLEntity) error {
req := a.c.raw.BucketAccessControls.Delete(a.bucket, string(entity))
a.configureCall(ctx, req)
return run(ctx, func() error {
return req.Do()
}, a.retry, false, req)
}, a.retry, false, setRetryHeaderHTTP(req))
}

func (a *ACLHandle) objectList(ctx context.Context) ([]ACLRule, error) {
Expand All @@ -187,7 +187,7 @@ func (a *ACLHandle) objectList(ctx context.Context) ([]ACLRule, error) {
err = run(ctx, func() error {
acls, err = req.Do()
return err
}, a.retry, true, req)
}, a.retry, true, setRetryHeaderHTTP(req))
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -215,15 +215,15 @@ func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRol
return run(ctx, func() error {
_, err := req.Do()
return err
}, a.retry, false, req)
}, a.retry, false, setRetryHeaderHTTP(req))
}

func (a *ACLHandle) objectDelete(ctx context.Context, entity ACLEntity) error {
req := a.c.raw.ObjectAccessControls.Delete(a.bucket, a.object, string(entity))
a.configureCall(ctx, req)
return run(ctx, func() error {
return req.Do()
}, a.retry, false, req)
}, a.retry, false, setRetryHeaderHTTP(req))
}

func (a *ACLHandle) configureCall(ctx context.Context, call interface{ Header() http.Header }) {
Expand Down
14 changes: 7 additions & 7 deletions storage/bucket.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *Buck
if attrs != nil && attrs.PredefinedDefaultObjectACL != "" {
req.PredefinedDefaultObjectAcl(attrs.PredefinedDefaultObjectACL)
}
return run(ctx, func() error { _, err := req.Context(ctx).Do(); return err }, b.retry, true, req)
return run(ctx, func() error { _, err := req.Context(ctx).Do(); return err }, b.retry, true, setRetryHeaderHTTP(req))
}

// Delete deletes the Bucket.
Expand All @@ -116,7 +116,7 @@ func (b *BucketHandle) Delete(ctx context.Context) (err error) {
return err
}

return run(ctx, func() error { return req.Context(ctx).Do() }, b.retry, true, req)
return run(ctx, func() error { return req.Context(ctx).Do() }, b.retry, true, setRetryHeaderHTTP(req))
}

func (b *BucketHandle) newDeleteCall() (*raw.BucketsDeleteCall, error) {
Expand Down Expand Up @@ -184,7 +184,7 @@ func (b *BucketHandle) Attrs(ctx context.Context) (attrs *BucketAttrs, err error
err = run(ctx, func() error {
resp, err = req.Context(ctx).Do()
return err
}, b.retry, true, req)
}, b.retry, true, setRetryHeaderHTTP(req))
var e *googleapi.Error
if ok := errors.As(err, &e); ok && e.Code == http.StatusNotFound {
return nil, ErrBucketNotExist
Expand Down Expand Up @@ -232,7 +232,7 @@ func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) (
return err
}

if err := run(ctx, call, b.retry, isIdempotent, req); err != nil {
if err := run(ctx, call, b.retry, isIdempotent, setRetryHeaderHTTP(req)); err != nil {
return nil, err
}
return newBucket(rawBucket)
Expand Down Expand Up @@ -1324,7 +1324,7 @@ func (b *BucketHandle) LockRetentionPolicy(ctx context.Context) error {
return run(ctx, func() error {
_, err := req.Context(ctx).Do()
return err
}, b.retry, true, req)
}, b.retry, true, setRetryHeaderHTTP(req))
}

// applyBucketConds modifies the provided call using the conditions in conds.
Expand Down Expand Up @@ -1982,7 +1982,7 @@ func (it *ObjectIterator) fetch(pageSize int, pageToken string) (string, error)
err = run(it.ctx, func() error {
resp, err = req.Context(it.ctx).Do()
return err
}, it.bucket.retry, true, req)
}, it.bucket.retry, true, setRetryHeaderHTTP(req))
if err != nil {
var e *googleapi.Error
if ok := errors.As(err, &e); ok && e.Code == http.StatusNotFound {
Expand Down Expand Up @@ -2069,7 +2069,7 @@ func (it *BucketIterator) fetch(pageSize int, pageToken string) (token string, e
err = run(it.ctx, func() error {
resp, err = req.Context(it.ctx).Do()
return err
}, it.client.retry, true, req)
}, it.client.retry, true, setRetryHeaderHTTP(req))
if err != nil {
return "", err
}
Expand Down
4 changes: 2 additions & 2 deletions storage/copy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ func (c *Copier) callRewrite(ctx context.Context, rawObj *raw.Object) (*raw.Rewr
retryCall := func() error { res, err = call.Do(); return err }
isIdempotent := c.dst.conds != nil && (c.dst.conds.GenerationMatch != 0 || c.dst.conds.DoesNotExist)

if err := run(ctx, retryCall, c.dst.retry, isIdempotent, call); err != nil {
if err := run(ctx, retryCall, c.dst.retry, isIdempotent, setRetryHeaderHTTP(call)); err != nil {
return nil, err
}
c.RewriteToken = res.RewriteToken
Expand Down Expand Up @@ -237,7 +237,7 @@ func (c *Composer) Run(ctx context.Context) (attrs *ObjectAttrs, err error) {
retryCall := func() error { obj, err = call.Do(); return err }
isIdempotent := c.dst.conds != nil && (c.dst.conds.GenerationMatch != 0 || c.dst.conds.DoesNotExist)

if err := run(ctx, retryCall, c.dst.retry, isIdempotent, call); err != nil {
if err := run(ctx, retryCall, c.dst.retry, isIdempotent, setRetryHeaderHTTP(call)); err != nil {
return nil, err
}
return newObject(obj), nil
Expand Down
20 changes: 10 additions & 10 deletions storage/grpc_client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ func (c *grpcStorageClient) GetServiceAccount(ctx context.Context, project strin
var err error
resp, err = c.raw.GetServiceAccount(ctx, req, s.gax...)
return err
}, s.retry, s.idempotent, nil)
}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))
if err != nil {
return "", err
}
Expand Down Expand Up @@ -153,7 +153,7 @@ func (c *grpcStorageClient) CreateBucket(ctx context.Context, project string, at
battrs = newBucketFromProto(res)

return err
}, s.retry, s.idempotent, nil)
}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))

return battrs, err
}
Expand Down Expand Up @@ -186,7 +186,7 @@ func (c *grpcStorageClient) ListBuckets(ctx context.Context, project string, opt
err = run(it.ctx, func() error {
buckets, next, err = gitr.InternalFetch(pageSize, pageToken)
return err
}, s.retry, s.idempotent, nil)
}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))
if err != nil {
return "", err
}
Expand Down Expand Up @@ -224,7 +224,7 @@ func (c *grpcStorageClient) DeleteBucket(ctx context.Context, bucket string, con

return run(ctx, func() error {
return c.raw.DeleteBucket(ctx, req, s.gax...)
}, s.retry, s.idempotent, nil)
}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))
}

func (c *grpcStorageClient) GetBucket(ctx context.Context, bucket string, conds *BucketConditions, opts ...storageOption) (*BucketAttrs, error) {
Expand All @@ -248,7 +248,7 @@ func (c *grpcStorageClient) GetBucket(ctx context.Context, bucket string, conds
battrs = newBucketFromProto(res)

return err
}, s.retry, s.idempotent, nil)
}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))

if s, ok := status.FromError(err); ok && s.Code() == codes.NotFound {
return nil, ErrBucketNotExist
Expand Down Expand Up @@ -328,7 +328,7 @@ func (c *grpcStorageClient) UpdateBucket(ctx context.Context, bucket string, uat
res, err := c.raw.UpdateBucket(ctx, req, s.gax...)
battrs = newBucketFromProto(res)
return err
}, s.retry, s.idempotent, nil)
}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))

return battrs, err
}
Expand Down Expand Up @@ -362,7 +362,7 @@ func (c *grpcStorageClient) ListObjects(ctx context.Context, bucket string, q *Q
err = run(it.ctx, func() error {
objects, token, err = gitr.InternalFetch(pageSize, pageToken)
return err
}, s.retry, s.idempotent, nil)
}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))
if err != nil {
if st, ok := status.FromError(err); ok && st.Code() == codes.NotFound {
err = ErrBucketNotExist
Expand Down Expand Up @@ -473,7 +473,7 @@ func (c *grpcStorageClient) GetIamPolicy(ctx context.Context, resource string, v
var err error
rp, err = c.raw.GetIamPolicy(ctx, req, s.gax...)
return err
}, s.retry, s.idempotent, nil)
}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))

return rp, err
}
Expand All @@ -490,7 +490,7 @@ func (c *grpcStorageClient) SetIamPolicy(ctx context.Context, resource string, p
return run(ctx, func() error {
_, err := c.raw.SetIamPolicy(ctx, req, s.gax...)
return err
}, s.retry, s.idempotent, nil)
}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))
}

func (c *grpcStorageClient) TestIamPermissions(ctx context.Context, resource string, permissions []string, opts ...storageOption) ([]string, error) {
Expand All @@ -505,7 +505,7 @@ func (c *grpcStorageClient) TestIamPermissions(ctx context.Context, resource str
var err error
res, err = c.raw.TestIamPermissions(ctx, req, s.gax...)
return err
}, s.retry, s.idempotent, nil)
}, s.retry, s.idempotent, setRetryHeaderGRPC(ctx))
if err != nil {
return nil, err
}
Expand Down
10 changes: 5 additions & 5 deletions storage/hmac.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ func (hkh *HMACKeyHandle) Get(ctx context.Context, opts ...HMACKeyOption) (*HMAC
err = run(ctx, func() error {
metadata, err = call.Context(ctx).Do()
return err
}, hkh.retry, true, call)
}, hkh.retry, true, setRetryHeaderHTTP(call))
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -159,7 +159,7 @@ func (hkh *HMACKeyHandle) Delete(ctx context.Context, opts ...HMACKeyOption) err

return run(ctx, func() error {
return delCall.Context(ctx).Do()
}, hkh.retry, true, delCall)
}, hkh.retry, true, setRetryHeaderHTTP(delCall))
}

func pbHmacKeyToHMACKey(pb *raw.HmacKey, updatedTimeCanBeNil bool) (*HMACKey, error) {
Expand Down Expand Up @@ -221,7 +221,7 @@ func (c *Client) CreateHMACKey(ctx context.Context, projectID, serviceAccountEma
h, err := call.Context(ctx).Do()
hkPb = h
return err
}, c.retry, false, call); err != nil {
}, c.retry, false, setRetryHeaderHTTP(call)); err != nil {
return nil, err
}

Expand Down Expand Up @@ -267,7 +267,7 @@ func (h *HMACKeyHandle) Update(ctx context.Context, au HMACKeyAttrsToUpdate, opt
err = run(ctx, func() error {
metadata, err = call.Context(ctx).Do()
return err
}, h.retry, isIdempotent, call)
}, h.retry, isIdempotent, setRetryHeaderHTTP(call))

if err != nil {
return nil, err
Expand Down Expand Up @@ -373,7 +373,7 @@ func (it *HMACKeysIterator) fetch(pageSize int, pageToken string) (token string,
err = run(it.ctx, func() error {
resp, err = call.Context(ctx).Do()
return err
}, it.retry, true, call)
}, it.retry, true, setRetryHeaderHTTP(call))
if err != nil {
return "", err
}
Expand Down
24 changes: 12 additions & 12 deletions storage/http_client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ func (c *httpStorageClient) GetServiceAccount(ctx context.Context, project strin
var err error
res, err = call.Context(ctx).Do()
return err
}, s.retry, s.idempotent, call)
}, s.retry, s.idempotent, setRetryHeaderHTTP(call))
if err != nil {
return "", err
}
Expand Down Expand Up @@ -182,7 +182,7 @@ func (c *httpStorageClient) CreateBucket(ctx context.Context, project string, at
}
battrs, err = newBucket(b)
return err
}, s.retry, s.idempotent, req)
}, s.retry, s.idempotent, setRetryHeaderHTTP(req))
return battrs, err
}

Expand All @@ -206,7 +206,7 @@ func (c *httpStorageClient) ListBuckets(ctx context.Context, project string, opt
err = run(it.ctx, func() error {
resp, err = req.Context(it.ctx).Do()
return err
}, s.retry, s.idempotent, req)
}, s.retry, s.idempotent, setRetryHeaderHTTP(req))
if err != nil {
return "", err
}
Expand Down Expand Up @@ -241,7 +241,7 @@ func (c *httpStorageClient) DeleteBucket(ctx context.Context, bucket string, con
req.UserProject(s.userProject)
}

return run(ctx, func() error { return req.Context(ctx).Do() }, s.retry, s.idempotent, req)
return run(ctx, func() error { return req.Context(ctx).Do() }, s.retry, s.idempotent, setRetryHeaderHTTP(req))
}

func (c *httpStorageClient) GetBucket(ctx context.Context, bucket string, conds *BucketConditions, opts ...storageOption) (*BucketAttrs, error) {
Expand All @@ -260,7 +260,7 @@ func (c *httpStorageClient) GetBucket(ctx context.Context, bucket string, conds
err = run(ctx, func() error {
resp, err = req.Context(ctx).Do()
return err
}, s.retry, s.idempotent, req)
}, s.retry, s.idempotent, setRetryHeaderHTTP(req))

var e *googleapi.Error
if ok := errors.As(err, &e); ok && e.Code == http.StatusNotFound {
Expand Down Expand Up @@ -294,7 +294,7 @@ func (c *httpStorageClient) UpdateBucket(ctx context.Context, bucket string, uat
err = run(ctx, func() error {
rawBucket, err = req.Context(ctx).Do()
return err
}, s.retry, s.idempotent, req)
}, s.retry, s.idempotent, setRetryHeaderHTTP(req))
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -341,7 +341,7 @@ func (c *httpStorageClient) ListObjects(ctx context.Context, bucket string, q *Q
err = run(it.ctx, func() error {
resp, err = req.Context(it.ctx).Do()
return err
}, s.retry, s.idempotent, req)
}, s.retry, s.idempotent, setRetryHeaderHTTP(req))
if err != nil {
var e *googleapi.Error
if ok := errors.As(err, &e); ok && e.Code == http.StatusNotFound {
Expand Down Expand Up @@ -392,7 +392,7 @@ func (c *httpStorageClient) ListDefaultObjectACLs(ctx context.Context, bucket st
err = run(ctx, func() error {
acls, err = req.Do()
return err
}, s.retry, true, req)
}, s.retry, true, setRetryHeaderHTTP(req))
if err != nil {
return nil, err
}
Expand All @@ -416,7 +416,7 @@ func (c *httpStorageClient) ListBucketACLs(ctx context.Context, bucket string, o
err = run(ctx, func() error {
acls, err = req.Do()
return err
}, s.retry, true, req)
}, s.retry, true, setRetryHeaderHTTP(req))
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -480,7 +480,7 @@ func (c *httpStorageClient) GetIamPolicy(ctx context.Context, resource string, v
var err error
rp, err = call.Context(ctx).Do()
return err
}, s.retry, s.idempotent, call)
}, s.retry, s.idempotent, setRetryHeaderHTTP(call))
if err != nil {
return nil, err
}
Expand All @@ -500,7 +500,7 @@ func (c *httpStorageClient) SetIamPolicy(ctx context.Context, resource string, p
return run(ctx, func() error {
_, err := call.Context(ctx).Do()
return err
}, s.retry, s.idempotent, call)
}, s.retry, s.idempotent, setRetryHeaderHTTP(call))
}

func (c *httpStorageClient) TestIamPermissions(ctx context.Context, resource string, permissions []string, opts ...storageOption) ([]string, error) {
Expand All @@ -515,7 +515,7 @@ func (c *httpStorageClient) TestIamPermissions(ctx context.Context, resource str
var err error
res, err = call.Context(ctx).Do()
return err
}, s.retry, s.idempotent, call)
}, s.retry, s.idempotent, setRetryHeaderHTTP(call))
if err != nil {
return nil, err
}
Expand Down
Loading