From 16ecfd150ff1982f03d207a80a82e934d1013874 Mon Sep 17 00:00:00 2001 From: Brenna N Epp Date: Wed, 13 Dec 2023 17:58:56 -0600 Subject: [PATCH] feat(storage): add object retention feature (#9072) --- storage/bucket.go | 47 +++++++-- storage/bucket_test.go | 8 +- storage/client.go | 13 ++- storage/client_test.go | 57 +++++------ storage/grpc_client.go | 39 ++++++-- storage/http_client.go | 31 ++++-- storage/integration_test.go | 194 ++++++++++++++++++++++++++++++++++++ storage/storage.go | 99 +++++++++++++++--- 8 files changed, 417 insertions(+), 71 deletions(-) diff --git a/storage/bucket.go b/storage/bucket.go index bc4bf101da1c..1059d4e8b7dd 100644 --- a/storage/bucket.go +++ b/storage/bucket.go @@ -41,13 +41,14 @@ import ( // BucketHandle provides operations on a Google Cloud Storage bucket. // Use Client.Bucket to get a handle. type BucketHandle struct { - c *Client - name string - acl ACLHandle - defaultObjectACL ACLHandle - conds *BucketConditions - userProject string // project for Requester Pays buckets - retry *retryConfig + c *Client + name string + acl ACLHandle + defaultObjectACL ACLHandle + conds *BucketConditions + userProject string // project for Requester Pays buckets + retry *retryConfig + enableObjectRetention *bool } // Bucket returns a BucketHandle, which provides operations on the named bucket. @@ -85,7 +86,8 @@ func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *Buck defer func() { trace.EndSpan(ctx, err) }() o := makeStorageOpts(true, b.retry, b.userProject) - if _, err := b.c.tc.CreateBucket(ctx, projectID, b.name, attrs, o...); err != nil { + + if _, err := b.c.tc.CreateBucket(ctx, projectID, b.name, attrs, b.enableObjectRetention, o...); err != nil { return err } return nil @@ -462,6 +464,15 @@ type BucketAttrs struct { // allows for the automatic selection of the best storage class // based on object access patterns. Autoclass *Autoclass + + // ObjectRetentionMode reports whether individual objects in the bucket can + // be configured with a retention policy. An empty value means that object + // retention is disabled. + // This field is read-only. Object retention can be enabled only by creating + // a bucket with SetObjectRetention set to true on the BucketHandle. It + // cannot be modified once the bucket is created. + // ObjectRetention cannot be configured or reported through the gRPC API. + ObjectRetentionMode string } // BucketPolicyOnly is an alias for UniformBucketLevelAccess. @@ -757,6 +768,7 @@ func newBucket(b *raw.Bucket) (*BucketAttrs, error) { if err != nil { return nil, err } + return &BucketAttrs{ Name: b.Name, Location: b.Location, @@ -771,6 +783,7 @@ func newBucket(b *raw.Bucket) (*BucketAttrs, error) { RequesterPays: b.Billing != nil && b.Billing.RequesterPays, Lifecycle: toLifecycle(b.Lifecycle), RetentionPolicy: rp, + ObjectRetentionMode: toBucketObjectRetention(b.ObjectRetention), CORS: toCORS(b.Cors), Encryption: toBucketEncryption(b.Encryption), Logging: toBucketLogging(b.Logging), @@ -1348,6 +1361,17 @@ func (b *BucketHandle) LockRetentionPolicy(ctx context.Context) error { return b.c.tc.LockBucketRetentionPolicy(ctx, b.name, b.conds, o...) } +// SetObjectRetention returns a new BucketHandle that will enable object retention +// on bucket creation. To enable object retention, you must use the returned +// handle to create the bucket. This has no effect on an already existing bucket. +// ObjectRetention is not enabled by default. +// ObjectRetention cannot be configured through the gRPC API. +func (b *BucketHandle) SetObjectRetention(enable bool) *BucketHandle { + b2 := *b + b2.enableObjectRetention = &enable + return &b2 +} + // applyBucketConds modifies the provided call using the conditions in conds. // call is something that quacks like a *raw.WhateverCall. func applyBucketConds(method string, conds *BucketConditions, call interface{}) error { @@ -1447,6 +1471,13 @@ func toRetentionPolicyFromProto(rp *storagepb.Bucket_RetentionPolicy) *Retention } } +func toBucketObjectRetention(or *raw.BucketObjectRetention) string { + if or == nil { + return "" + } + return or.Mode +} + func toRawCORS(c []CORS) []*raw.BucketCors { var out []*raw.BucketCors for _, v := range c { diff --git a/storage/bucket_test.go b/storage/bucket_test.go index eeb38097e2a0..6ba32daf1463 100644 --- a/storage/bucket_test.go +++ b/storage/bucket_test.go @@ -621,6 +621,9 @@ func TestNewBucket(t *testing.T) { RetentionPeriod: 3, EffectiveTime: aTime.Format(time.RFC3339), }, + ObjectRetention: &raw.BucketObjectRetention{ + Mode: "Enabled", + }, IamConfiguration: &raw.BucketIamConfiguration{ BucketPolicyOnly: &raw.BucketIamConfigurationBucketPolicyOnly{ Enabled: true, @@ -686,6 +689,7 @@ func TestNewBucket(t *testing.T) { EffectiveTime: aTime, RetentionPeriod: 3 * time.Second, }, + ObjectRetentionMode: "Enabled", BucketPolicyOnly: BucketPolicyOnly{Enabled: true, LockedTime: aTime}, UniformBucketLevelAccess: UniformBucketLevelAccess{Enabled: true, LockedTime: aTime}, CORS: []CORS{ @@ -714,7 +718,7 @@ func TestNewBucket(t *testing.T) { if err != nil { t.Fatal(err) } - if diff := testutil.Diff(got, want); diff != "" { + if diff := cmp.Diff(got, want); diff != "" { t.Errorf("got=-, want=+:\n%s", diff) } } @@ -817,7 +821,7 @@ func TestNewBucketFromProto(t *testing.T) { }, } got := newBucketFromProto(pb) - if diff := testutil.Diff(got, want); diff != "" { + if diff := cmp.Diff(got, want); diff != "" { t.Errorf("got=-, want=+:\n%s", diff) } } diff --git a/storage/client.go b/storage/client.go index 3bed9b64c2ba..4906b1d1f704 100644 --- a/storage/client.go +++ b/storage/client.go @@ -44,7 +44,7 @@ type storageClient interface { // Top-level methods. GetServiceAccount(ctx context.Context, project string, opts ...storageOption) (string, error) - CreateBucket(ctx context.Context, project, bucket string, attrs *BucketAttrs, opts ...storageOption) (*BucketAttrs, error) + CreateBucket(ctx context.Context, project, bucket string, attrs *BucketAttrs, enableObjectRetention *bool, opts ...storageOption) (*BucketAttrs, error) ListBuckets(ctx context.Context, project string, opts ...storageOption) *BucketIterator Close() error @@ -60,7 +60,7 @@ type storageClient interface { DeleteObject(ctx context.Context, bucket, object string, gen int64, conds *Conditions, opts ...storageOption) error GetObject(ctx context.Context, bucket, object string, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error) - UpdateObject(ctx context.Context, bucket, object string, uattrs *ObjectAttrsToUpdate, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error) + UpdateObject(ctx context.Context, params *updateObjectParams, opts ...storageOption) (*ObjectAttrs, error) // Default Object ACL methods. @@ -291,6 +291,15 @@ type newRangeReaderParams struct { readCompressed bool // Use accept-encoding: gzip. Only works for HTTP currently. } +type updateObjectParams struct { + bucket, object string + uattrs *ObjectAttrsToUpdate + gen int64 + encryptionKey []byte + conds *Conditions + overrideRetention *bool +} + type composeObjectRequest struct { dstBucket string dstObject destinationObject diff --git a/storage/client_test.go b/storage/client_test.go index a9a8a42ce270..6731e30f7940 100644 --- a/storage/client_test.go +++ b/storage/client_test.go @@ -40,7 +40,7 @@ func TestCreateBucketEmulated(t *testing.T) { LogBucket: bucket, }, } - got, err := client.CreateBucket(context.Background(), project, want.Name, want) + got, err := client.CreateBucket(context.Background(), project, want.Name, want, nil) if err != nil { t.Fatal(err) } @@ -63,7 +63,7 @@ func TestDeleteBucketEmulated(t *testing.T) { Name: bucket, } // Create the bucket that will be deleted. - _, err := client.CreateBucket(context.Background(), project, b.Name, b) + _, err := client.CreateBucket(context.Background(), project, b.Name, b, nil) if err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -81,7 +81,7 @@ func TestGetBucketEmulated(t *testing.T) { Name: bucket, } // Create the bucket that will be retrieved. - _, err := client.CreateBucket(context.Background(), project, want.Name, want) + _, err := client.CreateBucket(context.Background(), project, want.Name, want, nil) if err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -101,7 +101,7 @@ func TestUpdateBucketEmulated(t *testing.T) { Name: bucket, } // Create the bucket that will be updated. - _, err := client.CreateBucket(context.Background(), project, bkt.Name, bkt) + _, err := client.CreateBucket(context.Background(), project, bkt.Name, bkt, nil) if err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -194,7 +194,7 @@ func TestGetSetTestIamPolicyEmulated(t *testing.T) { transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) { battrs, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{ Name: bucket, - }) + }, nil) if err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -225,7 +225,7 @@ func TestDeleteObjectEmulated(t *testing.T) { // Populate test object that will be deleted. _, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{ Name: bucket, - }) + }, nil) if err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -252,7 +252,7 @@ func TestGetObjectEmulated(t *testing.T) { // Populate test object. _, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{ Name: bucket, - }) + }, nil) if err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -282,7 +282,7 @@ func TestRewriteObjectEmulated(t *testing.T) { // Populate test object. _, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{ Name: bucket, - }) + }, nil) if err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -327,7 +327,7 @@ func TestUpdateObjectEmulated(t *testing.T) { // Populate test object. _, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{ Name: bucket, - }) + }, nil) if err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -355,7 +355,8 @@ func TestUpdateObjectEmulated(t *testing.T) { CustomTime: ct.Add(10 * time.Hour), } - got, err := client.UpdateObject(context.Background(), bucket, o.Name, want, defaultGen, nil, &Conditions{MetagenerationMatch: 1}) + params := &updateObjectParams{bucket: bucket, object: o.Name, uattrs: want, gen: defaultGen, conds: &Conditions{MetagenerationMatch: 1}} + got, err := client.UpdateObject(context.Background(), params) if err != nil { t.Fatalf("client.UpdateObject: %v", err) } @@ -394,7 +395,7 @@ func TestListObjectsEmulated(t *testing.T) { // Populate test data. _, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{ Name: bucket, - }) + }, nil) if err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -452,7 +453,7 @@ func TestListObjectsWithPrefixEmulated(t *testing.T) { // Populate test data. _, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{ Name: bucket, - }) + }, nil) if err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -516,7 +517,7 @@ func TestListBucketsEmulated(t *testing.T) { } // Create the buckets that will be listed. for _, b := range want { - _, err := client.CreateBucket(context.Background(), project, b.Name, b) + _, err := client.CreateBucket(context.Background(), project, b.Name, b, nil) if err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -557,7 +558,7 @@ func TestListBucketACLsEmulated(t *testing.T) { PredefinedACL: "publicRead", } // Create the bucket that will be retrieved. - if _, err := client.CreateBucket(ctx, project, attrs.Name, attrs); err != nil { + if _, err := client.CreateBucket(ctx, project, attrs.Name, attrs, nil); err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -579,7 +580,7 @@ func TestUpdateBucketACLEmulated(t *testing.T) { PredefinedACL: "authenticatedRead", } // Create the bucket that will be retrieved. - if _, err := client.CreateBucket(ctx, project, attrs.Name, attrs); err != nil { + if _, err := client.CreateBucket(ctx, project, attrs.Name, attrs, nil); err != nil { t.Fatalf("client.CreateBucket: %v", err) } var listAcls []ACLRule @@ -615,7 +616,7 @@ func TestDeleteBucketACLEmulated(t *testing.T) { PredefinedACL: "publicRead", } // Create the bucket that will be retrieved. - if _, err := client.CreateBucket(ctx, project, attrs.Name, attrs); err != nil { + if _, err := client.CreateBucket(ctx, project, attrs.Name, attrs, nil); err != nil { t.Fatalf("client.CreateBucket: %v", err) } // Assert bucket has two BucketACL entities, including project owner and predefinedACL. @@ -649,7 +650,7 @@ func TestDefaultObjectACLCRUDEmulated(t *testing.T) { PredefinedDefaultObjectACL: "publicRead", } // Create the bucket that will be retrieved. - if _, err := client.CreateBucket(ctx, project, attrs.Name, attrs); err != nil { + if _, err := client.CreateBucket(ctx, project, attrs.Name, attrs, nil); err != nil { t.Fatalf("client.CreateBucket: %v", err) } // Assert bucket has 2 DefaultObjectACL entities, including project owner and PredefinedDefaultObjectACL. @@ -695,7 +696,7 @@ func TestObjectACLCRUDEmulated(t *testing.T) { ctx := context.Background() _, err := client.CreateBucket(ctx, project, bucket, &BucketAttrs{ Name: bucket, - }) + }, nil) if err != nil { t.Fatalf("CreateBucket: %v", err) } @@ -749,7 +750,7 @@ func TestOpenReaderEmulated(t *testing.T) { // Populate test data. _, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{ Name: bucket, - }) + }, nil) if err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -798,7 +799,7 @@ func TestOpenWriterEmulated(t *testing.T) { // Populate test data. _, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{ Name: bucket, - }) + }, nil) if err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -861,7 +862,7 @@ func TestListNotificationsEmulated(t *testing.T) { ctx := context.Background() _, err := client.CreateBucket(ctx, project, bucket, &BucketAttrs{ Name: bucket, - }) + }, nil) if err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -889,7 +890,7 @@ func TestCreateNotificationEmulated(t *testing.T) { ctx := context.Background() _, err := client.CreateBucket(ctx, project, bucket, &BucketAttrs{ Name: bucket, - }) + }, nil) if err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -915,7 +916,7 @@ func TestDeleteNotificationEmulated(t *testing.T) { ctx := context.Background() _, err := client.CreateBucket(ctx, project, bucket, &BucketAttrs{ Name: bucket, - }) + }, nil) if err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -987,7 +988,7 @@ func TestLockBucketRetentionPolicyEmulated(t *testing.T) { }, } // Create the bucket that will be locked. - _, err := client.CreateBucket(context.Background(), project, b.Name, b) + _, err := client.CreateBucket(context.Background(), project, b.Name, b, nil) if err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -1013,7 +1014,7 @@ func TestComposeEmulated(t *testing.T) { // Populate test data. _, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{ Name: bucket, - }) + }, nil) if err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -1178,7 +1179,7 @@ func TestObjectConditionsEmulated(t *testing.T) { ctx := context.Background() // Create test bucket - if _, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{Name: bucket}); err != nil { + if _, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{Name: bucket}, nil); err != nil { t.Fatalf("client.CreateBucket: %v", err) } @@ -1194,7 +1195,7 @@ func TestObjectConditionsEmulated(t *testing.T) { return fmt.Errorf("creating object: %w", err) } uattrs := &ObjectAttrsToUpdate{CustomTime: time.Now()} - _, err = client.UpdateObject(ctx, bucket, objName, uattrs, gen, nil, nil) + _, err = client.UpdateObject(ctx, &updateObjectParams{bucket: bucket, object: objName, uattrs: uattrs, gen: gen}) return err }, }, @@ -1210,7 +1211,7 @@ func TestObjectConditionsEmulated(t *testing.T) { GenerationMatch: gen, MetagenerationMatch: metaGen, } - _, err = client.UpdateObject(ctx, bucket, objName, uattrs, gen, nil, conds) + _, err = client.UpdateObject(ctx, &updateObjectParams{bucket: bucket, object: objName, uattrs: uattrs, gen: gen, conds: conds}) return err }, }, diff --git a/storage/grpc_client.go b/storage/grpc_client.go index 032a113aed6a..a51cf9c086b1 100644 --- a/storage/grpc_client.go +++ b/storage/grpc_client.go @@ -152,7 +152,12 @@ func (c *grpcStorageClient) GetServiceAccount(ctx context.Context, project strin return resp.EmailAddress, err } -func (c *grpcStorageClient) CreateBucket(ctx context.Context, project, bucket string, attrs *BucketAttrs, opts ...storageOption) (*BucketAttrs, error) { +func (c *grpcStorageClient) CreateBucket(ctx context.Context, project, bucket string, attrs *BucketAttrs, enableObjectRetention *bool, opts ...storageOption) (*BucketAttrs, error) { + if enableObjectRetention != nil { + // TO-DO: implement ObjectRetention once available - see b/308194853 + return nil, status.Errorf(codes.Unimplemented, "storage: object retention is not supported in gRPC") + } + s := callSettings(c.settings, opts...) b := attrs.toProtoBucket() b.Project = toProjectResource(project) @@ -507,25 +512,30 @@ func (c *grpcStorageClient) GetObject(ctx context.Context, bucket, object string return attrs, err } -func (c *grpcStorageClient) UpdateObject(ctx context.Context, bucket, object string, uattrs *ObjectAttrsToUpdate, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error) { +func (c *grpcStorageClient) UpdateObject(ctx context.Context, params *updateObjectParams, opts ...storageOption) (*ObjectAttrs, error) { + uattrs := params.uattrs + if params.overrideRetention != nil || uattrs.Retention != nil { + // TO-DO: implement ObjectRetention once available - see b/308194853 + return nil, status.Errorf(codes.Unimplemented, "storage: object retention is not supported in gRPC") + } s := callSettings(c.settings, opts...) - o := uattrs.toProtoObject(bucketResourceName(globalProjectAlias, bucket), object) + o := uattrs.toProtoObject(bucketResourceName(globalProjectAlias, params.bucket), params.object) // For Update, generation is passed via the object message rather than a field on the request. - if gen >= 0 { - o.Generation = gen + if params.gen >= 0 { + o.Generation = params.gen } req := &storagepb.UpdateObjectRequest{ Object: o, PredefinedAcl: uattrs.PredefinedACL, } - if err := applyCondsProto("grpcStorageClient.UpdateObject", defaultGen, conds, req); err != nil { + if err := applyCondsProto("grpcStorageClient.UpdateObject", defaultGen, params.conds, req); err != nil { return nil, err } if s.userProject != "" { ctx = setUserProjectMetadata(ctx, s.userProject) } - if encryptionKey != nil { - req.CommonObjectRequestParams = toProtoCommonObjectRequestParams(encryptionKey) + if params.encryptionKey != nil { + req.CommonObjectRequestParams = toProtoCommonObjectRequestParams(params.encryptionKey) } fieldMask := &fieldmaskpb.FieldMask{Paths: nil} @@ -739,7 +749,8 @@ func (c *grpcStorageClient) DeleteObjectACL(ctx context.Context, bucket, object } uattrs := &ObjectAttrsToUpdate{ACL: acl} // Call UpdateObject with the specified metageneration. - if _, err = c.UpdateObject(ctx, bucket, object, uattrs, defaultGen, nil, &Conditions{MetagenerationMatch: attrs.Metageneration}, opts...); err != nil { + params := &updateObjectParams{bucket: bucket, object: object, uattrs: uattrs, gen: defaultGen, conds: &Conditions{MetagenerationMatch: attrs.Metageneration}} + if _, err = c.UpdateObject(ctx, params, opts...); err != nil { return err } return nil @@ -769,7 +780,8 @@ func (c *grpcStorageClient) UpdateObjectACL(ctx context.Context, bucket, object acl = append(attrs.ACL, aclRule) uattrs := &ObjectAttrsToUpdate{ACL: acl} // Call UpdateObject with the specified metageneration. - if _, err = c.UpdateObject(ctx, bucket, object, uattrs, defaultGen, nil, &Conditions{MetagenerationMatch: attrs.Metageneration}, opts...); err != nil { + params := &updateObjectParams{bucket: bucket, object: object, uattrs: uattrs, gen: defaultGen, conds: &Conditions{MetagenerationMatch: attrs.Metageneration}} + if _, err = c.UpdateObject(ctx, params, opts...); err != nil { return err } return nil @@ -1049,6 +1061,13 @@ func (c *grpcStorageClient) OpenWriter(params *openWriterParams, opts ...storage return } + if params.attrs.Retention != nil { + // TO-DO: remove once ObjectRetention is available - see b/308194853 + err = status.Errorf(codes.Unimplemented, "storage: object retention is not supported in gRPC") + errorf(err) + pr.CloseWithError(err) + return + } // The chunk buffer is full, but there is no end in sight. This // means that either: // 1. A resumable upload will need to be used to send diff --git a/storage/http_client.go b/storage/http_client.go index b62f009da1d2..0e157e4ba994 100644 --- a/storage/http_client.go +++ b/storage/http_client.go @@ -159,7 +159,7 @@ func (c *httpStorageClient) GetServiceAccount(ctx context.Context, project strin return res.EmailAddress, nil } -func (c *httpStorageClient) CreateBucket(ctx context.Context, project, bucket string, attrs *BucketAttrs, opts ...storageOption) (*BucketAttrs, error) { +func (c *httpStorageClient) CreateBucket(ctx context.Context, project, bucket string, attrs *BucketAttrs, enableObjectRetention *bool, opts ...storageOption) (*BucketAttrs, error) { s := callSettings(c.settings, opts...) var bkt *raw.Bucket if attrs != nil { @@ -181,6 +181,9 @@ func (c *httpStorageClient) CreateBucket(ctx context.Context, project, bucket st if attrs != nil && attrs.PredefinedDefaultObjectACL != "" { req.PredefinedDefaultObjectAcl(attrs.PredefinedDefaultObjectACL) } + if enableObjectRetention != nil { + req.EnableObjectRetention(*enableObjectRetention) + } var battrs *BucketAttrs err := run(ctx, func(ctx context.Context) error { b, err := req.Context(ctx).Do() @@ -431,7 +434,8 @@ func (c *httpStorageClient) GetObject(ctx context.Context, bucket, object string return newObject(obj), nil } -func (c *httpStorageClient) UpdateObject(ctx context.Context, bucket, object string, uattrs *ObjectAttrsToUpdate, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error) { +func (c *httpStorageClient) UpdateObject(ctx context.Context, params *updateObjectParams, opts ...storageOption) (*ObjectAttrs, error) { + uattrs := params.uattrs s := callSettings(c.settings, opts...) var attrs ObjectAttrs @@ -496,11 +500,21 @@ func (c *httpStorageClient) UpdateObject(ctx context.Context, bucket, object str // we don't append to nullFields here. forceSendFields = append(forceSendFields, "Acl") } - rawObj := attrs.toRawObject(bucket) + if uattrs.Retention != nil { + // For ObjectRetention it's an error to send empty fields. + // Instead we send a null as the user's intention is to remove. + if uattrs.Retention.Mode == "" && uattrs.Retention.RetainUntil.IsZero() { + nullFields = append(nullFields, "Retention") + } else { + attrs.Retention = uattrs.Retention + forceSendFields = append(forceSendFields, "Retention") + } + } + rawObj := attrs.toRawObject(params.bucket) rawObj.ForceSendFields = forceSendFields rawObj.NullFields = nullFields - call := c.raw.Objects.Patch(bucket, object, rawObj).Projection("full") - if err := applyConds("Update", gen, conds, call); err != nil { + call := c.raw.Objects.Patch(params.bucket, params.object, rawObj).Projection("full") + if err := applyConds("Update", params.gen, params.conds, call); err != nil { return nil, err } if s.userProject != "" { @@ -509,9 +523,14 @@ func (c *httpStorageClient) UpdateObject(ctx context.Context, bucket, object str if uattrs.PredefinedACL != "" { call.PredefinedAcl(uattrs.PredefinedACL) } - if err := setEncryptionHeaders(call.Header(), encryptionKey, false); err != nil { + if err := setEncryptionHeaders(call.Header(), params.encryptionKey, false); err != nil { return nil, err } + + if params.overrideRetention != nil { + call.OverrideUnlockedRetention(*params.overrideRetention) + } + var obj *raw.Object var err error err = run(ctx, func(ctx context.Context) error { obj, err = call.Context(ctx).Do(); return err }, s.retry, s.idempotent) diff --git a/storage/integration_test.go b/storage/integration_test.go index 0293a89f2548..29766060a74a 100644 --- a/storage/integration_test.go +++ b/storage/integration_test.go @@ -3886,6 +3886,200 @@ func TestIntegration_LockBucket_MetagenerationRequired(t *testing.T) { }) } +func TestIntegration_BucketObjectRetention(t *testing.T) { + ctx := skipJSONReads(skipGRPC("not yet available in gRPC - b/308194853"), "no reads in test") + multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { + setTrue, setFalse := true, false + + for _, test := range []struct { + desc string + enable *bool + wantRetentionMode string + }{ + { + desc: "ObjectRetentionMode is not enabled by default", + wantRetentionMode: "", + }, + { + desc: "Enable retention", + enable: &setTrue, + wantRetentionMode: "Enabled", + }, + { + desc: "Set object retention to false", + enable: &setFalse, + wantRetentionMode: "", + }, + } { + t.Run(test.desc, func(t *testing.T) { + b := client.Bucket(prefix + uidSpace.New()) + if test.enable != nil { + b = b.SetObjectRetention(*test.enable) + } + + err := b.Create(ctx, testutil.ProjID(), nil) + if err != nil { + t.Fatalf("error creating bucket: %v", err) + } + t.Cleanup(func() { b.Delete(ctx) }) + + attrs, err := b.Attrs(ctx) + if err != nil { + t.Fatalf("b.Attrs: %v", err) + } + if got, want := attrs.ObjectRetentionMode, test.wantRetentionMode; got != want { + t.Errorf("expected ObjectRetentionMode to be %q, got %q", want, got) + } + }) + } + }) +} + +func TestIntegration_ObjectRetention(t *testing.T) { + ctx := skipJSONReads(skipGRPC("not yet available in gRPC - b/308194853"), "no reads in test") + multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { + h := testHelper{t} + + b := client.Bucket(prefix + uidSpace.New()).SetObjectRetention(true) + + if err := b.Create(ctx, testutil.ProjID(), nil); err != nil { + t.Fatalf("error creating bucket: %v", err) + } + t.Cleanup(func() { h.mustDeleteBucket(b) }) + + retentionUnlocked := &ObjectRetention{ + Mode: "Unlocked", + RetainUntil: time.Now().Add(time.Minute * 20).Truncate(time.Second), + } + retentionUnlockedExtended := &ObjectRetention{ + Mode: "Unlocked", + RetainUntil: time.Now().Add(time.Hour).Truncate(time.Second), + } + + // Create an object with future retain until time + o := b.Object("retention-on-create" + uidSpaceObjects.New()) + w := o.NewWriter(ctx) + w.Retention = retentionUnlocked + h.mustWrite(w, []byte("contents")) + t.Cleanup(func() { + if _, err := o.OverrideUnlockedRetention(true).Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{}}); err != nil { + t.Fatalf("failed to remove retention from object: %v", err) + } + h.mustDeleteObject(o) + }) + + if got, want := w.Attrs().Retention, retentionUnlocked; got.Mode != want.Mode || !got.RetainUntil.Equal(want.RetainUntil) { + t.Errorf("mismatching retention config, got: %+v, want:%+v", got, want) + } + + // Delete object under retention returns 403 + if err := o.Delete(ctx); err == nil || extractErrCode(err) != http.StatusForbidden { + t.Fatalf("delete should have failed with: %v, instead got:%v", http.StatusForbidden, err) + } + + // Extend retain until time of Unlocked object is possible + attrs, err := o.Update(ctx, ObjectAttrsToUpdate{Retention: retentionUnlockedExtended}) + if err != nil { + t.Fatalf("failed to add retention to object: %v", err) + } + + if got, want := attrs.Retention, retentionUnlockedExtended; got.Mode != want.Mode || !got.RetainUntil.Equal(want.RetainUntil) { + t.Errorf("mismatching retention config, got: %+v, want:%+v", got, want) + } + + // Reduce retain until time of Unlocked object without + // override_unlocked_retention=True returns 403 + _, err = o.Update(ctx, ObjectAttrsToUpdate{Retention: retentionUnlocked}) + if err == nil || extractErrCode(err) != http.StatusForbidden { + t.Fatalf("o.Update should have failed with: %v, instead got:%v", http.StatusBadRequest, err) + } + + // Remove retention of Unlocked object without + // override_unlocked_retention=True returns 403 + _, err = o.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{}}) + if err == nil || extractErrCode(err) != http.StatusForbidden { + t.Fatalf("o.Update should have failed with: %v, instead got:%v", http.StatusBadRequest, err) + } + + // Reduce retain until time of Unlocked object with override_unlocked_retention=True + attrs, err = o.OverrideUnlockedRetention(true).Update(ctx, ObjectAttrsToUpdate{ + Retention: retentionUnlocked, + }) + if err != nil { + t.Fatalf("failed to add retention to object: %v", err) + } + + if got, want := attrs.Retention, retentionUnlocked; got.Mode != want.Mode || !got.RetainUntil.Equal(want.RetainUntil) { + t.Errorf("mismatching retention config, got: %+v, want:%+v", got, want) + } + + // Create a new object + objectWithRetentionOnUpdate := b.Object("retention-on-update" + uidSpaceObjects.New()) + w = objectWithRetentionOnUpdate.NewWriter(ctx) + h.mustWrite(w, []byte("contents")) + + // Retention should not be set + if got := w.Attrs().Retention; got != nil { + t.Errorf("expected no ObjectRetention, got: %+v", got) + } + + // Update object with only one of (retain until time, retention mode) returns 400 + _, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{Mode: "Locked"}}) + if err == nil || extractErrCode(err) != http.StatusBadRequest { + t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err) + } + + _, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{RetainUntil: time.Now().Add(time.Second)}}) + if err == nil || extractErrCode(err) != http.StatusBadRequest { + t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err) + } + + // Update object with future retain until time + attrs, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: retentionUnlocked}) + if err != nil { + t.Errorf("o.Update: %v", err) + } + + if got, want := attrs.Retention, retentionUnlocked; got.Mode != want.Mode || !got.RetainUntil.Equal(want.RetainUntil) { + t.Errorf("mismatching retention config, got: %+v, want:%+v", got, want) + } + + // Update/Patch object with retain until time in the past returns 400 + _, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{RetainUntil: time.Now().Add(-time.Second)}}) + if err == nil || extractErrCode(err) != http.StatusBadRequest { + t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err) + } + + // Update object with only one of (retain until time, retention mode) returns 400 + _, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{Mode: "Locked"}}) + if err == nil || extractErrCode(err) != http.StatusBadRequest { + t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err) + } + + _, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{RetainUntil: time.Now().Add(time.Second)}}) + if err == nil || extractErrCode(err) != http.StatusBadRequest { + t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err) + } + + // Remove retention of Unlocked object with override_unlocked_retention=True + attrs, err = objectWithRetentionOnUpdate.OverrideUnlockedRetention(true).Update(ctx, ObjectAttrsToUpdate{ + Retention: &ObjectRetention{}, + }) + if err != nil { + t.Fatalf("failed to remove retention from object: %v", err) + } + + if got := attrs.Retention; got != nil { + t.Errorf("mismatching retention config, got: %+v, wanted nil", got) + } + + // We should be able to delete the object as normal since retention was removed + if err := objectWithRetentionOnUpdate.Delete(ctx); err != nil { + t.Errorf("object.Delete:%v", err) + } + }) +} + func TestIntegration_KMS(t *testing.T) { multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, prefix string, client *Client) { h := testHelper{t} diff --git a/storage/storage.go b/storage/storage.go index eb83597c840f..78ecbf0e892a 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -879,16 +879,17 @@ func signedURLV2(bucket, name string, opts *SignedURLOptions) (string, error) { // ObjectHandle provides operations on an object in a Google Cloud Storage bucket. // Use BucketHandle.Object to get a handle. type ObjectHandle struct { - c *Client - bucket string - object string - acl ACLHandle - gen int64 // a negative value indicates latest - conds *Conditions - encryptionKey []byte // AES-256 key - userProject string // for requester-pays buckets - readCompressed bool // Accept-Encoding: gzip - retry *retryConfig + c *Client + bucket string + object string + acl ACLHandle + gen int64 // a negative value indicates latest + conds *Conditions + encryptionKey []byte // AES-256 key + userProject string // for requester-pays buckets + readCompressed bool // Accept-Encoding: gzip + retry *retryConfig + overrideRetention *bool } // ACL provides access to the object's access control list. @@ -958,7 +959,15 @@ func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) ( } isIdempotent := o.conds != nil && o.conds.MetagenerationMatch != 0 opts := makeStorageOpts(isIdempotent, o.retry, o.userProject) - return o.c.tc.UpdateObject(ctx, o.bucket, o.object, &uattrs, o.gen, o.encryptionKey, o.conds, opts...) + return o.c.tc.UpdateObject(ctx, + &updateObjectParams{ + bucket: o.bucket, + object: o.object, + uattrs: &uattrs, + gen: o.gen, + encryptionKey: o.encryptionKey, + conds: o.conds, + overrideRetention: o.overrideRetention}, opts...) } // BucketName returns the name of the bucket. @@ -973,16 +982,19 @@ func (o *ObjectHandle) ObjectName() string { // ObjectAttrsToUpdate is used to update the attributes of an object. // Only fields set to non-nil values will be updated. -// For all fields except CustomTime, set the field to its zero value to delete -// it. CustomTime cannot be deleted or changed to an earlier time once set. +// For all fields except CustomTime and Retention, set the field to its zero +// value to delete it. CustomTime cannot be deleted or changed to an earlier +// time once set. Retention can be deleted (only if the Mode is Unlocked) by +// setting it to an empty value (not nil). // -// For example, to change ContentType and delete ContentEncoding and -// Metadata, use +// For example, to change ContentType and delete ContentEncoding, Metadata and +// Retention, use: // // ObjectAttrsToUpdate{ // ContentType: "text/html", // ContentEncoding: "", // Metadata: map[string]string{}, +// Retention: &ObjectRetention{}, // } type ObjectAttrsToUpdate struct { EventBasedHold optional.Bool @@ -999,6 +1011,12 @@ type ObjectAttrsToUpdate struct { // If not empty, applies a predefined set of access controls. ACL must be nil. // See https://cloud.google.com/storage/docs/json_api/v1/objects/patch. PredefinedACL string + + // Retention contains the retention configuration for this object. + // Operations other than setting the retention for the first time or + // extending the RetainUntil time on the object retention must be done + // on an ObjectHandle with OverrideUnlockedRetention set to true. + Retention *ObjectRetention } // Delete deletes the single specified object. @@ -1020,6 +1038,17 @@ func (o *ObjectHandle) ReadCompressed(compressed bool) *ObjectHandle { return &o2 } +// OverrideUnlockedRetention provides an option for overriding an Unlocked +// Retention policy. This must be set to true in order to change a policy +// from Unlocked to Locked, to set it to null, or to reduce its +// RetainUntil attribute. It is not required for setting the ObjectRetention for +// the first time nor for extending the RetainUntil time. +func (o *ObjectHandle) OverrideUnlockedRetention(override bool) *ObjectHandle { + o2 := *o + o2.overrideRetention = &override + return &o2 +} + // NewWriter returns a storage Writer that writes to the GCS object // associated with this ObjectHandle. // @@ -1109,6 +1138,7 @@ func (o *ObjectAttrs) toRawObject(bucket string) *raw.Object { Acl: toRawObjectACL(o.ACL), Metadata: o.Metadata, CustomTime: ct, + Retention: o.Retention.toRawObjectRetention(), } } @@ -1344,6 +1374,42 @@ type ObjectAttrs struct { // For non-composite objects, the value will be zero. // This field is read-only. ComponentCount int64 + + // Retention contains the retention configuration for this object. + // ObjectRetention cannot be configured or reported through the gRPC API. + Retention *ObjectRetention +} + +// ObjectRetention contains the retention configuration for this object. +type ObjectRetention struct { + // Mode is the retention policy's mode on this object. Valid values are + // "Locked" and "Unlocked". + // Locked retention policies cannot be changed. Unlocked policies require an + // override to change. + Mode string + + // RetainUntil is the time this object will be retained until. + RetainUntil time.Time +} + +func (r *ObjectRetention) toRawObjectRetention() *raw.ObjectRetention { + if r == nil { + return nil + } + return &raw.ObjectRetention{ + Mode: r.Mode, + RetainUntilTime: r.RetainUntil.Format(time.RFC3339), + } +} + +func toObjectRetention(r *raw.ObjectRetention) *ObjectRetention { + if r == nil { + return nil + } + return &ObjectRetention{ + Mode: r.Mode, + RetainUntil: convertTime(r.RetainUntilTime), + } } // convertTime converts a time in RFC3339 format to time.Time. @@ -1415,6 +1481,7 @@ func newObject(o *raw.Object) *ObjectAttrs { Etag: o.Etag, CustomTime: convertTime(o.CustomTime), ComponentCount: o.ComponentCount, + Retention: toObjectRetention(o.Retention), } } @@ -1587,6 +1654,7 @@ var attrToFieldMap = map[string]string{ "Etag": "etag", "CustomTime": "customTime", "ComponentCount": "componentCount", + "Retention": "retention", } // attrToProtoFieldMap maps the field names of ObjectAttrs to the underlying field @@ -1621,6 +1689,7 @@ var attrToProtoFieldMap = map[string]string{ "ComponentCount": "component_count", // MediaLink was explicitly excluded from the proto as it is an HTTP-ism. // "MediaLink": "mediaLink", + // TODO: add object retention - b/308194853 } // SetAttrSelection makes the query populate only specific attributes of