feat(storage): add bucket HierarchicalNamespace (#10315)

Add hierarchical namespace configuration field to bucket metadata.

Fixes #10146

storage/bucket.go

@@ -491,6 +491,13 @@ type BucketAttrs struct {
 	// 7 day retention duration. In order to fully disable soft delete, you need
 	// to set a policy with a RetentionDuration of 0.
 	SoftDeletePolicy *SoftDeletePolicy
+
+	// HierarchicalNamespace contains the bucket's hierarchical namespace
+	// configuration. Hierarchical namespace enabled buckets can contain
+	// [cloud.google.com/go/storage/control/apiv2/controlpb.Folder] resources.
+	// It cannot be modified after bucket creation time.
+	// UniformBucketLevelAccess must also also be enabled on the bucket.
+	HierarchicalNamespace *HierarchicalNamespace
 }
 
 // BucketPolicyOnly is an alias for UniformBucketLevelAccess.
@@ -792,6 +799,15 @@ type SoftDeletePolicy struct {
 	RetentionDuration time.Duration
 }
 
+// HierarchicalNamespace contains the bucket's hierarchical namespace
+// configuration. Hierarchical namespace enabled buckets can contain
+// [cloud.google.com/go/storage/control/apiv2/controlpb.Folder] resources.
+type HierarchicalNamespace struct {
+	// Enabled indicates whether hierarchical namespace features are enabled on
+	// the bucket. This can only be set at bucket creation time currently.
+	Enabled bool
+}
+
 func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
 	if b == nil {
 		return nil, nil
@@ -830,6 +846,7 @@ func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
 		CustomPlacementConfig:    customPlacementFromRaw(b.CustomPlacementConfig),
 		Autoclass:                toAutoclassFromRaw(b.Autoclass),
 		SoftDeletePolicy:         toSoftDeletePolicyFromRaw(b.SoftDeletePolicy),
+		HierarchicalNamespace:    toHierarchicalNamespaceFromRaw(b.HierarchicalNamespace),
 	}, nil
 }
 
@@ -864,6 +881,7 @@ func newBucketFromProto(b *storagepb.Bucket) *BucketAttrs {
 		ProjectNumber:            parseProjectNumber(b.GetProject()), // this can return 0 the project resource name is ID based
 		Autoclass:                toAutoclassFromProto(b.GetAutoclass()),
 		SoftDeletePolicy:         toSoftDeletePolicyFromProto(b.SoftDeletePolicy),
+		HierarchicalNamespace:    toHierarchicalNamespaceFromProto(b.HierarchicalNamespace),
 	}
 }
 
@@ -920,6 +938,7 @@ func (b *BucketAttrs) toRawBucket() *raw.Bucket {
 		CustomPlacementConfig: b.CustomPlacementConfig.toRawCustomPlacement(),
 		Autoclass:             b.Autoclass.toRawAutoclass(),
 		SoftDeletePolicy:      b.SoftDeletePolicy.toRawSoftDeletePolicy(),
+		HierarchicalNamespace: b.HierarchicalNamespace.toRawHierarchicalNamespace(),
 	}
 }
 
@@ -981,6 +1000,7 @@ func (b *BucketAttrs) toProtoBucket() *storagepb.Bucket {
 		CustomPlacementConfig: b.CustomPlacementConfig.toProtoCustomPlacement(),
 		Autoclass:             b.Autoclass.toProtoAutoclass(),
 		SoftDeletePolicy:      b.SoftDeletePolicy.toProtoSoftDeletePolicy(),
+		HierarchicalNamespace: b.HierarchicalNamespace.toProtoHierarchicalNamespace(),
 	}
 }
 
@@ -2145,6 +2165,42 @@ func toSoftDeletePolicyFromProto(p *storagepb.Bucket_SoftDeletePolicy) *SoftDele
 	}
 }
 
+func (hns *HierarchicalNamespace) toProtoHierarchicalNamespace() *storagepb.Bucket_HierarchicalNamespace {
+	if hns == nil {
+		return nil
+	}
+	return &storagepb.Bucket_HierarchicalNamespace{
+		Enabled: hns.Enabled,
+	}
+}
+
+func (hns *HierarchicalNamespace) toRawHierarchicalNamespace() *raw.BucketHierarchicalNamespace {
+	if hns == nil {
+		return nil
+	}
+	return &raw.BucketHierarchicalNamespace{
+		Enabled: hns.Enabled,
+	}
+}
+
+func toHierarchicalNamespaceFromProto(p *storagepb.Bucket_HierarchicalNamespace) *HierarchicalNamespace {
+	if p == nil {
+		return nil
+	}
+	return &HierarchicalNamespace{
+		Enabled: p.Enabled,
+	}
+}
+
+func toHierarchicalNamespaceFromRaw(r *raw.BucketHierarchicalNamespace) *HierarchicalNamespace {
+	if r == nil {
+		return nil
+	}
+	return &HierarchicalNamespace{
+		Enabled: r.Enabled,
+	}
+}
+
 // Objects returns an iterator over the objects in the bucket that match the
 // Query q. If q is nil, no filtering is done. Objects will be iterated over
 // lexicographically by name.

storage/bucket_test.go

@@ -62,11 +62,12 @@ func TestBucketAttrsToRawBucket(t *testing.T) {
 				ResponseHeaders: []string{"FOO"},
 			},
 		},
-		Encryption:       &BucketEncryption{DefaultKMSKeyName: "key"},
-		Logging:          &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
-		Website:          &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
-		Autoclass:        &Autoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
-		SoftDeletePolicy: &SoftDeletePolicy{RetentionDuration: time.Hour},
+		Encryption:            &BucketEncryption{DefaultKMSKeyName: "key"},
+		Logging:               &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
+		Website:               &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
+		Autoclass:             &Autoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
+		SoftDeletePolicy:      &SoftDeletePolicy{RetentionDuration: time.Hour},
+		HierarchicalNamespace: &HierarchicalNamespace{Enabled: true},
 		Lifecycle: Lifecycle{
 			Rules: []LifecycleRule{{
 				Action: LifecycleAction{
@@ -167,11 +168,12 @@ func TestBucketAttrsToRawBucket(t *testing.T) {
 				ResponseHeader: []string{"FOO"},
 			},
 		},
-		Encryption:       &raw.BucketEncryption{DefaultKmsKeyName: "key"},
-		Logging:          &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
-		Website:          &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
-		Autoclass:        &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
-		SoftDeletePolicy: &raw.BucketSoftDeletePolicy{RetentionDurationSeconds: 60 * 60},
+		Encryption:            &raw.BucketEncryption{DefaultKmsKeyName: "key"},
+		Logging:               &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
+		Website:               &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
+		Autoclass:             &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
+		SoftDeletePolicy:      &raw.BucketSoftDeletePolicy{RetentionDurationSeconds: 60 * 60},
+		HierarchicalNamespace: &raw.BucketHierarchicalNamespace{Enabled: true},
 		Lifecycle: &raw.BucketLifecycle{
 			Rule: []*raw.BucketLifecycleRule{{
 				Action: &raw.BucketLifecycleRuleAction{
@@ -665,6 +667,7 @@ func TestNewBucket(t *testing.T) {
 			EffectiveTime:            "2017-10-23T04:05:06Z",
 			RetentionDurationSeconds: 3600,
 		},
+		HierarchicalNamespace: &raw.BucketHierarchicalNamespace{Enabled: true},
 	}
 	want := &BucketAttrs{
 		Name:                  "name",
@@ -726,6 +729,7 @@ func TestNewBucket(t *testing.T) {
 			EffectiveTime:     time.Date(2017, 10, 23, 4, 5, 6, 0, time.UTC),
 			RetentionDuration: time.Hour,
 		},
+		HierarchicalNamespace: &HierarchicalNamespace{Enabled: true},
 	}
 	got, err := newBucket(rb)
 	if err != nil {
@@ -785,6 +789,9 @@ func TestNewBucketFromProto(t *testing.T) {
 			RetentionDuration: durationpb.New(3 * time.Hour),
 			EffectiveTime:     toProtoTimestamp(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)),
 		},
+		HierarchicalNamespace: &storagepb.Bucket_HierarchicalNamespace{
+			Enabled: true,
+		},
 		Lifecycle: &storagepb.Bucket_Lifecycle{
 			Rule: []*storagepb.Bucket_Lifecycle_Rule{
 				{
@@ -830,6 +837,9 @@ func TestNewBucketFromProto(t *testing.T) {
 			EffectiveTime:     time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC),
 			RetentionDuration: time.Hour * 3,
 		},
+		HierarchicalNamespace: &HierarchicalNamespace{
+			Enabled: true,
+		},
 		Lifecycle: Lifecycle{
 			Rules: []LifecycleRule{{
 				Action: LifecycleAction{
@@ -874,11 +884,12 @@ func TestBucketAttrsToProtoBucket(t *testing.T) {
 				ResponseHeaders: []string{"FOO"},
 			},
 		},
-		Encryption:       &BucketEncryption{DefaultKMSKeyName: "key"},
-		Logging:          &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
-		Website:          &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
-		Autoclass:        &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
-		SoftDeletePolicy: &SoftDeletePolicy{RetentionDuration: time.Hour * 2},
+		Encryption:            &BucketEncryption{DefaultKMSKeyName: "key"},
+		Logging:               &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
+		Website:               &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
+		Autoclass:             &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
+		SoftDeletePolicy:      &SoftDeletePolicy{RetentionDuration: time.Hour * 2},
+		HierarchicalNamespace: &HierarchicalNamespace{Enabled: true},
 		Lifecycle: Lifecycle{
 			Rules: []LifecycleRule{{
 				Action: LifecycleAction{
@@ -925,11 +936,12 @@ func TestBucketAttrsToProtoBucket(t *testing.T) {
 				ResponseHeader: []string{"FOO"},
 			},
 		},
-		Encryption:       &storagepb.Bucket_Encryption{DefaultKmsKey: "key"},
-		Logging:          &storagepb.Bucket_Logging{LogBucket: "projects/_/buckets/lb", LogObjectPrefix: "p"},
-		Website:          &storagepb.Bucket_Website{MainPageSuffix: "mps", NotFoundPage: "404"},
-		Autoclass:        &storagepb.Bucket_Autoclass{Enabled: true, TerminalStorageClass: &autoclassTSC},
-		SoftDeletePolicy: &storagepb.Bucket_SoftDeletePolicy{RetentionDuration: durationpb.New(2 * time.Hour)},
+		Encryption:            &storagepb.Bucket_Encryption{DefaultKmsKey: "key"},
+		Logging:               &storagepb.Bucket_Logging{LogBucket: "projects/_/buckets/lb", LogObjectPrefix: "p"},
+		Website:               &storagepb.Bucket_Website{MainPageSuffix: "mps", NotFoundPage: "404"},
+		Autoclass:             &storagepb.Bucket_Autoclass{Enabled: true, TerminalStorageClass: &autoclassTSC},
+		SoftDeletePolicy:      &storagepb.Bucket_SoftDeletePolicy{RetentionDuration: durationpb.New(2 * time.Hour)},
+		HierarchicalNamespace: &storagepb.Bucket_HierarchicalNamespace{Enabled: true},
 		Lifecycle: &storagepb.Bucket_Lifecycle{
 			Rule: []*storagepb.Bucket_Lifecycle_Rule{
 				{

storage/integration_test.go

@@ -1012,6 +1012,50 @@ func TestIntegration_ConditionalDelete(t *testing.T) {
 	})
 }
 
+func TestIntegration_HierarchicalNamespace(t *testing.T) {
+	ctx := skipJSONReads(context.Background(), "no reads in test")
+	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, prefix string, client *Client) {
+		h := testHelper{t}
+
+		// Create a bucket with HNS enabled.
+		hnsBucketName := prefix + uidSpace.New()
+		bkt := client.Bucket(hnsBucketName)
+		h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{
+			UniformBucketLevelAccess: UniformBucketLevelAccess{Enabled: true},
+			HierarchicalNamespace:    &HierarchicalNamespace{Enabled: true},
+		})
+		defer h.mustDeleteBucket(bkt)
+
+		attrs, err := bkt.Attrs(ctx)
+		if err != nil {
+			t.Fatalf("bkt(%q).Attrs: %v", hnsBucketName, err)
+		}
+
+		if got, want := (attrs.HierarchicalNamespace), (&HierarchicalNamespace{Enabled: true}); cmp.Diff(got, want) != "" {
+			t.Errorf("HierarchicalNamespace: got %+v, want %+v", got, want)
+		}
+
+		// Folder creation should work on HNS bucket, but not on standard bucket.
+		req := &controlpb.CreateFolderRequest{
+			Parent:   fmt.Sprintf("projects/_/buckets/%v", hnsBucketName),
+			FolderId: "foo/",
+			Folder:   &controlpb.Folder{},
+		}
+		if _, err := controlClient.CreateFolder(ctx, req); err != nil {
+			t.Errorf("creating folder in bucket %q: %v", hnsBucketName, err)
+		}
+
+		req2 := &controlpb.CreateFolderRequest{
+			Parent:   fmt.Sprintf("projects/_/buckets/%v", bucket),
+			FolderId: "foo/",
+			Folder:   &controlpb.Folder{},
+		}
+		if _, err := controlClient.CreateFolder(ctx, req2); status.Code(err) != codes.FailedPrecondition {
+			t.Errorf("creating folder in non-HNS bucket %q: got error %v, want FailedPrecondition", bucket, err)
+		}
+	})
+}
+
 func TestIntegration_ObjectsRangeReader(t *testing.T) {
 	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
 		bkt := client.Bucket(bucket)
@@ -6046,6 +6090,32 @@ func killBucket(ctx context.Context, client *Client, bucketName string) error {
 		}
 	}
 
+	// Delete any folders.
+	listFoldersReq := &controlpb.ListFoldersRequest{
+		Parent: fmt.Sprintf("projects/_/buckets/%s", bucketName),
+	}
+	folderIt := controlClient.ListFolders(ctx, listFoldersReq)
+	for {
+		resp, err := folderIt.Next()
+		if err == iterator.Done {
+			break
+		}
+		// Buckets without UBLA will return this error for Folder ops; skip.
+		if status.Code(err) == codes.FailedPrecondition {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		deleteFolderReq := &controlpb.DeleteFolderRequest{
+			Name: resp.Name,
+		}
+		err = controlClient.DeleteFolder(ctx, deleteFolderReq)
+		if err != nil {
+			return err
+		}
+	}
+
 	// GCS is eventually consistent, so this delete may fail because the
 	// replica still sees an object in the bucket. We log the error and expect
 	// a later test run to delete the bucket.