diff --git a/storage/grpc_client.go b/storage/grpc_client.go index a51cf9c086b1..e9e95993011b 100644 --- a/storage/grpc_client.go +++ b/storage/grpc_client.go @@ -420,6 +420,11 @@ func (c *grpcStorageClient) ListObjects(ctx context.Context, bucket string, q *Q ctx = setUserProjectMetadata(ctx, s.userProject) } fetch := func(pageSize int, pageToken string) (token string, err error) { + // IncludeFoldersAsPrefixes is not supported for gRPC + // TODO: remove this when support is added in the proto. + if it.query.IncludeFoldersAsPrefixes { + return "", status.Errorf(codes.Unimplemented, "storage: IncludeFoldersAsPrefixes is not supported in gRPC") + } var objects []*storagepb.Object var gitr *gapic.ObjectIterator err = run(it.ctx, func(ctx context.Context) error { diff --git a/storage/http_client.go b/storage/http_client.go index 0e157e4ba994..fe081b60b0ac 100644 --- a/storage/http_client.go +++ b/storage/http_client.go @@ -348,6 +348,7 @@ func (c *httpStorageClient) ListObjects(ctx context.Context, bucket string, q *Q req.Versions(it.query.Versions) req.IncludeTrailingDelimiter(it.query.IncludeTrailingDelimiter) req.MatchGlob(it.query.MatchGlob) + req.IncludeFoldersAsPrefixes(it.query.IncludeFoldersAsPrefixes) if selection := it.query.toFieldSelection(); selection != "" { req.Fields("nextPageToken", googleapi.Field(selection)) } diff --git a/storage/integration_test.go b/storage/integration_test.go index 2caf19a47c54..afcbb822b4f3 100644 --- a/storage/integration_test.go +++ b/storage/integration_test.go @@ -57,6 +57,7 @@ import ( "google.golang.org/api/iterator" itesting "google.golang.org/api/iterator/testing" "google.golang.org/api/option" + raw "google.golang.org/api/storage/v1" "google.golang.org/api/transport" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -1317,8 +1318,6 @@ func TestIntegration_ObjectIteration(t *testing.T) { } func TestIntegration_ObjectIterationMatchGlob(t *testing.T) { - // This is a separate test from the Object Iteration test above because - // MatchGlob is not yet implemented for gRPC. multiTransportTest(skipJSONReads(context.Background(), "no reads in test"), t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { // Reset testTime, 'cause object last modification time should be within 5 min // from test (test iteration if -count passed) start time. @@ -1377,6 +1376,113 @@ func TestIntegration_ObjectIterationMatchGlob(t *testing.T) { }) } +func TestIntegration_ObjectIterationManagedFolder(t *testing.T) { + ctx := skipGRPC("not yet implemented in gRPC") + multiTransportTest(skipJSONReads(ctx, "no reads in test"), t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { + newBucketName := prefix + uidSpace.New() + h := testHelper{t} + bkt := client.Bucket(newBucketName).Retryer(WithPolicy(RetryAlways)) + + // Create bucket with UBLA enabled as this is necessary for managed folders. + h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{ + UniformBucketLevelAccess: UniformBucketLevelAccess{ + Enabled: true, + }, + }) + + t.Cleanup(func() { + if err := killBucket(ctx, client, newBucketName); err != nil { + log.Printf("deleting %q: %v", newBucketName, err) + } + }) + const defaultType = "text/plain" + + // Populate object names and make a map for their contents. + objects := []string{ + "obj1", + "obj2", + "obj/with/slashes", + "obj/", + "other/obj1", + } + contents := make(map[string][]byte) + + // Test Writer. + for _, obj := range objects { + c := randomContents() + if err := writeObject(ctx, bkt.Object(obj), defaultType, c); err != nil { + t.Errorf("Write for %v failed with %v", obj, err) + } + contents[obj] = c + } + + // Create a managed folder. This requires using the Apiary client as this is not available + // in the veneer layer. + // TODO: change to use storage control client once available. + call := client.raw.ManagedFolders.Insert(newBucketName, &raw.ManagedFolder{Name: "mf"}) + mf, err := call.Context(ctx).Do() + if err != nil { + t.Fatalf("creating managed folder: %v", err) + } + + t.Cleanup(func() { + // TODO: add this cleanup logic to killBucket as well once gRPC support is available. + call := client.raw.ManagedFolders.Delete(newBucketName, mf.Name) + call.Context(ctx).Do() + }) + + // Test that managed folders are only included when IncludeFoldersAsPrefixes is set. + cases := []struct { + name string + query *Query + want []string + }{ + { + name: "include folders", + query: &Query{Delimiter: "/", IncludeFoldersAsPrefixes: true}, + want: []string{"mf/", "obj/", "other/"}, + }, + { + name: "no folders", + query: &Query{Delimiter: "/"}, + want: []string{"obj/", "other/"}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + var gotNames []string + var gotPrefixes []string + it := bkt.Objects(context.Background(), c.query) + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + t.Fatalf("iterator.Next: %v", err) + } + if attrs.Name != "" { + gotNames = append(gotNames, attrs.Name) + } + if attrs.Prefix != "" { + gotPrefixes = append(gotPrefixes, attrs.Prefix) + } + } + + sortedNames := []string{"obj1", "obj2"} + if !cmp.Equal(sortedNames, gotNames) { + t.Errorf("names = %v, want %v", gotNames, sortedNames) + } + + if !cmp.Equal(c.want, gotPrefixes) { + t.Errorf("prefixes = %v, want %v", gotPrefixes, c.want) + } + }) + } + }) +} + func TestIntegration_ObjectUpdate(t *testing.T) { ctx := skipJSONReads(context.Background(), "no reads in test") multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { diff --git a/storage/storage.go b/storage/storage.go index 7af32ac6fcd0..f047ef9cd4aa 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -1620,6 +1620,11 @@ type Query struct { // for syntax details. When Delimiter is set in conjunction with MatchGlob, // it must be set to /. MatchGlob string + + // IncludeFoldersAsPrefixes includes Folders and Managed Folders in the set of + // prefixes returned by the query. Only applicable if Delimiter is set to /. + // IncludeFoldersAsPrefixes is not yet implemented in the gRPC API. + IncludeFoldersAsPrefixes bool } // attrToFieldMap maps the field names of ObjectAttrs to the underlying field