From 206e894538245edbee3116a60938e11b54f49f87 Mon Sep 17 00:00:00 2001 From: Nitin Garg Date: Sat, 4 May 2024 19:33:20 +0000 Subject: [PATCH 1/7] Add utilities to filter unsupported objects --- internal/util/gcs_util.go | 76 ++++++++++++++++++++++ internal/util/gcs_util_test.go | 113 +++++++++++++++++++++++++++++++++ 2 files changed, 189 insertions(+) create mode 100644 internal/util/gcs_util.go create mode 100644 internal/util/gcs_util_test.go diff --git a/internal/util/gcs_util.go b/internal/util/gcs_util.go new file mode 100644 index 0000000000..8b94cd6433 --- /dev/null +++ b/internal/util/gcs_util.go @@ -0,0 +1,76 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "strings" + + "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" +) + +var ( + unsupportedObjectNameSubstrings = []string{"//"} + unsupportedObjectNamePrefixes = []string{"/"} + unsupportedObjectNames = []string{""} +) + +// isUnsupportedObjectName returns true if the passed +// string is a valid GCS object name or prefix, +// which is unsupported in GCSFuse. +func isUnsupportedObjectName(name string) bool { + for _, substring := range unsupportedObjectNameSubstrings { + if strings.Contains(name, substring) { + return true + } + } + for _, prefix := range unsupportedObjectNamePrefixes { + if strings.HasPrefix(name, prefix) { + return true + } + } + for _, unsupportedObjectName := range unsupportedObjectNames { + if name == unsupportedObjectName { + return true + } + } + return false +} + +// RemoveUnsupportedObjectsFromListing is a utility to ignore unsupported +// GCS object names such as those containing `//` in their names. +// As an example, GCS can have two different objects a//b and a/b at the same time +// in the same bucket. In linux FS however, both paths are same as a/b. +// So, GCSFuse will ignore objects with names like a//b to avoid causing `input/output error` in +// linux FS. +func RemoveUnsupportedObjectsFromListing(listing *gcs.Listing) (newListing *gcs.Listing, removedListing *gcs.Listing) { + newListing = &gcs.Listing{} + removedListing = &gcs.Listing{} + for _, collapsedRun := range listing.CollapsedRuns { + if !isUnsupportedObjectName(collapsedRun) { + newListing.CollapsedRuns = append(newListing.CollapsedRuns, collapsedRun) + } else { + removedListing.CollapsedRuns = append(removedListing.CollapsedRuns, collapsedRun) + } + } + for _, object := range listing.Objects { + if !isUnsupportedObjectName(object.Name) { + newListing.Objects = append(newListing.Objects, object) + } else { + removedListing.Objects = append(removedListing.Objects, object) + } + } + newListing.ContinuationToken = listing.ContinuationToken + return newListing, removedListing +} diff --git a/internal/util/gcs_util_test.go b/internal/util/gcs_util_test.go new file mode 100644 index 0000000000..9c4281c97e --- /dev/null +++ b/internal/util/gcs_util_test.go @@ -0,0 +1,113 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "fmt" + "testing" + + "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +//////////////////////////////////////////////////////////////////////// +// Boilerplate +//////////////////////////////////////////////////////////////////////// + +type GcsUtilTest struct { + suite.Suite +} + +func TestGcsUtil(t *testing.T) { + suite.Run(t, new(GcsUtilTest)) +} + +// ////////////////////////////////////////////////////////////////////// +// Tests +// ////////////////////////////////////////////////////////////////////// +func (ts *GcsUtilTest) TestIsUnsupportedObjectName() { + cases := []struct { + name string + isUnsupported bool + }{ + { + name: "foo", + isUnsupported: false, + }, + { + name: "foo/bar", + isUnsupported: false, + }, + { + name: "foo//bar", + isUnsupported: true, + }, + { + name: "abc/", + isUnsupported: false, + }, + { + name: "abc//", + isUnsupported: true, + }, + { + name: "/foo", + isUnsupported: true, + }, + { + name: "/", + isUnsupported: true, + }, + } + + for _, tc := range cases { + ts.Run(fmt.Sprintf("name=%s", tc.name), func() { + assert.Equal(ts.T(), tc.isUnsupported, isUnsupportedObjectName(tc.name)) + }) + } +} + +func (t *GcsUtilTest) Test_RemoveUnsupportedObjectsFromListing() { + createObject := func(name string) *gcs.Object { + return &gcs.Object{Name: name} + } + createObjects := func(names []string) []*gcs.Object { + objects := []*gcs.Object{} + for _, name := range names { + objects = append(objects, createObject(name)) + } + return objects + } + origGcsListing := &gcs.Listing{ + CollapsedRuns: []string{"/", "a/", "b//", "c/d/", "e//f/", "g/h//"}, + Objects: createObjects([]string{"a", "/b", "c/d", "e//f", "g/h//i"}), + ContinuationToken: "hfdwefo", + } + expectedNewGcsListing := &gcs.Listing{ + CollapsedRuns: []string{"a/", "c/d/"}, + Objects: createObjects([]string{"a", "c/d"}), + ContinuationToken: "hfdwefo", + } + expectedRemovedGcsListing := &gcs.Listing{ + CollapsedRuns: []string{"/", "b//", "e//f/", "g/h//"}, + Objects: createObjects([]string{"/b", "e//f", "g/h//i"}), + } + + newGcsListing, removedGcsListing := RemoveUnsupportedObjectsFromListing(origGcsListing) + + assert.Equal(t.T(), *expectedNewGcsListing, *newGcsListing) + assert.Equal(t.T(), *expectedRemovedGcsListing, *removedGcsListing) +} From 25484c3545f7a51cf28ed0c041d8b9722a4cd8bc Mon Sep 17 00:00:00 2001 From: Nitin Garg Date: Tue, 1 Oct 2024 06:17:20 +0000 Subject: [PATCH 2/7] remove unsupported objects from listing response --- internal/fs/inode/dir.go | 25 ++++++++++++++++ internal/fs/inode/dir_test.go | 56 +++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/internal/fs/inode/dir.go b/internal/fs/inode/dir.go index 52bc961c33..f0bb635f12 100644 --- a/internal/fs/inode/dir.go +++ b/internal/fs/inode/dir.go @@ -24,8 +24,10 @@ import ( "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/metadata" "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" "github.com/googlecloudplatform/gcsfuse/v2/internal/locker" + "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "github.com/googlecloudplatform/gcsfuse/v2/internal/util" "github.com/jacobsa/fuse/fuseops" "github.com/jacobsa/fuse/fuseutil" "github.com/jacobsa/timeutil" @@ -652,7 +654,24 @@ func (d *dirInode) ReadDescendants(ctx context.Context, limit int) (map[Name]*Co return descendants, nil } } +} + +func logUnsupportedListings(removedListings *gcs.Listing) { + if removedListings != nil { + if len(removedListings.CollapsedRuns) > 0 { + logger.Warnf("Ignored following unsupported prefixes: %v", removedListings.CollapsedRuns) + } + if len(removedListings.Objects) > 0 { + objectNames := []string{} + for _, object := range removedListings.Objects { + if object != nil { + objectNames = append(objectNames, object.Name) + } + } + logger.Warnf("Ignored following unsupported objects: %v", objectNames) + } + } } // LOCKS_REQUIRED(d) @@ -681,6 +700,12 @@ func (d *dirInode) readObjects( return } + // Remove unsupported prefixes/objects such as those + // containing '//' in them. + var removedListings *gcs.Listing + listing, removedListings = util.RemoveUnsupportedObjectsFromListing(listing) + logUnsupportedListings(removedListings) + cores = make(map[Name]*Core) defer func() { now := d.cacheClock.Now() diff --git a/internal/fs/inode/dir_test.go b/internal/fs/inode/dir_test.go index 3af10c5875..f6496688ad 100644 --- a/internal/fs/inode/dir_test.go +++ b/internal/fs/inode/dir_test.go @@ -956,6 +956,62 @@ func (t *DirTest) ReadEntries_TypeCaching() { AssertFalse(d.prevDirListingTimeStamp.IsZero()) } +func (t *DirTest) ReadEntries_IncludingUnsupportedObjects() { + var err error + var entry fuseutil.Dirent + + // Enable implicit dirs. + t.resetInode(true, false, true) + + // Set up contents. + objs := []string{ + dirInodeName + "supported_dir_explicit/", + dirInodeName + "supported_dir_implicit1/supported_file", + dirInodeName + "supported_dir_implicit2//unsupported_file", + dirInodeName + "/unsupported_dir_explicit/", + dirInodeName + "/unsupported_dir_implicit1/supported_file", + dirInodeName + "top_level_supported_file", + dirInodeName + "/top_level_unsupported_file", + } + + err = storageutil.CreateEmptyObjects(t.ctx, t.bucket, objs) + AssertEq(nil, err) + + // Nil prevDirListingTimeStamp + d := t.in.(*dirInode) + AssertNe(nil, d) + AssertTrue(d.prevDirListingTimeStamp.IsZero()) + + // Read entries. + entries, err := t.readAllEntries() + + AssertEq(nil, err) + AssertEq(4, len(entries)) + + entry = entries[0] + ExpectEq("supported_dir_explicit", entry.Name) + ExpectEq(fuseutil.DT_Directory, entry.Type) + ExpectEq(metadata.ExplicitDirType, t.getTypeFromCache("supported_dir_explicit")) + + entry = entries[1] + ExpectEq("supported_dir_implicit1", entry.Name) + ExpectEq(fuseutil.DT_Directory, entry.Type) + ExpectEq(metadata.ImplicitDirType, t.getTypeFromCache("supported_dir_implicit1")) + + entry = entries[2] + ExpectEq("supported_dir_implicit2", entry.Name) + ExpectEq(fuseutil.DT_Directory, entry.Type) + ExpectEq(metadata.ImplicitDirType, t.getTypeFromCache("supported_dir_implicit2")) + + entry = entries[3] + ExpectEq("top_level_supported_file", entry.Name) + ExpectEq(fuseutil.DT_File, entry.Type) + ExpectEq(metadata.RegularFileType, t.getTypeFromCache("top_level_supported_file")) + + // Make sure prevDirListingTimeStamp is not nil. + AssertFalse(d.prevDirListingTimeStamp.IsZero()) +} + func (t *DirTest) CreateChildFile_DoesntExist() { const name = "qux" objName := path.Join(dirInodeName, name) From 151af75fd9e3ca1f5433384ddfe1b5803c6f7044 Mon Sep 17 00:00:00 2001 From: Nitin Garg Date: Tue, 1 Oct 2024 06:18:35 +0000 Subject: [PATCH 3/7] [debug] add debug logs in dir_handle --- internal/fs/handle/dir_handle.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/fs/handle/dir_handle.go b/internal/fs/handle/dir_handle.go index 86b05117b6..1e17e20a0f 100644 --- a/internal/fs/handle/dir_handle.go +++ b/internal/fs/handle/dir_handle.go @@ -20,6 +20,7 @@ import ( "github.com/googlecloudplatform/gcsfuse/v2/internal/fs/inode" "github.com/googlecloudplatform/gcsfuse/v2/internal/locker" + "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" "github.com/jacobsa/fuse" "github.com/jacobsa/fuse/fuseops" "github.com/jacobsa/fuse/fuseutil" @@ -299,9 +300,11 @@ func (dh *DirHandle) ReadDir( return } + logger.Debugf("dirEntries for %q: ", dh.in.Name()) // We copy out entries until we run out of entries or space. for i := index; i < len(dh.entries); i++ { n := fuseutil.WriteDirent(op.Dst[op.BytesRead:], dh.entries[i]) + logger.Debugf("dirEntry[%d]= %q", i, dh.entries[i].Name) if n == 0 { break } From a7047b31a2b15917178ffc93799f7b3d59c3bd2e Mon Sep 17 00:00:00 2001 From: Nitin Garg Date: Tue, 1 Oct 2024 06:19:22 +0000 Subject: [PATCH 4/7] add composite test for unsupported objects --- internal/fs/implicit_dirs_test.go | 106 ++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/internal/fs/implicit_dirs_test.go b/internal/fs/implicit_dirs_test.go index e5000c53be..5e46807b8b 100644 --- a/internal/fs/implicit_dirs_test.go +++ b/internal/fs/implicit_dirs_test.go @@ -19,8 +19,12 @@ package fs_test import ( + "fmt" + "io/fs" "os" "path" + "path/filepath" + "strings" "syscall" "time" @@ -569,3 +573,105 @@ func (t *ImplicitDirsTest) AtimeCtimeAndMtime() { ExpectThat(ctime, timeutil.TimeNear(mountTime, delta)) ExpectThat(mtime, timeutil.TimeNear(mountTime, delta)) } + +func verifyInvalidPath(path string) { + _, err := os.Stat(path) + + AssertNe(nil, err, "Failed to get error in stat of %q", path) +} + +// Create objects with unsupported object names and +// verify the behavior of mount using os.Stat and WalkDir. +func (t *ImplicitDirsTest) UnsupportedDirNames() { + // Set up contents. + AssertEq( + nil, + t.createObjects( + map[string]string{ + "foo//1": "", // unsupported + "foo/2": "", // supported + "foo/3//4": "", // unsupported + "foo//3/5": "", // unsupported + "foo/3/6": "", // supported + "7": "", // supported + "/8": "", // unsupported + "/9/10": "", // unsupported + "/": "", // unsupported + "11/12": "", // supported + })) + + // Verify that the unsupported objects fail os.Stat. + verifyInvalidPath(path.Join(mntDir, "foo//1")) + verifyInvalidPath(path.Join(mntDir, "foo/3//4")) + verifyInvalidPath(path.Join(mntDir, "foo//3/5")) + verifyInvalidPath(path.Join(mntDir, "/8")) + verifyInvalidPath(path.Join(mntDir, "/9/10")) + + // Verify that the supported objects appear in WalkDir. + expectedWalkedEntries := []struct { + path string + name string + isDir bool + found bool + }{{ + path: mntDir, + name: mntDir[strings.LastIndex(mntDir, "/")+1:], + isDir: true, + }, { + path: path.Join(mntDir, "foo"), + name: "foo", + isDir: true, + }, { + path: path.Join(mntDir, "foo/2"), + name: "2", + }, { + path: path.Join(mntDir, "foo/3"), + name: "3", + isDir: true, + }, { + path: path.Join(mntDir, "foo/3/6"), + name: "6", + }, { + path: path.Join(mntDir, "7"), + name: "7", + }, { + path: path.Join(mntDir, "11"), + name: "11", + isDir: true, + }, { + path: path.Join(mntDir, "11/12"), + name: "12", + }, + } + + AssertEq(nil, filepath.WalkDir(mntDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + foundMatchingExpectedWalkingEntry := false + for i := range expectedWalkedEntries { + expectedWalkedEntry := &expectedWalkedEntries[i] + if expectedWalkedEntry.path == path && expectedWalkedEntry.name == d.Name() && d.IsDir() == expectedWalkedEntry.isDir { + if expectedWalkedEntry.found { + return fmt.Errorf("found duplicate walked entry: path=%s, name=%s, isDir=%v", path, d.Name(), d.IsDir()) + } + + foundMatchingExpectedWalkingEntry = true + expectedWalkedEntry.found = true + } + } + + if !foundMatchingExpectedWalkingEntry { + return fmt.Errorf("got unexpected walk entry: path=%s, name=%s, isDir=%v", path, d.Name(), d.IsDir()) + } + + return nil + })) + + for _, expectedWalkedEntry := range expectedWalkedEntries { + if !expectedWalkedEntry.found { + AddFailure("Missing walked entry: path=%s, name=%s, isDir=%v", expectedWalkedEntry.path, expectedWalkedEntry.name, expectedWalkedEntry.isDir) + } + } +} From d53cb827efc112ffa6f864608970a5700cdcd35f Mon Sep 17 00:00:00 2001 From: Nitin Garg Date: Tue, 1 Oct 2024 06:20:05 +0000 Subject: [PATCH 5/7] [debug] disable garbage collector --- internal/gcsx/bucket_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/gcsx/bucket_manager.go b/internal/gcsx/bucket_manager.go index 50ae19a9ee..79e1ff06c6 100644 --- a/internal/gcsx/bucket_manager.go +++ b/internal/gcsx/bucket_manager.go @@ -234,7 +234,7 @@ func (bm *bucketManager) SetUpBucket( } // Periodically garbage collect temporary objects - go garbageCollect(bm.gcCtx, bm.config.TmpObjectPrefix, sb) + //go garbageCollect(bm.gcCtx, bm.config.TmpObjectPrefix, sb) return } From 490fa908a59ccee7d1ea7a5bf20a8079d1ef0459 Mon Sep 17 00:00:00 2001 From: Nitin Garg Date: Tue, 1 Oct 2024 06:21:09 +0000 Subject: [PATCH 6/7] shorten code in bucket-handle unit tests --- internal/storage/bucket_handle_test.go | 59 +++++++++----------------- 1 file changed, 19 insertions(+), 40 deletions(-) diff --git a/internal/storage/bucket_handle_test.go b/internal/storage/bucket_handle_test.go index 671bfb7cfb..68ae584c2e 100644 --- a/internal/storage/bucket_handle_test.go +++ b/internal/storage/bucket_handle_test.go @@ -49,6 +49,16 @@ var ContentDisposition string = "ContentDisposition" // Hence, we are not writing tests for these flows. // https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/vendor/github.com/fsouza/fake-gcs-server/fakestorage/object.go#L515 +func objectsToObjectNames(objects []*gcs.Object) (objectNames []string) { + objectNames = make([]string, len(objects)) + for i, object := range objects { + if object != nil { + objectNames[i] = object.Name + } + } + return +} + type BucketHandleTest struct { suite.Suite bucketHandle *bucketHandle @@ -447,13 +457,9 @@ func (testSuite *BucketHandleTest) TestListObjectMethodWithPrefixObjectExist() { }) assert.Nil(testSuite.T(), err) - assert.Equal(testSuite.T(), 4, len(obj.Objects)) - assert.Equal(testSuite.T(), 1, len(obj.CollapsedRuns)) - assert.Equal(testSuite.T(), TestObjectRootFolderName, obj.Objects[0].Name) - assert.Equal(testSuite.T(), TestObjectSubRootFolderName, obj.Objects[1].Name) - assert.Equal(testSuite.T(), TestObjectName, obj.Objects[2].Name) + assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectSubRootFolderName, TestObjectName, TestGzipObjectName}, objectsToObjectNames(obj.Objects)) assert.Equal(testSuite.T(), TestObjectGeneration, obj.Objects[0].Generation) - assert.Equal(testSuite.T(), TestObjectSubRootFolderName, obj.CollapsedRuns[0]) + assert.Equal(testSuite.T(), []string{TestObjectSubRootFolderName}, obj.CollapsedRuns) } func (testSuite *BucketHandleTest) TestListObjectMethodWithPrefixObjectDoesNotExist() { @@ -484,12 +490,8 @@ func (testSuite *BucketHandleTest) TestListObjectMethodWithIncludeTrailingDelimi }) assert.Nil(testSuite.T(), err) - assert.Equal(testSuite.T(), 3, len(obj.Objects)) - assert.Equal(testSuite.T(), 1, len(obj.CollapsedRuns)) - assert.Equal(testSuite.T(), TestObjectRootFolderName, obj.Objects[0].Name) - assert.Equal(testSuite.T(), TestObjectName, obj.Objects[1].Name) - assert.Equal(testSuite.T(), TestGzipObjectName, obj.Objects[2].Name) - assert.Equal(testSuite.T(), TestObjectSubRootFolderName, obj.CollapsedRuns[0]) + assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectName, TestGzipObjectName}, objectsToObjectNames(obj.Objects)) + assert.Equal(testSuite.T(), []string{TestObjectSubRootFolderName}, obj.CollapsedRuns) } // If Delimiter is empty, all the objects will appear with same prefix. @@ -505,12 +507,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodWithEmptyDelimiter() { }) assert.Nil(testSuite.T(), err) - assert.Equal(testSuite.T(), 5, len(obj.Objects)) - assert.Equal(testSuite.T(), TestObjectRootFolderName, obj.Objects[0].Name) - assert.Equal(testSuite.T(), TestObjectSubRootFolderName, obj.Objects[1].Name) - assert.Equal(testSuite.T(), TestSubObjectName, obj.Objects[2].Name) - assert.Equal(testSuite.T(), TestObjectName, obj.Objects[3].Name) - assert.Equal(testSuite.T(), TestGzipObjectName, obj.Objects[4].Name) + assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectSubRootFolderName, TestSubObjectName, TestObjectName, TestGzipObjectName}, objectsToObjectNames(obj.Objects)) assert.Equal(testSuite.T(), TestObjectGeneration, obj.Objects[0].Generation) assert.Nil(testSuite.T(), obj.CollapsedRuns) } @@ -539,12 +536,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodForMaxResult() { // Validate that 5 objects are listed when MaxResults is passed 5. assert.Nil(testSuite.T(), err) - assert.Equal(testSuite.T(), 5, len(fiveObj.Objects)) - assert.Equal(testSuite.T(), TestObjectRootFolderName, fiveObj.Objects[0].Name) - assert.Equal(testSuite.T(), TestObjectSubRootFolderName, fiveObj.Objects[1].Name) - assert.Equal(testSuite.T(), TestSubObjectName, fiveObj.Objects[2].Name) - assert.Equal(testSuite.T(), TestObjectName, fiveObj.Objects[3].Name) - assert.Equal(testSuite.T(), TestGzipObjectName, fiveObj.Objects[4].Name) + assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectSubRootFolderName, TestSubObjectName, TestObjectName, TestGzipObjectName}, objectsToObjectNames(fiveObj.Objects)) assert.Nil(testSuite.T(), fiveObj.CollapsedRuns) // Note: The behavior is different in real GCS storage JSON API. In real API, @@ -554,10 +546,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodForMaxResult() { // This is because fake storage doesn'testSuite support pagination and hence maxResults // has no affect. assert.Nil(testSuite.T(), err2) - assert.Equal(testSuite.T(), 3, len(threeObj.Objects)) - assert.Equal(testSuite.T(), TestObjectRootFolderName, threeObj.Objects[0].Name) - assert.Equal(testSuite.T(), TestObjectName, threeObj.Objects[1].Name) - assert.Equal(testSuite.T(), TestGzipObjectName, threeObj.Objects[2].Name) + assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectName, TestGzipObjectName}, objectsToObjectNames(threeObj.Objects)) assert.Equal(testSuite.T(), 1, len(threeObj.CollapsedRuns)) } @@ -586,12 +575,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodWithMissingMaxResult() { // Validate that all objects (5) are listed when MaxResults is not passed. assert.Nil(testSuite.T(), err2) - assert.Equal(testSuite.T(), 5, len(fiveObjWithoutMaxResults.Objects)) - assert.Equal(testSuite.T(), TestObjectRootFolderName, fiveObjWithoutMaxResults.Objects[0].Name) - assert.Equal(testSuite.T(), TestObjectSubRootFolderName, fiveObjWithoutMaxResults.Objects[1].Name) - assert.Equal(testSuite.T(), TestSubObjectName, fiveObjWithoutMaxResults.Objects[2].Name) - assert.Equal(testSuite.T(), TestObjectName, fiveObjWithoutMaxResults.Objects[3].Name) - assert.Equal(testSuite.T(), TestGzipObjectName, fiveObjWithoutMaxResults.Objects[4].Name) + assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectSubRootFolderName, TestSubObjectName, TestObjectName, TestGzipObjectName}, objectsToObjectNames(fiveObjWithoutMaxResults.Objects)) assert.Nil(testSuite.T(), fiveObjWithoutMaxResults.CollapsedRuns) } @@ -622,12 +606,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodWithZeroMaxResult() { // Validate that all objects (5) are listed when MaxResults is 0. This has // same behavior as not passing MaxResults in request. assert.Nil(testSuite.T(), err2) - assert.Equal(testSuite.T(), 5, len(fiveObjWithZeroMaxResults.Objects)) - assert.Equal(testSuite.T(), TestObjectRootFolderName, fiveObjWithZeroMaxResults.Objects[0].Name) - assert.Equal(testSuite.T(), TestObjectSubRootFolderName, fiveObjWithZeroMaxResults.Objects[1].Name) - assert.Equal(testSuite.T(), TestSubObjectName, fiveObjWithZeroMaxResults.Objects[2].Name) - assert.Equal(testSuite.T(), TestObjectName, fiveObjWithZeroMaxResults.Objects[3].Name) - assert.Equal(testSuite.T(), TestGzipObjectName, fiveObjWithZeroMaxResults.Objects[4].Name) + assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectSubRootFolderName, TestSubObjectName, TestObjectName, TestGzipObjectName}, objectsToObjectNames(fiveObjWithZeroMaxResults.Objects)) assert.Nil(testSuite.T(), fiveObjWithZeroMaxResults.CollapsedRuns) } From 8635cfba7786a896782cc7f58f0f93a88c51e6ee Mon Sep 17 00:00:00 2001 From: Nitin Garg Date: Mon, 7 Oct 2024 11:40:40 +0000 Subject: [PATCH 7/7] ignore unsupported descendent objects during rmdir --- internal/fs/fs.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/internal/fs/fs.go b/internal/fs/fs.go index befb0b4175..a9fc7d3873 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -1893,7 +1893,18 @@ func (fs *fileSystem) RmDir( // Are there any entries? if len(entries) != 0 { - err = fuse.ENOTEMPTY + isThereAnySupportedObject := func(parent inode.DirInode, name string) bool { + // dummy implementation.. to be replaced with proper implementation. + return false + } + + for i := range len(entries) { + if isThereAnySupportedObject(childDir, entries[i].Name) { + err = fuse.ENOTEMPTY + } else { + logger.Tracef("******* Ignored prefix %q which only has unsupported objects *******", childDir.Name().GcsObjectName()+"/"+entries[i].Name) + } + } return }