From c88f532bd5ca35b0e0a84c7f7140b5c0a21dd780 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Wed, 21 Jun 2023 11:31:07 +0800 Subject: [PATCH 01/13] support GlusterFS as object store --- go.mod | 1 + go.sum | 2 + pkg/object/gluster.go | 294 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 297 insertions(+) create mode 100644 pkg/object/gluster.go diff --git a/go.mod b/go.mod index c2136d55279e..235fda07b8c3 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( github.com/jackc/pgx/v5 v5.3.1 github.com/jcmturner/gokrb5/v8 v8.4.4 github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d + github.com/juicedata/gogfapi v0.0.0-20230621032604-95097be84fb9 github.com/juju/ratelimit v1.0.2 github.com/ks3sdklib/aws-sdk-go v1.2.2 github.com/mattn/go-isatty v0.0.18 diff --git a/go.sum b/go.sum index b988f8bdc02f..f5d30224cdd7 100644 --- a/go.sum +++ b/go.sum @@ -627,6 +627,8 @@ github.com/juicedata/go-fuse/v2 v2.1.1-0.20230418090413-880e994b8a4c h1:AVWJgjm5 github.com/juicedata/go-fuse/v2 v2.1.1-0.20230418090413-880e994b8a4c/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc= github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d h1:kpQMvNZJKGY3PTt7OSoahYc4nM0HY67SvK0YyS0GLwA= github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d/go.mod h1:dlxKkLh3qAIPtgr2U/RVzsZJDuXA1ffg+Njikfmhvgw= +github.com/juicedata/gogfapi v0.0.0-20230621032604-95097be84fb9 h1:KaL6JKgcgu0CblgqIaUPlLQz6xuCl4TNf+1pqwyT724= +github.com/juicedata/gogfapi v0.0.0-20230621032604-95097be84fb9/go.mod h1:Ho5G4KgrgbMKW0buAJdOmYoJcOImkzznJQaLiATrsx4= github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible h1:2/ttSmYoX+QMegpNyAJR0Y6aHcVk57F7RJit5xN2T/s= github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible/go.mod h1:Ukwa8ffRQLV6QRwpqGioPjn2Wnf7TBDA4DbennDOqHE= github.com/juicedata/minio v0.0.0-20221113011458-8866d5c9df8c h1:w+4eiZLSLd6aQcy+7wn++hI1caDAm+rNOG7Me5qO7Sw= diff --git a/pkg/object/gluster.go b/pkg/object/gluster.go new file mode 100644 index 000000000000..d28ae36ae3fc --- /dev/null +++ b/pkg/object/gluster.go @@ -0,0 +1,294 @@ +//go:build gluster +// +build gluster + +/* + * JuiceFS, Copyright 2023 Juicedata, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package object + +import ( + "bytes" + "fmt" + "io" + "io/fs" + "math/rand" + "net/url" + "os" + "path" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/juicedata/gogfapi/gfapi" + "github.com/juicedata/juicefs/pkg/utils" +) + +type gluster struct { + DefaultObjectStorage + name string + vol *gfapi.Volume +} + +func (c *gluster) String() string { + return fmt.Sprintf("gluster://%s/", c.name) +} + +func (c *gluster) Create() error { + return notSupported +} + +func (c *gluster) Head(key string) (Object, error) { + fi, err := c.vol.Stat(key) + if err != nil { + return nil, err + } + return c.toFile(key, fi, false), nil +} + +func (d *gluster) toFile(key string, fi fs.FileInfo, isSymlink bool) *file { + size := fi.Size() + if fi.IsDir() { + size = 0 + } + owner, group := getOwnerGroup(fi) + return &file{ + obj{ + key, + size, + fi.ModTime(), + fi.IsDir(), + "", + }, + owner, + group, + fi.Mode(), + isSymlink, + } +} + +func (c *gluster) Get(key string, off, limit int64) (io.ReadCloser, error) { + f, err := c.vol.Open(key) + if err != nil { + return nil, err + } + + finfo, err := f.Stat() + if err != nil { + _ = f.Close() + return nil, err + } + if finfo.IsDir() || off > finfo.Size() { + _ = f.Close() + return io.NopCloser(bytes.NewBuffer([]byte{})), nil + } + + if limit > 0 { + return &SectionReaderCloser{ + SectionReader: io.NewSectionReader(f, off, limit), + Closer: f, + }, nil + } + return f, nil +} + +func (c *gluster) Put(key string, in io.Reader) error { + if strings.HasSuffix(key, dirSuffix) { + return c.vol.MkdirAll(key, os.FileMode(0777)) + } + p := key + tmp := filepath.Join(filepath.Dir(p), "."+filepath.Base(p)+".tmp"+strconv.Itoa(rand.Int())) + f, err := c.vol.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) + if err != nil && os.IsNotExist(err) { + if err := c.vol.MkdirAll(filepath.Dir(p), os.FileMode(0777)); err != nil { + return err + } + f, err = c.vol.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) + } + if err != nil { + return err + } + defer func() { + if err != nil { + _ = c.vol.Unlink(tmp) + } + }() + + buf := bufPool.Get().(*[]byte) + defer bufPool.Put(buf) + _, err = io.CopyBuffer(f, in, *buf) + if err != nil { + _ = f.Close() + return err + } + err = f.Close() + if err != nil { + return err + } + err = c.vol.Rename(tmp, p) + return err +} + +func (c *gluster) Delete(key string) error { + err := c.vol.Unlink(key) + if os.IsNotExist(err) { + err = nil + } + return err +} + +// readDirSorted reads the directory named by dirname and returns +// a sorted list of directory entries. +func (d *gluster) readDirSorted(dirname string) ([]*mEntry, error) { + f, err := d.vol.Open(dirname) + if err != nil { + return nil, err + } + defer f.Close() + entries, err := f.Readdir(0) + mEntries := make([]*mEntry, len(entries)) + + for i, e := range entries { + if e.IsDir() { + mEntries[i] = &mEntry{nil, e.Name() + dirSuffix, e, false} + } else if !e.Mode().IsRegular() { + // follow symlink + fi, err := os.Stat(filepath.Join(dirname, e.Name())) + if err != nil { + mEntries[i] = &mEntry{nil, e.Name(), e, true} + continue + } + name := e.Name() + if fi.IsDir() { + name = e.Name() + dirSuffix + } + mEntries[i] = &mEntry{nil, name, fi, true} + } else { + mEntries[i] = &mEntry{nil, e.Name(), e, false} + } + } + sort.Slice(mEntries, func(i, j int) bool { return mEntries[i].Name() < mEntries[j].Name() }) + return mEntries, err +} + +func (d *gluster) List(prefix, marker, delimiter string, limit int64) ([]Object, error) { + if delimiter != "/" { + return nil, notSupported + } + var dir string = prefix + var objs []Object + if !strings.HasSuffix(dir, dirSuffix) { + dir = path.Dir(dir) + if !strings.HasSuffix(dir, dirSuffix) { + dir += dirSuffix + } + } else if marker == "" { + obj, err := d.Head(prefix) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + objs = append(objs, obj) + } + entries, err := readDirSorted(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + for _, e := range entries { + p := filepath.Join(dir, e.Name()) + if e.IsDir() { + p = filepath.ToSlash(p + "/") + } + key := p + if !strings.HasPrefix(key, prefix) || (marker != "" && key <= marker) { + continue + } + info, err := e.Info() + if err != nil { + logger.Warnf("stat %s: %s", p, err) + continue + } + f := d.toFile(key, info, e.isSymlink) + objs = append(objs, f) + if len(objs) == int(limit) { + break + } + } + return objs, nil +} + +func (d *gluster) Chtimes(path string, mtime time.Time) error { + return d.vol.Chtimes(path, mtime) +} + +func (d *gluster) Chmod(path string, mode os.FileMode) error { + return d.vol.Chmod(path, mode) +} + +func (d *gluster) Chown(path string, owner, group string) error { + uid := utils.LookupUser(owner) + gid := utils.LookupGroup(group) + return d.vol.Chown(path, uid, gid) +} + +func (d *gluster) Symlink(oldName, newName string) error { + // TODO + return notSupported +} + +func (d *gluster) Readlink(name string) (string, error) { + // TODO + return "", notSupported +} + +func newGluster(endpoint, ak, sk, token string) (ObjectStorage, error) { + if !strings.Contains(endpoint, "://") { + endpoint = fmt.Sprintf("gluster://%s", endpoint) + } + uri, err := url.ParseRequestURI(endpoint) + if err != nil { + return nil, fmt.Errorf("Invalid endpoint %s: %s", endpoint, err) + } + ps := strings.Split(uri.Path, "/") + if len(ps) == 1 { + return nil, fmt.Errorf("no volume provided") + } + name := ps[1] + v := &gfapi.Volume{} + // TODO: support port in host + err = v.Init(name, strings.Split(uri.Host, ",")...) + if err != nil { + return nil, fmt.Errorf("init %s: %s", name, err) + } + err = v.Mount() + if err != nil { + return nil, fmt.Errorf("mount %s: %s", name, err) + } + return &gluster{ + name: name, + vol: v, + }, nil +} + +func init() { + Register("gluster", newGluster) +} From 95b450f2c0bba063559aab900f223d78a7fd4f14 Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Mon, 26 Jun 2023 11:06:03 +0800 Subject: [PATCH 02/13] remove unnecessary APIs --- Makefile | 3 +++ go.mod | 2 +- go.sum | 4 ++-- pkg/object/gluster.go | 9 ++++----- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index b6f45f7b5382..1ae6cccc236a 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,9 @@ juicefs.ceph: Makefile cmd/*.go pkg/*/*.go juicefs.fdb: Makefile cmd/*.go pkg/*/*.go go build -tags fdb -ldflags="$(LDFLAGS)" -o juicefs.fdb . +juicefs.gluster: Makefile cmd/*.go pkg/*/*.go + go build -tags gluster -ldflags="$(LDFLAGS)" -o juicefs.gluster . + # This is the script for compiling the Linux version on the MacOS platform. # Please execute the `brew install FiloSottile/musl-cross/musl-cross` command before using it. juicefs.linux: diff --git a/go.mod b/go.mod index 235fda07b8c3..6c0fed3dfe8b 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/jackc/pgx/v5 v5.3.1 github.com/jcmturner/gokrb5/v8 v8.4.4 github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d - github.com/juicedata/gogfapi v0.0.0-20230621032604-95097be84fb9 + github.com/juicedata/gogfapi v0.0.0-20230626022304-7a7ca87b71c4 github.com/juju/ratelimit v1.0.2 github.com/ks3sdklib/aws-sdk-go v1.2.2 github.com/mattn/go-isatty v0.0.18 diff --git a/go.sum b/go.sum index f5d30224cdd7..b4a53327e63d 100644 --- a/go.sum +++ b/go.sum @@ -627,8 +627,8 @@ github.com/juicedata/go-fuse/v2 v2.1.1-0.20230418090413-880e994b8a4c h1:AVWJgjm5 github.com/juicedata/go-fuse/v2 v2.1.1-0.20230418090413-880e994b8a4c/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc= github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d h1:kpQMvNZJKGY3PTt7OSoahYc4nM0HY67SvK0YyS0GLwA= github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d/go.mod h1:dlxKkLh3qAIPtgr2U/RVzsZJDuXA1ffg+Njikfmhvgw= -github.com/juicedata/gogfapi v0.0.0-20230621032604-95097be84fb9 h1:KaL6JKgcgu0CblgqIaUPlLQz6xuCl4TNf+1pqwyT724= -github.com/juicedata/gogfapi v0.0.0-20230621032604-95097be84fb9/go.mod h1:Ho5G4KgrgbMKW0buAJdOmYoJcOImkzznJQaLiATrsx4= +github.com/juicedata/gogfapi v0.0.0-20230626022304-7a7ca87b71c4 h1:CmQsMGJ4mu3ncQ7rv0FV1y5/ay1/BKLjGjlgWECOLGk= +github.com/juicedata/gogfapi v0.0.0-20230626022304-7a7ca87b71c4/go.mod h1:Ho5G4KgrgbMKW0buAJdOmYoJcOImkzznJQaLiATrsx4= github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible h1:2/ttSmYoX+QMegpNyAJR0Y6aHcVk57F7RJit5xN2T/s= github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible/go.mod h1:Ukwa8ffRQLV6QRwpqGioPjn2Wnf7TBDA4DbennDOqHE= github.com/juicedata/minio v0.0.0-20221113011458-8866d5c9df8c h1:w+4eiZLSLd6aQcy+7wn++hI1caDAm+rNOG7Me5qO7Sw= diff --git a/pkg/object/gluster.go b/pkg/object/gluster.go index d28ae36ae3fc..a799a8da0041 100644 --- a/pkg/object/gluster.go +++ b/pkg/object/gluster.go @@ -35,7 +35,6 @@ import ( "time" "github.com/juicedata/gogfapi/gfapi" - "github.com/juicedata/juicefs/pkg/utils" ) type gluster struct { @@ -237,7 +236,8 @@ func (d *gluster) List(prefix, marker, delimiter string, limit int64) ([]Object, } func (d *gluster) Chtimes(path string, mtime time.Time) error { - return d.vol.Chtimes(path, mtime) + // TODO + return notSupported } func (d *gluster) Chmod(path string, mode os.FileMode) error { @@ -245,9 +245,8 @@ func (d *gluster) Chmod(path string, mode os.FileMode) error { } func (d *gluster) Chown(path string, owner, group string) error { - uid := utils.LookupUser(owner) - gid := utils.LookupGroup(group) - return d.vol.Chown(path, uid, gid) + // TODO + return notSupported } func (d *gluster) Symlink(oldName, newName string) error { From 02a2ea6fd04b9764991cf7e7a22f55b3853ec2b0 Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Mon, 26 Jun 2023 14:44:47 +0800 Subject: [PATCH 03/13] call fsync in PUT method --- pkg/object/gluster.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/object/gluster.go b/pkg/object/gluster.go index a799a8da0041..7688e7e3e55c 100644 --- a/pkg/object/gluster.go +++ b/pkg/object/gluster.go @@ -134,8 +134,11 @@ func (c *gluster) Put(key string, in io.Reader) error { _ = f.Close() return err } - err = f.Close() - if err != nil { + if err = f.Sync(); err != nil { + _ = f.Close() + return err + } + if err = f.Close(); err != nil { return err } err = c.vol.Rename(tmp, p) From 06d1d50b93a1741b413e566c555e39ad13a14496 Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Mon, 26 Jun 2023 15:23:59 +0800 Subject: [PATCH 04/13] update gogfapi to support ReadAt --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6c0fed3dfe8b..3fe9c9ca67f2 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/jackc/pgx/v5 v5.3.1 github.com/jcmturner/gokrb5/v8 v8.4.4 github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d - github.com/juicedata/gogfapi v0.0.0-20230626022304-7a7ca87b71c4 + github.com/juicedata/gogfapi v0.0.0-20230626071140-fc28e5537825 github.com/juju/ratelimit v1.0.2 github.com/ks3sdklib/aws-sdk-go v1.2.2 github.com/mattn/go-isatty v0.0.18 diff --git a/go.sum b/go.sum index b4a53327e63d..b7d23fcb204d 100644 --- a/go.sum +++ b/go.sum @@ -627,8 +627,8 @@ github.com/juicedata/go-fuse/v2 v2.1.1-0.20230418090413-880e994b8a4c h1:AVWJgjm5 github.com/juicedata/go-fuse/v2 v2.1.1-0.20230418090413-880e994b8a4c/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc= github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d h1:kpQMvNZJKGY3PTt7OSoahYc4nM0HY67SvK0YyS0GLwA= github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d/go.mod h1:dlxKkLh3qAIPtgr2U/RVzsZJDuXA1ffg+Njikfmhvgw= -github.com/juicedata/gogfapi v0.0.0-20230626022304-7a7ca87b71c4 h1:CmQsMGJ4mu3ncQ7rv0FV1y5/ay1/BKLjGjlgWECOLGk= -github.com/juicedata/gogfapi v0.0.0-20230626022304-7a7ca87b71c4/go.mod h1:Ho5G4KgrgbMKW0buAJdOmYoJcOImkzznJQaLiATrsx4= +github.com/juicedata/gogfapi v0.0.0-20230626071140-fc28e5537825 h1:7KrwI4HPqvNLKVcfkfDMLQQmT0GrnCs8T9EX+XCdZnM= +github.com/juicedata/gogfapi v0.0.0-20230626071140-fc28e5537825/go.mod h1:Ho5G4KgrgbMKW0buAJdOmYoJcOImkzznJQaLiATrsx4= github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible h1:2/ttSmYoX+QMegpNyAJR0Y6aHcVk57F7RJit5xN2T/s= github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible/go.mod h1:Ukwa8ffRQLV6QRwpqGioPjn2Wnf7TBDA4DbennDOqHE= github.com/juicedata/minio v0.0.0-20221113011458-8866d5c9df8c h1:w+4eiZLSLd6aQcy+7wn++hI1caDAm+rNOG7Me5qO7Sw= From 9d72450bc40e3f892704b01145ede4821aa00f46 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Fri, 30 Jun 2023 10:36:20 +0800 Subject: [PATCH 05/13] Update gluster.go --- pkg/object/gluster.go | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/pkg/object/gluster.go b/pkg/object/gluster.go index 7688e7e3e55c..ac3895f091f6 100644 --- a/pkg/object/gluster.go +++ b/pkg/object/gluster.go @@ -47,10 +47,6 @@ func (c *gluster) String() string { return fmt.Sprintf("gluster://%s/", c.name) } -func (c *gluster) Create() error { - return notSupported -} - func (c *gluster) Head(key string) (Object, error) { fi, err := c.vol.Stat(key) if err != nil { @@ -238,30 +234,10 @@ func (d *gluster) List(prefix, marker, delimiter string, limit int64) ([]Object, return objs, nil } -func (d *gluster) Chtimes(path string, mtime time.Time) error { - // TODO - return notSupported -} - func (d *gluster) Chmod(path string, mode os.FileMode) error { return d.vol.Chmod(path, mode) } -func (d *gluster) Chown(path string, owner, group string) error { - // TODO - return notSupported -} - -func (d *gluster) Symlink(oldName, newName string) error { - // TODO - return notSupported -} - -func (d *gluster) Readlink(name string) (string, error) { - // TODO - return "", notSupported -} - func newGluster(endpoint, ak, sk, token string) (ObjectStorage, error) { if !strings.Contains(endpoint, "://") { endpoint = fmt.Sprintf("gluster://%s", endpoint) From 2f4c9c48cf259d94a5a9508cbd0f31a5b9def0d1 Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Fri, 30 Jun 2023 11:35:53 +0800 Subject: [PATCH 06/13] add unit test for gluster --- .github/workflows/unittests.yml | 9 ++++++++- Makefile | 3 ++- pkg/object/gluster_test.go | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 pkg/object/gluster_test.go diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 7f0b63ac951f..6c2d4bb3d965 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -37,6 +37,7 @@ jobs: MINIO_TEST_BUCKET: 127.0.0.1:9000/testbucket MINIO_ACCESS_KEY: testUser MINIO_SECRET_KEY: testUserPassword + GLUSTER_VOLUME: `hostname`/gv0 DISPLAY_PROGRESSBAR: false HDFS_ADDR: localhost:8020 SFTP_HOST: localhost:2222:/upload/ @@ -172,10 +173,16 @@ jobs: # minio docker run -d -p 9000:9000 -p 9001:9001 -e "MINIO_ROOT_USER=testUser" -e "MINIO_ROOT_PASSWORD=testUserPassword" quay.io/minio/minio:RELEASE.2022-01-25T19-56-04Z server /data --console-address ":9001" - # mc go install github.com/minio/mc@RELEASE.2022-01-07T06-01-38Z && mc config host add local http://127.0.0.1:9000 testUser testUserPassword && mc mb local/testbucket + # gluster + apt install -y glusterfs-server libgfapi0 + systemctl start glusterd.service + mkdir -p /gluster/gv0 + gluster volume create gv0 `hostname`:/gluster/gv0 force + gluster volume start gv0 + # webdav wget -O /home/travis/.m2/rclone-v1.57.0-linux-amd64.zip --no-check-certificate https://downloads.rclone.org/v1.57.0/rclone-v1.57.0-linux-amd64.zip unzip /home/travis/.m2/rclone-v1.57.0-linux-amd64.zip -d /home/travis/.m2/ diff --git a/Makefile b/Makefile index 1ae6cccc236a..24ef6f462aa9 100644 --- a/Makefile +++ b/Makefile @@ -80,9 +80,10 @@ test.meta.non-core: go test -v -cover -run='TestRedisCluster|TestPostgreSQLClient|TestLoadDumpSlow|TestEtcdClient|TestKeyDB' -count=1 -failfast -timeout=12m ./pkg/meta/... -coverprofile=cov.out test.pkg: - go test -v -cover -count=1 -failfast -timeout=12m $$(go list ./pkg/... | grep -v /meta) -coverprofile=cov.out + go test -tags gluster -v -cover -count=1 -failfast -timeout=12m $$(go list ./pkg/... | grep -v /meta) -coverprofile=cov.out test.cmd: sudo JFS_GC_SKIPPEDTIME=1 MINIO_ACCESS_KEY=testUser MINIO_SECRET_KEY=testUserPassword GOMAXPROCS=8 go test -v -count=1 -failfast -cover -timeout=8m ./cmd/... -coverprofile=cov.out -coverpkg=./pkg/...,./cmd/... + test.fdb: go test -v -cover -count=1 -failfast -timeout=4m ./pkg/meta/ -tags fdb -run=TestFdb -coverprofile=cov.out diff --git a/pkg/object/gluster_test.go b/pkg/object/gluster_test.go new file mode 100644 index 000000000000..e155cb7e5a9d --- /dev/null +++ b/pkg/object/gluster_test.go @@ -0,0 +1,33 @@ +//go:build gluster +// +build gluster + +/* + * JuiceFS, Copyright 2023 Juicedata, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package object + +import ( + "os" + "testing" +) + +func TestGluster(t *testing.T) { + if os.Getenv("GLUSTER_VOLUME") == "" { + t.SkipNow() + } + b, _ := newGluster(os.Getenv("GLUSTER_VOLUME"), "", "", "") + testStorage(t, b) +} From 86b4d0ec986cba9e75f042dfed8418876b2a99c4 Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Fri, 30 Jun 2023 11:50:35 +0800 Subject: [PATCH 07/13] trigger the unit test --- .github/workflows/unittests.yml | 2 +- pkg/meta/interface.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 6c2d4bb3d965..1a5dc6bcddd8 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -37,7 +37,7 @@ jobs: MINIO_TEST_BUCKET: 127.0.0.1:9000/testbucket MINIO_ACCESS_KEY: testUser MINIO_SECRET_KEY: testUserPassword - GLUSTER_VOLUME: `hostname`/gv0 + GLUSTER_VOLUME: hostname/gv0 DISPLAY_PROGRESSBAR: false HDFS_ADDR: localhost:8020 SFTP_HOST: localhost:2222:/upload/ diff --git a/pkg/meta/interface.go b/pkg/meta/interface.go index 507c60edfbe4..6aacdd22836b 100644 --- a/pkg/meta/interface.go +++ b/pkg/meta/interface.go @@ -442,7 +442,6 @@ type Meta interface { // Dump the tree under root, which may be modified by checkRoot DumpMeta(w io.Writer, root Ino, keepSecret bool) error LoadMeta(r io.Reader) error - // getBase return the base engine. getBase() *baseMeta InitMetrics(registerer prometheus.Registerer) From 61175fa5f4ea286660a33677a597db789f0f0b8b Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Fri, 30 Jun 2023 11:59:09 +0800 Subject: [PATCH 08/13] fix unit test --- .github/workflows/unittests.yml | 8 ++++---- pkg/meta/interface.go | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 1a5dc6bcddd8..63da7ee94e36 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -37,7 +37,7 @@ jobs: MINIO_TEST_BUCKET: 127.0.0.1:9000/testbucket MINIO_ACCESS_KEY: testUser MINIO_SECRET_KEY: testUserPassword - GLUSTER_VOLUME: hostname/gv0 + GLUSTER_VOLUME: jfstest/gv0 DISPLAY_PROGRESSBAR: false HDFS_ADDR: localhost:8020 SFTP_HOST: localhost:2222:/upload/ @@ -60,7 +60,7 @@ jobs: - name: Install Packages run: | - sudo .github/scripts/apt_install.sh g++-multilib redis-server libacl1-dev attr + sudo .github/scripts/apt_install.sh g++-multilib redis-server libacl1-dev attr glusterfs-server libgfapi0 sudo mkdir -p /home/travis/.m2/ - if: matrix.test == 'test.meta.non-core' @@ -177,10 +177,10 @@ jobs: go install github.com/minio/mc@RELEASE.2022-01-07T06-01-38Z && mc config host add local http://127.0.0.1:9000 testUser testUserPassword && mc mb local/testbucket # gluster - apt install -y glusterfs-server libgfapi0 systemctl start glusterd.service mkdir -p /gluster/gv0 - gluster volume create gv0 `hostname`:/gluster/gv0 force + hostname jfstest + gluster volume create gv0 jfstest:/gluster/gv0 force gluster volume start gv0 # webdav diff --git a/pkg/meta/interface.go b/pkg/meta/interface.go index 6aacdd22836b..507c60edfbe4 100644 --- a/pkg/meta/interface.go +++ b/pkg/meta/interface.go @@ -442,6 +442,7 @@ type Meta interface { // Dump the tree under root, which may be modified by checkRoot DumpMeta(w io.Writer, root Ino, keepSecret bool) error LoadMeta(r io.Reader) error + // getBase return the base engine. getBase() *baseMeta InitMetrics(registerer prometheus.Registerer) From b1e3f5827d62bf87a732682b44a638159c36541a Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Fri, 30 Jun 2023 12:04:16 +0800 Subject: [PATCH 09/13] add sudo --- .github/workflows/unittests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 63da7ee94e36..ab1a9a18f946 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -177,10 +177,10 @@ jobs: go install github.com/minio/mc@RELEASE.2022-01-07T06-01-38Z && mc config host add local http://127.0.0.1:9000 testUser testUserPassword && mc mb local/testbucket # gluster - systemctl start glusterd.service - mkdir -p /gluster/gv0 - hostname jfstest - gluster volume create gv0 jfstest:/gluster/gv0 force + sudo systemctl start glusterd.service + mkdir -p /tmp/gluster/gv0 + sudo hostname jfstest + gluster volume create gv0 jfstest:/tmp/gluster/gv0 force gluster volume start gv0 # webdav From bc568acf5fb9a34632623613c6796dd773f87ea3 Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Fri, 30 Jun 2023 14:20:13 +0800 Subject: [PATCH 10/13] enable upterm --- .github/workflows/unittests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index ab1a9a18f946..c7451af5f332 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -230,7 +230,7 @@ jobs: # files: ./cov.out - name: Setup upterm session - if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }} + if: ${{ failure() }} timeout-minutes: 120 uses: lhotari/action-upterm@v1 From bf59df8fd9c7ade8b1025f06ed85d57ae7dedda6 Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Fri, 30 Jun 2023 14:37:15 +0800 Subject: [PATCH 11/13] sudo for gluster command --- .github/workflows/unittests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index c7451af5f332..9118d722047c 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -60,7 +60,7 @@ jobs: - name: Install Packages run: | - sudo .github/scripts/apt_install.sh g++-multilib redis-server libacl1-dev attr glusterfs-server libgfapi0 + sudo .github/scripts/apt_install.sh g++-multilib redis-server libacl1-dev attr glusterfs-server libglusterfs-dev sudo mkdir -p /home/travis/.m2/ - if: matrix.test == 'test.meta.non-core' @@ -180,8 +180,8 @@ jobs: sudo systemctl start glusterd.service mkdir -p /tmp/gluster/gv0 sudo hostname jfstest - gluster volume create gv0 jfstest:/tmp/gluster/gv0 force - gluster volume start gv0 + sudo gluster volume create gv0 jfstest:/tmp/gluster/gv0 force + sudo gluster volume start gv0 # webdav wget -O /home/travis/.m2/rclone-v1.57.0-linux-amd64.zip --no-check-certificate https://downloads.rclone.org/v1.57.0/rclone-v1.57.0-linux-amd64.zip From b3b7f001b7ab632dd762365248c4cb11d1b908aa Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Fri, 30 Jun 2023 14:46:06 +0800 Subject: [PATCH 12/13] fix gluster.go --- .github/workflows/unittests.yml | 2 +- pkg/object/gluster.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 9118d722047c..61a0252fd136 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -230,7 +230,7 @@ jobs: # files: ./cov.out - name: Setup upterm session - if: ${{ failure() }} + if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }} timeout-minutes: 120 uses: lhotari/action-upterm@v1 diff --git a/pkg/object/gluster.go b/pkg/object/gluster.go index ac3895f091f6..24e96579c958 100644 --- a/pkg/object/gluster.go +++ b/pkg/object/gluster.go @@ -32,7 +32,6 @@ import ( "sort" "strconv" "strings" - "time" "github.com/juicedata/gogfapi/gfapi" ) From f4732e94b77d3e41f9be625d7400e66f3b47a10c Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Fri, 30 Jun 2023 21:37:05 +0800 Subject: [PATCH 13/13] fix gluster apis --- pkg/object/file.go | 5 ++++- pkg/object/gluster.go | 38 ++++++++++++++++++++++++++++---------- pkg/sync/sync.go | 4 ++-- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/pkg/object/file.go b/pkg/object/file.go index 8a605035f137..908b3b20cb28 100644 --- a/pkg/object/file.go +++ b/pkg/object/file.go @@ -231,8 +231,11 @@ func readDirSorted(dirname string) ([]*mEntry, error) { } defer f.Close() entries, err := f.ReadDir(-1) - mEntries := make([]*mEntry, len(entries)) + if err != nil { + return nil, err + } + mEntries := make([]*mEntry, len(entries)) for i, e := range entries { if e.IsDir() { mEntries[i] = &mEntry{e, e.Name() + dirSuffix, nil, false} diff --git a/pkg/object/gluster.go b/pkg/object/gluster.go index 24e96579c958..a633186012bc 100644 --- a/pkg/object/gluster.go +++ b/pkg/object/gluster.go @@ -32,6 +32,7 @@ import ( "sort" "strconv" "strings" + "time" "github.com/juicedata/gogfapi/gfapi" ) @@ -142,6 +143,9 @@ func (c *gluster) Put(key string, in io.Reader) error { func (c *gluster) Delete(key string) error { err := c.vol.Unlink(key) + if err != nil && strings.Contains(err.Error(), "is a directory") { + err = c.vol.Rmdir(key) + } if os.IsNotExist(err) { err = nil } @@ -157,25 +161,31 @@ func (d *gluster) readDirSorted(dirname string) ([]*mEntry, error) { } defer f.Close() entries, err := f.Readdir(0) - mEntries := make([]*mEntry, len(entries)) + if err != nil { + return nil, err + } - for i, e := range entries { + mEntries := make([]*mEntry, 0, len(entries)) + for _, e := range entries { + name := e.Name() + if name == "." || name == ".." { + continue + } if e.IsDir() { - mEntries[i] = &mEntry{nil, e.Name() + dirSuffix, e, false} + mEntries = append(mEntries, &mEntry{nil, name + dirSuffix, e, false}) } else if !e.Mode().IsRegular() { // follow symlink - fi, err := os.Stat(filepath.Join(dirname, e.Name())) + fi, err := os.Stat(filepath.Join(dirname, name)) if err != nil { - mEntries[i] = &mEntry{nil, e.Name(), e, true} + mEntries = append(mEntries, &mEntry{nil, name, e, true}) continue } - name := e.Name() if fi.IsDir() { - name = e.Name() + dirSuffix + name += dirSuffix } - mEntries[i] = &mEntry{nil, name, fi, true} + mEntries = append(mEntries, &mEntry{nil, name, fi, true}) } else { - mEntries[i] = &mEntry{nil, e.Name(), e, false} + mEntries = append(mEntries, &mEntry{nil, name, e, false}) } } sort.Slice(mEntries, func(i, j int) bool { return mEntries[i].Name() < mEntries[j].Name() }) @@ -203,7 +213,7 @@ func (d *gluster) List(prefix, marker, delimiter string, limit int64) ([]Object, } objs = append(objs, obj) } - entries, err := readDirSorted(dir) + entries, err := d.readDirSorted(dir) if err != nil { if os.IsNotExist(err) { return nil, nil @@ -233,10 +243,18 @@ func (d *gluster) List(prefix, marker, delimiter string, limit int64) ([]Object, return objs, nil } +func (d *gluster) Chtimes(path string, mtime time.Time) error { + return notSupported +} + func (d *gluster) Chmod(path string, mode os.FileMode) error { return d.vol.Chmod(path, mode) } +func (d *gluster) Chown(path string, owner, group string) error { + return notSupported +} + func newGluster(endpoint, ak, sk, token string) (ObjectStorage, error) { if !strings.Contains(endpoint, "://") { endpoint = fmt.Sprintf("gluster://%s", endpoint) diff --git a/pkg/sync/sync.go b/pkg/sync/sync.go index dcb90b38618a..a6697eef42f5 100644 --- a/pkg/sync/sync.go +++ b/pkg/sync/sync.go @@ -322,8 +322,8 @@ func checkSum(src, dst object.ObjectStorage, key string, size int64) (bool, erro return equal, err } -var streamRead = map[string]struct{}{"file": {}, "hdfs": {}, "sftp": {}} -var streamWrite = map[string]struct{}{"file": {}, "hdfs": {}, "sftp": {}, "gs": {}, "wasb": {}, "ceph": {}, "swift": {}, "webdav": {}, "upyun": {}, "jfs": {}} +var streamRead = map[string]struct{}{"file": {}, "hdfs": {}, "sftp": {}, "gluster": {}} +var streamWrite = map[string]struct{}{"file": {}, "hdfs": {}, "sftp": {}, "gs": {}, "wasb": {}, "ceph": {}, "swift": {}, "webdav": {}, "upyun": {}, "jfs": {}, "gluster": {}} var readInMem = map[string]struct{}{"mem": {}, "etcd": {}, "redis": {}, "tikv": {}, "mysql": {}, "postgres": {}, "sqlite3": {}} func inMap(obj object.ObjectStorage, m map[string]struct{}) bool {