support GlusterFS as object store

.github/workflows/unittests.yml

@@ -37,6 +37,7 @@ jobs:
       MINIO_TEST_BUCKET: 127.0.0.1:9000/testbucket
       MINIO_ACCESS_KEY: testUser
       MINIO_SECRET_KEY: testUserPassword
+      GLUSTER_VOLUME: jfstest/gv0
       DISPLAY_PROGRESSBAR: false
       HDFS_ADDR: localhost:8020
       SFTP_HOST: localhost:2222:/upload/
@@ -59,7 +60,7 @@ jobs:
 
       - name: Install Packages
         run: |
-          sudo .github/scripts/apt_install.sh g++-multilib redis-server libacl1-dev attr
+          sudo .github/scripts/apt_install.sh g++-multilib redis-server libacl1-dev attr glusterfs-server libglusterfs-dev
           sudo mkdir -p /home/travis/.m2/
 
       - if: matrix.test == 'test.meta.non-core'
@@ -172,10 +173,16 @@ jobs:
 
               # minio
               docker run -d -p 9000:9000 -p 9001:9001 -e "MINIO_ROOT_USER=testUser" -e "MINIO_ROOT_PASSWORD=testUserPassword" quay.io/minio/minio:RELEASE.2022-01-25T19-56-04Z server /data --console-address ":9001"
-
               # mc
               go install github.com/minio/[email protected] && mc config host add local http://127.0.0.1:9000 testUser testUserPassword && mc mb local/testbucket
 
+              # gluster
+              sudo systemctl start glusterd.service
+              mkdir -p /tmp/gluster/gv0
+              sudo hostname jfstest
+              sudo gluster volume create gv0 jfstest:/tmp/gluster/gv0 force
+              sudo gluster volume start gv0
+
               # webdav
               wget -O /home/travis/.m2/rclone-v1.57.0-linux-amd64.zip --no-check-certificate https://downloads.rclone.org/v1.57.0/rclone-v1.57.0-linux-amd64.zip
               unzip /home/travis/.m2/rclone-v1.57.0-linux-amd64.zip -d /home/travis/.m2/

Makefile

@@ -33,6 +33,9 @@ juicefs.ceph: Makefile cmd/*.go pkg/*/*.go
 juicefs.fdb: Makefile cmd/*.go pkg/*/*.go
 	go build -tags fdb -ldflags="$(LDFLAGS)"  -o juicefs.fdb .
 
+juicefs.gluster: Makefile cmd/*.go pkg/*/*.go
+	go build -tags gluster -ldflags="$(LDFLAGS)"  -o juicefs.gluster .
+
 # This is the script for compiling the Linux version on the MacOS platform.
 # Please execute the `brew install FiloSottile/musl-cross/musl-cross` command before using it.
 juicefs.linux:
@@ -77,9 +80,10 @@ test.meta.non-core:
 	go test -v -cover -run='TestRedisCluster|TestPostgreSQLClient|TestLoadDumpSlow|TestEtcdClient|TestKeyDB' -count=1  -failfast -timeout=12m ./pkg/meta/... -coverprofile=cov.out
 
 test.pkg:
-	go test -v -cover -count=1  -failfast -timeout=12m $$(go list ./pkg/... | grep -v /meta) -coverprofile=cov.out
+	go test -tags gluster -v -cover -count=1  -failfast -timeout=12m $$(go list ./pkg/... | grep -v /meta) -coverprofile=cov.out
 
 test.cmd:
 	sudo JFS_GC_SKIPPEDTIME=1 MINIO_ACCESS_KEY=testUser MINIO_SECRET_KEY=testUserPassword GOMAXPROCS=8 go test -v -count=1 -failfast -cover -timeout=8m ./cmd/... -coverprofile=cov.out -coverpkg=./pkg/...,./cmd/...
+
 test.fdb:
 	go test -v -cover -count=1  -failfast -timeout=4m ./pkg/meta/ -tags fdb -run=TestFdb -coverprofile=cov.out

go.mod

@@ -32,6 +32,7 @@ require (
 	github.com/jackc/pgx/v5 v5.3.1
 	github.com/jcmturner/gokrb5/v8 v8.4.4
 	github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d
+	github.com/juicedata/gogfapi v0.0.0-20230626071140-fc28e5537825
 	github.com/juju/ratelimit v1.0.2
 	github.com/ks3sdklib/aws-sdk-go v1.2.2
 	github.com/mattn/go-isatty v0.0.18

go.sum

@@ -627,6 +627,8 @@ github.com/juicedata/go-fuse/v2 v2.1.1-0.20230418090413-880e994b8a4c h1:AVWJgjm5
 github.com/juicedata/go-fuse/v2 v2.1.1-0.20230418090413-880e994b8a4c/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc=
 github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d h1:kpQMvNZJKGY3PTt7OSoahYc4nM0HY67SvK0YyS0GLwA=
 github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d/go.mod h1:dlxKkLh3qAIPtgr2U/RVzsZJDuXA1ffg+Njikfmhvgw=
+github.com/juicedata/gogfapi v0.0.0-20230626071140-fc28e5537825 h1:7KrwI4HPqvNLKVcfkfDMLQQmT0GrnCs8T9EX+XCdZnM=
+github.com/juicedata/gogfapi v0.0.0-20230626071140-fc28e5537825/go.mod h1:Ho5G4KgrgbMKW0buAJdOmYoJcOImkzznJQaLiATrsx4=
 github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible h1:2/ttSmYoX+QMegpNyAJR0Y6aHcVk57F7RJit5xN2T/s=
 github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible/go.mod h1:Ukwa8ffRQLV6QRwpqGioPjn2Wnf7TBDA4DbennDOqHE=
 github.com/juicedata/minio v0.0.0-20221113011458-8866d5c9df8c h1:w+4eiZLSLd6aQcy+7wn++hI1caDAm+rNOG7Me5qO7Sw=

pkg/object/file.go

@@ -231,8 +231,11 @@ func readDirSorted(dirname string) ([]*mEntry, error) {
 	}
 	defer f.Close()
 	entries, err := f.ReadDir(-1)
-	mEntries := make([]*mEntry, len(entries))
+	if err != nil {
+		return nil, err
+	}
 
+	mEntries := make([]*mEntry, len(entries))
 	for i, e := range entries {
 		if e.IsDir() {
 			mEntries[i] = &mEntry{e, e.Name() + dirSuffix, nil, false}

pkg/object/gluster.go

@@ -0,0 +1,289 @@
+//go:build gluster
+// +build gluster
+
+/*
+ * JuiceFS, Copyright 2023 Juicedata, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package object
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/fs"
+	"math/rand"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/juicedata/gogfapi/gfapi"
+)
+
+type gluster struct {
+	DefaultObjectStorage
+	name string
+	vol  *gfapi.Volume
+}
+
+func (c *gluster) String() string {
+	return fmt.Sprintf("gluster://%s/", c.name)
+}
+
+func (c *gluster) Head(key string) (Object, error) {
+	fi, err := c.vol.Stat(key)
+	if err != nil {
+		return nil, err
+	}
+	return c.toFile(key, fi, false), nil
+}
+
+func (d *gluster) toFile(key string, fi fs.FileInfo, isSymlink bool) *file {
+	size := fi.Size()
+	if fi.IsDir() {
+		size = 0
+	}
+	owner, group := getOwnerGroup(fi)
+	return &file{
+		obj{
+			key,
+			size,
+			fi.ModTime(),
+			fi.IsDir(),
+			"",
+		},
+		owner,
+		group,
+		fi.Mode(),
+		isSymlink,
+	}
+}
+
+func (c *gluster) Get(key string, off, limit int64) (io.ReadCloser, error) {
+	f, err := c.vol.Open(key)
+	if err != nil {
+		return nil, err
+	}
+
+	finfo, err := f.Stat()
+	if err != nil {
+		_ = f.Close()
+		return nil, err
+	}
+	if finfo.IsDir() || off > finfo.Size() {
+		_ = f.Close()
+		return io.NopCloser(bytes.NewBuffer([]byte{})), nil
+	}
+
+	if limit > 0 {
+		return &SectionReaderCloser{
+			SectionReader: io.NewSectionReader(f, off, limit),
+			Closer:        f,
+		}, nil
+	}
+	return f, nil
+}
+
+func (c *gluster) Put(key string, in io.Reader) error {
+	if strings.HasSuffix(key, dirSuffix) {
+		return c.vol.MkdirAll(key, os.FileMode(0777))
+	}
+	p := key
+	tmp := filepath.Join(filepath.Dir(p), "."+filepath.Base(p)+".tmp"+strconv.Itoa(rand.Int()))
+	f, err := c.vol.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
+	if err != nil && os.IsNotExist(err) {
+		if err := c.vol.MkdirAll(filepath.Dir(p), os.FileMode(0777)); err != nil {
+			return err
+		}
+		f, err = c.vol.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
+	}
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err != nil {
+			_ = c.vol.Unlink(tmp)
+		}
+	}()
+
+	buf := bufPool.Get().(*[]byte)
+	defer bufPool.Put(buf)
+	_, err = io.CopyBuffer(f, in, *buf)
+	if err != nil {
+		_ = f.Close()
+		return err
+	}
+	if err = f.Sync(); err != nil {
+		_ = f.Close()
+		return err
+	}
+	if err = f.Close(); err != nil {
+		return err
+	}
+	err = c.vol.Rename(tmp, p)
+	return err
+}
+
+func (c *gluster) Delete(key string) error {
+	err := c.vol.Unlink(key)
+	if err != nil && strings.Contains(err.Error(), "is a directory") {
+		err = c.vol.Rmdir(key)
+	}
+	if os.IsNotExist(err) {
+		err = nil
+	}
+	return err
+}
+
+// readDirSorted reads the directory named by dirname and returns
+// a sorted list of directory entries.
+func (d *gluster) readDirSorted(dirname string) ([]*mEntry, error) {
+	f, err := d.vol.Open(dirname)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	entries, err := f.Readdir(0)
+	if err != nil {
+		return nil, err
+	}
+
+	mEntries := make([]*mEntry, 0, len(entries))
+	for _, e := range entries {
+		name := e.Name()
+		if name == "." || name == ".." {
+			continue
+		}
+		if e.IsDir() {
+			mEntries = append(mEntries, &mEntry{nil, name + dirSuffix, e, false})
+		} else if !e.Mode().IsRegular() {
+			// follow symlink
+			fi, err := os.Stat(filepath.Join(dirname, name))
+			if err != nil {
+				mEntries = append(mEntries, &mEntry{nil, name, e, true})
+				continue
+			}
+			if fi.IsDir() {
+				name += dirSuffix
+			}
+			mEntries = append(mEntries, &mEntry{nil, name, fi, true})
+		} else {
+			mEntries = append(mEntries, &mEntry{nil, name, e, false})
+		}
+	}
+	sort.Slice(mEntries, func(i, j int) bool { return mEntries[i].Name() < mEntries[j].Name() })
+	return mEntries, err
+}
+
+func (d *gluster) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
+	if delimiter != "/" {
+		return nil, notSupported
+	}
+	var dir string = prefix
+	var objs []Object
+	if !strings.HasSuffix(dir, dirSuffix) {
+		dir = path.Dir(dir)
+		if !strings.HasSuffix(dir, dirSuffix) {
+			dir += dirSuffix
+		}
+	} else if marker == "" {
+		obj, err := d.Head(prefix)
+		if err != nil {
+			if os.IsNotExist(err) {
+				return nil, nil
+			}
+			return nil, err
+		}
+		objs = append(objs, obj)
+	}
+	entries, err := d.readDirSorted(dir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	for _, e := range entries {
+		p := filepath.Join(dir, e.Name())
+		if e.IsDir() {
+			p = filepath.ToSlash(p + "/")
+		}
+		key := p
+		if !strings.HasPrefix(key, prefix) || (marker != "" && key <= marker) {
+			continue
+		}
+		info, err := e.Info()
+		if err != nil {
+			logger.Warnf("stat %s: %s", p, err)
+			continue
+		}
+		f := d.toFile(key, info, e.isSymlink)
+		objs = append(objs, f)
+		if len(objs) == int(limit) {
+			break
+		}
+	}
+	return objs, nil
+}
+
+func (d *gluster) Chtimes(path string, mtime time.Time) error {
+	return notSupported
+}
+
+func (d *gluster) Chmod(path string, mode os.FileMode) error {
+	return d.vol.Chmod(path, mode)
+}
+
+func (d *gluster) Chown(path string, owner, group string) error {
+	return notSupported
+}
+
+func newGluster(endpoint, ak, sk, token string) (ObjectStorage, error) {
+	if !strings.Contains(endpoint, "://") {
+		endpoint = fmt.Sprintf("gluster://%s", endpoint)
+	}
+	uri, err := url.ParseRequestURI(endpoint)
+	if err != nil {
+		return nil, fmt.Errorf("Invalid endpoint %s: %s", endpoint, err)
+	}
+	ps := strings.Split(uri.Path, "/")
+	if len(ps) == 1 {
+		return nil, fmt.Errorf("no volume provided")
+	}
+	name := ps[1]
+	v := &gfapi.Volume{}
+	// TODO: support port in host
+	err = v.Init(name, strings.Split(uri.Host, ",")...)
+	if err != nil {
+		return nil, fmt.Errorf("init %s: %s", name, err)
+	}
+	err = v.Mount()
+	if err != nil {
+		return nil, fmt.Errorf("mount %s: %s", name, err)
+	}
+	return &gluster{
+		name: name,
+		vol:  v,
+	}, nil
+}
+
+func init() {
+	Register("gluster", newGluster)
+}