meta: slowdown dump process to reduce memory usage (#4253)

juicedata · Dec 12, 2023 · 6f2440c · 6f2440c
1 parent 963d81e
commit 6f2440c
Show file tree

Hide file tree

Showing 7 changed files with 29 additions and 9 deletions.
diff --git a/cmd/dump.go b/cmd/dump.go
@@ -55,6 +55,10 @@ Details: https://juicefs.com/docs/community/metadata_dump_load`,
 				Name:  "keep-secret-key",
 				Usage: "keep secret keys intact (WARNING: Be careful as they may be leaked)",
 			},
+			&cli.BoolFlag{
+				Name:  "fast",
+				Usage: "speedup dump by load all metadata into memory",
+			},
 		},
 	}
 }
@@ -101,7 +105,7 @@ func dump(ctx *cli.Context) (err error) {
 	if st := m.Chroot(meta.Background, metaConf.Subdir); st != 0 {
 		return st
 	}
-	if err := m.DumpMeta(w, 1, ctx.Bool("keep-secret-key")); err != nil {
+	if err := m.DumpMeta(w, 1, ctx.Bool("keep-secret-key"), ctx.Bool("fast")); err != nil {
 		return err
 	}
 	logger.Infof("Dump metadata into %s succeed", dst)

diff --git a/pkg/meta/interface.go b/pkg/meta/interface.go
@@ -444,7 +444,7 @@ type Meta interface {
 	HandleQuota(ctx Context, cmd uint8, dpath string, quotas map[string]*Quota, strict, repair bool) error
 
 	// Dump the tree under root, which may be modified by checkRoot
-	DumpMeta(w io.Writer, root Ino, keepSecret bool) error
+	DumpMeta(w io.Writer, root Ino, keepSecret, fast bool) error
 	LoadMeta(r io.Reader) error
 
 	// getBase return the base engine.

diff --git a/pkg/meta/load_dump_test.go b/pkg/meta/load_dump_test.go
@@ -248,7 +248,7 @@ func testDump(t *testing.T, m Meta, root Ino, expect, result string) {
 	if _, err = m.Load(true); err != nil {
 		t.Fatalf("load setting: %s", err)
 	}
-	if err = m.DumpMeta(fp, root, false); err != nil {
+	if err = m.DumpMeta(fp, root, false, true); err != nil {
 		t.Fatalf("dump meta: %s", err)
 	}
 	cmd := exec.Command("diff", expect, result)

diff --git a/pkg/meta/redis.go b/pkg/meta/redis.go
@@ -3845,7 +3845,7 @@ func (m *redisMeta) dumpDir(inode Ino, tree *DumpedEntry, bw *bufio.Writer, dept
 	return nil
 }
 
-func (m *redisMeta) DumpMeta(w io.Writer, root Ino, keepSecret bool) (err error) {
+func (m *redisMeta) DumpMeta(w io.Writer, root Ino, keepSecret, fast bool) (err error) {
 	defer func() {
 		if p := recover(); p != nil {
 			if e, ok := p.(error); ok {

diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go
@@ -3524,7 +3524,7 @@ func (m *dbMeta) makeSnap(ses *xorm.Session, bar *utils.Bar) error {
 	return nil
 }
 
-func (m *dbMeta) DumpMeta(w io.Writer, root Ino, keepSecret bool) (err error) {
+func (m *dbMeta) DumpMeta(w io.Writer, root Ino, keepSecret, fast bool) (err error) {
 	defer func() {
 		if p := recover(); p != nil {
 			if e, ok := p.(error); ok {
@@ -3539,7 +3539,7 @@ func (m *dbMeta) DumpMeta(w io.Writer, root Ino, keepSecret bool) (err error) {
 	var tree, trash *DumpedEntry
 	root = m.checkRoot(root)
 	return m.roTxn(func(s *xorm.Session) error {
-		if root == RootInode {
+		if root == RootInode && fast {
 			defer func() { m.snap = nil }()
 			bar := progress.AddCountBar("Snapshot keys", 0)
 			if err = m.makeSnap(s, bar); err != nil {
@@ -3552,6 +3552,11 @@ func (m *dbMeta) DumpMeta(w io.Writer, root Ino, keepSecret bool) (err error) {
 			if tree, err = m.dumpEntry(s, root, TypeDirectory); err != nil {
 				return err
 			}
+			if root == 1 {
+				if trash, err = m.dumpEntry(s, TrashInode, TypeDirectory); err != nil {
+					return err
+				}
+			}
 		}
 		if tree == nil {
 			return errors.New("The entry of the root inode was not found")

diff --git a/pkg/meta/tkv.go b/pkg/meta/tkv.go
@@ -2956,7 +2956,7 @@ func (m *kvMeta) dumpDir(inode Ino, tree *DumpedEntry, bw *bufio.Writer, depth i
 	return nil
 }
 
-func (m *kvMeta) DumpMeta(w io.Writer, root Ino, keepSecret bool) (err error) {
+func (m *kvMeta) DumpMeta(w io.Writer, root Ino, keepSecret, fast bool) (err error) {
 	defer func() {
 		if p := recover(); p != nil {
 			debug.PrintStack()
@@ -2986,7 +2986,7 @@ func (m *kvMeta) DumpMeta(w io.Writer, root Ino, keepSecret bool) (err error) {
 	var tree, trash *DumpedEntry
 	root = m.checkRoot(root)
 
-	if root == 1 { // make snap
+	if root == 1 && fast { // make snap
 		m.snap = make(map[Ino]*DumpedEntry)
 		defer func() {
 			m.snap = nil
@@ -3072,6 +3072,17 @@ func (m *kvMeta) DumpMeta(w io.Writer, root Ino, keepSecret bool) (err error) {
 		if err = m.dumpEntry(root, tree); err != nil {
 			return err
 		}
+		if root == 1 {
+			trash = &DumpedEntry{
+				Attr: &DumpedAttr{
+					Inode: TrashInode,
+					Type:  "directory",
+				},
+			}
+			if err = m.dumpEntry(TrashInode, trash); err != nil {
+				return err
+			}
+		}
 	}
 
 	if tree == nil || tree.Attr == nil {

diff --git a/pkg/vfs/backup.go b/pkg/vfs/backup.go
@@ -83,7 +83,7 @@ func backup(m meta.Meta, blob object.ObjectStorage, now time.Time) error {
 	defer os.Remove(fp.Name())
 	defer fp.Close()
 	zw := gzip.NewWriter(fp)
-	err = m.DumpMeta(zw, 0, false) // force dump the whole tree
+	err = m.DumpMeta(zw, 0, false, false) // force dump the whole tree
 	_ = zw.Close()
 	if err != nil {
 		return err