From 014a68d5810fbbbe2c9f0c9cd994751e7eaaa673 Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Fri, 27 Sep 2024 14:03:47 +0800 Subject: [PATCH] cmd/gc: fix the issue that pending delete objects are misclassified as leaked (#5198) --- cmd/fsck.go | 2 +- cmd/gc.go | 27 +++++++++++++++++++++------ pkg/meta/base_test.go | 6 +++--- pkg/meta/interface.go | 2 +- pkg/meta/redis.go | 27 +++++++++++++++++++++++++-- pkg/meta/sql.go | 18 ++++++++++++++++-- pkg/meta/tkv.go | 16 ++++++++++++++-- 7 files changed, 81 insertions(+), 17 deletions(-) diff --git a/cmd/fsck.go b/cmd/fsck.go index 53eb5a97d198..aed50d35e8a5 100644 --- a/cmd/fsck.go +++ b/cmd/fsck.go @@ -142,7 +142,7 @@ func fsck(ctx *cli.Context) error { // List all slices in metadata engine sliceCSpin := progress.AddCountSpinner("Listed slices") slices := make(map[meta.Ino][]meta.Slice) - r := m.ListSlices(c, slices, false, sliceCSpin.Increment) + r := m.ListSlices(c, slices, false, false, sliceCSpin.Increment) if r != 0 { logger.Fatalf("list all slices: %s", r) } diff --git a/cmd/gc.go b/cmd/gc.go index e14c4e1ab843..30e7d9740f88 100644 --- a/cmd/gc.go +++ b/cmd/gc.go @@ -210,7 +210,7 @@ func gc(ctx *cli.Context) error { // List all slices in metadata engine slices := make(map[meta.Ino][]meta.Slice) - r := m.ListSlices(c, slices, delete, sliceCSpin.Increment) + r := m.ListSlices(c, slices, true, delete, sliceCSpin.Increment) if r != 0 { logger.Fatalf("list all slices: %s", r) } @@ -247,9 +247,16 @@ func gc(ctx *cli.Context) error { logger.Fatalf("list all blocks: %s", err) } vkeys := make(map[uint64]uint32) + pkeys := make(map[uint64]uint32) ckeys := make(map[uint64]uint32) var total int64 var totalBytes uint64 + for _, s := range slices[0] { + pkeys[s.Id] = s.Size + total += int64(int(s.Size-1)/chunkConf.BlockSize) + 1 + totalBytes += uint64(s.Size) + } + slices[0] = nil for _, s := range slices[1] { ckeys[s.Id] = s.Size total += int64(int(s.Size-1)/chunkConf.BlockSize) + 1 @@ -269,6 +276,7 @@ func gc(ctx *cli.Context) error { bar := progress.AddCountBar("Scanned objects", total) valid := progress.AddDoubleSpinnerTwo("Valid objects", "Valid data") + pending := progress.AddDoubleSpinnerTwo("Pending delete objects", "Pending delete data") compacted := progress.AddDoubleSpinnerTwo("Compacted objects", "Compacted data") leaked := progress.AddDoubleSpinnerTwo("Leaked objects", "Leaked data") skipped := progress.AddDoubleSpinnerTwo("Skipped objects", "Skipped data") @@ -321,10 +329,12 @@ func gc(ctx *cli.Context) error { bar.Increment() cid, _ := strconv.Atoi(parts[0]) size := vkeys[uint64(cid)] - var cobj bool + var pobj, cobj bool + if size == 0 { + size, pobj = pkeys[uint64(cid)] + } if size == 0 { - size = ckeys[uint64(cid)] - cobj = true + size, cobj = ckeys[uint64(cid)] } if size == 0 { logger.Debugf("find leaked object: %s, size: %d", obj.Key(), obj.Size()) @@ -337,6 +347,8 @@ func gc(ctx *cli.Context) error { if (indx+1)*csize > int(size) { logger.Warnf("size of slice %d is larger than expected: %d > %d", cid, indx*chunkConf.BlockSize+csize, size) foundLeaked(obj) + } else if pobj { + pending.IncrInt64(obj.Size()) } else if cobj { compacted.IncrInt64(obj.Size()) } else { @@ -346,6 +358,8 @@ func gc(ctx *cli.Context) error { if indx*chunkConf.BlockSize+csize != int(size) { logger.Warnf("size of slice %d is %d, but expect %d", cid, indx*chunkConf.BlockSize+csize, size) foundLeaked(obj) + } else if pobj { + pending.IncrInt64(obj.Size()) } else if cobj { compacted.IncrInt64(obj.Size()) } else { @@ -371,13 +385,14 @@ func gc(ctx *cli.Context) error { progress.Done() vc, _ := valid.Current() + pc, pb := pending.Current() cc, cb := compacted.Current() lc, lb := leaked.Current() sc, sb := skipped.Current() dsc, dsb := cleanedSliceSpin.Current() fc, fb := cleanedFileSpin.Current() - logger.Infof("scanned %d objects, %d valid, %d compacted (%d bytes), %d leaked (%d bytes), %d delslices (%d bytes), %d delfiles (%d bytes), %d skipped (%d bytes)", - bar.Current(), vc, cc, cb, lc, lb, dsc, dsb, fc, fb, sc, sb) + logger.Infof("scanned %d objects, %d valid, %d pending delete (%d bytes), %d compacted (%d bytes), %d leaked (%d bytes), %d delslices (%d bytes), %d delfiles (%d bytes), %d skipped (%d bytes)", + bar.Current(), vc, pc, pb, cc, cb, lc, lb, dsc, dsb, fc, fb, sc, sb) if lc > 0 && !delete { logger.Infof("Please add `--delete` to clean leaked objects") } diff --git a/pkg/meta/base_test.go b/pkg/meta/base_test.go index e1447b2b0d14..a77eabfce02f 100644 --- a/pkg/meta/base_test.go +++ b/pkg/meta/base_test.go @@ -1486,7 +1486,7 @@ func testCompaction(t *testing.T, m Meta, trash bool) { } p.Done() sliceMap := make(map[Ino][]Slice) - if st := m.ListSlices(ctx, sliceMap, false, nil); st != 0 { + if st := m.ListSlices(ctx, sliceMap, false, false, nil); st != 0 { t.Fatalf("list all slices: %s", st) } @@ -1694,7 +1694,7 @@ func testTruncateAndDelete(t *testing.T, m Meta) { } var total int64 slices := make(map[Ino][]Slice) - m.ListSlices(ctx, slices, false, func() { total++ }) + m.ListSlices(ctx, slices, false, false, func() { total++ }) var totalSlices int for _, ss := range slices { totalSlices += len(ss) @@ -1709,7 +1709,7 @@ func testTruncateAndDelete(t *testing.T, m Meta) { time.Sleep(time.Millisecond * 100) slices = make(map[Ino][]Slice) - m.ListSlices(ctx, slices, false, nil) + m.ListSlices(ctx, slices, false, false, nil) totalSlices = 0 for _, ss := range slices { totalSlices += len(ss) diff --git a/pkg/meta/interface.go b/pkg/meta/interface.go index 3fc721179140..3d97cf588ded 100644 --- a/pkg/meta/interface.go +++ b/pkg/meta/interface.go @@ -418,7 +418,7 @@ type Meta interface { Compact(ctx Context, inode Ino, concurrency int, preFunc, postFunc func()) syscall.Errno // ListSlices returns all slices used by all files. - ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, showProgress func()) syscall.Errno + ListSlices(ctx Context, slices map[Ino][]Slice, scanPending, delete bool, showProgress func()) syscall.Errno // Remove all files and directories recursively. // count represents the number of attempted deletions of entries (even if failed). Remove(ctx Context, parent Ino, name string, count *uint64) syscall.Errno diff --git a/pkg/meta/redis.go b/pkg/meta/redis.go index 24f84d8e0c11..67674bdd40d2 100644 --- a/pkg/meta/redis.go +++ b/pkg/meta/redis.go @@ -3041,7 +3041,7 @@ func (m *redisMeta) hscan(ctx context.Context, key string, f func([]string) erro return nil } -func (m *redisMeta) ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, showProgress func()) syscall.Errno { +func (m *redisMeta) ListSlices(ctx Context, slices map[Ino][]Slice, scanPending, delete bool, showProgress func()) syscall.Errno { m.cleanupLeakedInodes(delete) m.cleanupLeakedChunks(delete) m.cleanupOldSliceRefs(delete) @@ -3083,10 +3083,33 @@ func (m *redisMeta) ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, } return nil }) - if err != nil || m.getFormat().TrashDays == 0 { + if err != nil { + logger.Warnf("scan chunks: %s", err) return errno(err) } + if scanPending { + _ = m.hscan(Background, m.sliceRefs(), func(keys []string) error { + for i := 0; i < len(keys); i += 2 { + key, val := keys[i], keys[i+1] + if strings.HasPrefix(val, "-") { // < 0 + ps := strings.Split(key, "_") + if len(ps) == 2 { + id, _ := strconv.ParseUint(ps[0][1:], 10, 64) + size, _ := strconv.ParseUint(ps[1], 10, 32) + if id > 0 && size > 0 { + slices[0] = append(slices[0], Slice{Id: id, Size: uint32(size)}) + } + } + } + } + return nil + }) + } + + if m.getFormat().TrashDays == 0 { + return 0 + } return errno(m.scanTrashSlices(ctx, func(ss []Slice, _ int64) (bool, error) { slices[1] = append(slices[1], ss...) if showProgress != nil { diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index 9803051fe54d..634ac6f16382 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -2851,7 +2851,7 @@ func (m *dbMeta) scanAllChunks(ctx Context, ch chan<- cchunk, bar *utils.Bar) er }) } -func (m *dbMeta) ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, showProgress func()) syscall.Errno { +func (m *dbMeta) ListSlices(ctx Context, slices map[Ino][]Slice, scanPending, delete bool, showProgress func()) syscall.Errno { if delete { m.doCleanupSlices() } @@ -2881,10 +2881,24 @@ func (m *dbMeta) ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, sh if err != nil { return errno(err) } + + if scanPending { + _ = m.roTxn(func(s *xorm.Session) error { + var cks []sliceRef + err := s.Where("refs <= 0").Find(&cks) + if err != nil { + return err + } + for _, ck := range cks { + slices[0] = append(slices[0], Slice{Id: ck.Id, Size: ck.Size}) + } + return nil + }) + } + if m.getFormat().TrashDays == 0 { return 0 } - return errno(m.scanTrashSlices(ctx, func(ss []Slice, _ int64) (bool, error) { slices[1] = append(slices[1], ss...) if showProgress != nil { diff --git a/pkg/meta/tkv.go b/pkg/meta/tkv.go index e9f6b0718b31..df921bcfb4d2 100644 --- a/pkg/meta/tkv.go +++ b/pkg/meta/tkv.go @@ -2392,7 +2392,7 @@ func (m *kvMeta) scanAllChunks(ctx Context, ch chan<- cchunk, bar *utils.Bar) er }) } -func (m *kvMeta) ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, showProgress func()) syscall.Errno { +func (m *kvMeta) ListSlices(ctx Context, slices map[Ino][]Slice, scanPending, delete bool, showProgress func()) syscall.Errno { if delete { m.doCleanupSlices() } @@ -2421,10 +2421,22 @@ func (m *kvMeta) ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, sh } } } + + if scanPending { + // slice refs: Kccccccccnnnn + klen = 1 + 8 + 4 + result, _ = m.scanValues(m.fmtKey("K"), -1, func(k, v []byte) bool { + return len(k) == klen && len(v) == 8 && parseCounter(v) < 0 + }) + for k := range result { + rb := utils.FromBuffer([]byte(k)[1:]) + slices[0] = append(slices[0], Slice{Id: rb.Get64(), Size: rb.Get32()}) + } + } + if m.getFormat().TrashDays == 0 { return 0 } - return errno(m.scanTrashSlices(ctx, func(ss []Slice, _ int64) (bool, error) { slices[1] = append(slices[1], ss...) if showProgress != nil {