From c8cc4d03ba017746b2fb096904f751bf3888769a Mon Sep 17 00:00:00 2001 From: tbs60 Date: Fri, 13 Sep 2024 17:25:44 +0800 Subject: [PATCH] fix: opt pump cache, issue: #296 --- .../controller/pkg/manager/remote/utils.go | 103 ++++++++++++++---- 1 file changed, 84 insertions(+), 19 deletions(-) diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/remote/utils.go b/src/backend/booster/bk_dist/controller/pkg/manager/remote/utils.go index 30c29448..d4e7528e 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/remote/utils.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/remote/utils.go @@ -13,7 +13,6 @@ import ( "fmt" "net" "os" - "path/filepath" "strings" "syscall" @@ -111,30 +110,30 @@ func workerSideCache(sandbox *dcSyscall.Sandbox) bool { func isCaredNetError(err error) bool { netErr, ok := err.(net.Error) if !ok { - blog.Infof("remote uitl: error[%v] is not net.Error", err) + blog.Infof("remote util: error[%v] is not net.Error", err) return false } if netErr.Timeout() { - blog.Infof("remote uitl: error[%v] is Timeout()", err) + blog.Infof("remote util: error[%v] is Timeout()", err) return true } opErr, ok := netErr.(*net.OpError) if !ok { - blog.Infof("remote uitl: error[%v] is not net.OpError", netErr) + blog.Infof("remote util: error[%v] is not net.OpError", netErr) return false } else { - blog.Infof("remote uitl: error[%v] is net.OpError[%+v]", err, opErr) + blog.Infof("remote util: error[%v] is net.OpError[%+v]", err, opErr) } switch t := opErr.Err.(type) { case *net.DNSError: - blog.Infof("remote uitl: error[%v] is net.DNSError", opErr.Err) + blog.Infof("remote util: error[%v] is net.DNSError", opErr.Err) return true case *os.SyscallError: if errno, ok := t.Err.(syscall.Errno); ok { - blog.Infof("remote uitl: error[%v] got syscall.Errno[%d]", err, errno) + blog.Infof("remote util: error[%v] got syscall.Errno[%d]", err, errno) switch errno { // syscall.WSAECONNABORTED(10053) syscall.WSAECONNRESET(10054) case syscall.ECONNREFUSED, syscall.ECONNRESET, syscall.ECONNABORTED, 10053, 10054: @@ -156,11 +155,48 @@ func calculateDependencies(fileDetails []*types.FilesDetails) [][]int { dependencies = append(dependencies, make([]int, 0, 0)) } + // 记录需要排除的依赖关系，比如 + // /a/b/xx1.h -> /a/b/c/xx.h /a/b/xx2.h -> /a/b/c/xx.h + // 如果有循环依赖，则链接关系优先，去掉目录依赖 + excludedepend := make(map[int]*[]int) + // 遍历字符串数组，计算依赖关系 for i, s1 := range fileDetails { for j, s2 := range fileDetails { - if i != j && depend(s1, s2) { - dependencies[i] = append(dependencies[i], j) + if i != j { + if dirDepend(s1, s2) { + dependencies[i] = append(dependencies[i], j) + } else if linkDepend(s1, s2) { + dependencies[i] = append(dependencies[i], j) + if dirDepend(s2, s1) { + v, ok := excludedepend[j] + if ok { + *v = append(*v, i) + } else { + excludedepend[j] = &[]int{i} + } + } + } + } + } + } + + // 清掉排除的依赖 + if len(excludedepend) > 0 { + for k, arrv := range excludedepend { + for _, v := range *arrv { + blog.Infof("remote util: clean depend:%s to %s", + fileDetails[k].File.FilePath, + fileDetails[v].File.FilePath) + if len(dependencies[k]) > 0 { + for i := range dependencies[k] { + if dependencies[k][i] == v { + dependencies[k][i] = -1 + blog.Infof("remote util: set depend:%s %d to -1", + fileDetails[k].File.FilePath, i) + } + } + } } } } @@ -174,14 +210,14 @@ func isSubString(s1, s2 string) bool { s2 != "/" } -// depend 检查 s1 是否依赖 s2 -func depend(s1, s2 *types.FilesDetails) bool { +// dirDepend 检查 s1的目录 是否依赖 s2的目录 +func dirDepend(s1, s2 *types.FilesDetails) bool { is1File := s1.File.Priority == sdk.RealFilePriority || s1.File.Priority == sdk.LinkFilePriority is2File := s2.File.Priority == sdk.RealFilePriority || s2.File.Priority == sdk.LinkFilePriority // 如果s1是文件，s2是目录 if is1File { if is2File { // 如果s1是文件，s2是文件 - if isSubString(filepath.Dir(s1.File.FilePath), filepath.Dir(s2.File.FilePath)) { + if isSubString(s1.File.Targetrelativepath, s2.File.Targetrelativepath) { return true } } else { // 如果s1是文件，s2是目录 @@ -191,7 +227,7 @@ func depend(s1, s2 *types.FilesDetails) bool { } } else { if is2File { // 如果s1是目录，s2是文件 - if isSubString(s1.File.FilePath, filepath.Dir(s2.File.FilePath)) { + if isSubString(s1.File.FilePath, s2.File.Targetrelativepath) { return true } } else { // 如果s1是目录，s2是目录 @@ -201,7 +237,11 @@ func depend(s1, s2 *types.FilesDetails) bool { } } - // 如果s1是链接，并且指向s2，则s1依赖s2 + return false +} + +// linkDepend 检查 s1 是否链接到了 s2 +func linkDepend(s1, s2 *types.FilesDetails) bool { if s1.File.LinkTarget != "" && s1.File.LinkTarget == s2.File.FilePath { return true } @@ -224,9 +264,13 @@ func freshPriority(fileDetails []*types.FilesDetails) error { tryNum := 0 for { tryNum++ + if tryNum >= maxTry { + printLeftDepend(fileDetails, dependencies) + } + allok := true for i := range fileDetails { - // 如果超过遍历次数，则剩余的全部赋值，避免死循环 + // 如果超过遍历次数上限，则剩余的全部赋值，避免死循环 if tryNum >= maxTry { if fileDetails[i].File.Priority < 0 { fileDetails[i].File.Priority = sdk.FileDescPriority(maxPriority + 1) @@ -241,9 +285,13 @@ func freshPriority(fileDetails []*types.FilesDetails) error { } else { maxDependPriority := -1 alldependok := true - for _, v := range dependencies[i] { + for j, v := range dependencies[i] { + if v < 0 { + continue + } dependPriority := int(fileDetails[v].File.Priority) if dependPriority >= 0 { + dependencies[i][j] = -1 if dependPriority > maxDependPriority { maxDependPriority = dependPriority } @@ -251,7 +299,7 @@ func freshPriority(fileDetails []*types.FilesDetails) error { maxPriority = dependPriority } } else { - blog.Debugf("remote uitl: %s wait %s", + blog.Debugf("remote util: %s wait %s", fileDetails[i].File.FilePath, fileDetails[v].File.FilePath) alldependok = false } @@ -259,7 +307,7 @@ func freshPriority(fileDetails []*types.FilesDetails) error { if alldependok { dependencies[i] = nil fileDetails[i].File.Priority = sdk.FileDescPriority(maxDependPriority + 1) - blog.Debugf("remote uitl: %s set Priority to %d", + blog.Debugf("remote util: %s set Priority to %d", fileDetails[i].File.FilePath, maxDependPriority+1) } else { allok = false @@ -268,10 +316,27 @@ func freshPriority(fileDetails []*types.FilesDetails) error { } if allok || tryNum >= maxTry { - blog.Infof("remote uitl: finished set Priority after %d try", tryNum) + blog.Infof("remote util: finished set Priority after %d try", tryNum) break } } return nil } + +func printLeftDepend(fileDetails []*types.FilesDetails, dependencies [][]int) { + for i := range fileDetails { + if fileDetails[i].File.Priority < 0 { + dependfiles := []string{} + for _, v := range dependencies[i] { + dependPriority := int(fileDetails[v].File.Priority) + if dependPriority < 0 { + dependfiles = append(dependfiles, fileDetails[v].File.FilePath) + } + } + blog.Warnf("remote util: after max try, %s wait %s", + fileDetails[i].File.FilePath, + strings.Join(dependfiles, " ")) + } + } +}