Skip to content

Commit

Permalink
fix: opt pump cache, issue: TencentBlueKing#296
Browse files Browse the repository at this point in the history
  • Loading branch information
@tbs60
tbs60 committed Sep 13, 2024
1 parent f06aa06 commit c8cc4d0
Showing 1 changed file with 84 additions and 19 deletions.
103 changes: 84 additions & 19 deletions src/backend/booster/bk_dist/controller/pkg/manager/remote/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"fmt"
"net"
"os"
"path/filepath"
"strings"
"syscall"

Expand Down Expand Up @@ -111,30 +110,30 @@ func workerSideCache(sandbox *dcSyscall.Sandbox) bool {
func isCaredNetError(err error) bool {
netErr, ok := err.(net.Error)
if !ok {
blog.Infof("remote uitl: error[%v] is not net.Error", err)
blog.Infof("remote util: error[%v] is not net.Error", err)
return false
}

if netErr.Timeout() {
blog.Infof("remote uitl: error[%v] is Timeout()", err)
blog.Infof("remote util: error[%v] is Timeout()", err)
return true
}

opErr, ok := netErr.(*net.OpError)
if !ok {
blog.Infof("remote uitl: error[%v] is not net.OpError", netErr)
blog.Infof("remote util: error[%v] is not net.OpError", netErr)
return false
} else {
blog.Infof("remote uitl: error[%v] is net.OpError[%+v]", err, opErr)
blog.Infof("remote util: error[%v] is net.OpError[%+v]", err, opErr)
}

switch t := opErr.Err.(type) {
case *net.DNSError:
blog.Infof("remote uitl: error[%v] is net.DNSError", opErr.Err)
blog.Infof("remote util: error[%v] is net.DNSError", opErr.Err)
return true
case *os.SyscallError:
if errno, ok := t.Err.(syscall.Errno); ok {
blog.Infof("remote uitl: error[%v] got syscall.Errno[%d]", err, errno)
blog.Infof("remote util: error[%v] got syscall.Errno[%d]", err, errno)
switch errno {
// syscall.WSAECONNABORTED(10053) syscall.WSAECONNRESET(10054)
case syscall.ECONNREFUSED, syscall.ECONNRESET, syscall.ECONNABORTED, 10053, 10054:
Expand All @@ -156,11 +155,48 @@ func calculateDependencies(fileDetails []*types.FilesDetails) [][]int {
dependencies = append(dependencies, make([]int, 0, 0))
}

// 记录需要排除的依赖关系，比如
// /a/b/xx1.h -> /a/b/c/xx.h /a/b/xx2.h -> /a/b/c/xx.h
// 如果有循环依赖，则链接关系优先，去掉目录依赖
excludedepend := make(map[int]*[]int)

// 遍历字符串数组，计算依赖关系
for i, s1 := range fileDetails {
for j, s2 := range fileDetails {
if i != j && depend(s1, s2) {
dependencies[i] = append(dependencies[i], j)
if i != j {
if dirDepend(s1, s2) {
dependencies[i] = append(dependencies[i], j)
} else if linkDepend(s1, s2) {
dependencies[i] = append(dependencies[i], j)
if dirDepend(s2, s1) {
v, ok := excludedepend[j]
if ok {
*v = append(*v, i)
} else {
excludedepend[j] = &[]int{i}
}
}
}
}
}
}

// 清掉排除的依赖
if len(excludedepend) > 0 {
for k, arrv := range excludedepend {
for _, v := range *arrv {
blog.Infof("remote util: clean depend:%s to %s",
fileDetails[k].File.FilePath,
fileDetails[v].File.FilePath)
if len(dependencies[k]) > 0 {
for i := range dependencies[k] {
if dependencies[k][i] == v {
dependencies[k][i] = -1
blog.Infof("remote util: set depend:%s %d to -1",
fileDetails[k].File.FilePath, i)
}
}
}
}
}
}
Expand All @@ -174,14 +210,14 @@ func isSubString(s1, s2 string) bool {
s2 != "/"
}

// depend 检查 s1 是否依赖 s2
func depend(s1, s2 *types.FilesDetails) bool {
// dirDepend 检查 s1的目录 是否依赖 s2的目录
func dirDepend(s1, s2 *types.FilesDetails) bool {
is1File := s1.File.Priority == sdk.RealFilePriority || s1.File.Priority == sdk.LinkFilePriority
is2File := s2.File.Priority == sdk.RealFilePriority || s2.File.Priority == sdk.LinkFilePriority
// 如果s1是文件，s2是目录
if is1File {
if is2File { // 如果s1是文件，s2是文件
if isSubString(filepath.Dir(s1.File.FilePath), filepath.Dir(s2.File.FilePath)) {
if isSubString(s1.File.Targetrelativepath, s2.File.Targetrelativepath) {
return true
}
} else { // 如果s1是文件，s2是目录
Expand All @@ -191,7 +227,7 @@ func depend(s1, s2 *types.FilesDetails) bool {
}
} else {
if is2File { // 如果s1是目录，s2是文件
if isSubString(s1.File.FilePath, filepath.Dir(s2.File.FilePath)) {
if isSubString(s1.File.FilePath, s2.File.Targetrelativepath) {
return true
}
} else { // 如果s1是目录，s2是目录
Expand All @@ -201,7 +237,11 @@ func depend(s1, s2 *types.FilesDetails) bool {
}
}

// 如果s1是链接，并且指向s2，则s1依赖s2
return false
}

// linkDepend 检查 s1 是否链接到了 s2
func linkDepend(s1, s2 *types.FilesDetails) bool {
if s1.File.LinkTarget != "" && s1.File.LinkTarget == s2.File.FilePath {
return true
}
Expand All @@ -224,9 +264,13 @@ func freshPriority(fileDetails []*types.FilesDetails) error {
tryNum := 0
for {
tryNum++
if tryNum >= maxTry {
printLeftDepend(fileDetails, dependencies)
}

allok := true
for i := range fileDetails {
// 如果超过遍历次数，则剩余的全部赋值，避免死循环
// 如果超过遍历次数上限，则剩余的全部赋值，避免死循环
if tryNum >= maxTry {
if fileDetails[i].File.Priority < 0 {
fileDetails[i].File.Priority = sdk.FileDescPriority(maxPriority + 1)
Expand All @@ -241,25 +285,29 @@ func freshPriority(fileDetails []*types.FilesDetails) error {
} else {
maxDependPriority := -1
alldependok := true
for _, v := range dependencies[i] {
for j, v := range dependencies[i] {
if v < 0 {
continue
}
dependPriority := int(fileDetails[v].File.Priority)
if dependPriority >= 0 {
dependencies[i][j] = -1
if dependPriority > maxDependPriority {
maxDependPriority = dependPriority
}
if dependPriority > maxPriority {
maxPriority = dependPriority
}
} else {
blog.Debugf("remote uitl: %s wait %s",
blog.Debugf("remote util: %s wait %s",
fileDetails[i].File.FilePath, fileDetails[v].File.FilePath)
alldependok = false
}
}
if alldependok {
dependencies[i] = nil
fileDetails[i].File.Priority = sdk.FileDescPriority(maxDependPriority + 1)
blog.Debugf("remote uitl: %s set Priority to %d",
blog.Debugf("remote util: %s set Priority to %d",
fileDetails[i].File.FilePath, maxDependPriority+1)
} else {
allok = false
Expand All @@ -268,10 +316,27 @@ func freshPriority(fileDetails []*types.FilesDetails) error {
}

if allok || tryNum >= maxTry {
blog.Infof("remote uitl: finished set Priority after %d try", tryNum)
blog.Infof("remote util: finished set Priority after %d try", tryNum)
break
}
}

return nil
}

func printLeftDepend(fileDetails []*types.FilesDetails, dependencies [][]int) {
for i := range fileDetails {
if fileDetails[i].File.Priority < 0 {
dependfiles := []string{}
for _, v := range dependencies[i] {
dependPriority := int(fileDetails[v].File.Priority)
if dependPriority < 0 {
dependfiles = append(dependfiles, fileDetails[v].File.FilePath)
}
}
blog.Warnf("remote util: after max try, %s wait %s",
fileDetails[i].File.FilePath,
strings.Join(dependfiles, " "))
}
}
}

0 comments on commit c8cc4d0

Please sign in to comment.