diff --git a/src/backend/booster/bk_dist/common/env/env.go b/src/backend/booster/bk_dist/common/env/env.go index 83313817b..1c633565a 100644 --- a/src/backend/booster/bk_dist/common/env/env.go +++ b/src/backend/booster/bk_dist/common/env/env.go @@ -65,6 +65,16 @@ const ( KeyExecutorUseWebSocket = "USE_WEBSOCKET" KeyExecutorNewShader = "NEW_SHADER" KeyExecutorSearchToolchain = "SEARCH_TOOLCHAIN" + KeyExecutorUECLNotUseLocal = "UE_CL_NOT_USE_LOCAL" + KeyExecutorUECCNotUseLocal = "UE_CC_NOT_USE_LOCAL" + KeyExecutorUELibNotUseLocal = "UE_LIB_NOT_USE_LOCAL" + KeyExecutorUELinkNotUseLocal = "UE_LINK_NOT_USE_LOCAL" + KeyExecutorUEShaderNotUseLocal = "UE_SHADER_NOT_USE_LOCAL" + KeyExecutorUECLLocalCPUWeight = "UE_CL_LOCAL_CPU_WEIGHT" + KeyExecutorUECCLocalCPUWeight = "UE_CC_LOCAL_CPU_WEIGHT" + KeyExecutorUELibLocalCPUWeight = "UE_LIB_LOCAL_CPU_WEIGHT" + KeyExecutorUELinkLocalCPUWeight = "UE_LINK_LOCAL_CPU_WEIGHT" + KeyExecutorUEShaderLocalCPUWeight = "UE_SHADER_LOCAL_CPU_WEIGHT" KeyUserDefinedLogLevel = "USER_DEFINED_LOG_LEVEL" KeyUserDefinedExecutorLogLevel = "USER_DEFINED_EXECUTOR_LOG_LEVEL" diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go b/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go index d7d2d3c73..4721b7b86 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go @@ -523,3 +523,11 @@ func (e *executor) retryAndSuccessTooManyAndDegradeDirectly() bool { return false } + +func (e *executor) canExecuteWithLocalIdleResource() bool { + if e.handler != nil { + return e.handler.CanExecuteWithLocalIdleResource(e.req.Commands) + } + + return false +} diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/local/mgr.go b/src/backend/booster/bk_dist/controller/pkg/manager/local/mgr.go index 011421d07..427618717 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/local/mgr.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/local/mgr.go @@ -108,7 +108,8 @@ func (m *Mgr) GetPumpCache() (*analyser.FileCache, *analyser.RootCache) { func (m *Mgr) ExecuteTask( req *types.LocalTaskExecuteRequest, globalWork *types.Work, - withlocalresource bool) (*types.LocalTaskExecuteResult, error) { + canUseLocalIdleResource bool, + f types.CallbackCheckLocalResource) (*types.LocalTaskExecuteResult, error) { blog.Infof("local: try to execute task(%s) for work(%s) from pid(%d) in env(%v) dir(%s)", strings.Join(req.Commands, " "), m.work.ID(), req.Pid, req.Environments, req.Dir) @@ -136,11 +137,14 @@ func (m *Mgr) ExecuteTask( return e.executeLocalTask(), nil } + // TODO : 本地空闲资源执行任务需要更多条件判断 // 该任务已确定用本地资源运行，则直接走本地执行 - if withlocalresource { - blog.Infof("local: execute task for work(%s) from pid(%d) degrade to local for with local resource", - m.work.ID(), req.Pid) - return e.executeLocalTask(), nil + if canUseLocalIdleResource { + if e.canExecuteWithLocalIdleResource() && f() { + blog.Infof("local: execute task [%s] for work(%s) from pid(%d) degrade to local with local idle resource", + req.Commands[0], m.work.ID(), req.Pid) + return e.executeLocalTask(), nil + } } // 优化没有远程资源转本地的逻辑； 如果没有远程资源，则先获取本地锁，然后转本地执行 diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/manager.go b/src/backend/booster/bk_dist/controller/pkg/manager/manager.go index c8186f586..c35b8bed4 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/manager.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/manager.go @@ -407,11 +407,23 @@ func (m *mgr) ExecuteLocalTask( dcSDK.StatsTimeNow(&req.Stats.EnterTime) defer dcSDK.StatsTimeNow(&req.Stats.LeaveTime) work.Basic().UpdateJobStats(req.Stats) - withlocalresource := m.checkRunWithLocalResource(work) + + // TODO : 是否转本地执行的判断，下放到basic执行，结合handle的信息 + // 另外，本地锁需要加上本地权重 + // withlocalresource := m.checkRunWithLocalResource(work) + // if withlocalresource { + // defer m.decLocalResourceTask() + // } + var withlocalresource bool + result, err := work.Local().ExecuteTask(req, globalWork, m.canUseLocalIdleResource(), func() bool { + withlocalresource = m.checkRunWithLocalResource(work) + blog.Infof("mgr: check run with local resource for work(%s) from pid(%d) got %v", + workID, req.Pid, withlocalresource) + return withlocalresource + }) if withlocalresource { - defer m.decLocalResourceTask() + m.decLocalResourceTask() } - result, err := work.Local().ExecuteTask(req, globalWork, withlocalresource) if err != nil { if result == nil { result = &types.LocalTaskExecuteResult{Result: &dcSDK.LocalTaskResult{ @@ -821,6 +833,10 @@ func sdkToolChain2Types(sdkToolChain *dcSDK.OneToolChain) *types.ToolChain { } } +func (m *mgr) canUseLocalIdleResource() bool { + return m.conf.UseLocalCPUPercent > 0 && m.conf.UseLocalCPUPercent <= 100 +} + func (m *mgr) checkRunWithLocalResource(work *types.Work) bool { if m.conf.UseLocalCPUPercent <= 0 || m.conf.UseLocalCPUPercent > 100 { return false @@ -880,6 +896,7 @@ func (m *mgr) checkRunWithLocalResource(work *types.Work) bool { work.Basic().Info().WorkID()) atomic.AddInt32(&m.localResourceTaskNum, 1) + return true } diff --git a/src/backend/booster/bk_dist/controller/pkg/types/interface.go b/src/backend/booster/bk_dist/controller/pkg/types/interface.go index 518824816..628b2d59f 100644 --- a/src/backend/booster/bk_dist/controller/pkg/types/interface.go +++ b/src/backend/booster/bk_dist/controller/pkg/types/interface.go @@ -112,6 +112,8 @@ type RemoteMgr interface { DecRemoteJobs() } +type CallbackCheckLocalResource func() bool + // LocalMgr describe a manager for handling all actions with local execution for work type LocalMgr interface { // init handler @@ -131,7 +133,8 @@ type LocalMgr interface { // do task execution ExecuteTask(req *LocalTaskExecuteRequest, globalWork *Work, - withlocalresource bool) (*LocalTaskExecuteResult, error) + canUseLocalIdleResource bool, + f CallbackCheckLocalResource) (*LocalTaskExecuteResult, error) // get caches in pump mode GetPumpCache() (*analyser.FileCache, *analyser.RootCache) diff --git a/src/backend/booster/bk_dist/handler/cc/handler.go b/src/backend/booster/bk_dist/handler/cc/handler.go index a70e24be5..0e8ca25b4 100644 --- a/src/backend/booster/bk_dist/handler/cc/handler.go +++ b/src/backend/booster/bk_dist/handler/cc/handler.go @@ -127,6 +127,10 @@ func (cc *TaskCC) InitSandbox(sandbox *dcSyscall.Sandbox) { cc.sandbox = sandbox } +func (cc *TaskCC) CanExecuteWithLocalIdleResource(command []string) bool { + return true +} + // PreExecuteNeedLock 如果编译本身是预处理, 那么不需要pre-lock, 因为它会在pre-execute中转本地, 不会真正地执行预处理 func (cc *TaskCC) PreExecuteNeedLock(command []string) bool { for _, arg := range command { diff --git a/src/backend/booster/bk_dist/handler/cc/utils.go b/src/backend/booster/bk_dist/handler/cc/utils.go index 2286b3941..e35fbc956 100644 --- a/src/backend/booster/bk_dist/handler/cc/utils.go +++ b/src/backend/booster/bk_dist/handler/cc/utils.go @@ -756,12 +756,12 @@ func scanArgs(args []string, sandbox *dcSyscall.Sandbox) (*ccArgs, error) { return nil, ErrorNotSupportDr } - // ++ by tomtian 2021-05-18 - if strings.HasPrefix(arg, "-fsanitize") { - blog.Warnf("cc: scan args: clang option %s need read origin source file; running locally", arg) - return nil, ErrorNotSupportFsanitize - } - // -- + // // ++ by tomtian 2021-05-18 + // if strings.HasPrefix(arg, "-fsanitize") { + // blog.Warnf("cc: scan args: clang option %s need read origin source file; running locally", arg) + // return nil, ErrorNotSupportFsanitize + // } + // // -- if strings.HasPrefix(arg, "-I") { // if -I just a prefix, save the remain of this line. diff --git a/src/backend/booster/bk_dist/handler/custom/handler.go b/src/backend/booster/bk_dist/handler/custom/handler.go index 501833923..7f3002cef 100644 --- a/src/backend/booster/bk_dist/handler/custom/handler.go +++ b/src/backend/booster/bk_dist/handler/custom/handler.go @@ -139,6 +139,10 @@ func (c *Custom) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDi return c.innerHandler.PreExecute(command) } +func (c *Custom) CanExecuteWithLocalIdleResource(command []string) bool { + return true +} + // PreExecuteNeedLock 决定是否需要在执行PreExecute之前获取一个pre-lock func (c *Custom) PreExecuteNeedLock(command []string) bool { return c.innerHandler.PreExecuteNeedLock(command) diff --git a/src/backend/booster/bk_dist/handler/echo/handler.go b/src/backend/booster/bk_dist/handler/echo/handler.go index d29597ebc..a3f965ef3 100644 --- a/src/backend/booster/bk_dist/handler/echo/handler.go +++ b/src/backend/booster/bk_dist/handler/echo/handler.go @@ -107,6 +107,10 @@ func (c *Echo) GetFilterRules() ([]dcSDK.FilterRuleItem, error) { return nil, nil } +func (c *Echo) CanExecuteWithLocalIdleResource(command []string) bool { + return true +} + // PreExecuteNeedLock decide whether should lock when executor do the pre-process func (c *Echo) PreExecuteNeedLock(command []string) bool { return false diff --git a/src/backend/booster/bk_dist/handler/find/handler.go b/src/backend/booster/bk_dist/handler/find/handler.go index b656f8d15..2a1226058 100644 --- a/src/backend/booster/bk_dist/handler/find/handler.go +++ b/src/backend/booster/bk_dist/handler/find/handler.go @@ -152,6 +152,10 @@ func (c *Finder) GetFilterRules() ([]dcSDK.FilterRuleItem, error) { return nil, nil } +func (c *Finder) CanExecuteWithLocalIdleResource(command []string) bool { + return true +} + // PreExecuteNeedLock decide whether should lock when executor do the pre-process func (c *Finder) PreExecuteNeedLock(command []string) bool { return false diff --git a/src/backend/booster/bk_dist/handler/handler.go b/src/backend/booster/bk_dist/handler/handler.go index ac109f9e8..5c5928ded 100644 --- a/src/backend/booster/bk_dist/handler/handler.go +++ b/src/backend/booster/bk_dist/handler/handler.go @@ -40,6 +40,10 @@ type Handler interface { // GetFilterRules return files which will be used by multi-executor, send only once GetFilterRules() ([]dcSDK.FilterRuleItem, error) + // CanExecuteWithLocalIdleResource return whether can execute this with local idle resource, + // true by default + CanExecuteWithLocalIdleResource(command []string) bool + // PreExecuteNeedLock decide whether executor should lock before pre execution PreExecuteNeedLock(command []string) bool diff --git a/src/backend/booster/bk_dist/handler/tc/handler.go b/src/backend/booster/bk_dist/handler/tc/handler.go index de685ecf6..acc5d021c 100644 --- a/src/backend/booster/bk_dist/handler/tc/handler.go +++ b/src/backend/booster/bk_dist/handler/tc/handler.go @@ -142,6 +142,10 @@ func (tc *TextureCompressor) GetFilterRules() ([]dcSDK.FilterRuleItem, error) { return nil, nil } +func (c *TextureCompressor) CanExecuteWithLocalIdleResource(command []string) bool { + return true +} + // PreExecuteNeedLock no need func (tc *TextureCompressor) PreExecuteNeedLock(command []string) bool { return false diff --git a/src/backend/booster/bk_dist/handler/ue4/astc/handler.go b/src/backend/booster/bk_dist/handler/ue4/astc/handler.go index 4899cdf34..d6f46a06d 100644 --- a/src/backend/booster/bk_dist/handler/ue4/astc/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/astc/handler.go @@ -120,6 +120,10 @@ func (tc *TextureCompressor) GetFilterRules() ([]dcSDK.FilterRuleItem, error) { return nil, nil } +func (cc *TextureCompressor) CanExecuteWithLocalIdleResource(command []string) bool { + return true +} + // PreExecuteNeedLock no need func (tc *TextureCompressor) PreExecuteNeedLock(command []string) bool { return false diff --git a/src/backend/booster/bk_dist/handler/ue4/cc/handler.go b/src/backend/booster/bk_dist/handler/ue4/cc/handler.go index 90c578357..f214e0ce0 100644 --- a/src/backend/booster/bk_dist/handler/ue4/cc/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/cc/handler.go @@ -127,6 +127,14 @@ func (cc *TaskCC) InitSandbox(sandbox *dcSyscall.Sandbox) { cc.sandbox = sandbox } +func (cc *TaskCC) CanExecuteWithLocalIdleResource(command []string) bool { + if cc.sandbox.Env.GetEnv(env.KeyExecutorUECCNotUseLocal) == "true" { + return false + } + + return true +} + // PreExecuteNeedLock 需要pre-lock来保证预处理不会跑满本地资源 func (cc *TaskCC) PreExecuteNeedLock(command []string) bool { return true @@ -154,6 +162,14 @@ func (cc *TaskCC) LocalExecuteNeed(command []string) bool { // LocalLockWeight decide local-execute lock weight, default 1 func (cc *TaskCC) LocalLockWeight(command []string) int32 { + envvalue := cc.sandbox.Env.GetEnv(env.KeyExecutorUECCLocalCPUWeight) + if envvalue != "" { + w, err := strconv.Atoi(envvalue) + if err == nil && w > 0 && w <= runtime.NumCPU() { + return int32(w) + } + } + return 1 } @@ -1129,7 +1145,7 @@ func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { if r.Results[0].RetCode == 0 { blog.Infof("cc: success done post execute for: %v", cc.originArgs) // set output to inputFile - r.Results[0].OutputMessage = []byte(filepath.Base(cc.inputFile)) + // r.Results[0].OutputMessage = []byte(filepath.Base(cc.inputFile)) // if remote succeed with pump,do not need copy head file if cc.pumpremote { cc.needcopypumpheadfile = false diff --git a/src/backend/booster/bk_dist/handler/ue4/cl/handler.go b/src/backend/booster/bk_dist/handler/ue4/cl/handler.go index 92d9709f5..d51af3118 100644 --- a/src/backend/booster/bk_dist/handler/ue4/cl/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/cl/handler.go @@ -223,6 +223,14 @@ func (cl *TaskCL) getPreLoadConfigPath(config dcType.BoosterConfig) string { return dcConfig.GetFile(hookConfigPathCCCommon) } +func (cl *TaskCL) CanExecuteWithLocalIdleResource(command []string) bool { + if cl.sandbox.Env.GetEnv(env.KeyExecutorUECLNotUseLocal) == "true" { + return false + } + + return true +} + // PreExecuteNeedLock 防止预处理跑满本机CPU func (cl *TaskCL) PreExecuteNeedLock(command []string) bool { return true @@ -280,6 +288,14 @@ func (cl *TaskCL) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.B // LocalLockWeight decide local-execute lock weight, default 1 func (cl *TaskCL) LocalLockWeight(command []string) int32 { + envvalue := cl.sandbox.Env.GetEnv(env.KeyExecutorUECLLocalCPUWeight) + if envvalue != "" { + w, err := strconv.Atoi(envvalue) + if err == nil && w > 0 && w <= runtime.NumCPU() { + return int32(w) + } + } + return 1 } @@ -1071,7 +1087,7 @@ func (cl *TaskCL) postExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { } } else { // simulate output with inputFile - r.Results[0].OutputMessage = []byte(filepath.Base(cl.inputFile)) + // r.Results[0].OutputMessage = []byte(filepath.Base(cl.inputFile)) } // if remote succeed with pump,do not need copy head file diff --git a/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go b/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go index 79af3a1b7..39b7aafb3 100644 --- a/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go @@ -90,6 +90,14 @@ func (cf *TaskCLFilter) GetPreloadConfig(config dcType.BoosterConfig) (*dcSDK.Pr return nil, nil } +func (cf *TaskCLFilter) CanExecuteWithLocalIdleResource(command []string) bool { + if cf.clhandle != nil { + return cf.clhandle.CanExecuteWithLocalIdleResource(command) + } + + return true +} + // PreExecuteNeedLock 防止预处理跑满本机CPU func (cf *TaskCLFilter) PreExecuteNeedLock(command []string) bool { return true diff --git a/src/backend/booster/bk_dist/handler/ue4/handler.go b/src/backend/booster/bk_dist/handler/ue4/handler.go index d8d0987ae..6aaeacdea 100644 --- a/src/backend/booster/bk_dist/handler/ue4/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/handler.go @@ -150,6 +150,20 @@ func (u *UE4) GetFilterRules() ([]dcSDK.FilterRuleItem, error) { }, nil } +func (u *UE4) CanExecuteWithLocalIdleResource(command []string) bool { + if u.innerhandler == nil { + u.initInnerHandle(command) + } + if u.innerhandler != nil { + if u.sandbox != nil { + u.innerhandler.InitSandbox(u.sandbox.Fork()) + } + return u.innerhandler.CanExecuteWithLocalIdleResource(command) + } + + return true +} + // PreExecuteNeedLock 防止预处理跑满本机CPU func (u *UE4) PreExecuteNeedLock(command []string) bool { return true diff --git a/src/backend/booster/bk_dist/handler/ue4/lib/handler.go b/src/backend/booster/bk_dist/handler/ue4/lib/handler.go index 851a911b2..78f490aec 100644 --- a/src/backend/booster/bk_dist/handler/ue4/lib/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/lib/handler.go @@ -12,6 +12,7 @@ package lib import ( "fmt" "path/filepath" + "runtime" "strconv" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/env" @@ -79,6 +80,14 @@ func (l *TaskLib) GetPreloadConfig(config dcType.BoosterConfig) (*dcSDK.PreloadC return nil, nil } +func (l *TaskLib) CanExecuteWithLocalIdleResource(command []string) bool { + if l.sandbox.Env.GetEnv(env.KeyExecutorUELibNotUseLocal) == "true" { + return false + } + + return true +} + // PreExecuteNeedLock 没有在本地执行的预处理步骤, 无需pre-lock func (l *TaskLib) PreExecuteNeedLock(command []string) bool { return false @@ -136,6 +145,14 @@ func (l *TaskLib) LocalExecuteNeed(command []string) bool { // LocalLockWeight decide local-execute lock weight, default 1 func (l *TaskLib) LocalLockWeight(command []string) int32 { + envvalue := l.sandbox.Env.GetEnv(env.KeyExecutorUELibLocalCPUWeight) + if envvalue != "" { + w, err := strconv.Atoi(envvalue) + if err == nil && w > 0 && w <= runtime.NumCPU() { + return int32(w) + } + } + return 1 } diff --git a/src/backend/booster/bk_dist/handler/ue4/link/handler.go b/src/backend/booster/bk_dist/handler/ue4/link/handler.go index f6c855adc..063afadfd 100644 --- a/src/backend/booster/bk_dist/handler/ue4/link/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/link/handler.go @@ -12,6 +12,7 @@ package link import ( "fmt" "path/filepath" + "runtime" "strconv" "strings" @@ -95,6 +96,14 @@ func (l *TaskLink) GetPreloadConfig(config dcType.BoosterConfig) (*dcSDK.Preload return nil, nil } +func (l *TaskLink) CanExecuteWithLocalIdleResource(command []string) bool { + if l.sandbox.Env.GetEnv(env.KeyExecutorUELinkNotUseLocal) == "true" { + return false + } + + return true +} + // PreExecuteNeedLock 没有在本地执行的预处理步骤, 无需pre-lock func (l *TaskLink) PreExecuteNeedLock(command []string) bool { return false @@ -152,6 +161,14 @@ func (l *TaskLink) LocalExecuteNeed(command []string) bool { // LocalLockWeight decide local-execute lock weight, default 1 func (l *TaskLink) LocalLockWeight(command []string) int32 { + envvalue := l.sandbox.Env.GetEnv(env.KeyExecutorUELinkLocalCPUWeight) + if envvalue != "" { + w, err := strconv.Atoi(envvalue) + if err == nil && w > 0 && w <= runtime.NumCPU() { + return int32(w) + } + } + return 1 } diff --git a/src/backend/booster/bk_dist/handler/ue4/linkfilter/handler.go b/src/backend/booster/bk_dist/handler/ue4/linkfilter/handler.go index 5798804a4..70db2fa3b 100644 --- a/src/backend/booster/bk_dist/handler/ue4/linkfilter/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/linkfilter/handler.go @@ -80,6 +80,14 @@ func (lf *TaskLinkFilter) GetPreloadConfig(config dcType.BoosterConfig) (*dcSDK. return nil, nil } +func (lf *TaskLinkFilter) CanExecuteWithLocalIdleResource(command []string) bool { + if lf.handle != nil { + return lf.handle.CanExecuteWithLocalIdleResource(command) + } + + return true +} + // PreExecuteNeedLock 防止预处理跑满本机CPU func (lf *TaskLinkFilter) PreExecuteNeedLock(command []string) bool { return true diff --git a/src/backend/booster/bk_dist/handler/ue4/shader/handler.go b/src/backend/booster/bk_dist/handler/ue4/shader/handler.go index ff8a34872..14a620747 100644 --- a/src/backend/booster/bk_dist/handler/ue4/shader/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/shader/handler.go @@ -14,6 +14,7 @@ import ( "os" "path/filepath" "runtime" + "strconv" "strings" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/env" @@ -83,6 +84,17 @@ func (u *UE4Shader) GetFilterRules() ([]dcSDK.FilterRuleItem, error) { return nil, nil } +func (u *UE4Shader) CanExecuteWithLocalIdleResource(command []string) bool { + // only for debug + blog.Infof("shader: BK_DIST_UE_SHADER_NOT_USE_LOCAL=[%v]", u.sandbox.Env.GetEnv(env.KeyExecutorUEShaderNotUseLocal)) + + if u.sandbox.Env.GetEnv(env.KeyExecutorUEShaderNotUseLocal) == "true" { + return false + } + + return true +} + // PreExecuteNeedLock 没有在本地执行的预处理步骤, 无需pre-lock func (u *UE4Shader) PreExecuteNeedLock(command []string) bool { return false @@ -366,6 +378,14 @@ func (u *UE4Shader) LocalExecuteNeed(command []string) bool { // LocalLockWeight decide local-execute lock weight, default 1 func (u *UE4Shader) LocalLockWeight(command []string) int32 { + envvalue := u.sandbox.Env.GetEnv(env.KeyExecutorUEShaderLocalCPUWeight) + if envvalue != "" { + w, err := strconv.Atoi(envvalue) + if err == nil && w > 0 && w <= runtime.NumCPU() { + return int32(w) + } + } + /* // default setting of ue 4.26 ; Make sure we don't starve loading threads NumUnusedShaderCompilingThreads=3 diff --git a/src/backend/booster/bk_dist/handler/winclangcl/handler.go b/src/backend/booster/bk_dist/handler/winclangcl/handler.go index 201c82552..04108a494 100644 --- a/src/backend/booster/bk_dist/handler/winclangcl/handler.go +++ b/src/backend/booster/bk_dist/handler/winclangcl/handler.go @@ -69,6 +69,10 @@ func (cc *WinClangCl) RenderArgs(config dcType.BoosterConfig, originArgs string) return originArgs } +func (cc *WinClangCl) CanExecuteWithLocalIdleResource(command []string) bool { + return true +} + func (cc *WinClangCl) PreExecuteNeedLock(command []string) bool { return true diff --git a/src/backend/booster/bk_dist/shadertool/common/types.go b/src/backend/booster/bk_dist/shadertool/common/types.go index 02caa77e1..ff59c8b6f 100644 --- a/src/backend/booster/bk_dist/shadertool/common/types.go +++ b/src/backend/booster/bk_dist/shadertool/common/types.go @@ -86,6 +86,7 @@ type ApplyParameters struct { MaxLocalPostJobs int `json:"max_Local_post_jobs"` Env map[string]string `json:"env"` ContinueOnError bool `json:"continue_on_error" usage:"if false, the program will stop on error immediately"` + ControllerUseLocalCPUPercent int `json:"controller_use_local_cpu_percent"` } // Actionresult define action result diff --git a/src/backend/booster/bk_dist/shadertool/pkg/shadertool.go b/src/backend/booster/bk_dist/shadertool/pkg/shadertool.go index cb20a4f44..43e98fd47 100644 --- a/src/backend/booster/bk_dist/shadertool/pkg/shadertool.go +++ b/src/backend/booster/bk_dist/shadertool/pkg/shadertool.go @@ -195,6 +195,7 @@ func (h *ShaderTool) getControllerConfig() dcSDK.ControllerConfig { LongTCP: h.settings.ControllerLongTCP, DynamicPort: h.settings.ControllerDynamicPort, WorkerOfferSlot: h.settings.ControllerWorkerOfferSlot, + UseLocalCPUPercent: h.settings.ControllerUseLocalCPUPercent, } return *h.controllerconfig