From f2cb3513671253f160ed6ff36805ebd26628f86e Mon Sep 17 00:00:00 2001 From: Matt Johnson Date: Tue, 9 Jun 2020 13:22:15 -0700 Subject: [PATCH] allow running ctest using multiple jobs using --jobs option This splits the original "numJobs" property on the build context into separate "numBuildJobs" and "numTestJobs" properties. When --jobs is not provided, "numBuildJobs" continues to default to the number of CPUs on the machine, while "numTestJobs" defaults to 1 to continue to run tests serially. When --jobs is explicitly provided, that number of jobs will be used both for building from source and for running tests. This allows more gradual adoption of parallel test execution using the existing command line argument. --- build.py | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/build.py b/build.py index b932284107..0f3c08505e 100644 --- a/build.py +++ b/build.py @@ -119,6 +119,12 @@ def GetCPUCount(): except NotImplementedError: return 1 +def GetDefaultNumBuildJobs(): + return GetCPUCount() + +def GetDefaultNumTestJobs(): + return 1 + def Run(context, cmd): """Run the specified command in a subprocess.""" PrintInfo('Running "{cmd}"'.format(cmd=cmd)) @@ -171,7 +177,7 @@ def BuildVariant(context): return "RelWithDebInfo" return "RelWithDebInfo" -def FormatMultiProcs(numJobs, generator): +def FormatMultiProcs(numBuildJobs, generator): tag = "-j" if generator: if "Visual Studio" in generator: @@ -179,7 +185,7 @@ def FormatMultiProcs(numJobs, generator): elif "Xcode" in generator: tag = "-j " - return "{tag}{procs}".format(tag=tag, procs=numJobs) + return "{tag}{procs}".format(tag=tag, procs=numBuildJobs) def onerror(func, path, exc_info): """ @@ -282,23 +288,21 @@ def RunCMake(context, extraArgs=None, stages=None): Run(context, "cmake --build . --config {variant} {installArg} -- {multiproc}" .format(variant=variant, installArg=installArg, - multiproc=FormatMultiProcs(context.numJobs, generator))) + multiproc=FormatMultiProcs(context.numBuildJobs, generator))) def RunCTest(context, extraArgs=None): buildDir = context.buildDir variant = BuildVariant(context) - #TODO we can't currently run tests in parallel, something to revisit. - numJobs = 1 with CurrentWorkingDirectory(buildDir): Run(context, 'ctest ' '--output-on-failure ' '--timeout 300 ' - '-j {numJobs} ' + '-j {numTestJobs} ' '-C {variant} ' '{extraArgs} ' - .format(numJobs=numJobs, + .format(numTestJobs=context.numTestJobs, variant=variant, extraArgs=(" ".join(extraArgs) if extraArgs else ""))) @@ -457,10 +461,10 @@ def Package(context): parser.add_argument("--stages", type=str, nargs="*", default=['clean','configure','build','install'], help=("Comma-separated list of stages to execute.(possible stages: clean, configure, build, install, test, package)")) -parser.add_argument("-j", "--jobs", type=int, default=GetCPUCount(), - help=("Number of build jobs to run in parallel. " - "(default: # of processors [{0}])" - .format(GetCPUCount()))) +parser.add_argument("-j", "--jobs", type=int, + help=("Number of jobs to run in parallel. " + "(Build default: # of processors [{0}], Test default: {1})" + .format(GetDefaultNumBuildJobs(), GetDefaultNumTestJobs()))) parser.add_argument("--redirect-outstream-file", type=distutils.util.strtobool, dest="redirect_outstream_file", default=True, help="Redirect output stream to a file. Set this flag to false to redirect output stream to console instead.") @@ -503,9 +507,15 @@ def __init__(self, args): self.cmakeGenerator = args.generator # Number of jobs - self.numJobs = args.jobs - if self.numJobs <= 0: - raise ValueError("Number of jobs must be greater than 0") + if (args.jobs is not None): + if args.jobs <= 0: + raise ValueError("Number of jobs must be greater than 0") + + self.numBuildJobs = args.jobs + self.numTestJobs = args.jobs + else: + self.numBuildJobs = GetDefaultNumBuildJobs() + self.numTestJobs = GetDefaultNumTestJobs() # Maya Location self.mayaLocation = (os.path.abspath(args.maya_location)