Skip to content

Commit

Permalink
allow running ctest using multiple jobs using --jobs option
Browse files Browse the repository at this point in the history
This splits the original "numJobs" property on the build context into separate
"numBuildJobs" and "numTestJobs" properties. When --jobs is not provided,
"numBuildJobs" continues to default to the number of CPUs on the machine, while
"numTestJobs" defaults to 1 to continue to run tests serially.

When --jobs is explicitly provided, that number of jobs will be used both for
building from source and for running tests. This allows more gradual adoption
of parallel test execution using the existing command line argument.
  • Loading branch information
mattyjams committed Jun 17, 2020
1 parent 1a5d926 commit f2cb351
Showing 1 changed file with 24 additions and 14 deletions.
38 changes: 24 additions & 14 deletions build.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,12 @@ def GetCPUCount():
except NotImplementedError:
return 1

def GetDefaultNumBuildJobs():
return GetCPUCount()

def GetDefaultNumTestJobs():
return 1

def Run(context, cmd):
"""Run the specified command in a subprocess."""
PrintInfo('Running "{cmd}"'.format(cmd=cmd))
Expand Down Expand Up @@ -171,15 +177,15 @@ def BuildVariant(context):
return "RelWithDebInfo"
return "RelWithDebInfo"

def FormatMultiProcs(numJobs, generator):
def FormatMultiProcs(numBuildJobs, generator):
tag = "-j"
if generator:
if "Visual Studio" in generator:
tag = "/M:"
elif "Xcode" in generator:
tag = "-j "

return "{tag}{procs}".format(tag=tag, procs=numJobs)
return "{tag}{procs}".format(tag=tag, procs=numBuildJobs)

def onerror(func, path, exc_info):
"""
Expand Down Expand Up @@ -282,23 +288,21 @@ def RunCMake(context, extraArgs=None, stages=None):
Run(context, "cmake --build . --config {variant} {installArg} -- {multiproc}"
.format(variant=variant,
installArg=installArg,
multiproc=FormatMultiProcs(context.numJobs, generator)))
multiproc=FormatMultiProcs(context.numBuildJobs, generator)))

def RunCTest(context, extraArgs=None):
buildDir = context.buildDir
variant = BuildVariant(context)
#TODO we can't currently run tests in parallel, something to revisit.
numJobs = 1

with CurrentWorkingDirectory(buildDir):
Run(context,
'ctest '
'--output-on-failure '
'--timeout 300 '
'-j {numJobs} '
'-j {numTestJobs} '
'-C {variant} '
'{extraArgs} '
.format(numJobs=numJobs,
.format(numTestJobs=context.numTestJobs,
variant=variant,
extraArgs=(" ".join(extraArgs) if extraArgs else "")))

Expand Down Expand Up @@ -457,10 +461,10 @@ def Package(context):
parser.add_argument("--stages", type=str, nargs="*", default=['clean','configure','build','install'],
help=("Comma-separated list of stages to execute.(possible stages: clean, configure, build, install, test, package)"))

parser.add_argument("-j", "--jobs", type=int, default=GetCPUCount(),
help=("Number of build jobs to run in parallel. "
"(default: # of processors [{0}])"
.format(GetCPUCount())))
parser.add_argument("-j", "--jobs", type=int,
help=("Number of jobs to run in parallel. "
"(Build default: # of processors [{0}], Test default: {1})"
.format(GetDefaultNumBuildJobs(), GetDefaultNumTestJobs())))

parser.add_argument("--redirect-outstream-file", type=distutils.util.strtobool, dest="redirect_outstream_file", default=True,
help="Redirect output stream to a file. Set this flag to false to redirect output stream to console instead.")
Expand Down Expand Up @@ -503,9 +507,15 @@ def __init__(self, args):
self.cmakeGenerator = args.generator

# Number of jobs
self.numJobs = args.jobs
if self.numJobs <= 0:
raise ValueError("Number of jobs must be greater than 0")
if (args.jobs is not None):
if args.jobs <= 0:
raise ValueError("Number of jobs must be greater than 0")

self.numBuildJobs = args.jobs
self.numTestJobs = args.jobs
else:
self.numBuildJobs = GetDefaultNumBuildJobs()
self.numTestJobs = GetDefaultNumTestJobs()

# Maya Location
self.mayaLocation = (os.path.abspath(args.maya_location)
Expand Down

0 comments on commit f2cb351

Please sign in to comment.