From a41c5abea1a4d17ec34db06d5938b67f17f3a5f2 Mon Sep 17 00:00:00 2001 From: Michael Dawson Date: Wed, 11 May 2016 17:54:42 -0400 Subject: [PATCH 1/4] test: add --repeat option to tools/test.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I often want to run a test many times to see if a failure can be recreated and I believe this is a common use case. We even have this job in the CI https://ci.nodejs.org/job/node-stress-single-test/configure but often you want to run it on a specific machine. This patch adds the --repeat option so that you can repeat the selected set of tests a number of times. Given existing options in test.py this will allow you to run one or more tests for the number of repeats specified. For example: tools/test.py -j8 --repeat 1000 parallel/test-process-exec-argv runs the test-process-exec-argv test 1000 times, running 8 copies in parallel tools/test.py --repeat 2 would run the entire test suite twice.  --- tools/test.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tools/test.py b/tools/test.py index 68141ee1ecb58f..a71a1ed0d73e70 100755 --- a/tools/test.py +++ b/tools/test.py @@ -721,13 +721,14 @@ def GetConfiguration(self, context): def GetBuildRequirements(self, path, context): return self.GetConfiguration(context).GetBuildRequirements() - def AddTestsToList(self, result, current_path, path, context, arch, mode): + def AddTestsToList(self, result, current_path, path, context, + arch, mode, repeat): for v in VARIANT_FLAGS: tests = self.GetConfiguration(context).ListTests(current_path, path, arch, mode) for t in tests: t.variant_flags = v - result += tests - + for rep in range(0, repeat): + result += tests def GetTestStatus(self, context, sections, defs): self.GetConfiguration(context).GetTestStatus(sections, defs) @@ -747,14 +748,16 @@ def GetBuildRequirements(self, path, context): result += test.GetBuildRequirements(rest, context) return result - def ListTests(self, current_path, path, context, arch, mode): + def ListTests(self, current_path, path, context, + arch, mode, repeat): (name, rest) = CarCdr(path) result = [ ] for test in self.tests: test_name = test.GetName() if not name or name.match(test_name): full_path = current_path + [test_name] - test.AddTestsToList(result, full_path, path, context, arch, mode) + test.AddTestsToList(result, full_path, path, context, + arch, mode, repeat) result.sort(cmp=lambda a, b: cmp(a.GetName(), b.GetName())) return result @@ -1324,6 +1327,9 @@ def BuildOptions(): default="") result.add_option('--temp-dir', help='Optional path to change directory used for tests', default=False) + result.add_option('--repeat', + help='how many times to repeat each test', + default=1, type="int") return result @@ -1534,7 +1540,7 @@ def Main(): 'system': utils.GuessOS(), 'arch': vmArch, } - test_list = root.ListTests([], path, context, arch, mode) + test_list = root.ListTests([], path, context, arch, mode, options.repeat) unclassified_tests += test_list (cases, unused_rules, all_outcomes) = ( config.ClassifyTests(test_list, env)) From abb3e63236ea6cebb5015d989d256563c9342855 Mon Sep 17 00:00:00 2001 From: Michael Dawson Date: Thu, 12 May 2016 09:23:30 -0400 Subject: [PATCH 2/4] address first round of comments --- tools/test.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tools/test.py b/tools/test.py index a71a1ed0d73e70..cf2d94e6d631db 100755 --- a/tools/test.py +++ b/tools/test.py @@ -721,13 +721,12 @@ def GetConfiguration(self, context): def GetBuildRequirements(self, path, context): return self.GetConfiguration(context).GetBuildRequirements() - def AddTestsToList(self, result, current_path, path, context, - arch, mode, repeat): + def AddTestsToList(self, result, current_path, path, context, arch, mode): for v in VARIANT_FLAGS: tests = self.GetConfiguration(context).ListTests(current_path, path, arch, mode) for t in tests: t.variant_flags = v - for rep in range(0, repeat): + for rep in range(0, context.repeat): result += tests def GetTestStatus(self, context, sections, defs): @@ -748,16 +747,14 @@ def GetBuildRequirements(self, path, context): result += test.GetBuildRequirements(rest, context) return result - def ListTests(self, current_path, path, context, - arch, mode, repeat): + def ListTests(self, current_path, path, context, arch, mode): (name, rest) = CarCdr(path) result = [ ] for test in self.tests: test_name = test.GetName() if not name or name.match(test_name): full_path = current_path + [test_name] - test.AddTestsToList(result, full_path, path, context, - arch, mode, repeat) + test.AddTestsToList(result, full_path, path, context, arch, mode) result.sort(cmp=lambda a, b: cmp(a.GetName(), b.GetName())) return result @@ -783,7 +780,7 @@ def GetTestStatus(self, context, sections, defs): class Context(object): def __init__(self, workspace, buildspace, verbose, vm, args, expect_fail, - timeout, processor, suppress_dialogs, store_unexpected_output): + timeout, processor, suppress_dialogs, store_unexpected_output, repeat): self.workspace = workspace self.buildspace = buildspace self.verbose = verbose @@ -794,6 +791,7 @@ def __init__(self, workspace, buildspace, verbose, vm, args, expect_fail, self.processor = processor self.suppress_dialogs = suppress_dialogs self.store_unexpected_output = store_unexpected_output + self.repeat = repeat def GetVm(self, arch, mode): if arch == 'none': @@ -1495,7 +1493,8 @@ def Main(): options.timeout, processor, options.suppress_dialogs, - options.store_unexpected_output) + options.store_unexpected_output, + options.repeat) # First build the required targets if not options.no_build: reqs = [ ] @@ -1540,7 +1539,7 @@ def Main(): 'system': utils.GuessOS(), 'arch': vmArch, } - test_list = root.ListTests([], path, context, arch, mode, options.repeat) + test_list = root.ListTests([], path, context, arch, mode) unclassified_tests += test_list (cases, unused_rules, all_outcomes) = ( config.ClassifyTests(test_list, env)) From f36f59d15b179f7245902ea439d8fa134721f2e3 Mon Sep 17 00:00:00 2001 From: Michael Dawson Date: Thu, 12 May 2016 09:31:25 -0400 Subject: [PATCH 3/4] address rest of comments --- tools/test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/test.py b/tools/test.py index cf2d94e6d631db..4646377fe82cd2 100755 --- a/tools/test.py +++ b/tools/test.py @@ -726,8 +726,7 @@ def AddTestsToList(self, result, current_path, path, context, arch, mode): tests = self.GetConfiguration(context).ListTests(current_path, path, arch, mode) for t in tests: t.variant_flags = v - for rep in range(0, context.repeat): - result += tests + result += tests * context.repeat def GetTestStatus(self, context, sections, defs): self.GetConfiguration(context).GetTestStatus(sections, defs) From efad8e9ddc01507625747a6e66ac122590e30f47 Mon Sep 17 00:00:00 2001 From: Michael Dawson Date: Thu, 12 May 2016 12:56:48 -0400 Subject: [PATCH 4/4] address another comment --- tools/test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/test.py b/tools/test.py index 4646377fe82cd2..6784f1d21839f5 100755 --- a/tools/test.py +++ b/tools/test.py @@ -779,7 +779,8 @@ def GetTestStatus(self, context, sections, defs): class Context(object): def __init__(self, workspace, buildspace, verbose, vm, args, expect_fail, - timeout, processor, suppress_dialogs, store_unexpected_output, repeat): + timeout, processor, suppress_dialogs, + store_unexpected_output, repeat): self.workspace = workspace self.buildspace = buildspace self.verbose = verbose