From a2f1a2e48c1eb4ee918fcb591fe97738fdbfe66a Mon Sep 17 00:00:00 2001 From: Sebastian Wilzbach Date: Tue, 10 Apr 2018 14:59:58 +0200 Subject: [PATCH] Use D to run the testsuite --- .circleci/run.sh | 2 +- src/osmodel.mak | 2 +- test/README.md | 8 +- test/run.d | 342 ++++++++++++++++++++++++++++++++++++ test/run_individual_tests.d | 44 ----- test/tools/d_do_test.d | 6 +- 6 files changed, 352 insertions(+), 52 deletions(-) create mode 100755 test/run.d delete mode 100755 test/run_individual_tests.d diff --git a/.circleci/run.sh b/.circleci/run.sh index eeeab28ac948..82ce7ad658b4 100755 --- a/.circleci/run.sh +++ b/.circleci/run.sh @@ -168,7 +168,7 @@ check_clean_git() check_run_individual() { local build_path=generated/linux/release/$MODEL - "${build_path}/dmd" -i -run ./test/run_individual_tests.d test/runnable/template2962.d ./test/compilable/test14275.d + "${build_path}/dmd" -i -run ./test/run.d test/runnable/template2962.d ./test/compilable/test14275.d } codecov() diff --git a/src/osmodel.mak b/src/osmodel.mak index dfbcdc3a3b94..6b4681f32bda 100644 --- a/src/osmodel.mak +++ b/src/osmodel.mak @@ -2,7 +2,7 @@ # # Detects and sets the macros: # -# OS = one of {osx,linux,freebsd,openbsd,netbsd,solaris} +# OS = one of {osx,linux,freebsd,openbsd,netbsd,dragonflybsd,solaris} # MODEL = one of { 32, 64 } # MODEL_FLAG = one of { -m32, -m64 } # diff --git a/test/README.md b/test/README.md index b85ba0f631ef..27f8e82aacc9 100644 --- a/test/README.md +++ b/test/README.md @@ -35,15 +35,15 @@ make -j10 run_runnable_tests ### Run only an individual test ```sh -./run_individual_tests.d fail_compilation/diag10089.d +./run.d fail_compilation/diag10089.d ``` Multiple arguments are supported too. -You can use `./run_individual_tests.d` to quickly run a custom subset of tests. +You can use `./run.d` to quickly run a custom subset of tests. For example, all diagnostic tests in `fail_compilation`: ```sh -./run_individual_tests.d fail_compilation/diag*.d +./run.d fail_compilation/diag*.d ``` ### Automatically update the `TEST_OUTPUT` segments @@ -58,7 +58,7 @@ AUTO_UPDATE=1 make run_fail_compilation_tests -j10 Updating the `TEST_OUTPUT` can also be done for a custom subset of tests: ```sh -AUTO_UPDATE=1 ./run_individual_tests.d fail_compilation/diag*.d +AUTO_UPDATE=1 ./run.d fail_compilation/diag*.d ``` Note: diff --git a/test/run.d b/test/run.d new file mode 100755 index 000000000000..7e61a5ef16b0 --- /dev/null +++ b/test/run.d @@ -0,0 +1,342 @@ +#!/usr/bin/env rdmd +/** +DMD testsuite runner + +Usage: + ./run.d ... + +Example: + ./run.d runnable/template2962.d fail_compilation/fail282.d + +See the README.md for all available test targets +*/ + +import std.algorithm, std.conv, std.datetime, std.exception, std.file, std.format, + std.getopt, std.parallelism, std.path, std.process, std.range, std.stdio, std.string; +import core.stdc.stdlib : exit; + +const scriptDir = __FILE_FULL_PATH__.dirName; +string resultsDir = scriptDir.buildPath("test_results"); +immutable testDirs = ["runnable", "compilable", "fail_compilation"]; +shared bool verbose; // output verbose logging +shared bool force; // always run all tests (ignores timestamp checking) +shared string hostDMD; // path to host DMD binary (used for building the tools) + +void main(string[] args) +{ + int jobs = totalCPUs; + auto res = getopt(args, + "j|jobs", "Specifies the number of jobs (commands) to run simultaneously (default: %d)".format(totalCPUs), &jobs, + "v", "Verbose command output", (cast(bool*) &verbose), + "f", "Force run (ignore timestamps and always run all tests)", (cast(bool*) &force), + ); + if (res.helpWanted) + { + defaultGetoptPrinter(`./run.d ... + +Examples: + + ./run.d runnable/template2962.d # runs a specific tests + ./run.d runnable/template2962.d fail_compilation/fail282.d # runs multiple specific tests + ./run.d fail_compilation # runs all tests in fail_compilation + ./run.d all # runs all tests + ./run.d clean # remove all test results + +Options: +`, res.options); + "\nSee the README.md for a more in-depth explanation of the test-runner.".writeln; + return; + } + + // parse arguments + args.popFront; + args2Environment(args); + + // allow overwrites from the environment + resultsDir = environment.get("RESULTS_DIR", resultsDir); + hostDMD = environment.get("HOST_DMD", "dmd"); + + // bootstrap all needed environment variables + auto env = getEnvironment; + + // default target + if (!args.length) + args = ["all"]; + + alias normalizeTestName = f => f.absolutePath.dirName.baseName.buildPath(f.baseName); + auto targets = args + .predefinedTargets // preprocess + .map!normalizeTestName + .array + .filterTargets; + + if (targets.length > 0) + { + if (verbose) + { + log("================================================================================"); + foreach (key, value; env) + log("%s=%s", key, value); + log("================================================================================"); + } + auto taskPool = new TaskPool(jobs); + scope(exit) taskPool.finish(); + ensureToolsExists; + foreach (target; taskPool.parallel(targets, 1)) + { + auto args = [resultsDir.buildPath("d_do_test"), target]; + log("run: %-(%s %)", args); + spawnProcess(args, env, Config.none, scriptDir).wait; + } + } +} + +/** +Builds the binary of the tools required by the testsuite. +Does nothing if the tools already exist and are newer than their source. +*/ +void ensureToolsExists() +{ + static toolsDir = scriptDir.buildPath("tools"); + resultsDir.mkdirRecurse; + auto tools = [ + "d_do_test", + "sanitize_json", + ]; + foreach (tool; tools.parallel(1)) + { + auto targetBin = resultsDir.buildPath(tool); + auto sourceFile = toolsDir.buildPath(tool ~ ".d"); + if (targetBin.timeLastModified.ifThrown(SysTime.init) > sourceFile.timeLastModified) + writefln("%s is already up-to-date", tool); + else + { + auto command = [hostDMD, "-of"~targetBin, sourceFile]; + writefln("Executing: %-(%s %)", command); + spawnProcess(command).wait; + } + } + + // ensure output directories exist + foreach (dir; testDirs) + resultsDir.buildPath(dir).mkdirRecurse; +} + +/** +Goes through the target list and replaces short-hand targets with their expanded version. +Special targets: +- clean -> removes resultsDir + immediately stops the runner +*/ +auto predefinedTargets(string[] targets) +{ + static findFiles(string dir) + { + return scriptDir.buildPath(dir).dirEntries("*{.d,.sh}", SpanMode.shallow).map!(e => e.name); + } + + Appender!(string[]) newTargets; + foreach (t; targets) + { + t = t.buildNormalizedPath; // remove trailing slashes + switch (t) + { + case "clean": + resultsDir.rmdirRecurse; + exit(0); + break; + + case "run_runnable_tests", "runnable": + newTargets.put(findFiles("runnable")); + break; + + case "run_fail_compilation_tests", "fail_compilation", "fail": + newTargets.put(findFiles("fail_compilation")); + break; + + case "run_compilable_tests", "compilable", "compile": + newTargets.put(findFiles("compilable")); + break; + + case "all": + foreach (testDir; testDirs) + newTargets.put(findFiles(testDir)); + break; + + default: + newTargets ~= t; + } + } + return newTargets.data; +} + +// Removes targets that do not need updating (i.e. their .out file exists and is newer than the source file) +auto filterTargets(string[] targets) +{ + bool error; + foreach (target; targets) + { + if (!scriptDir.buildPath(target).exists) + { + writefln("Warning: %s can't be found", target); + error = true; + } + } + if (error) + exit(1); + + string[] targetsThatNeedUpdating; + foreach (t; targets) + { + if (!force && resultsDir.buildPath(t ~ ".out").timeLastModified.ifThrown(SysTime.init) > + scriptDir.buildPath(t).timeLastModified) + writefln("%s is already up-to-date", t); + else + targetsThatNeedUpdating ~= t; + } + return targetsThatNeedUpdating; +} + +// Add additional make-like assignments to the environment +// e.g. ./run.d ARGS=foo -> sets ARGS to 'foo' +void args2Environment(ref string[] args) +{ + bool tryToAdd(string arg) + { + if (!arg.canFind("=")) + return false; + + auto sp = arg.splitter("="); + environment[sp.front] = sp.dropOne.front; + return true; + } + args = args.filter!(a => !tryToAdd(a)).array; +} + +/** +Checks whether the environment already contains a value for key and if so, sets +the found value to the new environment object. +Otherwise uses the `default_` value as fallback. + +Params: + env = environment to write the check to + key = key to check for existence and write into the new env + default_ = fallback value if the key doesn't exist in the global environment +*/ +auto getDefault(string[string] env, string key, string default_) +{ + if (key in environment) + env[key] = environment[key]; + else + env[key] = default_; + + return env[key]; +} + +// Sets the environment variables required by d_do_test and sh_do_test.sh +string[string] getEnvironment() +{ + string[string] env; + + env["RESULTS_DIR"] = resultsDir; + auto os = env.getDefault("OS", detectOS); + auto build = env.getDefault("BUILD", "release"); + env.getDefault("DMD_TEST_COVERAGE", "0"); + + version(Windows) + { + env.getDefault("ARGS", "-inline -release -g -O"); + auto exe = env["EXE"] = ".exe"; + env["OBJ"] = ".obj"; + env["DSEP"] = `\\`; + env["SEP"] = `\`; + auto druntimePath = environment.get("DRUNTIME_PATH", `..\..\druntime`); + auto phobosPath = environment.get("PHOBOS_PATH", `..\..\phobos`); + env["DFLAGS"] = `-I%s\import -I%s`.format(druntimePath, phobosPath); + env["LIB"] = phobosPath; + + // auto-tester might run the testsuite with a different $(MODEL) than DMD + // has been compiled with. Hence we manually check which binary exists. + // For windows the $(OS) during build is: `windows` + int dmdModel = "../generated/windows/%s/64/dmd%s".format(build, exe).exists ? 64 : 32; + env.getDefault("MODEL", dmdModel.text); + env["DMD"] = "../generated/windows/%s/%d/dmd%s".format(build, dmdModel, exe); + } + else + { + env.getDefault("ARGS", "-inline -release -g -O -fPIC"); + env["EXE"] = ""; + env["OBJ"] = ".o"; + env["DSEP"] = "/"; + env["SEP"] = "/"; + auto druntimePath = environment.get("DRUNTIME_PATH", scriptDir ~ `/../../druntime`); + auto phobosPath = environment.get("PHOBOS_PATH", scriptDir ~ `/../../phobos`); + + // auto-tester might run the testsuite with a different $(MODEL) than DMD + // has been compiled with. Hence we manually check which binary exists. + int dmdModel = scriptDir ~ "../generated/%s/%s/64/dmd".format(os, build).exists ? 64 : 32; + env.getDefault("MODEL", dmdModel.text); + + auto generatedSuffix = "generated/%s/%s/%s".format(os, build, dmdModel); + env["DMD"] = scriptDir ~ "/../" ~ generatedSuffix ~ "/dmd"; + + // default to PIC on x86_64, use PIC=1/0 to en-/disable PIC. + // Note that shared libraries and C files are always compiled with PIC. + bool pic; + version(X86_64) + pic = true; + else version(X86) + pic = false; + if (environment.get("PIC", "0") == "1") + pic = true; + + env["PIC_FLAGS"] = pic ? "-fPIC" : ""; + env["DFLAGS"] = "-I%s/import -I%s".format(druntimePath, phobosPath) + ~ " -L-L%s/%s".format(phobosPath, generatedSuffix); + bool isShared = os.among("linux", "freebsd") >= 0; + if (isShared) + env["DFLAGS"] = env["DFLAGS"] ~ " -defaultlib=libphobos2.so -L-rpath=%s/%s".format(phobosPath, generatedSuffix); + + env["REQUIRED_ARGS"] = environment.get("REQUIRED_ARGS") ~ env["PIC_FLAGS"]; + + version(OSX) + version(X86_64) + env["D_OBJC"] = 1; + } + return env; +} + +/* +Detects the host OS. + +Returns: a string from `{windows, osx,linux,freebsd,openbsd,netbsd,dragonflybsd,solaris}` +*/ +string detectOS() +{ + version(Windows) + return "windows"; + else version(OSX) + return "osx"; + else version(linux) + return "linux"; + else version(FreeBSD) + return "freebsd"; + else version(OpenBSD) + return "openbsd"; + else version(NetBSD) + return "netbsd"; + else version(DragonFlyBSD) + return "dragonflybsd"; + else version(Solaris) + return "solaris"; + else version(SunOS) + return "solaris"; + else + static assert(0, "Unrecognized or unsupported OS."); +} + +// Logging primitive +auto log(T...)(T args) +{ + if (verbose) + writefln(args); +} diff --git a/test/run_individual_tests.d b/test/run_individual_tests.d deleted file mode 100755 index 14bb664b1488..000000000000 --- a/test/run_individual_tests.d +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env rdmd -/** -Allows running tests individually - -Usage: - ./run_individual_tests.d ... - -Example: - ./run_individual_tests.d runnable/template2962.d fail_compilation/fail282.d - -See the README.md for all available test targets -*/ - -void main(string[] args) -{ - import std.algorithm, std.conv, std.format, std.getopt, std.path, std.process, std.range, std.stdio, std.string; - import std.parallelism : totalCPUs; - - const scriptDir = __FILE_FULL_PATH__.dirName; - int jobs = totalCPUs; - auto res = getopt(args, - "j|jobs", "Specifies the number of jobs (commands) to run simultaneously (default: %d)".format(totalCPUs), &jobs, - ); - if (res.helpWanted || args.length < 2) - { - defaultGetoptPrinter(`./run_individual_tests.d ... - -Examples: - - ./run_individual_tests.d runnable/template2962.d - ./run_individual_tests.d runnable/template2962.d fail_compilation/fail282.d - -Options: -`, res.options); - "\nSee the README.md for a more in-depth explanation of the test-runner.".writeln; - return; - } - - auto makeArguments = ["make", "--jobs=".text(jobs)] - .chain(args.dropOne.map!(f => - format!"test_results/%s/%s.out"(f.absolutePath.dirName.baseName, f.baseName))) - .array; - spawnProcess(makeArguments, null, Config.none, scriptDir).wait; -} diff --git a/test/tools/d_do_test.d b/test/tools/d_do_test.d index b9509ad4ffcd..84f2ee95e747 100755 --- a/test/tools/d_do_test.d +++ b/test/tools/d_do_test.d @@ -16,6 +16,8 @@ import std.stdio; import std.string; import core.sys.posix.sys.wait; +const scriptDir = __FILE_FULL_PATH__.dirName.dirName; + version(Win32) { extern(C) int putenv(const char*); @@ -895,11 +897,11 @@ int runBashTest(string input_dir, string test_name) version(Windows) { auto process = spawnShell(format("bash %s %s %s", - buildPath("tools", "sh_do_test.sh"), input_dir, test_name)); + buildPath(scriptDir, "tools", "sh_do_test.sh"), input_dir, test_name)); } else { - auto process = spawnProcess(["./tools/sh_do_test.sh", input_dir, test_name]); + auto process = spawnProcess([scriptDir.buildPath("tools", "sh_do_test.sh"), input_dir, test_name]); } return process.wait(); }