From e96007833553f383b3a63cc5cbf552bc42c931fb Mon Sep 17 00:00:00 2001 From: Hamid Zare Date: Fri, 5 Jun 2020 10:20:33 -0700 Subject: [PATCH 1/6] feat: write master logs to disk --- webui/tests/bin/e2e-tests.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/webui/tests/bin/e2e-tests.py b/webui/tests/bin/e2e-tests.py index c75c3586c32..3acb64a1fbb 100755 --- a/webui/tests/bin/e2e-tests.py +++ b/webui/tests/bin/e2e-tests.py @@ -9,7 +9,6 @@ import sys from typing import List -RESULTS_DIR_NAME = "results" logger = logging.getLogger("e2e-tests") root = subprocess.check_output( @@ -18,6 +17,7 @@ root_path = pathlib.Path(root) webui_dir = root_path.joinpath("webui") tests_dir = webui_dir.joinpath("tests") +results_dir = tests_dir.joinpath("results") CLUSTER_CMD_PREFIX = ["make", "-C", "test-cluster"] @@ -32,7 +32,10 @@ def run(cmd: List[str], config) -> None: def run_forget(cmd: List[str], config) -> None: - return subprocess.Popen(cmd, stdout=subprocess.DEVNULL) + return subprocess.Popen( + cmd, + stdout=str(results_dir.joinpath("cluster.stdout.logs")), + ) def run_ignore_failure(cmd: List[str], config): @@ -69,7 +72,7 @@ def det_cluster(config): def pre_e2e_tests(config): - run_ignore_failure(["rm", "-r", str(tests_dir.joinpath(RESULTS_DIR_NAME))], config) + run_ignore_failure(["rm", "-r", str(results_dir)], config) # TODO add a check for cluster condition run( ["python", str(tests_dir.joinpath("bin", "createUserAndExperiments.py"))], @@ -168,7 +171,11 @@ def main(): help_msg = f"operation must be in {sorted(operation_to_fn.keys())}" parser.add_argument("operation", help=help_msg) parser.add_argument("--det-port", default="8081", help="det master port") - parser.add_argument("--det-host", default="localhost", help="det master address eg localhost or 192.168.1.2") + parser.add_argument( + "--det-host", + default="localhost", + help="det master address eg localhost or 192.168.1.2", + ) parser.add_argument("--cypress-default-command-timeout", default="4000") parser.add_argument("--cypress-args", help="other cypress arguments") parser.add_argument("--log-level") From cedc9d9f3f0e9413d75de6d8e2bd1716fe91bafe Mon Sep 17 00:00:00 2001 From: Hamid Zare Date: Fri, 5 Jun 2020 10:21:04 -0700 Subject: [PATCH 2/6] update try-for-flake script --- webui/tests/bin/try-for-flake.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/webui/tests/bin/try-for-flake.sh b/webui/tests/bin/try-for-flake.sh index 6f04102fda7..6cfbb56039b 100755 --- a/webui/tests/bin/try-for-flake.sh +++ b/webui/tests/bin/try-for-flake.sh @@ -1,23 +1,22 @@ -#!/bin/bash +#!/bin/bash -e # if tests are re-runnable re_runnable=${1:-false} -cmd_prefix=docker- c=0 if $re_runnable; then - make pre-e2e-tests + ./bin/e2e-tests.py pre-e2e-tests while true; do c=$((c+1)) echo "run #$c" - make ${cmd_prefix}run-e2e-tests || break + ./bin/e2e-tests.py run-e2e-tests done else # run the whole suite while true; do c=$((c+1)) echo "run #$c" - make ${cmd_prefix}e2e-tests || break + make test done fi From 2a5e5b3a2183dca65cbf84a6f4788d4a10d26b4d Mon Sep 17 00:00:00 2001 From: Hamid Zare Date: Fri, 5 Jun 2020 10:39:06 -0700 Subject: [PATCH 3/6] give stdout a file handle --- webui/tests/bin/e2e-tests.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/webui/tests/bin/e2e-tests.py b/webui/tests/bin/e2e-tests.py index 3acb64a1fbb..1fc8261ffa6 100755 --- a/webui/tests/bin/e2e-tests.py +++ b/webui/tests/bin/e2e-tests.py @@ -31,11 +31,8 @@ def run(cmd: List[str], config) -> None: return subprocess.check_call(cmd, env=config["env"]) -def run_forget(cmd: List[str], config) -> None: - return subprocess.Popen( - cmd, - stdout=str(results_dir.joinpath("cluster.stdout.logs")), - ) +def run_forget(cmd: List[str], logfile, config) -> None: + return subprocess.Popen(cmd, stdout=logfile) def run_ignore_failure(cmd: List[str], config): @@ -45,10 +42,10 @@ def run_ignore_failure(cmd: List[str], config): pass -def setup_cluster(config): +def setup_cluster(logfile, config): logger.info("setting up the cluster..") run(CLUSTER_CMD_PREFIX + ["start-db"], config) - cluster_process = run_forget(CLUSTER_CMD_PREFIX + ["run"], config) + cluster_process = run_forget(CLUSTER_CMD_PREFIX + ["run"], logfile, config) time.sleep(5) # FIXME add a ready check for master logger.info(f"cluster pid: {cluster_process.pid}") return cluster_process @@ -66,7 +63,9 @@ def teardown_cluster(config): @contextmanager def det_cluster(config): try: - yield setup_cluster(config) + with open(str(results_dir.joinpath("cluster.stdout.logs")), "w") as f: + yield setup_cluster(f, config) + finally: teardown_cluster(config) From 76987708f73902854adcbf90e05e27f2442c190b Mon Sep 17 00:00:00 2001 From: Hamid Zare Date: Fri, 5 Jun 2020 10:50:47 -0700 Subject: [PATCH 4/6] add test development and debugging help --- webui/tests/README.md | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/webui/tests/README.md b/webui/tests/README.md index db267405adf..d865752f71d 100644 --- a/webui/tests/README.md +++ b/webui/tests/README.md @@ -6,7 +6,7 @@ two SPAs, Elm and React, and in some sense the whole det cluster. We provide a test script to take care of depdencies around Cypress commands. Let's call this the "test script" for the rest of the this document. -## Requirements +## Running the Tests When the tester loads up it loads the WebUI through a given master address (`$HOST`) and port (`$PORT`) @@ -28,7 +28,7 @@ Once the cluster is ready and accessible run the tests: Note that it's is important that one immediately follows the other since the pre-e2e-tests target starts some experiments. -### Using the bundled cluster manager +### Using the Bundled Cluster Manager For ease of use and quicker local testing we provide an option to set up and tear down such a cluster through the test script. @@ -37,8 +37,30 @@ Before the tests can be started we need to build the cluster including the WebUI to make sure the served WebUI and cluster in general are up to date. Issue the following command: -`./bin/e2e-tests.py e2e-tests` which in turn will: +`./bin/e2e-tests.py e2e-tests` (or `make test`) which in turn will: 1. Set up a test cluster 2. Run the Cyrpess tests `Cypress run` 3. Tear down the cluster and clean up in case of errors + +## Test Development + +Use `make dev-tests` to set up for test development and then proceed to add new +tests suites or update and rebuild the WebUI artifacts to see changes in tests. + +### Debugging Test Issues + +For reproducing and catching test flakes there is a simple helper script `./bin/try-for-flake.sh`. +Just executing the script will run `make test` over and over until it hits a error. + +To speed up this process you can: + +1. try to avoid the whole cluster set up and tear down on each iteration +2. limit the test scope: + - use `.skip` on each unwanted test suite or `.only` on the target suite. + - temporarily delete the unwanted test suites + +If through this process you can get the tests to a state where they are re-runnable +without the need for the cluster to be reset, pass in `true` to the test flake +script to instruct it to skip the cluster set up and tear down. + From 418430de4c52cd97f22bda987841a844cd04c8a7 Mon Sep 17 00:00:00 2001 From: Hamid Zare Date: Mon, 8 Jun 2020 15:47:17 -0700 Subject: [PATCH 5/6] set up results dir and cluster log location --- webui/tests/.gitignore | 1 + webui/tests/bin/e2e-tests.py | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/webui/tests/.gitignore b/webui/tests/.gitignore index d1f24e07fc8..346f28a154a 100644 --- a/webui/tests/.gitignore +++ b/webui/tests/.gitignore @@ -1,2 +1,3 @@ node_modules/ results/ +*.log diff --git a/webui/tests/bin/e2e-tests.py b/webui/tests/bin/e2e-tests.py index 1fc8261ffa6..30bd04d6c2f 100755 --- a/webui/tests/bin/e2e-tests.py +++ b/webui/tests/bin/e2e-tests.py @@ -18,8 +18,9 @@ webui_dir = root_path.joinpath("webui") tests_dir = webui_dir.joinpath("tests") results_dir = tests_dir.joinpath("results") +test_cluster_dir = tests_dir.joinpath("test-cluster") -CLUSTER_CMD_PREFIX = ["make", "-C", "test-cluster"] +CLUSTER_CMD_PREFIX = ["make", "-C", str(test_cluster_dir)] CLEAR = "\033[39m" BLUE = "\033[94m" @@ -42,6 +43,11 @@ def run_ignore_failure(cmd: List[str], config): pass +def setup_results_dir(config): + run_ignore_failure(["rm", "-r", str(results_dir)], config) + run(["mkdir", "-p", str(results_dir)], config) + + def setup_cluster(logfile, config): logger.info("setting up the cluster..") run(CLUSTER_CMD_PREFIX + ["start-db"], config) @@ -63,7 +69,8 @@ def teardown_cluster(config): @contextmanager def det_cluster(config): try: - with open(str(results_dir.joinpath("cluster.stdout.logs")), "w") as f: + log_path = str(test_cluster_dir.joinpath("cluster.stdout.log")) + with open(log_path, "w") as f: yield setup_cluster(f, config) finally: @@ -71,8 +78,8 @@ def det_cluster(config): def pre_e2e_tests(config): - run_ignore_failure(["rm", "-r", str(results_dir)], config) # TODO add a check for cluster condition + setup_results_dir(config) run( ["python", str(tests_dir.joinpath("bin", "createUserAndExperiments.py"))], config, @@ -122,6 +129,7 @@ def cypress_open(config): def e2e_tests(config): + setup_results_dir(config) with det_cluster(config): pre_e2e_tests(config) run_e2e_tests(config) From ba326f5c235885b960eeb38c3cb3235de882dc99 Mon Sep 17 00:00:00 2001 From: Hamid Zare Date: Mon, 8 Jun 2020 16:13:06 -0700 Subject: [PATCH 6/6] add more description around the rerunnable tests --- webui/tests/README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/webui/tests/README.md b/webui/tests/README.md index d865752f71d..69538c2d2e1 100644 --- a/webui/tests/README.md +++ b/webui/tests/README.md @@ -55,12 +55,16 @@ Just executing the script will run `make test` over and over until it hits a err To speed up this process you can: -1. try to avoid the whole cluster set up and tear down on each iteration +1. try to avoid the whole cluster set up and tear down on each iteration. If the suspected tests are +re-runable or you have a way of setting up and tearing down the cluster that is faster than the provided +`test-cluster` refer to the "Re-runnable Tests" section. 2. limit the test scope: - use `.skip` on each unwanted test suite or `.only` on the target suite. - temporarily delete the unwanted test suites -If through this process you can get the tests to a state where they are re-runnable -without the need for the cluster to be reset, pass in `true` to the test flake -script to instruct it to skip the cluster set up and tear down. +#### Re-runnable Tests +By default the tests are not re-runnable without meeting the conditions in the "Running the Tests" section. +However if you can get the tests to a state where they are re-runnable without the need for the cluster to be +reset, pass in `true` to the test flake script to instruct it to skip the cluster set up and tear down and +greatly speed up the process