test: remove WebUI e2e-tests dependency on det-deploy

.circleci/config.yml

@@ -416,7 +416,7 @@ commands:
             -w /webui/tests \
             --env DET_MASTER=determined-master:8080 \
             cypress/included:4.3.0 \
-            --config-file cypress-docker.json \
+            --config-file cypress-ci.json \
             --config "defaultCommandTimeout=30000,baseUrl=http://determined-master:8080" \
             --browser chrome \
             --headless

webui/tests/Makefile

@@ -1,41 +1,30 @@
-.PHONY: get-deps clean e2e-tests docker-e2e-tests
-
-export INTEGRATIONS_HOST_PORT ?= 5300
-
+.PHONY: get-deps
 get-deps:
 	for i in `seq 3`; do \
 		npm ci --no-audit && break ; \
 	done
 
+.PHONY: clean-test-artifacts
 clean-test-artifacts:
 	rm -rf cypress/screenshots/*
 	rm -rf cypress/videos/*
 
+.PHONY: clean
 clean: clean-test-artifacts
 	rm -rf node_modules/
 
-test: docker-e2e-tests
+.PHONY: test
+test:
+	python ./bin/e2e-tests.py e2e-tests
 
+.PHONY: check
 check:
 	yarn run lint
 
+.PHONY: fmt
 fmt:
 	yarn run lint --fix
 
-pre-e2e-tests:
-	python ./bin/e2e-tests.py pre-e2e-tests --integrations-host-port $(INTEGRATIONS_HOST_PORT)
-
-post-e2e-tests:
-	python ./bin/e2e-tests.py post-e2e-tests --integrations-host-port $(INTEGRATIONS_HOST_PORT)
-
-run-e2e-tests:
-	python ./bin/e2e-tests.py run-e2e-tests --integrations-host-port $(INTEGRATIONS_HOST_PORT)
-
-e2e-tests:
-	python ./bin/e2e-tests.py e2e-tests --integrations-host-port $(INTEGRATIONS_HOST_PORT)
-
-docker-run-e2e-tests:
-	python ./bin/e2e-tests.py docker-run-e2e-tests --integrations-host-port $(INTEGRATIONS_HOST_PORT)
-
-docker-e2e-tests:
-	python ./bin/e2e-tests.py docker-e2e-tests --integrations-host-port $(INTEGRATIONS_HOST_PORT)
+.PHONY: dev-tests
+dev-tests:
+	python ./bin/e2e-tests.py dev-tests

webui/tests/README.md

@@ -0,0 +1,44 @@
+# WebUI End-to-end Tests
+
+We use [Cypress](https://www.cypress.io/) to enable end-to-end testing of our
+two SPAs, Elm and React, and in some sense the whole det cluster.
+
+We provide a test script to take care of depdencies around Cypress commands.
+Let's call this the "test script" for the rest of the this document.
+
+## Requirements
+
+When the tester loads up it loads the WebUI through a given master address
+(`$HOST`) and port (`$PORT`)
+
+Based the tester requirements and ease of test development the following
+assumptions are made:
+
+- It's a brand new cluster
+- The cluster is isolated. Meaning this tester is the only entity interacting
+with the cluster
+
+Once the cluster is ready and accessible run the tests:
+
+```
+./bin/e2e-tests.py --det-port $PORT pre-e2e-tests && \
+./bin/e2e-tests.py --det-port $PORT run-e2e-tests
+```
+
+Note that it's is important that one immediately follows the other since the
+pre-e2e-tests target starts some experiments.
+
+### Using the bundled cluster manager
+
+For ease of use and quicker local testing we provide an option to set up and
+tear down such a cluster through the test script.
+
+Before the tests can be started we need to build the cluster including the WebUI
+to make sure the served WebUI and cluster in general are up to date.
+
+Issue the following command:
+`./bin/e2e-tests.py e2e-tests` which in turn will:
+
+1. Set up a test cluster
+2. Run the Cyrpess tests `Cypress run`
+3. Tear down the cluster and clean up in case of errors

webui/tests/bin/createUserAndExperiments.py

@@ -8,6 +8,10 @@
 USER_WITH_PASSWORD_PASSWORD = "special-pw"
 USER_WITHOUT_PASSWORD_USERNAME = "user-wo-pw"
 
+determined_root_dir = pathlib.Path(__file__).absolute().parents[3]
+noop_dir = determined_root_dir.joinpath("e2e_tests", "tests", "fixtures", "no_op")
+noop_config = "single-very-many-long-steps.yaml"
+
 
 def wait_for_process(process):
     # Avoid hang on macOS
@@ -43,6 +47,7 @@ def create_user(username, password=""):
         wait_for_process(p)
 
 
+print("setting up users..")
 # First login as admin to avoid having to authenticate downstream
 login_as("admin")
 
@@ -55,18 +60,21 @@ def create_user(username, password=""):
 # Login as non-default user with password
 login_as(USER_WITH_PASSWORD_USERNAME, USER_WITH_PASSWORD_PASSWORD)
 
+
 # Create experiments
-determined_root_dir = pathlib.Path(__file__).absolute().parents[3]
-experiment_dir = determined_root_dir.joinpath("e2e_tests", "tests", "fixtures", "no_op")
-
-for _ in range(4):
-    subprocess.run(
-        [
-            "det",
-            "experiment",
-            "create",
-            str(experiment_dir.joinpath("single-very-many-long-steps.yaml")),
-            str(experiment_dir),
-        ],
-        check=True,
-    )
+def createExperiment(directory, config_file, count):
+    cmd = [
+        "det",
+        "experiment",
+        "create",
+        str(directory.joinpath(config_file)),
+        str(directory),
+    ]
+
+    procs = [subprocess.Popen(cmd) for _ in range(count)]
+    for p in procs:
+        p.wait()
+
+
+print("creating experiments..")
+createExperiment(noop_dir, noop_config, 4)