ui: reintroduce end-to-end UI tests with cypress

A recent commit [1] removed the stale, unused end-to-end tests. In their
place, add newly-written tests that can run headlessly in TeamCity (via
Docker) as a quick-failing health check and as a more exhaustive suite.

[1] 3d171b2 (ui: remove unused end-to-end UI tests, 2022-07-25)

Release note: None

WORKSPACE

@@ -8,6 +8,7 @@ workspace(
         "@npm_protos": ["pkg/ui/workspaces/db-console/src/js/node_modules"],
         "@npm_cluster_ui": ["pkg/ui/workspaces/cluster_ui/node_modules"],
         "@npm_db_console": ["pkg/ui/workspaces/db-console/node_modules"],
+        "@npm_e2e_tests": ["pkg/ui/workspaces/e2e-tests/node_modules"],
     },
 )
 
@@ -263,6 +264,21 @@ yarn_install(
     symlink_node_modules = True,
 )
 
+yarn_install(
+    name = "npm_e2e_tests",
+    args = [
+        "--offline",
+    ],
+    data = [
+      "//pkg/ui:.yarnrc",
+      "@yarn_cache//:.seed",
+    ],
+    package_json = "//pkg/ui/workspaces/e2e-tests:package.json",
+    strict_visibility = False,
+    yarn_lock = "//pkg/ui/workspaces/e2e-tests:yarn.lock",
+    symlink_node_modules = True,
+)
+
 yarn_install(
     name = "npm_protos",
     args = [

build/deploy/cockroach.sh

@@ -15,7 +15,7 @@
 # set -m: enable job control.
 set -eum
 
-cockroach_entrypoint="/cockroach/cockroach"
+cockroach_entrypoint=${COCKROACH:-"/cockroach/cockroach"}
 
 certs_dir="certs"
 url=

build/teamcity/cockroach/ci/tests/ui_e2e_test_all.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+build="$(dirname $(dirname $(dirname $(dirname $(dirname "${0}")))))"
+# for tc_prepare, tc_start_block, and friends
+source "$build/teamcity-support.sh"
+# for build_docker_image and run_tests
+source "$build/teamcity/cockroach/ci/tests/ui_e2e_test_impl.sh"
+
+tc_prepare
+
+tc_start_block "Load cockroachdb/cockroach-ci image"
+load_cockroach_docker_image
+tc_end_block "Load cockroachdb/cockroach-ci image"
+
+tc_start_block "Run Cypress health checks"
+cd $root/pkg/ui/workspaces/e2e-tests
+run_tests health
+tc_end_block "Run Cypress health checks"

build/teamcity/cockroach/ci/tests/ui_e2e_test_health.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+build="$(dirname $(dirname $(dirname $(dirname $(dirname "${0}")))))"
+# for tc_prepare, tc_start_block, and friends
+source "$build/teamcity-support.sh"
+# for build_docker_image and run_tests
+source "$build/teamcity/cockroach/ci/tests/ui_e2e_test_impl.sh"
+
+tc_prepare
+
+tc_start_block "Load cockroachdb/cockroach-ci image"
+load_cockroach_docker_image
+tc_end_block "Load cockroachdb/cockroach-ci image"
+
+tc_start_block "Run Cypress health checks"
+cd $root/pkg/ui/workspaces/e2e-tests
+run_tests health
+tc_end_block "Run Cypress health checks"

build/teamcity/cockroach/ci/tests/ui_e2e_test_impl.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+function load_cockroach_docker_image() {
+  docker load --input upstream_artifacts/cockroach-docker-image.tar.gz &> artifacts/docker-load.log || (cat artifacts/docker-load.log && false)
+  rm artifacts/docker-load.log
+}
+
+function run_tests() {
+  SPEC_ARG=""
+  if [ "health" = "${1:-'EMPTY'}" ]; then
+    SPEC_ARG="--spec 'cypress/e2e/health-check/**'"
+  fi
+
+  run docker compose run cypress -- $SPEC_ARG
+}

dev

@@ -8,7 +8,7 @@ fi
 set -euo pipefail
 
 # Bump this counter to force rebuilding `dev` on all machines.
-DEV_VERSION=52
+DEV_VERSION=53
 
 THIS_DIR=$(cd "$(dirname "$0")" && pwd)
 BINARY_DIR=$THIS_DIR/bin/dev-versions

pkg/cmd/dev/testdata/datadriven/ui

@@ -145,3 +145,30 @@ rm -rf crdb-checkout/pkg/ui/node_modules
 rm -rf crdb-checkout/pkg/ui/workspaces/db-console/node_modules
 rm -rf crdb-checkout/pkg/ui/workspaces/db-console/src/js/node_modules
 rm -rf crdb-checkout/pkg/ui/workspaces/cluster-ui/node_modules
+
+exec
+dev ui e2e
+----
+bazel info workspace --color=no
+bazel info workspace --color=no
+bazel run @yarn//:yarn -- --silent --cwd crdb-checkout/pkg/ui/workspaces/e2e-tests install
+bazel build //pkg/cmd/cockroach:cockroach --config=with_ui
+crdb-checkout/pkg/ui/workspaces/e2e-tests/build/start-crdb-then.sh bazel run @yarn//:yarn -- --silent --cwd crdb-checkout/pkg/ui/workspaces/e2e-tests cy:run
+
+exec
+dev ui e2e ./foo/bar
+----
+bazel info workspace --color=no
+bazel info workspace --color=no
+bazel run @yarn//:yarn -- --silent --cwd crdb-checkout/pkg/ui/workspaces/e2e-tests install
+bazel build //pkg/cmd/cockroach:cockroach --config=with_ui
+crdb-checkout/pkg/ui/workspaces/e2e-tests/build/start-crdb-then.sh bazel run @yarn//:yarn -- --silent --cwd crdb-checkout/pkg/ui/workspaces/e2e-tests cy:run ./foo/bar
+
+exec
+dev ui e2e --headed
+----
+bazel info workspace --color=no
+bazel info workspace --color=no
+bazel run @yarn//:yarn -- --silent --cwd crdb-checkout/pkg/ui/workspaces/e2e-tests install
+bazel build //pkg/cmd/cockroach:cockroach --config=with_ui
+crdb-checkout/pkg/ui/workspaces/e2e-tests/build/start-crdb-then.sh bazel run @yarn//:yarn -- --silent --cwd crdb-checkout/pkg/ui/workspaces/e2e-tests cy:debug

pkg/cmd/dev/ui.go

@@ -17,6 +17,7 @@ import (
 	"os/signal"
 	"path"
 	"path/filepath"
+	"strings"
 
 	"github.com/spf13/cobra"
 )
@@ -39,6 +40,7 @@ func makeUICmd(d *dev) *cobra.Command {
 	uiCmd.AddCommand(makeUILintCmd(d))
 	uiCmd.AddCommand(makeUITestCmd(d))
 	uiCmd.AddCommand(makeUIWatchCmd(d))
+	uiCmd.AddCommand(makeUIE2eCmd(d))
 
 	return uiCmd
 }
@@ -49,6 +51,8 @@ type UIDirectories struct {
 	clusterUI string
 	// dbConsole is the absolute path to ./pkg/ui/workspaces/db-console.
 	dbConsole string
+	// e2eTests is the absolute path to ./pkg/ui/workspaces/e2e-tests.
+	e2eTests string
 	// eslintPlugin is the absolute path to ./pkg/ui/workspaces/eslint-plugin-crdb.
 	eslintPlugin string
 }
@@ -63,6 +67,7 @@ func getUIDirs(d *dev) (*UIDirectories, error) {
 	return &UIDirectories{
 		clusterUI:    path.Join(workspace, "./pkg/ui/workspaces/cluster-ui"),
 		dbConsole:    path.Join(workspace, "./pkg/ui/workspaces/db-console"),
+		e2eTests:     path.Join(workspace, "./pkg/ui/workspaces/e2e-tests"),
 		eslintPlugin: path.Join(workspace, "./pkg/ui/workspaces/eslint-plugin-crdb"),
 	}, nil
 }
@@ -520,6 +525,83 @@ Replaces 'make ui-test' and 'make ui-test-watch'.`,
 	return testCmd
 }
 
+func makeUIE2eCmd(d *dev) *cobra.Command {
+	const headedFlag = "headed"
+
+	e2eTestCmd := &cobra.Command{
+		Use:   "e2e -- [args passed to Cypress ...]",
+		Short: "run e2e (Cypress) tests",
+		Long: strings.TrimSpace(`
+Run end-to-end tests with Cypress, spinning up a real local cluster and
+launching test in a real browser. Extra flags are passed directly to the
+'cypress' binary".
+`),
+		Args: cobra.ArbitraryArgs,
+		RunE: func(cmd *cobra.Command, commandLine []string) error {
+			ctx := cmd.Context()
+
+			isHeaded := mustGetFlagBool(cmd, headedFlag)
+			var yarnTarget string = "cy:run"
+			if isHeaded {
+				yarnTarget = "cy:debug"
+			}
+
+			uiDirs, err := getUIDirs(d)
+			if err != nil {
+				return err
+			}
+
+			workspace, err := d.getWorkspace(ctx)
+			if err != nil {
+				return err
+			}
+
+			// Ensure e2e-tests dependencies are installed
+			installArgv := buildBazelYarnArgv("--silent", "--cwd", uiDirs.e2eTests, "install")
+			logCommand("bazel", installArgv...)
+			err = d.exec.CommandContextInheritingStdStreams(ctx, "bazel", installArgv...)
+			if err != nil {
+				return fmt.Errorf("unable to install NPM dependencies: %w", err)
+			}
+
+			// Build cockroach (relying on Bazel to short-circuit repeated builds)
+			buildCockroachArgv := []string{
+				"build",
+				"//pkg/cmd/cockroach:cockroach",
+				"--config=with_ui",
+			}
+			logCommand("bazel", buildCockroachArgv...)
+			err = d.exec.CommandContextInheritingStdStreams(ctx, "bazel", buildCockroachArgv...)
+			if err != nil {
+				return fmt.Errorf("unable to build cockroach with UI: %w", err)
+			}
+
+			// Run Cypress tests, passing any extra args through to 'cypress'
+			startCrdbThenSh := path.Join(uiDirs.e2eTests, "build/start-crdb-then.sh")
+			runCypressArgv := append(
+				[]string{"bazel"},
+				buildBazelYarnArgv("--silent", "--cwd", uiDirs.e2eTests, yarnTarget)...,
+			)
+			runCypressArgv = append(runCypressArgv, cmd.Flags().Args()...)
+
+			logCommand(startCrdbThenSh, runCypressArgv...)
+			env := append(
+				os.Environ(),
+				fmt.Sprintf("COCKROACH=%s", path.Join(workspace, "cockroach")),
+			)
+			err = d.exec.CommandContextWithEnv(ctx, env, startCrdbThenSh, runCypressArgv...)
+			if err != nil {
+				return fmt.Errorf("error while running Cypress tests: %w", err)
+			}
+
+			return nil
+		},
+	}
+	e2eTestCmd.Flags().Bool(headedFlag /* default */, false, "run tests in the interactive Cypres GUI")
+
+	return e2eTestCmd
+}
+
 // buildBazelYarnArgv returns the provided argv formatted so it can be run with
 // the bazel-provided version of yarn via `d.exec.CommandContextWithEnv`, e.g.:
 //

pkg/ui/BUILD.bazel

@@ -49,6 +49,7 @@ test_suite(
     tests = [
         "//pkg/ui/workspaces/cluster-ui:lint",
         "//pkg/ui/workspaces/db-console:lint",
+        "//pkg/ui/workspaces/e2e-tests:lint",
     ],
 )
 

pkg/ui/not-yarn-workspace.sh

@@ -7,6 +7,7 @@ set -euo pipefail
   yarn --cwd workspaces/db-console/src/js install
   yarn --cwd workspaces/cluster-ui install
   yarn --cwd workspaces/db-console install
+  yarn --cwd workspaces/e2e-tests install
 )
 
 cat << "EOF"

pkg/ui/workspaces/e2e-tests/.eslintrc.json

@@ -0,0 +1,10 @@
+{
+  "root": true,
+  "extends": "@cockroachlabs/eslint-config",
+  "settings": {
+    "react": {
+      "pragma": "React",
+      "version": "17.0"
+    }
+  }
+}

pkg/ui/workspaces/e2e-tests/.gitignore

@@ -0,0 +1,3 @@
+artifacts
+videos
+screenshots

pkg/ui/workspaces/e2e-tests/BUILD.bazel

@@ -0,0 +1,25 @@
+load("@npm_e2e_tests//eslint:index.bzl", "eslint_test")
+
+eslint_test(
+    name = "lint",
+    data = [
+        ".eslintrc.json",
+        "cypress",
+        "cypress.config.ts",
+        "//pkg/ui/workspaces/eslint-plugin-crdb",
+        "@npm_e2e_tests//@cockroachlabs/eslint-config",
+        "@npm_e2e_tests//@typescript-eslint/eslint-plugin",
+        "@npm_e2e_tests//@typescript-eslint/parser",
+        "@npm_e2e_tests//eslint-plugin-prettier",
+        "@npm_e2e_tests//eslint-plugin-react",
+        "@npm_e2e_tests//eslint-plugin-react-hooks",
+        "@npm_e2e_tests//prettier",
+    ],
+    templated_args = [
+        "--ext .ts",
+        "-c",
+        "$$(rlocation $(rootpath .eslintrc.json))",
+        "$$(rlocation $(rootpath cypress))",
+        "$$(rlocation $(rootpath cypress.config.ts))",
+    ],
+)

pkg/ui/workspaces/e2e-tests/README.md

@@ -0,0 +1,60 @@
+# End-to-end Tests for CockroachDB UI
+This package contains end-to-end tests for the CockroachDB web UI served (by
+default) on port 8080. Tests are run via [Cypress](https://cypress.io) and
+use a combination of the default Cypress API and the dom-testing-library queries
+provided by [Cypress Testing
+Library](https://testing-library.com/docs/cypress-testing-library/intro).
+
+## Running Tests Locally
+### Via `yarn`
+If you've already built a `cockroach` binary and installed the dependencies in
+this directory with `yarn`, these tests can be run with `yarn` directly:
+
+```sh
+yarn test # run all tests
+yarn test ./path/to/some/file.cy.ts # run only a single test file
+```
+
+For "headed" test development, in which the Cypress UI and a browser are
+visible, run `yarn test:debug` and choose a test file in the Cypress UI.
+
+### Via `dev`
+The standard `dev` CLI used for other CockroachDB tasks can be used to run all
+(or a selected subset) of end-to-end tests via:
+
+```sh
+dev ui e2e # run all tests
+dev ui e2e ./path/to/some/file.cy.ts # run only a single test file
+```
+
+This will ensure the relevant NPM dependencies are installed and build
+CockroachDB (if necessary) before running any tests, which is especially helpful
+during a bisect. For more information, see `dev ui e2e --help`. 
+
+For "headed" test development, in which the Cypress UI and a browser are
+visible, run `dev ui e2e --headed` and choose a test file in the Cypress UI.
+
+## Running Tests in TeamCity
+For CI purposes, these tests are also run in TeamCity via Docker's [Compose
+V2](https://docs.docker.com/compose) by combining the [CockroachDB Docker
+image](https://github.com/cockroachdb/cockroach/tree/master/build/deploy) from a
+[previous build
+step](https://teamcity.cockroachdb.com/buildConfiguration/Cockroach_ScratchProjectPutTcExperimentsInHere_JaneDockerImage)
+with an [upstream Cypress docker
+image](https://hub.docker.com/r/cypress/browsers). Testing this configuration
+requires a bit of manual file movement and is out of scope for this document.
+
+## Cluster Management in Tests
+Regardless of how tests are launched, each test suite spins up a single-node
+cluster with an empty 'movr' database  that listens only on localhost. The test
+scripts waits for the HTTP server to be listening on port 8080 before invoking
+`cypress`, and tears down the cluster when `cypress` exits.
+
+## Health Checks vs All Tests
+Each CockroachDB pull request results in only the tests in
+`./cypress/e2e/health-checks` being executed. These are intended to confirm only
+the most basic functionality of the UI, to ensure it hasn't been completely
+broken or accidentally removed.  A more comprehensive suite including all tests
+is run in a separate TeamCity job.
+
+In general: put only simple, short-lived tests in the health-checks tree.

pkg/ui/workspaces/e2e-tests/build/compose-entrypoint.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -uemo pipefail
+
+set -x
+
+# Copy the /e2e tree into /scratch and remove any existing node_modules, to
+# prevent files with unexpected owners from escaping
+mkdir /scratch
+cp --recursive --no-target-directory --no-preserve=owner /e2e /scratch
+cd /scratch
+
+# Write a minimal .yarnrc mirroring the one in cockroach.git/pkg/ui, to avoid
+# mounting that entire tree.
+echo "yarn-offline-mirror /yarn-vendor" > .yarnrc
+
+# Remove and reinstall any node_modules that may have been copied, since they're
+# potentially specific to the host platform.
+rm -rf node_modules/
+yarn install --force --offline
+
+# Run tests, passing extra CLI arguments through to Cypress
+yarn cy:run "$@"