test: add e2e workflow

.github/workflows/e2e.yml

@@ -1,17 +1,34 @@
 name: E2E Tests
 
 on:
-  workflow_dispatch # disabled pending db seeding
-  # push:
-  #   branches: [ main ]
-  # pull_request:
-  #   branches: [ main ]
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
 
 jobs:
-  build:
+  e2e:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: build containers and run cypress e2e tests
-        # TODO: seed the database with a sparse(r) dump so there are pages to test..
-        run: make e2e
+
+      - name: deploy services in e2e mode
+        run: make deploy-e2e
+
+      - name: run e2e tests
+        uses: cypress-io/github-action@v6
+        with:
+          working-directory: e2e
+          wait-on: "http://localhost:8000"
+          wait-on-timeout: 300
+
+      - name: display service logs
+        if: failure()
+        run: docker compose -f docker-compose.yml -f e2e.yml logs
+
+      - name: upload cypress videos
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cypress-videos
+          path: e2e/cypress/videos

Makefile

@@ -82,7 +82,7 @@ release-version: .env
 .PHONY: docker-compose.yml
 docker-compose.yml: base.yml dev.yml staging.yml prod.yml config.mk $(PGPASS_PATH) release-version
 	case "$(DEPLOY_ENVIRONMENT)" in \
-	  dev|staging|e2e) docker compose -f base.yml -f $(DEPLOY_ENVIRONMENT).yml config > docker-compose.yml;; \
+	  dev|staging) docker compose -f base.yml -f $(DEPLOY_ENVIRONMENT).yml config > docker-compose.yml;; \
 	  prod) docker compose -f base.yml -f staging.yml -f $(DEPLOY_ENVIRONMENT).yml config > docker-compose.yml;; \
 	  *) echo "invalid environment. must be either dev, staging or prod" 1>&2; exit 1;; \
 	esac
@@ -138,10 +138,21 @@ clean_deploy: clean
 test: build
 	docker compose run --rm server /code/deploy/test.sh
 
+# e2e testing setup
+
+E2E_SHARED_DIR=${DOCKER_SHARED_DIR}/e2e
+E2E_BACKUPS_PATH=${E2E_SHARED_DIR}/backups
+E2E_REPO_PATH=${E2E_BACKUPS_PATH}/repo
+
+$(E2E_REPO_PATH):
+	mkdir -p $(E2E_BACKUPS_PATH)
+	wget -c ${BORG_REPO_URL} -P $(E2E_BACKUPS_PATH)
+	tar -Jxf $(E2E_BACKUPS_PATH)/repo.tar.xz -C $(E2E_BACKUPS_PATH)
+
 .PHONY: e2e
-e2e: DEPLOY_ENVIRONMENT=e2e
-e2e: build
-	docker compose up -d
-	docker compose exec server inv collectstatic
-	docker compose run --rm --no-deps e2e npm run test
-	docker compose down > /dev/null 2>&1
+e2e: $(E2E_REPO_PATH)
+	docker compose -f docker-compose.yml -f e2e.yml up -d --build
+	docker compose -f docker-compose.yml -f e2e.yml exec server bash -c "\
+		inv borg.restore --force && \
+		./manage.py migrate && \
+		inv prepare"

django/core/settings/e2e.py

@@ -1,5 +1,25 @@
-from .dev import *
+from .test import *
 
-DEPLOY_ENVIRONMENT = Environment.TEST
+DEBUG = True
 
 DJANGO_VITE_DEV_MODE = False
+
+SHARE_DIR = path.realpath("/shared/e2e")
+LIBRARY_ROOT = path.join(SHARE_DIR, "library")
+LIBRARY_PREVIOUS_ROOT = path.join(SHARE_DIR, ".latest")
+REPOSITORY_ROOT = path.join(BASE_DIR, "repository")
+BACKUP_ROOT = path.join(SHARE_DIR, "backups")
+BORG_ROOT = path.join(BACKUP_ROOT, "repo")
+EXTRACT_ROOT = path.join(SHARE_DIR, "extract")
+MEDIA_ROOT = path.join(SHARE_DIR, "media")
+
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": os.getenv("DB_NAME"),
+        "USER": os.getenv("DB_USER"),
+        "PASSWORD": read_secret("db_password", os.getenv("DB_PASSWORD")),
+        "HOST": "e2edb",
+        "PORT": os.getenv("DB_PORT"),
+    }
+}

django/curator/invoke_tasks/borg.py

@@ -203,14 +203,19 @@ def restore_database(
 
 @task()
 def restore(
-    ctx, repo=settings.BORG_ROOT, archive=None, target_database=db._DEFAULT_DATABASE
+    ctx,
+    repo=settings.BORG_ROOT,
+    archive=None,
+    target_database=db._DEFAULT_DATABASE,
+    force=False,
 ):
     """Restore the library files, media files and database to the state given in the borg repo at path REPO
     using archive ARCHIVE. The target_database argument is for testing so a different database can be used to
     make sure the database is getting restored properly"""
-    confirm(
-        "Are you sure you want to restore the database and all file content (y/n)? "
-    )
+    if not force:
+        confirm(
+            "Are you sure you want to restore the database and all file content (y/n)? "
+        )
     with tempfile.TemporaryDirectory(dir=settings.SHARE_DIR) as working_directory:
         _restore(
             ctx,

django/curator/invoke_tasks/database.py

@@ -42,7 +42,7 @@ def create_pgpass_file(ctx, db_key=_DEFAULT_DATABASE, force=False):
     if os.path.isfile(pgpass_path) and not force:
         return
     with open(pgpass_path, "w+") as pgpass:
-        pgpass.write("db:*:*:{db_user}:{db_password}\n".format(**db_config))
+        pgpass.write("{db_host}:*:*:{db_user}:{db_password}\n".format(**db_config))
         ctx.run("chmod 0600 ~/.pgpass")
 
 
@@ -132,7 +132,7 @@ def restore_from_dump(
             cat_cmd = "zcat"
         drop(ctx, database=target_database, create=True)
         ctx.run(
-            "{cat_cmd} {dumpfile} | psql -w -q -o restore-from-dump-log.txt -h db {db_name} {db_user}".format(
+            "{cat_cmd} {dumpfile} | psql -w -q -o restore-from-dump-log.txt -h {db_host} {db_name} {db_user}".format(
                 cat_cmd=cat_cmd, dumpfile=dumpfile, **db_config
             ),
             echo=True,

e2e.yml

@@ -1,31 +1,43 @@
 services:
-  e2e:
-    image: comses/cypress
-    build: e2e
-    environment:
-      - CYPRESS_baseUrl=http://server:8000
+  db:
+    image: alpine # disable the normal db container
+    command: tail -f /dev/null
+    healthcheck: null
+  e2edb:
+    image: postgis/postgis:15-3.4
+    secrets:
+      - db_password # re-using the same db password
     volumes:
-      - ./e2e/cypress:/code/cypress
-    depends_on:
-      server:
-        condition: service_healthy
+      - ./build/secrets/db_password:/run/secrets/db_password
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${DB_USER} -d ${DB_NAME}"]
+      interval: 30s
+      timeout: 5s
+      retries: 5
+    environment:
+      POSTGRES_USER: "${DB_USER}"
+      POSTGRES_DB: "${DB_NAME}"
+      POSTGRES_PASSWORD_FILE: /run/secrets/db_password
   vite:
-    volumes:
-      - ./frontend:/code
     command: ["npm", "run", "build"]
     environment:
       NODE_ENV: "e2e"
   server:
-    image: comses/server:dev
-    volumes:
-      - ./django:/code
-      - ./docs:/docs
+    depends_on:
+      db:
+        condition: service_started
+      e2edb:
+        condition: service_healthy
+      elasticsearch:
+        condition: service_started
+      redis:
+        condition: service_started
+      vite:
+        condition: service_started
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8000"]
       interval: 30s
       timeout: 10s
       retries: 5
     environment:
       DJANGO_SETTINGS_MODULE: "core.settings.e2e"
-    ports:
-      - "127.0.0.1:8000:8000"