From dfa185318eeac1a899622c78c3cdb8e5db73ce81 Mon Sep 17 00:00:00 2001 From: sgfost Date: Tue, 23 Jul 2024 18:44:57 -0700 Subject: [PATCH] test: add e2e workflow --- .github/workflows/e2e.yml | 35 ++++++++++++++----- Makefile | 25 ++++++++++---- django/core/settings/e2e.py | 24 +++++++++++-- django/curator/invoke_tasks/borg.py | 13 ++++--- django/curator/invoke_tasks/database.py | 4 +-- e2e.yml | 46 ++++++++++++++++--------- e2e/Dockerfile | 12 ------- 7 files changed, 106 insertions(+), 53 deletions(-) delete mode 100644 e2e/Dockerfile diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e6287f3a0..520b3d6b5 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,17 +1,34 @@ name: E2E Tests on: - workflow_dispatch # disabled pending db seeding - # push: - # branches: [ main ] - # pull_request: - # branches: [ main ] + push: + branches: [ main ] + pull_request: + branches: [ main ] jobs: - build: + e2e: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: build containers and run cypress e2e tests - # TODO: seed the database with a sparse(r) dump so there are pages to test.. - run: make e2e + + - name: deploy services in e2e mode + run: make deploy-e2e + + - name: run e2e tests + uses: cypress-io/github-action@v6 + with: + working-directory: e2e + wait-on: "http://localhost:8000" + wait-on-timeout: 300 + + - name: display service logs + if: failure() + run: docker compose -f docker-compose.yml -f e2e.yml logs + + - name: upload cypress videos + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cypress-videos + path: e2e/cypress/videos diff --git a/Makefile b/Makefile index 6bfa2b9c9..40541d5b5 100644 --- a/Makefile +++ b/Makefile @@ -82,7 +82,7 @@ release-version: .env .PHONY: docker-compose.yml docker-compose.yml: base.yml dev.yml staging.yml prod.yml config.mk $(PGPASS_PATH) release-version case "$(DEPLOY_ENVIRONMENT)" in \ - dev|staging|e2e) docker compose -f base.yml -f $(DEPLOY_ENVIRONMENT).yml config > docker-compose.yml;; \ + dev|staging) docker compose -f base.yml -f $(DEPLOY_ENVIRONMENT).yml config > docker-compose.yml;; \ prod) docker compose -f base.yml -f staging.yml -f $(DEPLOY_ENVIRONMENT).yml config > docker-compose.yml;; \ *) echo "invalid environment. must be either dev, staging or prod" 1>&2; exit 1;; \ esac @@ -138,10 +138,21 @@ clean_deploy: clean test: build docker compose run --rm server /code/deploy/test.sh +# e2e testing setup + +E2E_SHARED_DIR=${DOCKER_SHARED_DIR}/e2e +E2E_BACKUPS_PATH=${E2E_SHARED_DIR}/backups +E2E_REPO_PATH=${E2E_BACKUPS_PATH}/repo + +$(E2E_REPO_PATH): + mkdir -p $(E2E_BACKUPS_PATH) + wget -c ${BORG_REPO_URL} -P $(E2E_BACKUPS_PATH) + tar -Jxf $(E2E_BACKUPS_PATH)/repo.tar.xz -C $(E2E_BACKUPS_PATH) + .PHONY: e2e -e2e: DEPLOY_ENVIRONMENT=e2e -e2e: build - docker compose up -d - docker compose exec server inv collectstatic - docker compose run --rm --no-deps e2e npm run test - docker compose down > /dev/null 2>&1 +e2e: $(E2E_REPO_PATH) + docker compose -f docker-compose.yml -f e2e.yml up -d --build + docker compose -f docker-compose.yml -f e2e.yml exec server bash -c "\ + inv borg.restore --force && \ + ./manage.py migrate && \ + inv prepare" diff --git a/django/core/settings/e2e.py b/django/core/settings/e2e.py index 6ba62d430..a7bc69622 100644 --- a/django/core/settings/e2e.py +++ b/django/core/settings/e2e.py @@ -1,5 +1,25 @@ -from .dev import * +from .test import * -DEPLOY_ENVIRONMENT = Environment.TEST +DEBUG = True DJANGO_VITE_DEV_MODE = False + +SHARE_DIR = path.realpath("/shared/e2e") +LIBRARY_ROOT = path.join(SHARE_DIR, "library") +LIBRARY_PREVIOUS_ROOT = path.join(SHARE_DIR, ".latest") +REPOSITORY_ROOT = path.join(BASE_DIR, "repository") +BACKUP_ROOT = path.join(SHARE_DIR, "backups") +BORG_ROOT = path.join(BACKUP_ROOT, "repo") +EXTRACT_ROOT = path.join(SHARE_DIR, "extract") +MEDIA_ROOT = path.join(SHARE_DIR, "media") + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": os.getenv("DB_NAME"), + "USER": os.getenv("DB_USER"), + "PASSWORD": read_secret("db_password", os.getenv("DB_PASSWORD")), + "HOST": "e2edb", + "PORT": os.getenv("DB_PORT"), + } +} diff --git a/django/curator/invoke_tasks/borg.py b/django/curator/invoke_tasks/borg.py index 56408c564..2db1922ad 100644 --- a/django/curator/invoke_tasks/borg.py +++ b/django/curator/invoke_tasks/borg.py @@ -203,14 +203,19 @@ def restore_database( @task() def restore( - ctx, repo=settings.BORG_ROOT, archive=None, target_database=db._DEFAULT_DATABASE + ctx, + repo=settings.BORG_ROOT, + archive=None, + target_database=db._DEFAULT_DATABASE, + force=False, ): """Restore the library files, media files and database to the state given in the borg repo at path REPO using archive ARCHIVE. The target_database argument is for testing so a different database can be used to make sure the database is getting restored properly""" - confirm( - "Are you sure you want to restore the database and all file content (y/n)? " - ) + if not force: + confirm( + "Are you sure you want to restore the database and all file content (y/n)? " + ) with tempfile.TemporaryDirectory(dir=settings.SHARE_DIR) as working_directory: _restore( ctx, diff --git a/django/curator/invoke_tasks/database.py b/django/curator/invoke_tasks/database.py index b2427073c..43035704e 100644 --- a/django/curator/invoke_tasks/database.py +++ b/django/curator/invoke_tasks/database.py @@ -42,7 +42,7 @@ def create_pgpass_file(ctx, db_key=_DEFAULT_DATABASE, force=False): if os.path.isfile(pgpass_path) and not force: return with open(pgpass_path, "w+") as pgpass: - pgpass.write("db:*:*:{db_user}:{db_password}\n".format(**db_config)) + pgpass.write("{db_host}:*:*:{db_user}:{db_password}\n".format(**db_config)) ctx.run("chmod 0600 ~/.pgpass") @@ -132,7 +132,7 @@ def restore_from_dump( cat_cmd = "zcat" drop(ctx, database=target_database, create=True) ctx.run( - "{cat_cmd} {dumpfile} | psql -w -q -o restore-from-dump-log.txt -h db {db_name} {db_user}".format( + "{cat_cmd} {dumpfile} | psql -w -q -o restore-from-dump-log.txt -h {db_host} {db_name} {db_user}".format( cat_cmd=cat_cmd, dumpfile=dumpfile, **db_config ), echo=True, diff --git a/e2e.yml b/e2e.yml index 9aece4b7c..8b406f2f1 100644 --- a/e2e.yml +++ b/e2e.yml @@ -1,25 +1,39 @@ services: - e2e: - image: comses/cypress - build: e2e - environment: - - CYPRESS_baseUrl=http://server:8000 + db: + image: alpine # disable the normal db container + command: tail -f /dev/null + healthcheck: null + e2edb: + image: postgis/postgis:15-3.4 + secrets: + - db_password # re-using the same db password volumes: - - ./e2e/cypress:/code/cypress - depends_on: - server: - condition: service_healthy + - ./build/secrets/db_password:/run/secrets/db_password + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER} -d ${DB_NAME}"] + interval: 30s + timeout: 5s + retries: 5 + environment: + POSTGRES_USER: "${DB_USER}" + POSTGRES_DB: "${DB_NAME}" + POSTGRES_PASSWORD_FILE: /run/secrets/db_password vite: - volumes: - - ./frontend:/code command: ["npm", "run", "build"] environment: NODE_ENV: "e2e" server: - image: comses/server:dev - volumes: - - ./django:/code - - ./docs:/docs + depends_on: + db: + condition: service_started + e2edb: + condition: service_healthy + elasticsearch: + condition: service_started + redis: + condition: service_started + vite: + condition: service_started healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8000"] interval: 30s @@ -27,5 +41,3 @@ services: retries: 5 environment: DJANGO_SETTINGS_MODULE: "core.settings.e2e" - ports: - - "127.0.0.1:8000:8000" diff --git a/e2e/Dockerfile b/e2e/Dockerfile deleted file mode 100644 index 9e6ee2b5d..000000000 --- a/e2e/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM cypress/base:18.14.1 -WORKDIR /code - -# only install dependencies if package.json or package-lock.json has changed -COPY package.json . -COPY package-lock.json . -COPY cypress.config.js . - -# suppress most of the cypress messages -ENV CI=1 - -RUN npm install