Merge pull request #1758 from bcgov/1734-registration2-ci-cd

1734 registration2 ci cd

.github/actions/nx-app-build/action.yaml

@@ -0,0 +1,72 @@
+name: "Build and push Docker image for Nx frontend app in /bciers directory"
+description: "Reusable action to build and push a Docker image for an Nx app in the /bciers directory"
+
+inputs:
+  project:
+    description: "Nx project name"
+    required: true
+  image_url:
+    description: "Docker image URL"
+    required: true
+  github_token:
+    description: "GitHub token"
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Setup Docker Buildx
+      uses: docker/setup-buildx-action@v3
+    - name: Docker metadata
+      id: meta
+      uses: docker/metadata-action@v5
+      with:
+        images: ${{ inputs.image_url }}
+        tags: |
+          type=sha,format=long,prefix=
+          latest
+          type=ref,event=pr
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ inputs.github_token }}
+    - name: Install dependencies
+      shell: bash
+      run: |
+        corepack enable
+        yarn install --immutable
+        cd bciers && yarn install --immutable
+    - uses: actions/setup-node@v4
+    - name: Derive appropriate SHAs for base and head for `nx affected` commands
+      uses: nrwl/nx-set-shas@v4
+      with:
+        main-branch-name: "develop"
+    - name: Cache Docker layers
+      uses: actions/cache@v4
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-bciers-${{ inputs.project }}${{ github.sha }}
+        restore-keys: |
+          ${{ runner.os }}-buildx-bciers-${{ inputs.project }}
+    - name: Build images
+      shell: bash
+      env:
+        INPUT_GITHUB_TOKEN: ${{ inputs.github_token }}
+        INPUT_PUSH: true
+        INPUT_TAGS: ${{ steps.meta.outputs.tags }}
+        INPUT_LABELS: ${{ steps.meta.output.labels }}
+        INPUT_CACHE_FROM: type=local,src=/tmp/.buildx-cache
+        INPUT_CACHE_TO: type=local,dest=/tmp/.buildx-cache-new
+      run: |
+        npx nx container ${{ inputs.project }} --skip-nx-cache
+      working-directory: ./bciers
+      # Temp fix
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+    - name: Move cache
+      shell: bash
+      run: |
+        rm -rf /tmp/.buildx-cache
+        mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/actions/run-backend/action.yml

@@ -0,0 +1,14 @@
+name: "Run Django backend"
+description: "Runs the Django backend locally"
+
+inputs:
+  django_secret_key:
+    description: "Django secret key"
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: start backend
+      shell: bash
+      run: docker run -d --network=host -e "DB_USER=postgres" -e "DB_NAME=registration" -e "DB_PORT=5432" -e "DB_HOST=localhost" -e "DJANGO_SECRET_KEY=${{ inputs.django_secret_key }}" -e "ALLOWED_HOSTS=localhost,0.0.0.0,127.0.0.1" -e "ENVIRONMENT=develop" -e "CI=true" ghcr.io/bcgov/cas-reg-backend:${{ github.sha }}

.github/actions/local-app-run/action.yml → .github/actions/run-registration1-app/action.yml

@@ -19,8 +19,9 @@ runs:
   using: composite
   steps:
     - name: start backend
-      shell: bash
-      run: docker run -d --network=host -e "DB_USER=postgres" -e "DB_NAME=registration" -e "DB_PORT=5432" -e "DB_HOST=localhost" -e "DJANGO_SECRET_KEY=${{ inputs.django_secret_key }}" -e "ALLOWED_HOSTS=localhost,0.0.0.0,127.0.0.1" -e "ENVIRONMENT=develop" -e "CI=true" ghcr.io/bcgov/cas-reg-backend:${{ github.sha }}
+      uses: ./.github/actions/run-backend
+      with:
+        django_secret_key: ${{ inputs.django_secret_key }}
     - name: start frontend
       shell: bash
-      run: docker run -d --network=host -e "NEXTAUTH_URL_INTERNAL=http://localhost:3000/" -e "NEXTAUTH_URL=http://localhost:3000/" -e "NEXTAUTH_SECRET=${{ inputs.nextauth_secret }}" -e "API_URL=http://127.0.0.1:8000/api/" -e "KEYCLOAK_LOGIN_URL=https://dev.loginproxy.gov.bc.ca/auth/realms/standard" -e "KEYCLOAK_CLIENT_SECRET=${{ inputs.keycloak_client_secret }}" -e "KEYCLOAK_CLIENT_ID=${{ inputs.keycloak_client_id }}" ghcr.io/bcgov/cas-reg-frontend:${{ github.sha }}
+      run: docker run -d --network=host -e "NEXTAUTH_URL_INTERNAL=http://localhost:3000/" -e "NEXTAUTH_URL=http://localhost:3000/" -e "NEXTAUTH_SECRET=${{ inputs.nextauth_secret }}" -e "API_URL=http://127.0.0.1:8000/api/" -e "KEYCLOAK_LOGIN_URL=https://dev.loginproxy.gov.bc.ca/auth/realms/standard" -e "KEYCLOAK_CLIENT_SECRET=${{ inputs.keycloak_client_secret }}" -e "KEYCLOAK_CLIENT_ID=${{ inputs.keycloak_client_id }}" ghcr.io/bcgov/cas-reg1-frontend:${{ github.sha }}

.github/workflows/build-backend.yaml

@@ -0,0 +1,55 @@
+name: Build BCIERS backend Docker container
+
+on:
+  workflow_call:
+
+jobs:
+  docker-build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          install: true
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/bcgov/cas-reg-backend
+          tags: |
+            type=sha,format=long,prefix=
+            latest
+            type=ref,event=pr
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Cache Docker layers
+        uses: actions/cache@v4
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-bc_obps-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-bc_obps
+      - name: Build image
+        uses: docker/build-push-action@v5
+        with:
+          context: bc_obps
+          builder: ${{ steps.buildx.outputs.name }}
+          push: true
+          file: bc_obps/Dockerfile
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+        # Temp fix
+        # https://github.com/docker/build-push-action/issues/252
+        # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/build-registration.yaml

@@ -0,0 +1,19 @@
+name: Build Registration Docker container
+
+on:
+  workflow_call:
+
+jobs:
+  docker-build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Build registration container
+        uses: ./.github/actions/nx-app-build
+        with:
+          project: registration
+          image_url: ghcr.io/bcgov/cas-reg-frontend
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/build-registration1.yaml

@@ -0,0 +1,19 @@
+name: Build Registration1 Docker container
+
+on:
+  workflow_call:
+
+jobs:
+  docker-build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Build registration1 container
+        uses: ./.github/actions/nx-app-build
+        with:
+          project: registration1
+          image_url: ghcr.io/bcgov/cas-reg1-frontend
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/build-reporting.yaml

@@ -0,0 +1,19 @@
+name: Build Reporting Docker container
+
+on:
+  workflow_call:
+
+jobs:
+  docker-build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Build reporting container
+        uses: ./.github/actions/nx-app-build
+        with:
+          project: reporting
+          image_url: ghcr.io/bcgov/cas-rep-frontend
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/codeql.yaml

@@ -12,21 +12,12 @@
 name: "CodeQL"
 
 on:
-  push:
-    branches: [main, develop]
-    tags:
-      - frontend
-      - backend
-      - configs
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [main, develop]
+  workflow_call:
   schedule:
     - cron: "19 23 * * 5"
 
 jobs:
   analyze:
-    name: Analyze
     runs-on: ubuntu-latest
     permissions:
       actions: read

.github/workflows/main.yaml

@@ -0,0 +1,54 @@
+# This is the main workflow which will orchestrate the other workflows:
+
+name: main
+
+on:
+  push:
+    branches: [develop, main]
+  pull_request:
+    branches: [develop, main]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  PGUSER: postgres
+
+jobs:
+  install-dev-tools:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: dev env setup
+        uses: ./.github/actions/dev-env-setup
+      - run: yarn install --immutable
+        working-directory: ./bciers
+  build-backend:
+    uses: ./.github/workflows/build-backend.yaml
+  build-registration:
+    uses: ./.github/workflows/build-registration.yaml
+  build-registration1:
+    uses: ./.github/workflows/build-registration1.yaml
+  build-reporting:
+    uses: ./.github/workflows/build-reporting.yaml
+  test-code:
+    needs: [install-dev-tools]
+    uses: ./.github/workflows/test-code.yaml
+  test-backend:
+    needs: [build-backend, install-dev-tools]
+    uses: ./.github/workflows/test-backend.yaml
+    secrets: inherit
+  test-e2e:
+    needs: [build-registration1, install-dev-tools]
+    uses: ./.github/workflows/test-e2e.yaml
+    secrets: inherit
+  zap-owasp:
+    # We will need to set up registration part 2 and reporting to get zap scanned
+    needs: [build-backend, build-registration1, install-dev-tools]
+    uses: ./.github/workflows/zap-owasp.yaml
+  trivy:
+    uses: ./.github/workflows/trivy.yaml
+  codeql:
+    uses: ./.github/workflows/codeql.yaml

.github/workflows/test-backend.yaml

@@ -0,0 +1,26 @@
+# This file is used for tests that use the backend container
+# For tests that don't require containers to be built first, see test-code.yaml
+
+name: Test BCIERS backend container
+
+on:
+  workflow_call:
+
+env:
+  PGUSER: postgres
+  DJANGO_SECRET_KEY: ${{ secrets.DJANGO_SECRET_KEY }}
+
+jobs:
+  pythontests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: dev env setup
+        uses: ./.github/actions/dev-env-setup
+      - name: start backend
+        uses: ./.github/actions/run-backend
+        with:
+          django_secret_key: ${{ env.DJANGO_SECRET_KEY }}
+      - name: Run pytest
+        working-directory: ./bc_obps
+        run: make pythontests

.github/workflows/test-code.yaml

@@ -0,0 +1,76 @@
+# This workflow is to run all tests that don't require containers to be built
+
+name: Test BCIERS code
+
+on:
+  workflow_call:
+
+env:
+  PGUSER: postgres
+  DJANGO_SECRET_KEY: ${{ secrets.DJANGO_SECRET_KEY }}
+  KEYCLOAK_CLIENT_ID: ${{ secrets.KEYCLOAK_CLIENT_ID }}
+  KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }}
+  NEXTAUTH_SECRET: ${{ secrets.NEXTAUTH_SECRET }}
+
+jobs:
+  nx-tests:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./bciers
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: dev env setup
+        uses: ./.github/actions/dev-env-setup
+      - name: Run Nx Affected Tests with Remote Caching
+        id: nx-tests
+        run: |
+          yarn nx affected --base=origin/develop --target=test --parallel
+        shell: bash
+
+  yarn-audit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: dev env setup
+        uses: ./.github/actions/dev-env-setup
+      - run: yarn npm audit
+        working-directory: ./bciers
+
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: dev env setup
+        uses: ./.github/actions/dev-env-setup
+      - name: set pre-commit cache directory
+        run: |
+          echo "PRE_COMMIT_HOME=$GITHUB_WORKSPACE/.pre-commit-cache" >> $GITHUB_ENV
+      - name: set PY
+        run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
+      - name: create commitlint COMMIT_EDITMSG if not exists
+        run: |
+          if test -f ".git/COMMIT_EDITMSG"; then
+              echo "COMMIT_EDITMSG EXISTS, skipping"
+          else
+              touch .git/COMMIT_EDITMSG
+          fi
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ./.pre-commit-cache
+          key: pre-commit-${{ env.PY }}-${{ hashFiles('.pre-commit-config.yaml') }}-v4
+      - run: pip install -r requirements.txt
+      - uses: pre-commit/[email protected]
+
+  check-migrations:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: dev env setup
+        uses: ./.github/actions/dev-env-setup
+      - run: poetry run python manage.py makemigrations --check --dry-run
+        working-directory: ./bc_obps