ACR, Workflow UX, gh act, azure batch automation

.github/workflows/build-and-push-docker.yaml

@@ -0,0 +1,154 @@
+# This GitHub Actions workflow builds a Docker image for the
+# cfa-epinow2-pipeline-docker project. It consists of two jobs:
+# build_image_dependencies and build_image.
+#
+# - The `build_image_dependencies` job carries out the first part of a
+#   multi-stage build. It downloads and install all the dependencies
+#   listed in the `DESCRIPTION` file. It uses the `Dockerfile-dependencies`
+#   file to build the image.
+#
+#   The built image is then pushed to the corresponding registry.
+#
+#   The process is cached to avoid rebuilding the image if the dependencies
+#   have not changed. This is by hashing the `DESCRIPTION` file and the
+#   `Dockerfile-dependencies` file.
+#
+# - The build_image job builds the final image using the `Dockerfile` file.
+#   It uses the image built in the previous job as a base image.
+#
+#   During the build process, the package is installed and built. Furthermore
+#   the package is checked using `R CMD check` to ensure that it is working
+#   correctly.
+#
+#   Once the image is built, it is pushed to the corresponding registry.
+
+name: Build and Push Docker image
+
+on:
+  push:
+    branches:
+      - main
+      - jk-azure-readiness
+  workflow_dispatch:
+
+env:
+  # Together, these form: cfaprdbatchcr.azurecr.io/cfa-epinow2-pipeline
+  REGISTRY: cfaprdbatchcr.azurecr.io
+  IMAGE_NAME: cfa-epinow2-pipeline
+
+jobs:
+  build_image_dependencies:
+    runs-on: cfa-cdcgov
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      # - name: Check cache
+      #   uses: actions/cache@v3
+      #   id: cache
+      #   with:
+      #     key: docker-dependencies-${{ runner.os }}-${{ hashFiles('./DESCRIPTION', './Dockerfile-dependencies') }}
+      #     path:
+      #       ./DESCRIPTION
+
+      - name: Login to the Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: "cfaprdbatchcr.azurecr.io"
+          username: "cfaprdbatchcr"
+          password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
+
+      # if: steps.cache.outputs.cache-hit != 'true'
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          push: true
+          tags: |
+            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:latest
+            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:${{ github.sha }}
+          file: ./Dockerfile-dependencies
+
+  build_image:
+    needs: build_image_dependencies
+    runs-on: cfa-cdcgov
+    steps:
+
+      - name: Login to the Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: "cfaprdbatchcr.azurecr.io"
+          username: "cfaprdbatchcr"
+          password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          push: true
+          tags: |
+            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:latest
+            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:${{ github.sha }}
+          file: ./Dockerfile
+
+  create-and-launch-batch-pools:
+    runs-on: cfa-cdcgov
+    needs:
+      - build_image
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout Repo
+        id: checkout_repo
+        uses: actions/checkout@v4
+
+      - name: Login to Azure with NNH Service Principal
+        id: azure_login_2
+        uses: azure/login@v2
+        with:
+        # managed by EDAV. Contact Amit Mantri or Jon Kislin if you have issues.
+          creds: ${{ secrets.EDAV_CFA_PREDICT_NNHT_SP }}
+
+      - name: Download Configuration Tomls from cfadatalakeprd/cfapredict
+        id: download_config_tomls
+        run: |
+          rm -rf ./ci_config_tomls/
+          az storage blob download-batch \
+            --source https://cfadatalakeprd.blob.core.windows.net/cfapredict/ \
+            --pattern "./ci_config_tomls/*" \
+            --destination ./ \
+            --auth-mode "login"
+
+      - name: Delete Existing Batch Pool
+        id: delete_existing_batch_pool
+        run: |
+
+          export pool_id="cfa-epinow2-pipeline-ci"
+
+          # Check if the batch pool exists
+          EPINOW2_POOL_EXISTS=$(az batch pool show \
+            --account-name "cfaprdba" \
+            --account-endpoint "https://cfaprdba.eastus.batch.azure.com" \
+            --pool-id "$pool_id" \
+            --query "id" --output tsv 2>/dev/null)
+
+          if [ -z "$EPINOW2_POOL_EXISTS" ]; then
+              echo -e "Batch pool $pool_id does not exist."
+          else
+              echo -e "Batch pool $pool_id exists. Deleting it now..."
+              az batch pool delete \
+                --account-name cfaprdba \
+                --account-endpoint https://cfaprdba.eastus.batch.azure.com \
+                --pool-id "$pool_id" \
+                --yes
+              echo "Batch pool $pool_id has been deleted."
+          fi
+
+      - name: Create cfa-epinow2-pipeline Pools
+        id: create_nhsn_nssp_pools
+        run: |
+          echo 'hello world'
+
+

.secrets.baseline

@@ -1,5 +1,5 @@
 {
-  "version": "1.4.0",
+  "version": "1.5.0",
   "plugins_used": [
     {
       "name": "ArtifactoryDetector"
@@ -26,6 +26,9 @@
     {
       "name": "GitHubTokenDetector"
     },
+    {
+      "name": "GitLabTokenDetector"
+    },
     {
       "name": "HexHighEntropyString",
       "limit": 3.0
@@ -36,6 +39,9 @@
     {
       "name": "IbmCosHmacDetector"
     },
+    {
+      "name": "IPPublicDetector"
+    },
     {
       "name": "JwtTokenDetector"
     },
@@ -49,9 +55,15 @@
     {
       "name": "NpmDetector"
     },
+    {
+      "name": "OpenAIDetector"
+    },
     {
       "name": "PrivateKeyDetector"
     },
+    {
+      "name": "PypiTokenDetector"
+    },
     {
       "name": "SendGridDetector"
     },
@@ -67,6 +79,9 @@
     {
       "name": "StripeDetector"
     },
+    {
+      "name": "TelegramBotTokenDetector"
+    },
     {
       "name": "TwilioKeyDetector"
     }
@@ -108,5 +123,5 @@
     }
   ],
   "results": {},
-  "generated_at": "2023-09-24T19:52:08Z"
+  "generated_at": "2024-09-20T17:50:20Z"
 }

Dockerfile

@@ -1,4 +1,5 @@
-FROM gvegayon/cfa-epinow2-pipeline-dependencies:latest
+# This requires access to the Azure Container Registry
+FROM cfaprdbatchcr.azurecr.io/cfa-epinow2-pipeline-dependencies:latest
 
 # Will copy the package to the container preserving the directory structure
 COPY . pkg/

NEWS.md

@@ -1,5 +1,5 @@
 # CFAEpiNow2Pipeline (development version)
-
+* Now uses CFA Azure ACR and images in the workflows and Dockerfiles, etc.
 * Added Docker image with all the requirements to build the package.
 * Bump pre-commit hooks
 * Fix bug in warning message for incomplete data read (h/t @damonbayer)