ACR, Workflow UX, gh act, azure batch automation

.github/workflows/1-Build-Dependency-Image.yaml

@@ -0,0 +1,82 @@
+name: Build Dependencies Container Image
+run-name: (1) Dependencies Image Build - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
+
+# This GitHub Actions workflow builds a Docker image for the
+# cfa-epinow2-pipeline-docker project. It consists of two jobs:
+# build_image_dependencies and build_image.
+#
+# - The `build_image_dependencies` job carries out the first part of a
+#   multi-stage build. It downloads and install all the dependencies
+#   listed in the `DESCRIPTION` file. It uses the `Dockerfile-dependencies`
+#   file to build the image.
+#
+#   The built image is then pushed to the corresponding registry.
+#
+#   The process is cached to avoid rebuilding the image if the dependencies
+#   have not changed. This is by hashing the `DESCRIPTION` file and the
+#   `Dockerfile-dependencies` file.
+#
+# - The build_image job builds the final image using the `Dockerfile` file.
+#   It uses the image built in the previous job as a base image.
+#
+#   During the build process, the package is installed and built. Furthermore
+#   the package is checked using `R CMD check` to ensure that it is working
+#   correctly.
+#
+#   Once the image is built, it is pushed to the corresponding registry.
+
+on:
+  push:
+    branches:
+      - main
+      - jk-azure-readiness # temporary
+    paths:
+      - "./Dockerfile-dependencies" # the dockerfile this workflow builds from
+      - "./.github/workflows/build-dependency-image.yaml" # this workflow
+
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "./Dockerfile-dependencies" # the dockerfile this workflow builds from
+      - "./.github/workflows/build-dependency-image.yaml" # this workflow
+
+  workflow_dispatch:
+
+env:
+  # Together, these form: cfaprdbatchcr.azurecr.io/cfa-epinow2-pipeline
+  REGISTRY: cfaprdbatchcr.azurecr.io
+  IMAGE_NAME: cfa-epinow2-pipeline
+
+jobs:
+  Job01-build_image_dependencies:
+    runs-on: cfa-cdcgov # VM based runner serving CFA's cdcgov repos (as opposed to cdcent)
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      # - name: Check cache
+      #   uses: actions/cache@v3
+      #   id: cache
+      #   with:
+      #     key: docker-dependencies-${{ runner.os }}-${{ hashFiles('./DESCRIPTION', './Dockerfile-dependencies') }}
+      #     path:
+      #       ./DESCRIPTION
+
+      - name: Login to the Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: "cfaprdbatchcr.azurecr.io"
+          username: "cfaprdbatchcr"
+          password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
+
+      # if: steps.cache.outputs.cache-hit != 'true'
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          push: true
+          tags: |
+            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:latest
+            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:${{ github.sha }}
+          file: ./Dockerfile-dependencies

.github/workflows/2-Run-Epinow2-Pipeline.yaml

@@ -0,0 +1,140 @@
+name: Run Epinow2 Pipeline
+run-name: (2) Pipeline - "${{ github.event.head_commit.message }}" - ${{ github.ref_name }}
+
+# This GitHub Actions workflow builds a Docker image for the
+# cfa-epinow2-pipeline-docker project. It consists of two jobs:
+# build_image_dependencies and build_image.
+#
+# - The `build_image_dependencies` job carries out the first part of a
+#   multi-stage build. It downloads and install all the dependencies
+#   listed in the `DESCRIPTION` file. It uses the `Dockerfile-dependencies`
+#   file to build the image.
+#
+#   The built image is then pushed to the corresponding registry.
+#
+#   The process is cached to avoid rebuilding the image if the dependencies
+#   have not changed. This is by hashing the `DESCRIPTION` file and the
+#   `Dockerfile-dependencies` file.
+#
+# - The build_image job builds the final image using the `Dockerfile` file.
+#   It uses the image built in the previous job as a base image.
+#
+#   During the build process, the package is installed and built. Furthermore
+#   the package is checked using `R CMD check` to ensure that it is working
+#   correctly.
+#
+#   Once the image is built, it is pushed to the corresponding registry.
+
+
+on:
+  push:
+    branches:
+      - main
+      - jk-azure-readiness # temporary
+  workflow_dispatch:
+
+env:
+  # Together, these form: cfaprdbatchcr.azurecr.io/cfa-epinow2-pipeline
+  REGISTRY: cfaprdbatchcr.azurecr.io
+  IMAGE_NAME: cfa-epinow2-pipeline
+
+jobs:
+
+  _01_build-model-image:
+    runs-on: cfa-cdcgov #
+    steps:
+
+      - name: Login to the Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: "cfaprdbatchcr.azurecr.io"
+          username: "cfaprdbatchcr"
+          password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
+
+      # Comment out if you want to just test the pool creation without waiting for build and push
+      - name: Build and push model pipeline image for Azure batch # second stage, the actual payload
+        id: build_and_push_model_image
+        uses: docker/build-push-action@v6
+        with:
+          push: true
+          tags: |
+            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:latest
+            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:${{ github.sha }}
+          file: ./Dockerfile # second stage Dockerfile
+
+  _02_create-batch-pool-and-submit-jobs:
+    runs-on: cfa-cdcgov
+    needs:
+      - _01_build-model-image
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout Repo
+        id: checkout_repo
+        uses: actions/checkout@v4
+
+      - name: Login to Azure with NNH Service Principal
+        id: azure_login_2
+        uses: azure/login@v2
+        with:
+        # managed by EDAV. Contact Amit Mantri or Jon Kislin if you have issues.
+          creds: ${{ secrets.EDAV_CFA_PREDICT_NNHT_SP }}
+
+
+      - name: Get Github Short sha
+        run: |
+          shortSHA=$(git rev-parse --short ${{ github.sha }})
+          echo "SHORT_SHA=$shortSHA" >> $GITHUB_ENV
+
+      - name: Create cfa-epinow2-pipeline Pool
+        id: create_batch_pool
+
+        # Every Azure Batch Pool parameter can simply go here,
+        # no python module or config toml necessary
+        env:
+          POOL_ID:           "cfa-epinow2-${{ env.SHORT_SHA }}" # version the pools with the github sha
+          BATCH_ACCOUNT:     "cfaprdba"
+          BATCH_ENDPOINT:    "https://cfaprdba.eastus.batch.azure.com/"
+          VM_IMAGE_TAG:      "canonical:0001-com-ubuntu-server-focal:20_04-lts"
+          NODE_AGENT_SKU_ID: "batch.node.ubuntu 20.04"
+          VM_SIZE:           "standard_a4m_v2"
+          RESOURCE_GROUP:    ${{ secrets.PRD_RESOURCE_GROUP }}
+
+
+        # The call to the az cli that actually generates the pool
+        run: |
+          az batch account login \
+          --resource-group ${{ secrets.PRD_RESOURCE_GROUP }} \
+          --name "${{ env.BATCH_ACCOUNT }}" \
+
+          az batch pool create \
+          --account-endpoint "${{ env.BATCH_ENDPOINT }}" \
+          --id "${{ env.POOL_ID }}" \
+          --image "${{ env.VM_IMAGE_TAG }}" \
+          --node-agent-sku-id "${{ env.NODE_AGENT_SKU_ID }}" \
+          --vm-size "${{ env.VM_SIZE }}" \
+
+          az batch pool autoscale enable \
+          --pool-id ${{ env.POOL_ID }} \
+          --auto-scale-formula "$(cat './batch-autoscale-formula.txt')"
+
+          # Let's get this POOL_ID var for the next step too
+          echo "POOL_ID=${{ env.POOL_ID}}" >> $GITHUB_ENV
+
+      # - name: legacy_test
+      #   run: |
+      #     az batch pool create --json-file ./cfa-epinow2-batch-pool-config.json
+
+
+      # - name: Submit azure batch job
+      #   id: submit_batch_job
+      #   env:
+      #     POOL_ID: "$POOL_ID"
+      #     BATCH_JOB_ID: "cfa-epinow2-pipeline-job-${{ env.SHORT_SHA }}"
+      #   run: |
+      #     az batch job create \
+      #     --pool-id "${{ env.POOL_ID }}" \
+      #     --id "${{ env.BATCH_JOB_ID }}"

.github/workflows/check-news-md.yaml → .github/workflows/3-check-news-md.yaml

@@ -1,4 +1,8 @@
+# All PRs into main MUST be deliberately labelled in the NEWS.md with a succint but informative entry
+# describing the changes made - this workflow checks to make sure that this has been done.
+
 name: Check NEWS.md Update
+run-name: (3) Check News.md for Compliance - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
 
 on:
   pull_request:

.github/workflows/R-CMD-check.yaml → .github/workflows/4-R-CMD-check.yaml

@@ -1,14 +1,16 @@
 # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
 # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
+
+name: R-CMD-check
+run-name: (4) R-CMD-check - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
+
 on:
   pull_request:
     branches: [main]
   push:
     branches:
       - main
 
-name: R-CMD-check
-
 jobs:
   R-CMD-check:
     runs-on: ${{ matrix.config.os }}

.github/workflows/block-fixup.yaml → .github/workflows/5-block-fixup.yaml

@@ -1,4 +1,6 @@
-name: Git Checks
+name: Block Fix-up (Git Check)
+run-name: (5) Block Fix-up / Git Check - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
+
 
 on: [pull_request]
 

.github/workflows/pkgdown.yaml → .github/workflows/6-pkgdown.yaml

@@ -10,6 +10,7 @@ on:
   workflow_dispatch:
 
 name: pkgdown
+run-name: (6) pkgdown - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
 
 jobs:
   pkgdown:

.github/workflows/test-coverage.yaml → .github/workflows/7-test-coverage.yaml

@@ -7,6 +7,7 @@ on:
     branches: [main]
 
 name: test-coverage
+run-name: (7) Test Coverage - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
 
 jobs:
   test-coverage:
@@ -27,14 +28,14 @@ jobs:
           needs: coverage
 
       - name: Test coverage
+        shell: Rscript {0}
         run: |
           covr::codecov(
             quiet = FALSE,
             clean = FALSE,
             install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package"),
             token = "${{ secrets.CODECOV_TOKEN }}"
           )
-        shell: Rscript {0}
 
       - name: Show testthat output
         if: always()

.github/workflows/pr-commands.yaml → .github/workflows/8-pr-commands.yaml

@@ -5,7 +5,9 @@ on:
     types: [created]
   workflow_dispatch:
 
-name: Commands
+name: PR Commands
+run-name: (8) PR Commands - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
+
 
 jobs:
   document:

.github/workflows/gh-act/2-dry.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Requires that you have first run 'gh extension install nektos/gh-act'
+# as well as having installed the docker engine and added your user to the docker group
+
+# This checks syntax before you push to Github Actions, helping with debug hell
+# To run the entire pipeline locally, see 2-full.sh
+
+gh act -P cfa-cdcgov=... -n -W '.github/workflows/2-Run-Epinow2-Pipeline.yaml'

.github/workflows/gh-act/2-full.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Requires that you have first run 'gh extension install nektos/gh-act'
+# as well as having installed the docker engine and added your user to the docker group
+
+# This runs the github actions workflow locally
+
+gh act -P cfa-cdcgov=catthehacker/ubuntu:full-20.04 -W '.github/workflows/2-Run-Epinow2-Pipeline.yaml'