From 5892d2365995b8a06823604b3a0c62b964d72d73 Mon Sep 17 00:00:00 2001
From: Jon Kislin <136386616+jkislin@users.noreply.github.com>
Date: Wed, 18 Sep 2024 21:55:30 +0000
Subject: [PATCH 01/53] Changes for CFA Azure ACR

---
 ...docker.yaml => build-and-push-docker.yaml} | 44 +++++++++++++------
 Dockerfile                                    |  3 +-
 2 files changed, 32 insertions(+), 15 deletions(-)
 rename .github/workflows/{build-docker.yaml => build-and-push-docker.yaml} (62%)

diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-and-push-docker.yaml
similarity index 62%
rename from .github/workflows/build-docker.yaml
rename to .github/workflows/build-and-push-docker.yaml
index 2cd6653..5ccfc6a 100644
--- a/.github/workflows/build-docker.yaml
+++ b/.github/workflows/build-and-push-docker.yaml
@@ -22,7 +22,7 @@
 #
 #   Once the image is built, it is pushed to the corresponding registry.
 
-name: Builds Docker image
+name: Builds and Pushes Docker image
 
 on:
   push:
@@ -31,12 +31,13 @@ on:
   workflow_dispatch:
 
 env:
-  IMAGE_NAME: gvegayon/cfa-epinow2-pipeline
-  REGISTRY: docker.io
-
+  # Together, these form: cfaprdbatchcr.azurecr.io/cfa-epinow2-pipeline
+  REGISTRY: cfaprdbatchcr.azurecr.io
+  IMAGE_NAME: cfa-epinow2-pipeline
+  
 jobs:
   build_image_dependencies:
-    runs-on: ubuntu-latest
+    runs-on: cfa-linux-ext
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -49,25 +50,40 @@ jobs:
           path:
             ./DESCRIPTION
 
+      - name: Login to the Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: "cfaprdbatchcr.azurecr.io"
+          username: "cfaprdbatchcr"
+          password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
+
       - name: Build and push
         uses: docker/build-push-action@v6
         if: steps.cache.outputs.cache-hit != 'true'
         with:
-          push: false
-          tags: ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:latest
+          push: true
+          tags: |
+            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:latest
+            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:${{ github.sha }}
           file: ./Dockerfile-dependencies
 
-    outputs:
-      IMAGE_NAME: ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:latest
-
   build_image:
     needs: build_image_dependencies
-    runs-on: ubuntu-latest
-
+    runs-on: cfa-linux-ext
     steps:
+      
+      - name: Login to the Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: "cfaprdbatchcr.azurecr.io"
+          username: "cfaprdbatchcr"
+          password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
+          
       - name: Build and push
         uses: docker/build-push-action@v6
         with:
-          push: false
-          tags: ${{ needs.build_image_dependencies.outputs.IMAGE_NAME }}
+          push: true
+          tags: |
+            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:latest
+            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:${{ github.sha }}
           file: ./Dockerfile
diff --git a/Dockerfile b/Dockerfile
index 22adfd2..ac300ed 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,5 @@
-FROM gvegayon/cfa-epinow2-pipeline-dependencies:latest
+# This requires access to the Azure Container Registry
+FROM cfaprdbatchcr.azurecr.io/cfa-epinow2-pipeline-dependencies:latest
 
 # Will copy the package to the container preserving the directory structure
 COPY . pkg/

From 96c64c96923f667d0eed8bde68987dd8b0171f4e Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 18 Sep 2024 21:59:36 +0000
Subject: [PATCH 02/53] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .github/workflows/build-and-push-docker.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build-and-push-docker.yaml b/.github/workflows/build-and-push-docker.yaml
index 5ccfc6a..fb66e42 100644
--- a/.github/workflows/build-and-push-docker.yaml
+++ b/.github/workflows/build-and-push-docker.yaml
@@ -34,7 +34,7 @@ env:
   # Together, these form: cfaprdbatchcr.azurecr.io/cfa-epinow2-pipeline
   REGISTRY: cfaprdbatchcr.azurecr.io
   IMAGE_NAME: cfa-epinow2-pipeline
-  
+
 jobs:
   build_image_dependencies:
     runs-on: cfa-linux-ext
@@ -71,14 +71,14 @@ jobs:
     needs: build_image_dependencies
     runs-on: cfa-linux-ext
     steps:
-      
+
       - name: Login to the Container Registry
         uses: docker/login-action@v3
         with:
           registry: "cfaprdbatchcr.azurecr.io"
           username: "cfaprdbatchcr"
           password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
-          
+
       - name: Build and push
         uses: docker/build-push-action@v6
         with:

From db86b6c0294465adedf4816189f60ff7748d358b Mon Sep 17 00:00:00 2001
From: Jon Kislin <136386616+jkislin@users.noreply.github.com>
Date: Wed, 18 Sep 2024 22:02:57 +0000
Subject: [PATCH 03/53] added to news.md

---
 NEWS.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/NEWS.md b/NEWS.md
index 285022d..250dd9b 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,5 +1,5 @@
 # CFAEpiNow2Pipeline (development version)
-
+* Now uses CFA Azure ACR and images in the workflows and Dockerfiles, etc.
 * Added Docker image with all the requirements to build the package.
 * Bump pre-commit hooks
 * Fix bug in warning message for incomplete data read (h/t @damonbayer)

From 444b42389564773fc0308b9e942297a6c0c2af39 Mon Sep 17 00:00:00 2001
From: Jon Kislin <136386616+jkislin@users.noreply.github.com>
Date: Fri, 20 Sep 2024 01:29:43 +0000
Subject: [PATCH 04/53] testing image on jk branch

---
 .github/workflows/build-and-push-docker.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build-and-push-docker.yaml b/.github/workflows/build-and-push-docker.yaml
index fb66e42..ed61dab 100644
--- a/.github/workflows/build-and-push-docker.yaml
+++ b/.github/workflows/build-and-push-docker.yaml
@@ -28,6 +28,7 @@ on:
   push:
     branches:
       - main
+      - jk-azure-readiness
   workflow_dispatch:
 
 env:

From 5cfe5d3f3e6610b38c41ff6e635c206ad30ff4f2 Mon Sep 17 00:00:00 2001
From: Jon Kislin <136386616+jkislin@users.noreply.github.com>
Date: Fri, 20 Sep 2024 01:32:25 +0000
Subject: [PATCH 05/53] update workflow name; resubmit job

---
 .github/workflows/build-and-push-docker.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-push-docker.yaml b/.github/workflows/build-and-push-docker.yaml
index ed61dab..ed9fe1a 100644
--- a/.github/workflows/build-and-push-docker.yaml
+++ b/.github/workflows/build-and-push-docker.yaml
@@ -22,7 +22,7 @@
 #
 #   Once the image is built, it is pushed to the corresponding registry.
 
-name: Builds and Pushes Docker image
+name: Build and Push Docker image
 
 on:
   push:

From 4023f651acc5fa7d9a657754a009cadd052dceb9 Mon Sep 17 00:00:00 2001
From: Jon Kislin <136386616+jkislin@users.noreply.github.com>
Date: Fri, 20 Sep 2024 01:34:58 +0000
Subject: [PATCH 06/53] change runs-on to new cdcgov runner

---
 .github/workflows/build-and-push-docker.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-push-docker.yaml b/.github/workflows/build-and-push-docker.yaml
index ed9fe1a..2a515e4 100644
--- a/.github/workflows/build-and-push-docker.yaml
+++ b/.github/workflows/build-and-push-docker.yaml
@@ -38,7 +38,7 @@ env:
 
 jobs:
   build_image_dependencies:
-    runs-on: cfa-linux-ext
+    runs-on: cfa-cdcgov
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -70,7 +70,7 @@ jobs:
 
   build_image:
     needs: build_image_dependencies
-    runs-on: cfa-linux-ext
+    runs-on: cfa-cdcgov
     steps:
 
       - name: Login to the Container Registry

From 42845112ba690447d666d47a4fb6ffec6f7cbbf1 Mon Sep 17 00:00:00 2001
From: Jon Kislin <136386616+jkislin@users.noreply.github.com>
Date: Fri, 20 Sep 2024 01:39:58 +0000
Subject: [PATCH 07/53] removed unworking cache check

---
 .github/workflows/build-and-push-docker.yaml | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/build-and-push-docker.yaml b/.github/workflows/build-and-push-docker.yaml
index 2a515e4..df03a87 100644
--- a/.github/workflows/build-and-push-docker.yaml
+++ b/.github/workflows/build-and-push-docker.yaml
@@ -43,13 +43,13 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Check cache
-        uses: actions/cache@v3
-        id: cache
-        with:
-          key: docker-dependencies-${{ runner.os }}-${{ hashFiles('./DESCRIPTION', './Dockerfile-dependencies') }}
-          path:
-            ./DESCRIPTION
+      # - name: Check cache
+      #   uses: actions/cache@v3
+      #   id: cache
+      #   with:
+      #     key: docker-dependencies-${{ runner.os }}-${{ hashFiles('./DESCRIPTION', './Dockerfile-dependencies') }}
+      #     path:
+      #       ./DESCRIPTION
 
       - name: Login to the Container Registry
         uses: docker/login-action@v3
@@ -58,9 +58,10 @@ jobs:
           username: "cfaprdbatchcr"
           password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
 
+      # if: steps.cache.outputs.cache-hit != 'true'
+      
       - name: Build and push
         uses: docker/build-push-action@v6
-        if: steps.cache.outputs.cache-hit != 'true'
         with:
           push: true
           tags: |

From 5c4ec1071717339950b9afc5c615c4b32e85f9fe Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 20 Sep 2024 01:40:28 +0000
Subject: [PATCH 08/53] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .github/workflows/build-and-push-docker.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-push-docker.yaml b/.github/workflows/build-and-push-docker.yaml
index df03a87..0dcdcdd 100644
--- a/.github/workflows/build-and-push-docker.yaml
+++ b/.github/workflows/build-and-push-docker.yaml
@@ -59,7 +59,7 @@ jobs:
           password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
 
       # if: steps.cache.outputs.cache-hit != 'true'
-      
+
       - name: Build and push
         uses: docker/build-push-action@v6
         with:

From 9777717fdd326350ddb7070f386085b6ddb8d391 Mon Sep 17 00:00:00 2001
From: Jon Kislin <136386616+jkislin@users.noreply.github.com>
Date: Sat, 21 Sep 2024 02:30:29 +0000
Subject: [PATCH 09/53] azure batch scaffolding (maybe not necessary here)

---
 .github/workflows/build-and-push-docker.yaml | 63 ++++++++++++++++++++
 .secrets.baseline                            | 19 +++++-
 2 files changed, 80 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-push-docker.yaml b/.github/workflows/build-and-push-docker.yaml
index df03a87..ab5722f 100644
--- a/.github/workflows/build-and-push-docker.yaml
+++ b/.github/workflows/build-and-push-docker.yaml
@@ -89,3 +89,66 @@ jobs:
             ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:latest
             ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:${{ github.sha }}
           file: ./Dockerfile
+
+  create-and-launch-batch-pools:
+    runs-on: cfa-cdcgov
+    needs: 
+      - build_image
+        
+    permissions:
+      contents: read
+      packages: write
+        
+    steps:   
+      - name: Checkout Repo
+        id: checkout_repo
+        uses: actions/checkout@v4
+       
+      - name: Login to Azure with NNH Service Principal
+        id: azure_login_2
+        uses: azure/login@v2
+        with:
+        # managed by EDAV. Contact Amit Mantri or Jon Kislin if you have issues.
+          creds: ${{ secrets.EDAV_CFA_PREDICT_NNHT_SP }} 
+
+      - name: Download Configuration Tomls from cfadatalakeprd/cfapredict
+        id: download_config_tomls
+        run: |
+          rm -rf ./ci_config_tomls/
+          az storage blob download-batch \
+            --source https://cfadatalakeprd.blob.core.windows.net/cfapredict/ \
+            --pattern "./ci_config_tomls/*" \
+            --destination ./ \
+            --auth-mode "login"
+
+      - name: Delete Existing Batch Pool
+        id: delete_existing_batch_pool
+        run: |
+
+          export pool_id="cfa-epinow2-pipeline-ci"
+
+          # Check if the batch pool exists
+          EPINOW2_POOL_EXISTS=$(az batch pool show \
+            --account-name "cfaprdba" \
+            --account-endpoint "https://cfaprdba.eastus.batch.azure.com" \
+            --pool-id "$pool_id" \
+            --query "id" --output tsv 2>/dev/null)
+
+          if [ -z "$EPINOW2_POOL_EXISTS" ]; then
+              echo -e "Batch pool $pool_id does not exist."
+          else
+              echo -e "Batch pool $pool_id exists. Deleting it now..."  
+              az batch pool delete \
+                --account-name cfaprdba \
+                --account-endpoint https://cfaprdba.eastus.batch.azure.com \
+                --pool-id "$pool_id" \
+                --yes
+              echo "Batch pool $pool_id has been deleted."
+          fi
+
+      - name: Create cfa-epinow2-pipeline Pools
+        id: create_nhsn_nssp_pools
+        run: |
+          echo 'hello world'
+
+     
diff --git a/.secrets.baseline b/.secrets.baseline
index 63f42f9..4fc48f2 100644
--- a/.secrets.baseline
+++ b/.secrets.baseline
@@ -1,5 +1,5 @@
 {
-  "version": "1.4.0",
+  "version": "1.5.0",
   "plugins_used": [
     {
       "name": "ArtifactoryDetector"
@@ -26,6 +26,9 @@
     {
       "name": "GitHubTokenDetector"
     },
+    {
+      "name": "GitLabTokenDetector"
+    },
     {
       "name": "HexHighEntropyString",
       "limit": 3.0
@@ -36,6 +39,9 @@
     {
       "name": "IbmCosHmacDetector"
     },
+    {
+      "name": "IPPublicDetector"
+    },
     {
       "name": "JwtTokenDetector"
     },
@@ -49,9 +55,15 @@
     {
       "name": "NpmDetector"
     },
+    {
+      "name": "OpenAIDetector"
+    },
     {
       "name": "PrivateKeyDetector"
     },
+    {
+      "name": "PypiTokenDetector"
+    },
     {
       "name": "SendGridDetector"
     },
@@ -67,6 +79,9 @@
     {
       "name": "StripeDetector"
     },
+    {
+      "name": "TelegramBotTokenDetector"
+    },
     {
       "name": "TwilioKeyDetector"
     }
@@ -108,5 +123,5 @@
     }
   ],
   "results": {},
-  "generated_at": "2023-09-24T19:52:08Z"
+  "generated_at": "2024-09-20T17:50:20Z"
 }

From b2e279f9c2bce305270f0137dfec7c01cffeb9d7 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sat, 21 Sep 2024 02:36:10 +0000
Subject: [PATCH 10/53] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .github/workflows/build-and-push-docker.yaml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/build-and-push-docker.yaml b/.github/workflows/build-and-push-docker.yaml
index 850209d..6c8f073 100644
--- a/.github/workflows/build-and-push-docker.yaml
+++ b/.github/workflows/build-and-push-docker.yaml
@@ -92,24 +92,24 @@ jobs:
 
   create-and-launch-batch-pools:
     runs-on: cfa-cdcgov
-    needs: 
+    needs:
       - build_image
-        
+
     permissions:
       contents: read
       packages: write
-        
-    steps:   
+
+    steps:
       - name: Checkout Repo
         id: checkout_repo
         uses: actions/checkout@v4
-       
+
       - name: Login to Azure with NNH Service Principal
         id: azure_login_2
         uses: azure/login@v2
         with:
         # managed by EDAV. Contact Amit Mantri or Jon Kislin if you have issues.
-          creds: ${{ secrets.EDAV_CFA_PREDICT_NNHT_SP }} 
+          creds: ${{ secrets.EDAV_CFA_PREDICT_NNHT_SP }}
 
       - name: Download Configuration Tomls from cfadatalakeprd/cfapredict
         id: download_config_tomls
@@ -137,7 +137,7 @@ jobs:
           if [ -z "$EPINOW2_POOL_EXISTS" ]; then
               echo -e "Batch pool $pool_id does not exist."
           else
-              echo -e "Batch pool $pool_id exists. Deleting it now..."  
+              echo -e "Batch pool $pool_id exists. Deleting it now..."
               az batch pool delete \
                 --account-name cfaprdba \
                 --account-endpoint https://cfaprdba.eastus.batch.azure.com \
@@ -151,4 +151,4 @@ jobs:
         run: |
           echo 'hello world'
 
-     
+

From e955773b3e46f20c498327a910d21b29761fefb7 Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Wed, 25 Sep 2024 16:33:51 +0000
Subject: [PATCH 11/53] another attempt at cacheing and simplification;
 splitting the buiild/dependencies workflows

---
 .github/workflows/build-dependency-image.yaml | 81 ++++++++++++++++++
 ...-docker.yaml => run-epinow2-pipeline.yaml} | 85 ++++++++-----------
 2 files changed, 115 insertions(+), 51 deletions(-)
 create mode 100644 .github/workflows/build-dependency-image.yaml
 rename .github/workflows/{build-and-push-docker.yaml => run-epinow2-pipeline.yaml} (63%)

diff --git a/.github/workflows/build-dependency-image.yaml b/.github/workflows/build-dependency-image.yaml
new file mode 100644
index 0000000..74bca33
--- /dev/null
+++ b/.github/workflows/build-dependency-image.yaml
@@ -0,0 +1,81 @@
+# This GitHub Actions workflow builds a Docker image for the
+# cfa-epinow2-pipeline-docker project. It consists of two jobs:
+# build_image_dependencies and build_image.
+#
+# - The `build_image_dependencies` job carries out the first part of a
+#   multi-stage build. It downloads and install all the dependencies
+#   listed in the `DESCRIPTION` file. It uses the `Dockerfile-dependencies`
+#   file to build the image.
+#
+#   The built image is then pushed to the corresponding registry.
+#
+#   The process is cached to avoid rebuilding the image if the dependencies
+#   have not changed. This is by hashing the `DESCRIPTION` file and the
+#   `Dockerfile-dependencies` file.
+#
+# - The build_image job builds the final image using the `Dockerfile` file.
+#   It uses the image built in the previous job as a base image.
+#
+#   During the build process, the package is installed and built. Furthermore
+#   the package is checked using `R CMD check` to ensure that it is working
+#   correctly.
+#
+#   Once the image is built, it is pushed to the corresponding registry.
+
+name: Build Dependencies Container Image
+
+on:
+  push:
+    branches:
+      - main
+      - jk-azure-readiness # temporary
+    paths:
+      - "./Dockerfile-dependencies" # the dockerfile this workflow builds from
+      - "./.github/workflows/build-dependency-image.yaml" # this workflow
+
+  pull_request:
+    branches:
+      - main    
+    paths:
+      - "./Dockerfile-dependencies" # the dockerfile this workflow builds from
+      - "./.github/workflows/build-dependency-image.yaml" # this workflow
+      
+  workflow_dispatch:
+
+env:
+  # Together, these form: cfaprdbatchcr.azurecr.io/cfa-epinow2-pipeline
+  REGISTRY: cfaprdbatchcr.azurecr.io
+  IMAGE_NAME: cfa-epinow2-pipeline
+
+jobs:
+  Job01-build_image_dependencies:
+    runs-on: cfa-cdcgov # VM based runner serving CFA's cdcgov repos (as opposed to cdcent)
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Check cache
+        uses: actions/cache@v3
+        id: cache
+        with:
+          key: docker-dependencies-${{ runner.os }}-${{ hashFiles('./DESCRIPTION', './Dockerfile-dependencies') }}
+          path:
+            ./DESCRIPTION
+
+      - name: Login to the Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: "cfaprdbatchcr.azurecr.io"
+          username: "cfaprdbatchcr"
+          password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
+
+      # if: steps.cache.outputs.cache-hit != 'true'
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          push: true
+          tags: |
+            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:latest
+            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:${{ github.sha }}
+          file: ./Dockerfile-dependencies
\ No newline at end of file
diff --git a/.github/workflows/build-and-push-docker.yaml b/.github/workflows/run-epinow2-pipeline.yaml
similarity index 63%
rename from .github/workflows/build-and-push-docker.yaml
rename to .github/workflows/run-epinow2-pipeline.yaml
index 6c8f073..fcdfcb1 100644
--- a/.github/workflows/build-and-push-docker.yaml
+++ b/.github/workflows/run-epinow2-pipeline.yaml
@@ -22,7 +22,7 @@
 #
 #   Once the image is built, it is pushed to the corresponding registry.
 
-name: Build and Push Docker image
+name: Run Epinow2 Pipeline
 
 on:
   push:
@@ -37,20 +37,12 @@ env:
   IMAGE_NAME: cfa-epinow2-pipeline
 
 jobs:
-  build_image_dependencies:
+  Job01_build-image-dependencies:
     runs-on: cfa-cdcgov
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      # - name: Check cache
-      #   uses: actions/cache@v3
-      #   id: cache
-      #   with:
-      #     key: docker-dependencies-${{ runner.os }}-${{ hashFiles('./DESCRIPTION', './Dockerfile-dependencies') }}
-      #     path:
-      #       ./DESCRIPTION
-
       - name: Login to the Container Registry
         uses: docker/login-action@v3
         with:
@@ -58,8 +50,6 @@ jobs:
           username: "cfaprdbatchcr"
           password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
 
-      # if: steps.cache.outputs.cache-hit != 'true'
-
       - name: Build and push
         uses: docker/build-push-action@v6
         with:
@@ -68,9 +58,10 @@ jobs:
             ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:latest
             ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:${{ github.sha }}
           file: ./Dockerfile-dependencies
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
-  build_image:
-    needs: build_image_dependencies
+  Job02_build-model-image:
     runs-on: cfa-cdcgov
     steps:
 
@@ -89,11 +80,13 @@ jobs:
             ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:latest
             ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:${{ github.sha }}
           file: ./Dockerfile
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
-  create-and-launch-batch-pools:
+  Job03_create-and-launch-batch-pools:
     runs-on: cfa-cdcgov
     needs:
-      - build_image
+      - Job02_build-model-image
 
     permissions:
       contents: read
@@ -111,44 +104,34 @@ jobs:
         # managed by EDAV. Contact Amit Mantri or Jon Kislin if you have issues.
           creds: ${{ secrets.EDAV_CFA_PREDICT_NNHT_SP }}
 
-      - name: Download Configuration Tomls from cfadatalakeprd/cfapredict
-        id: download_config_tomls
-        run: |
-          rm -rf ./ci_config_tomls/
-          az storage blob download-batch \
-            --source https://cfadatalakeprd.blob.core.windows.net/cfapredict/ \
-            --pattern "./ci_config_tomls/*" \
-            --destination ./ \
-            --auth-mode "login"
-
-      - name: Delete Existing Batch Pool
-        id: delete_existing_batch_pool
-        run: |
-
-          export pool_id="cfa-epinow2-pipeline-ci"
-
-          # Check if the batch pool exists
-          EPINOW2_POOL_EXISTS=$(az batch pool show \
-            --account-name "cfaprdba" \
-            --account-endpoint "https://cfaprdba.eastus.batch.azure.com" \
-            --pool-id "$pool_id" \
-            --query "id" --output tsv 2>/dev/null)
-
-          if [ -z "$EPINOW2_POOL_EXISTS" ]; then
-              echo -e "Batch pool $pool_id does not exist."
-          else
-              echo -e "Batch pool $pool_id exists. Deleting it now..."
-              az batch pool delete \
-                --account-name cfaprdba \
-                --account-endpoint https://cfaprdba.eastus.batch.azure.com \
-                --pool-id "$pool_id" \
-                --yes
-              echo "Batch pool $pool_id has been deleted."
-          fi
+      # - name: Delete Existing Batch Pool
+      #   id: delete_existing_batch_pool
+      #   run: |
+
+      #     export pool_id="cfa-epinow2-pipeline-ci"
+
+      #     # Check if the batch pool exists
+      #     EPINOW2_POOL_EXISTS=$(az batch pool show \
+      #       --account-name "cfaprdba" \
+      #       --account-endpoint "https://cfaprdba.eastus.batch.azure.com" \
+      #       --pool-id "$pool_id" \
+      #       --query "id" --output tsv 2>/dev/null)
+
+      #     if [ -z "$EPINOW2_POOL_EXISTS" ]; then
+      #         echo -e "Batch pool $pool_id does not exist."
+      #     else
+      #         echo -e "Batch pool $pool_id exists. Deleting it now..."
+      #         az batch pool delete \
+      #           --account-name cfaprdba \
+      #           --account-endpoint https://cfaprdba.eastus.batch.azure.com \
+      #           --pool-id "$pool_id" \
+      #           --yes
+      #         echo "Batch pool $pool_id has been deleted."
+      #     fi
 
       - name: Create cfa-epinow2-pipeline Pools
         id: create_nhsn_nssp_pools
-        run: |
+        run: 
           echo 'hello world'
 
 

From f4de2a6a18e4413c8c99f9a8f22a50e859f2d3ef Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 25 Sep 2024 16:34:28 +0000
Subject: [PATCH 12/53] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .github/workflows/build-dependency-image.yaml | 6 +++---
 .github/workflows/run-epinow2-pipeline.yaml   | 4 +---
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build-dependency-image.yaml b/.github/workflows/build-dependency-image.yaml
index 74bca33..b18fb6d 100644
--- a/.github/workflows/build-dependency-image.yaml
+++ b/.github/workflows/build-dependency-image.yaml
@@ -35,11 +35,11 @@ on:
 
   pull_request:
     branches:
-      - main    
+      - main
     paths:
       - "./Dockerfile-dependencies" # the dockerfile this workflow builds from
       - "./.github/workflows/build-dependency-image.yaml" # this workflow
-      
+
   workflow_dispatch:
 
 env:
@@ -78,4 +78,4 @@ jobs:
           tags: |
             ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:latest
             ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:${{ github.sha }}
-          file: ./Dockerfile-dependencies
\ No newline at end of file
+          file: ./Dockerfile-dependencies
diff --git a/.github/workflows/run-epinow2-pipeline.yaml b/.github/workflows/run-epinow2-pipeline.yaml
index fcdfcb1..922bae2 100644
--- a/.github/workflows/run-epinow2-pipeline.yaml
+++ b/.github/workflows/run-epinow2-pipeline.yaml
@@ -131,7 +131,5 @@ jobs:
 
       - name: Create cfa-epinow2-pipeline Pools
         id: create_nhsn_nssp_pools
-        run: 
+        run:
           echo 'hello world'
-
-

From 604069fe2c554acc793a1c75b176d7f175c687ee Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Wed, 25 Sep 2024 16:44:02 +0000
Subject: [PATCH 13/53] better github actions ux via build names

---
 .../{build-dependency-image.yaml => Build-Dependency-Image.yaml} | 1 +
 .../{run-epinow2-pipeline.yaml => Run-Epinow2-Pipeline.yaml}     | 1 +
 2 files changed, 2 insertions(+)
 rename .github/workflows/{build-dependency-image.yaml => Build-Dependency-Image.yaml} (96%)
 rename .github/workflows/{run-epinow2-pipeline.yaml => Run-Epinow2-Pipeline.yaml} (97%)

diff --git a/.github/workflows/build-dependency-image.yaml b/.github/workflows/Build-Dependency-Image.yaml
similarity index 96%
rename from .github/workflows/build-dependency-image.yaml
rename to .github/workflows/Build-Dependency-Image.yaml
index b18fb6d..8f67bcd 100644
--- a/.github/workflows/build-dependency-image.yaml
+++ b/.github/workflows/Build-Dependency-Image.yaml
@@ -23,6 +23,7 @@
 #   Once the image is built, it is pushed to the corresponding registry.
 
 name: Build Dependencies Container Image
+run-name: Dependencies Image Build - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
 
 on:
   push:
diff --git a/.github/workflows/run-epinow2-pipeline.yaml b/.github/workflows/Run-Epinow2-Pipeline.yaml
similarity index 97%
rename from .github/workflows/run-epinow2-pipeline.yaml
rename to .github/workflows/Run-Epinow2-Pipeline.yaml
index 922bae2..571e6de 100644
--- a/.github/workflows/run-epinow2-pipeline.yaml
+++ b/.github/workflows/Run-Epinow2-Pipeline.yaml
@@ -23,6 +23,7 @@
 #   Once the image is built, it is pushed to the corresponding registry.
 
 name: Run Epinow2 Pipeline
+run-name: Epinow2 Pipeline - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
 
 on:
   push:

From 8403153783f46d63a6ac507f2f98ac3d19507046 Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Wed, 25 Sep 2024 16:49:15 +0000
Subject: [PATCH 14/53] organized workflows for contributor/tester ux

---
 ...ld-Dependency-Image.yaml => 1-Build-Dependency-Image.yaml} | 0
 ...{Run-Epinow2-Pipeline.yaml => 2-Run-Epinow2-Pipeline.yaml} | 0
 .../workflows/{check-news-md.yaml => 3-check-news-md.yaml}    | 0
 .github/workflows/{R-CMD-check.yaml => 4-R-CMD-check.yaml}    | 0
 .github/workflows/{block-fixup.yaml => 5-block-fixup.yaml}    | 0
 .github/workflows/{pkgdown.yaml => 6-pkgdown.yaml}            | 0
 .../workflows/{test-coverage.yaml => 7-test-coverage.yaml}    | 4 ++--
 .github/workflows/{pr-commands.yaml => 8-pr-commands.yaml}    | 0
 8 files changed, 2 insertions(+), 2 deletions(-)
 rename .github/workflows/{Build-Dependency-Image.yaml => 1-Build-Dependency-Image.yaml} (100%)
 rename .github/workflows/{Run-Epinow2-Pipeline.yaml => 2-Run-Epinow2-Pipeline.yaml} (100%)
 rename .github/workflows/{check-news-md.yaml => 3-check-news-md.yaml} (100%)
 rename .github/workflows/{R-CMD-check.yaml => 4-R-CMD-check.yaml} (100%)
 rename .github/workflows/{block-fixup.yaml => 5-block-fixup.yaml} (100%)
 rename .github/workflows/{pkgdown.yaml => 6-pkgdown.yaml} (100%)
 rename .github/workflows/{test-coverage.yaml => 7-test-coverage.yaml} (99%)
 rename .github/workflows/{pr-commands.yaml => 8-pr-commands.yaml} (100%)

diff --git a/.github/workflows/Build-Dependency-Image.yaml b/.github/workflows/1-Build-Dependency-Image.yaml
similarity index 100%
rename from .github/workflows/Build-Dependency-Image.yaml
rename to .github/workflows/1-Build-Dependency-Image.yaml
diff --git a/.github/workflows/Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
similarity index 100%
rename from .github/workflows/Run-Epinow2-Pipeline.yaml
rename to .github/workflows/2-Run-Epinow2-Pipeline.yaml
diff --git a/.github/workflows/check-news-md.yaml b/.github/workflows/3-check-news-md.yaml
similarity index 100%
rename from .github/workflows/check-news-md.yaml
rename to .github/workflows/3-check-news-md.yaml
diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/4-R-CMD-check.yaml
similarity index 100%
rename from .github/workflows/R-CMD-check.yaml
rename to .github/workflows/4-R-CMD-check.yaml
diff --git a/.github/workflows/block-fixup.yaml b/.github/workflows/5-block-fixup.yaml
similarity index 100%
rename from .github/workflows/block-fixup.yaml
rename to .github/workflows/5-block-fixup.yaml
diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/6-pkgdown.yaml
similarity index 100%
rename from .github/workflows/pkgdown.yaml
rename to .github/workflows/6-pkgdown.yaml
diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/7-test-coverage.yaml
similarity index 99%
rename from .github/workflows/test-coverage.yaml
rename to .github/workflows/7-test-coverage.yaml
index e8a8471..75af0a5 100644
--- a/.github/workflows/test-coverage.yaml
+++ b/.github/workflows/7-test-coverage.yaml
@@ -27,6 +27,7 @@ jobs:
           needs: coverage
 
       - name: Test coverage
+        shell: Rscript {0}
         run: |
           covr::codecov(
             quiet = FALSE,
@@ -34,8 +35,7 @@ jobs:
             install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package"),
             token = "${{ secrets.CODECOV_TOKEN }}"
           )
-        shell: Rscript {0}
-
+        
       - name: Show testthat output
         if: always()
         run: |
diff --git a/.github/workflows/pr-commands.yaml b/.github/workflows/8-pr-commands.yaml
similarity index 100%
rename from .github/workflows/pr-commands.yaml
rename to .github/workflows/8-pr-commands.yaml

From 59163187cb978556d364e577f228416cdc081a29 Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Wed, 25 Sep 2024 16:53:36 +0000
Subject: [PATCH 15/53] removed cacheing and made names easier to read

---
 .../workflows/1-Build-Dependency-Image.yaml   | 14 ++++++-------
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 20 +++++++++++--------
 2 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/1-Build-Dependency-Image.yaml b/.github/workflows/1-Build-Dependency-Image.yaml
index 8f67bcd..89c65b2 100644
--- a/.github/workflows/1-Build-Dependency-Image.yaml
+++ b/.github/workflows/1-Build-Dependency-Image.yaml
@@ -55,13 +55,13 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Check cache
-        uses: actions/cache@v3
-        id: cache
-        with:
-          key: docker-dependencies-${{ runner.os }}-${{ hashFiles('./DESCRIPTION', './Dockerfile-dependencies') }}
-          path:
-            ./DESCRIPTION
+      # - name: Check cache
+      #   uses: actions/cache@v3
+      #   id: cache
+      #   with:
+      #     key: docker-dependencies-${{ runner.os }}-${{ hashFiles('./DESCRIPTION', './Dockerfile-dependencies') }}
+      #     path:
+      #       ./DESCRIPTION
 
       - name: Login to the Container Registry
         uses: docker/login-action@v3
diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 571e6de..6248be4 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -38,6 +38,11 @@ env:
   IMAGE_NAME: cfa-epinow2-pipeline
 
 jobs:
+
+  # We now use a two-stage docker build (essentially), split across two dockerfiles.
+  # TODO: Only run this if necessary, will need conditional job syntax 
+  # SEE also; ./.github/workflows/1-Build-Dependency-Image.yml
+
   Job01_build-image-dependencies:
     runs-on: cfa-cdcgov
     steps:
@@ -51,16 +56,16 @@ jobs:
           username: "cfaprdbatchcr"
           password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
 
-      - name: Build and push
+      - name: Build and push dependencies image # first stage
+        id: build_and_push_dependencies_image
         uses: docker/build-push-action@v6
         with:
           push: true
           tags: |
             ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:latest
             ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:${{ github.sha }}
-          file: ./Dockerfile-dependencies
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          file: ./Dockerfile-dependencies # first stage Dockerfile
+
 
   Job02_build-model-image:
     runs-on: cfa-cdcgov
@@ -73,16 +78,15 @@ jobs:
           username: "cfaprdbatchcr"
           password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
 
-      - name: Build and push
+      - name: Build and push model pipeline image for Azure batch # second stage, the actual payload
+        id: build_and_push_model_image
         uses: docker/build-push-action@v6
         with:
           push: true
           tags: |
             ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:latest
             ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:${{ github.sha }}
-          file: ./Dockerfile
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          file: ./Dockerfile # second stage Dockerfile
 
   Job03_create-and-launch-batch-pools:
     runs-on: cfa-cdcgov

From e5071a733c6df1a66ead32fedd732661e910a805 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 25 Sep 2024 16:55:16 +0000
Subject: [PATCH 16/53] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 2 +-
 .github/workflows/7-test-coverage.yaml        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 6248be4..54f421b 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -40,7 +40,7 @@ env:
 jobs:
 
   # We now use a two-stage docker build (essentially), split across two dockerfiles.
-  # TODO: Only run this if necessary, will need conditional job syntax 
+  # TODO: Only run this if necessary, will need conditional job syntax
   # SEE also; ./.github/workflows/1-Build-Dependency-Image.yml
 
   Job01_build-image-dependencies:
diff --git a/.github/workflows/7-test-coverage.yaml b/.github/workflows/7-test-coverage.yaml
index 75af0a5..8c25776 100644
--- a/.github/workflows/7-test-coverage.yaml
+++ b/.github/workflows/7-test-coverage.yaml
@@ -35,7 +35,7 @@ jobs:
             install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package"),
             token = "${{ secrets.CODECOV_TOKEN }}"
           )
-        
+
       - name: Show testthat output
         if: always()
         run: |

From bfc8eedb299d39ffc8927046f3157ed16414257f Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Wed, 25 Sep 2024 16:59:00 +0000
Subject: [PATCH 17/53] comprehensive workflow renaming for ux/trackability

---
 .github/workflows/1-Build-Dependency-Image.yaml | 6 +++---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml   | 7 ++++---
 .github/workflows/3-check-news-md.yaml          | 4 ++++
 .github/workflows/4-R-CMD-check.yaml            | 6 ++++--
 .github/workflows/5-block-fixup.yaml            | 4 +++-
 .github/workflows/6-pkgdown.yaml                | 1 +
 .github/workflows/7-test-coverage.yaml          | 1 +
 .github/workflows/8-pr-commands.yaml            | 4 +++-
 8 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/1-Build-Dependency-Image.yaml b/.github/workflows/1-Build-Dependency-Image.yaml
index 89c65b2..beb723e 100644
--- a/.github/workflows/1-Build-Dependency-Image.yaml
+++ b/.github/workflows/1-Build-Dependency-Image.yaml
@@ -1,3 +1,6 @@
+name: Build Dependencies Container Image
+run-name: (1) Dependencies Image Build - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
+
 # This GitHub Actions workflow builds a Docker image for the
 # cfa-epinow2-pipeline-docker project. It consists of two jobs:
 # build_image_dependencies and build_image.
@@ -22,9 +25,6 @@
 #
 #   Once the image is built, it is pushed to the corresponding registry.
 
-name: Build Dependencies Container Image
-run-name: Dependencies Image Build - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
-
 on:
   push:
     branches:
diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 6248be4..981a4ca 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -1,3 +1,6 @@
+name: Run Epinow2 Pipeline
+run-name: (2) Epinow2 Pipeline - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
+
 # This GitHub Actions workflow builds a Docker image for the
 # cfa-epinow2-pipeline-docker project. It consists of two jobs:
 # build_image_dependencies and build_image.
@@ -22,14 +25,12 @@
 #
 #   Once the image is built, it is pushed to the corresponding registry.
 
-name: Run Epinow2 Pipeline
-run-name: Epinow2 Pipeline - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
 
 on:
   push:
     branches:
       - main
-      - jk-azure-readiness
+      - jk-azure-readiness # temporary
   workflow_dispatch:
 
 env:
diff --git a/.github/workflows/3-check-news-md.yaml b/.github/workflows/3-check-news-md.yaml
index 07ae7de..83028a5 100644
--- a/.github/workflows/3-check-news-md.yaml
+++ b/.github/workflows/3-check-news-md.yaml
@@ -1,4 +1,8 @@
+# All PRs into main MUST be deliberately labelled in the NEWS.md with a succint but informative entry
+# describing the changes made - this workflow checks to make sure that this has been done.
+
 name: Check NEWS.md Update
+run-name: (3) Check News.md for Compliance - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
 
 on:
   pull_request:
diff --git a/.github/workflows/4-R-CMD-check.yaml b/.github/workflows/4-R-CMD-check.yaml
index 0489bb7..0aac7d1 100644
--- a/.github/workflows/4-R-CMD-check.yaml
+++ b/.github/workflows/4-R-CMD-check.yaml
@@ -1,5 +1,9 @@
 # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
 # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
+
+name: R-CMD-check
+run-name: (4) R-CMD-check - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
+
 on:
   pull_request:
     branches: [main]
@@ -7,8 +11,6 @@ on:
     branches:
       - main
 
-name: R-CMD-check
-
 jobs:
   R-CMD-check:
     runs-on: ${{ matrix.config.os }}
diff --git a/.github/workflows/5-block-fixup.yaml b/.github/workflows/5-block-fixup.yaml
index b42aeb4..c88a4f5 100644
--- a/.github/workflows/5-block-fixup.yaml
+++ b/.github/workflows/5-block-fixup.yaml
@@ -1,4 +1,6 @@
-name: Git Checks
+name: Block Fix-up (Git Check)
+run-name: (5) Block Fix-up / Git Check - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
+
 
 on: [pull_request]
 
diff --git a/.github/workflows/6-pkgdown.yaml b/.github/workflows/6-pkgdown.yaml
index a7276e8..86e18ca 100644
--- a/.github/workflows/6-pkgdown.yaml
+++ b/.github/workflows/6-pkgdown.yaml
@@ -10,6 +10,7 @@ on:
   workflow_dispatch:
 
 name: pkgdown
+run-name: (6) pkgdown - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
 
 jobs:
   pkgdown:
diff --git a/.github/workflows/7-test-coverage.yaml b/.github/workflows/7-test-coverage.yaml
index 75af0a5..7daf819 100644
--- a/.github/workflows/7-test-coverage.yaml
+++ b/.github/workflows/7-test-coverage.yaml
@@ -7,6 +7,7 @@ on:
     branches: [main]
 
 name: test-coverage
+run-name: (7) Test Coverage - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
 
 jobs:
   test-coverage:
diff --git a/.github/workflows/8-pr-commands.yaml b/.github/workflows/8-pr-commands.yaml
index c9914e1..cb11c80 100644
--- a/.github/workflows/8-pr-commands.yaml
+++ b/.github/workflows/8-pr-commands.yaml
@@ -5,7 +5,9 @@ on:
     types: [created]
   workflow_dispatch:
 
-name: Commands
+name: PR Commands
+run-name: (8) PR Commands - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
+
 
 jobs:
   document:

From 4f9dbad65f49072d81d4a35e56cc1b25c56e9f8f Mon Sep 17 00:00:00 2001
From: Zachary Susswein <utb2@cdc.gov>
Date: Wed, 25 Sep 2024 17:42:06 +0000
Subject: [PATCH 18/53] Explicit date casting in queries

To fix test failures caused by DuckDB v1.1.1 release
---
 R/parameters.R |  2 +-
 R/read_data.R  | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/R/parameters.R b/R/parameters.R
index 63fd530..7da2229 100644
--- a/R/parameters.R
+++ b/R/parameters.R
@@ -141,7 +141,7 @@ read_interval_pmf <- function(path,
       AND parameter = ?
       AND disease = ?
       AND start_date < ? :: DATE
-      AND (end_date > ? OR end_date IS NULL)
+      AND (CAST(end_date AS DATE) > ? :: DATE OR end_date IS NULL)
     "
   parameters <- list(
     path,
diff --git a/R/read_data.R b/R/read_data.R
index d5f8eec..385f5f1 100644
--- a/R/read_data.R
+++ b/R/read_data.R
@@ -71,9 +71,9 @@ read_data <- function(data_path,
     WHERE 1=1
       AND disease = ?
       AND metric = 'count_ed_visits'
-      AND reference_date >= ?
-      AND reference_date <= ?
-      AND report_date = ?
+      AND reference_date >= ? :: DATE
+      AND reference_date <= ? :: DATE
+      AND report_date = ? :: DATE
     GROUP BY reference_date, report_date, disease
     ORDER BY reference_date
    "
@@ -90,9 +90,9 @@ read_data <- function(data_path,
   WHERE 1=1
     AND disease = ?
     AND metric = 'count_ed_visits'
-    AND reference_date >= ?
-    AND reference_date <= ?
-    AND report_date = ?
+    AND reference_date >= ? :: DATE
+    AND reference_date <= ? :: DATE
+    AND report_date = ? :: DATE
     AND geo_value = ?
   GROUP BY geo_value, reference_date, report_date, disease
   ORDER BY reference_date

From 6745c571dff000ac29ec2f5461377ea12fde0ffc Mon Sep 17 00:00:00 2001
From: Zachary Susswein <utb2@cdc.gov>
Date: Wed, 25 Sep 2024 18:04:56 +0000
Subject: [PATCH 19/53] Bump NEWS

---
 NEWS.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/NEWS.md b/NEWS.md
index 285022d..821858a 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -17,3 +17,4 @@
 * Updated DESCRIPTION and added guidelines for package authorship
 * Set up README with explanation of purpose and scope
 * Removed `add.R` placeholder
+* Fix bugs in date casting caused by DuckDB v1.1.1 release

From 915fa76d52f2bee2ce3100a116abf8d0af81dd2a Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Wed, 25 Sep 2024 21:54:45 +0000
Subject: [PATCH 20/53] pipeline with batch code - not yet working

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 54 +++++++++----------
 .gitignore                                    |  4 ++
 2 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index b1eec81..2bb5711 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -110,32 +110,28 @@ jobs:
         # managed by EDAV. Contact Amit Mantri or Jon Kislin if you have issues.
           creds: ${{ secrets.EDAV_CFA_PREDICT_NNHT_SP }}
 
-      # - name: Delete Existing Batch Pool
-      #   id: delete_existing_batch_pool
-      #   run: |
-
-      #     export pool_id="cfa-epinow2-pipeline-ci"
-
-      #     # Check if the batch pool exists
-      #     EPINOW2_POOL_EXISTS=$(az batch pool show \
-      #       --account-name "cfaprdba" \
-      #       --account-endpoint "https://cfaprdba.eastus.batch.azure.com" \
-      #       --pool-id "$pool_id" \
-      #       --query "id" --output tsv 2>/dev/null)
-
-      #     if [ -z "$EPINOW2_POOL_EXISTS" ]; then
-      #         echo -e "Batch pool $pool_id does not exist."
-      #     else
-      #         echo -e "Batch pool $pool_id exists. Deleting it now..."
-      #         az batch pool delete \
-      #           --account-name cfaprdba \
-      #           --account-endpoint https://cfaprdba.eastus.batch.azure.com \
-      #           --pool-id "$pool_id" \
-      #           --yes
-      #         echo "Batch pool $pool_id has been deleted."
-      #     fi
-
-      - name: Create cfa-epinow2-pipeline Pools
-        id: create_nhsn_nssp_pools
-        run:
-          echo 'hello world'
+      - name: Create cfa-epinow2-pipeline Pool
+        id: create_batch_pool
+
+        # Every Azure Batch Pool parameter can simply go here, no python module necessary
+        env:
+          POOL_ID: "cfa-epinow2-${{ github.sha }}"
+          BATCH_ENDPOINT: "https://cfaprdba.eastus.batch.azure.com/pools/cfa-epinow2"
+          VM_IMAGE_TAG: "canonical:0001-com-ubuntu-server-focal:20_04-lts"
+          NODE_AGENT_SKU_ID: "batch.node.ubuntu 20.04"
+          # TARGET_DEDICATED_NODES: "0" # This is overriden by Autoscale Formula
+          AUTO_SCALE_FORMULA: 
+          VM_SIZE: "standard_a4m_v2"
+
+        # The call to the az cli that actually generates the pool
+        run: |
+          az batch pool create \
+          --id ${{ env.POOL_ID }} \
+          --image ${{ env.VM_IMAGE_TAG }} \
+          --node-agent-sku-id ${{ env.NODE_AGENT_SKU_ID}} \
+          --target-dedicated-nodes ${{ env.TARGET_DEDICATED_NODES}} \
+          --auto-scale-formula ${{ env.AUTOSCALE_FORMULA}}
+          --vm-size ${{ env.VM_SIZE}}
+      
+      - name: legacy_test
+        run: az batch pool create --json-file ./cfa-epinow2-batch-pool-config.json
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 2998db1..6073966 100644
--- a/.gitignore
+++ b/.gitignore
@@ -380,3 +380,7 @@ docs/site/
 # applications that require a static environment.
 #Manifest.toml
 docs
+
+# cfa-epinow2-pool-config.json
+# for now... will have to gpg encry[pt
+cfa-epinow2-batch-pool-config.json

From 1961bc21ebe88d763dfca2db119ad148bacd949a Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 25 Sep 2024 21:55:39 +0000
Subject: [PATCH 21/53] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 2bb5711..862589f 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -120,7 +120,7 @@ jobs:
           VM_IMAGE_TAG: "canonical:0001-com-ubuntu-server-focal:20_04-lts"
           NODE_AGENT_SKU_ID: "batch.node.ubuntu 20.04"
           # TARGET_DEDICATED_NODES: "0" # This is overriden by Autoscale Formula
-          AUTO_SCALE_FORMULA: 
+          AUTO_SCALE_FORMULA:
           VM_SIZE: "standard_a4m_v2"
 
         # The call to the az cli that actually generates the pool
@@ -132,6 +132,6 @@ jobs:
           --target-dedicated-nodes ${{ env.TARGET_DEDICATED_NODES}} \
           --auto-scale-formula ${{ env.AUTOSCALE_FORMULA}}
           --vm-size ${{ env.VM_SIZE}}
-      
+
       - name: legacy_test
-        run: az batch pool create --json-file ./cfa-epinow2-batch-pool-config.json
\ No newline at end of file
+        run: az batch pool create --json-file ./cfa-epinow2-batch-pool-config.json

From cd9b198aec2c32cde8d21c7f1120623b7d404bdf Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Wed, 25 Sep 2024 22:02:09 +0000
Subject: [PATCH 22/53] commenting out the job01 dependencies build to claw us
 back some test cycle time

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 44 +++++++++----------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 2bb5711..1ee7982 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -44,28 +44,28 @@ jobs:
   # TODO: Only run this if necessary, will need conditional job syntax
   # SEE also; ./.github/workflows/1-Build-Dependency-Image.yml
 
-  Job01_build-image-dependencies:
-    runs-on: cfa-cdcgov
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Login to the Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: "cfaprdbatchcr.azurecr.io"
-          username: "cfaprdbatchcr"
-          password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
-
-      - name: Build and push dependencies image # first stage
-        id: build_and_push_dependencies_image
-        uses: docker/build-push-action@v6
-        with:
-          push: true
-          tags: |
-            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:latest
-            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:${{ github.sha }}
-          file: ./Dockerfile-dependencies # first stage Dockerfile
+  # Job01_build-image-dependencies:
+  #   runs-on: cfa-cdcgov
+  #   steps:
+  #     - name: Checkout code
+  #       uses: actions/checkout@v4
+
+  #     - name: Login to the Container Registry
+  #       uses: docker/login-action@v3
+  #       with:
+  #         registry: "cfaprdbatchcr.azurecr.io"
+  #         username: "cfaprdbatchcr"
+  #         password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
+
+  #     - name: Build and push dependencies image # first stage
+  #       id: build_and_push_dependencies_image
+  #       uses: docker/build-push-action@v6
+  #       with:
+  #         push: true
+  #         tags: |
+  #           ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:latest
+  #           ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:${{ github.sha }}
+  #         file: ./Dockerfile-dependencies # first stage Dockerfile
 
 
   Job02_build-model-image:

From 718d372aeb5221c9d0c62a729387df08be2bcaa7 Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Thu, 26 Sep 2024 02:14:52 +0000
Subject: [PATCH 23/53] nektos gh-act tests and pool creation code

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 60 +++++++++++++++----
 .github/workflows/gh-act/2-dry.sh             |  9 +++
 .github/workflows/gh-act/2-full.sh            |  8 +++
 3 files changed, 64 insertions(+), 13 deletions(-)
 create mode 100644 .github/workflows/gh-act/2-dry.sh
 create mode 100644 .github/workflows/gh-act/2-full.sh

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index beb5e25..102140a 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -89,7 +89,7 @@ jobs:
             ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:${{ github.sha }}
           file: ./Dockerfile # second stage Dockerfile
 
-  Job03_create-and-launch-batch-pools:
+  Job03_create-batch-pool-and-submit-jobs:
     runs-on: cfa-cdcgov
     needs:
       - Job02_build-model-image
@@ -113,15 +113,36 @@ jobs:
       - name: Create cfa-epinow2-pipeline Pool
         id: create_batch_pool
 
-        # Every Azure Batch Pool parameter can simply go here, no python module necessary
+        # Every Azure Batch Pool parameter can simply go here, 
+        # no python module or config toml necessary
         env:
-          POOL_ID: "cfa-epinow2-${{ github.sha }}"
-          BATCH_ENDPOINT: "https://cfaprdba.eastus.batch.azure.com/pools/cfa-epinow2"
-          VM_IMAGE_TAG: "canonical:0001-com-ubuntu-server-focal:20_04-lts"
+          POOL_ID:           "cfa-epinow2-${{ github.sha }}" # version the pools with the github sha
+          BATCH_ACCOUNT:     "cfaprdba"
+          BATCH_ENDPOINT:    "https://cfaprdba.eastus.batch.azure.com/pools/cfa-epinow2"
+          VM_IMAGE_TAG:      "canonical:0001-com-ubuntu-server-focal:20_04-lts"
           NODE_AGENT_SKU_ID: "batch.node.ubuntu 20.04"
-          # TARGET_DEDICATED_NODES: "0" # This is overriden by Autoscale Formula
-          AUTO_SCALE_FORMULA:
-          VM_SIZE: "standard_a4m_v2"
+          VM_SIZE:           "standard_a4m_v2"
+
+          AUTO_SCALE_FORMULA: |
+            "// In this example, the pool size is adjusted based on the number of tasks in the queue.
+             // Note that both comments and line breaks are acceptable in formula strings.
+             
+             // Get pending tasks for the past 15 minutes.
+             $samples = $ActiveTasks.GetSamplePercent(TimeInterval_Minute * 15);
+             
+             // If we have fewer than 70 percent data points, we use the last sample point, otherwise we use the maximum of last sample point and the history average.
+             $tasks = $samples < 70 ? max(0, $ActiveTasks.GetSample(1)) :
+             max( $ActiveTasks.GetSample(1), avg($ActiveTasks.GetSample(TimeInterval_Minute * 15)));
+             
+             // If number of pending tasks is not 0, set targetVM to pending tasks, otherwise half of current dedicated.
+             $targetVMs = $tasks > 0 ? $tasks : max(0, $TargetDedicatedNodes / 2);
+             
+             // The pool size is capped at 120, if target VM value is more than that, set it to 120.
+             cappedPoolSize = 120;
+             $TargetDedicatedNodes = max(0, min($targetVMs, cappedPoolSize));
+             
+             // Set node deallocation mode - keep nodes active only until tasks finish
+             $NodeDeallocationOption = taskcompletion;"
 
         # The call to the az cli that actually generates the pool
         run: |
@@ -129,9 +150,22 @@ jobs:
           --id ${{ env.POOL_ID }} \
           --image ${{ env.VM_IMAGE_TAG }} \
           --node-agent-sku-id ${{ env.NODE_AGENT_SKU_ID}} \
-          --target-dedicated-nodes ${{ env.TARGET_DEDICATED_NODES}} \
-          --auto-scale-formula ${{ env.AUTOSCALE_FORMULA}}
-          --vm-size ${{ env.VM_SIZE}}
+          --auto-scale-formula ${{ env.AUTOSCALE_FORMULA}} \
+          --vm-size ${{ env.VM_SIZE }} \
+          --account-name ${{ env.BATCH_ACCOUNT }}
+          --account-endpoint ${{ env.BATCH_ENDPOINT }}
 
-      - name: legacy_test
-        run: az batch pool create --json-file ./cfa-epinow2-batch-pool-config.json
+          # Let's get this var for the next step too
+          echo "POOL_ID=${{ env.POOL_ID}}" >> $GITHUB_ENV
+
+      # - name: legacy_test
+      #   run: |
+      #     az batch pool create --json-file ./cfa-epinow2-batch-pool-config.json
+
+
+      - name: Submit azure batch job
+        id: submit_batch_job
+        run: |
+          az batch job create --pool-id ${{ github.env.POOL_ID }}
+
+        
diff --git a/.github/workflows/gh-act/2-dry.sh b/.github/workflows/gh-act/2-dry.sh
new file mode 100644
index 0000000..711c866
--- /dev/null
+++ b/.github/workflows/gh-act/2-dry.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Requires that you have first run 'gh extension install nektos/gh-act'
+# as well as having installed the docker engine and added your user to the docker group
+
+# This checks syntax before you push to Github Actions, helping with debug hell
+# To run the entire pipeline locally, see 2-full.sh
+
+gh act -P cfa-cdcgov=... -n -W '.github/workflows/2-Run-Epinow2-Pipeline.yaml'
\ No newline at end of file
diff --git a/.github/workflows/gh-act/2-full.sh b/.github/workflows/gh-act/2-full.sh
new file mode 100644
index 0000000..3675070
--- /dev/null
+++ b/.github/workflows/gh-act/2-full.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Requires that you have first run 'gh extension install nektos/gh-act'
+# as well as having installed the docker engine and added your user to the docker group
+
+# This runs the github actions workflow locally
+
+gh act -P cfa-cdcgov=catthehacker/ubuntu:full-20.04 -W '.github/workflows/2-Run-Epinow2-Pipeline.yaml'
\ No newline at end of file

From 929fdff810322bedb2a32a6105fdeab66b792d87 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 26 Sep 2024 02:15:58 +0000
Subject: [PATCH 24/53] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 14 +++++++-------
 .github/workflows/gh-act/2-dry.sh             |  2 +-
 .github/workflows/gh-act/2-full.sh            |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 102140a..661b061 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -113,7 +113,7 @@ jobs:
       - name: Create cfa-epinow2-pipeline Pool
         id: create_batch_pool
 
-        # Every Azure Batch Pool parameter can simply go here, 
+        # Every Azure Batch Pool parameter can simply go here,
         # no python module or config toml necessary
         env:
           POOL_ID:           "cfa-epinow2-${{ github.sha }}" # version the pools with the github sha
@@ -126,21 +126,21 @@ jobs:
           AUTO_SCALE_FORMULA: |
             "// In this example, the pool size is adjusted based on the number of tasks in the queue.
              // Note that both comments and line breaks are acceptable in formula strings.
-             
+
              // Get pending tasks for the past 15 minutes.
              $samples = $ActiveTasks.GetSamplePercent(TimeInterval_Minute * 15);
-             
+
              // If we have fewer than 70 percent data points, we use the last sample point, otherwise we use the maximum of last sample point and the history average.
              $tasks = $samples < 70 ? max(0, $ActiveTasks.GetSample(1)) :
              max( $ActiveTasks.GetSample(1), avg($ActiveTasks.GetSample(TimeInterval_Minute * 15)));
-             
+
              // If number of pending tasks is not 0, set targetVM to pending tasks, otherwise half of current dedicated.
              $targetVMs = $tasks > 0 ? $tasks : max(0, $TargetDedicatedNodes / 2);
-             
+
              // The pool size is capped at 120, if target VM value is more than that, set it to 120.
              cappedPoolSize = 120;
              $TargetDedicatedNodes = max(0, min($targetVMs, cappedPoolSize));
-             
+
              // Set node deallocation mode - keep nodes active only until tasks finish
              $NodeDeallocationOption = taskcompletion;"
 
@@ -168,4 +168,4 @@ jobs:
         run: |
           az batch job create --pool-id ${{ github.env.POOL_ID }}
 
-        
+
diff --git a/.github/workflows/gh-act/2-dry.sh b/.github/workflows/gh-act/2-dry.sh
index 711c866..f147536 100644
--- a/.github/workflows/gh-act/2-dry.sh
+++ b/.github/workflows/gh-act/2-dry.sh
@@ -6,4 +6,4 @@
 # This checks syntax before you push to Github Actions, helping with debug hell
 # To run the entire pipeline locally, see 2-full.sh
 
-gh act -P cfa-cdcgov=... -n -W '.github/workflows/2-Run-Epinow2-Pipeline.yaml'
\ No newline at end of file
+gh act -P cfa-cdcgov=... -n -W '.github/workflows/2-Run-Epinow2-Pipeline.yaml'
diff --git a/.github/workflows/gh-act/2-full.sh b/.github/workflows/gh-act/2-full.sh
index 3675070..cfafa9a 100644
--- a/.github/workflows/gh-act/2-full.sh
+++ b/.github/workflows/gh-act/2-full.sh
@@ -5,4 +5,4 @@
 
 # This runs the github actions workflow locally
 
-gh act -P cfa-cdcgov=catthehacker/ubuntu:full-20.04 -W '.github/workflows/2-Run-Epinow2-Pipeline.yaml'
\ No newline at end of file
+gh act -P cfa-cdcgov=catthehacker/ubuntu:full-20.04 -W '.github/workflows/2-Run-Epinow2-Pipeline.yaml'

From 9ad0060d5bf75bbf661d0e999f068a00748cef4e Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Thu, 26 Sep 2024 02:23:54 +0000
Subject: [PATCH 25/53] fix some env variables for auto scale formula

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 60 ++++++-------------
 1 file changed, 17 insertions(+), 43 deletions(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 102140a..b22f31e 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -40,35 +40,7 @@ env:
 
 jobs:
 
-  # We now use a two-stage docker build (essentially), split across two dockerfiles.
-  # TODO: Only run this if necessary, will need conditional job syntax
-  # SEE also; ./.github/workflows/1-Build-Dependency-Image.yml
-
-  # Job01_build-image-dependencies:
-  #   runs-on: cfa-cdcgov
-  #   steps:
-  #     - name: Checkout code
-  #       uses: actions/checkout@v4
-
-  #     - name: Login to the Container Registry
-  #       uses: docker/login-action@v3
-  #       with:
-  #         registry: "cfaprdbatchcr.azurecr.io"
-  #         username: "cfaprdbatchcr"
-  #         password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
-
-  #     - name: Build and push dependencies image # first stage
-  #       id: build_and_push_dependencies_image
-  #       uses: docker/build-push-action@v6
-  #       with:
-  #         push: true
-  #         tags: |
-  #           ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:latest
-  #           ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:${{ github.sha }}
-  #         file: ./Dockerfile-dependencies # first stage Dockerfile
-
-
-  Job02_build-model-image:
+  _01_build-model-image:
     runs-on: cfa-cdcgov
     steps:
 
@@ -79,17 +51,18 @@ jobs:
           username: "cfaprdbatchcr"
           password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
 
-      - name: Build and push model pipeline image for Azure batch # second stage, the actual payload
-        id: build_and_push_model_image
-        uses: docker/build-push-action@v6
-        with:
-          push: true
-          tags: |
-            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:latest
-            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:${{ github.sha }}
-          file: ./Dockerfile # second stage Dockerfile
-
-  Job03_create-batch-pool-and-submit-jobs:
+      # Comment out if you want to just test the pool creation without waiting for build and push
+      # - name: Build and push model pipeline image for Azure batch # second stage, the actual payload
+      #   id: build_and_push_model_image
+      #   uses: docker/build-push-action@v6
+      #   with:
+      #     push: true
+      #     tags: |
+      #       ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:latest
+      #       ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:${{ github.sha }}
+      #     file: ./Dockerfile # second stage Dockerfile
+
+  _02_create-batch-pool-and-submit-jobs:
     runs-on: cfa-cdcgov
     needs:
       - Job02_build-model-image
@@ -130,7 +103,8 @@ jobs:
              // Get pending tasks for the past 15 minutes.
              $samples = $ActiveTasks.GetSamplePercent(TimeInterval_Minute * 15);
              
-             // If we have fewer than 70 percent data points, we use the last sample point, otherwise we use the maximum of last sample point and the history average.
+             // If we have fewer than 70 percent data points, we use the last sample point, 
+             // otherwise we use the maximum of last sample point and the history average.
              $tasks = $samples < 70 ? max(0, $ActiveTasks.GetSample(1)) :
              max( $ActiveTasks.GetSample(1), avg($ActiveTasks.GetSample(TimeInterval_Minute * 15)));
              
@@ -149,8 +123,8 @@ jobs:
           az batch pool create \
           --id ${{ env.POOL_ID }} \
           --image ${{ env.VM_IMAGE_TAG }} \
-          --node-agent-sku-id ${{ env.NODE_AGENT_SKU_ID}} \
-          --auto-scale-formula ${{ env.AUTOSCALE_FORMULA}} \
+          --node-agent-sku-id ${{ env.NODE_AGENT_SKU_ID }} \
+          --auto-scale-formula ${{ env.AUTO_SCALE_FORMULA }} \
           --vm-size ${{ env.VM_SIZE }} \
           --account-name ${{ env.BATCH_ACCOUNT }}
           --account-endpoint ${{ env.BATCH_ENDPOINT }}

From 73efd5a868aea679628637006505bad58d2ed8d2 Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Thu, 26 Sep 2024 02:26:08 +0000
Subject: [PATCH 26/53] more cowbell

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 4f546a5..af8ae65 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -41,7 +41,7 @@ env:
 jobs:
 
   _01_build-model-image:
-    runs-on: cfa-cdcgov
+    runs-on: cfa-cdcgov # 
     steps:
 
       - name: Login to the Container Registry

From 4aa8bdcdf7f4fd983e3236100cc49c26723dced3 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 26 Sep 2024 02:26:41 +0000
Subject: [PATCH 27/53] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index af8ae65..3e75df1 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -41,7 +41,7 @@ env:
 jobs:
 
   _01_build-model-image:
-    runs-on: cfa-cdcgov # 
+    runs-on: cfa-cdcgov #
     steps:
 
       - name: Login to the Container Registry
@@ -102,8 +102,8 @@ jobs:
 
              // Get pending tasks for the past 15 minutes.
              $samples = $ActiveTasks.GetSamplePercent(TimeInterval_Minute * 15);
-             
-             // If we have fewer than 70 percent data points, we use the last sample point, 
+
+             // If we have fewer than 70 percent data points, we use the last sample point,
              // otherwise we use the maximum of last sample point and the history average.
              $tasks = $samples < 70 ? max(0, $ActiveTasks.GetSample(1)) :
              max( $ActiveTasks.GetSample(1), avg($ActiveTasks.GetSample(TimeInterval_Minute * 15)));
@@ -141,5 +141,3 @@ jobs:
         id: submit_batch_job
         run: |
           az batch job create --pool-id ${{ github.env.POOL_ID }}
-
-

From d672938a95de2c5524c194625539991eaca41e5e Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Thu, 26 Sep 2024 02:27:10 +0000
Subject: [PATCH 28/53] fix needs issue

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index af8ae65..73976b1 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -65,7 +65,7 @@ jobs:
   _02_create-batch-pool-and-submit-jobs:
     runs-on: cfa-cdcgov
     needs:
-      - Job02_build-model-image
+      - _01_build-model-image
 
     permissions:
       contents: read

From ee0e5e9edf7af3fc07063f57949e7fe74ab9e3f4 Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Thu, 26 Sep 2024 02:31:30 +0000
Subject: [PATCH 29/53] we need quotes around var names"

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 94b81cb..a7fcfb4 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -121,13 +121,13 @@ jobs:
         # The call to the az cli that actually generates the pool
         run: |
           az batch pool create \
-          --id ${{ env.POOL_ID }} \
-          --image ${{ env.VM_IMAGE_TAG }} \
-          --node-agent-sku-id ${{ env.NODE_AGENT_SKU_ID }} \
-          --auto-scale-formula ${{ env.AUTO_SCALE_FORMULA }} \
-          --vm-size ${{ env.VM_SIZE }} \
-          --account-name ${{ env.BATCH_ACCOUNT }}
-          --account-endpoint ${{ env.BATCH_ENDPOINT }}
+          --id "${{ env.POOL_ID }}" \
+          --image "${{ env.VM_IMAGE_TAG }}" \
+          --node-agent-sku-id "${{ env.NODE_AGENT_SKU_ID }}" \
+          --auto-scale-formula "${{ env.AUTO_SCALE_FORMULA }}" \
+          --vm-size "${{ env.VM_SIZE }}" \
+          --account-name "${{ env.BATCH_ACCOUNT }}" \
+          --account-endpoint "${{ env.BATCH_ENDPOINT }}"
 
           # Let's get this var for the next step too
           echo "POOL_ID=${{ env.POOL_ID}}" >> $GITHUB_ENV

From 21c70cadc5b666b8b9effb578d9931e04559baac Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Thu, 26 Sep 2024 02:41:49 +0000
Subject: [PATCH 30/53] autoscale formula as cat'd variable

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 24 +------------------
 batch-autoscale-formula.txt                   | 20 ++++++++++++++++
 2 files changed, 21 insertions(+), 23 deletions(-)
 create mode 100644 batch-autoscale-formula.txt

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index a7fcfb4..db79589 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -96,35 +96,13 @@ jobs:
           NODE_AGENT_SKU_ID: "batch.node.ubuntu 20.04"
           VM_SIZE:           "standard_a4m_v2"
 
-          AUTO_SCALE_FORMULA: |
-            "// In this example, the pool size is adjusted based on the number of tasks in the queue.
-             // Note that both comments and line breaks are acceptable in formula strings.
-
-             // Get pending tasks for the past 15 minutes.
-             $samples = $ActiveTasks.GetSamplePercent(TimeInterval_Minute * 15);
-
-             // If we have fewer than 70 percent data points, we use the last sample point,
-             // otherwise we use the maximum of last sample point and the history average.
-             $tasks = $samples < 70 ? max(0, $ActiveTasks.GetSample(1)) :
-             max( $ActiveTasks.GetSample(1), avg($ActiveTasks.GetSample(TimeInterval_Minute * 15)));
-
-             // If number of pending tasks is not 0, set targetVM to pending tasks, otherwise half of current dedicated.
-             $targetVMs = $tasks > 0 ? $tasks : max(0, $TargetDedicatedNodes / 2);
-
-             // The pool size is capped at 120, if target VM value is more than that, set it to 120.
-             cappedPoolSize = 120;
-             $TargetDedicatedNodes = max(0, min($targetVMs, cappedPoolSize));
-
-             // Set node deallocation mode - keep nodes active only until tasks finish
-             $NodeDeallocationOption = taskcompletion;"
-
         # The call to the az cli that actually generates the pool
         run: |
           az batch pool create \
           --id "${{ env.POOL_ID }}" \
           --image "${{ env.VM_IMAGE_TAG }}" \
           --node-agent-sku-id "${{ env.NODE_AGENT_SKU_ID }}" \
-          --auto-scale-formula "${{ env.AUTO_SCALE_FORMULA }}" \
+          --auto-scale-formula "$(cat './batch-autoscale-formula.txt')"
           --vm-size "${{ env.VM_SIZE }}" \
           --account-name "${{ env.BATCH_ACCOUNT }}" \
           --account-endpoint "${{ env.BATCH_ENDPOINT }}"
diff --git a/batch-autoscale-formula.txt b/batch-autoscale-formula.txt
new file mode 100644
index 0000000..7213fc8
--- /dev/null
+++ b/batch-autoscale-formula.txt
@@ -0,0 +1,20 @@
+// In this example, the pool size is adjusted based on the number of tasks in the queue.
+// Note that both comments and line breaks are acceptable in formula strings.
+
+// Get pending tasks for the past 15 minutes.
+$samples = $ActiveTasks.GetSamplePercent(TimeInterval_Minute * 15);
+
+// If we have fewer than 70 percent data points, we use the last sample point,
+// otherwise we use the maximum of last sample point and the history average.
+$tasks = $samples < 70 ? max(0, $ActiveTasks.GetSample(1)) :
+max( $ActiveTasks.GetSample(1), avg($ActiveTasks.GetSample(TimeInterval_Minute * 15)));
+
+// If number of pending tasks is not 0, set targetVM to pending tasks, otherwise half of current dedicated.
+$targetVMs = $tasks > 0 ? $tasks : max(0, $TargetDedicatedNodes / 2);
+
+// The pool size is capped at 120, if target VM value is more than that, set it to 120.
+cappedPoolSize = 120;
+$TargetDedicatedNodes = max(0, min($targetVMs, cappedPoolSize));
+
+// Set node deallocation mode - keep nodes active only until tasks finish
+$NodeDeallocationOption = taskcompletion;
\ No newline at end of file

From f71a1572f9fe5e23104d43da8f2de2900ed89c7f Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 26 Sep 2024 02:42:21 +0000
Subject: [PATCH 31/53] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 batch-autoscale-formula.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/batch-autoscale-formula.txt b/batch-autoscale-formula.txt
index 7213fc8..c98a8ee 100644
--- a/batch-autoscale-formula.txt
+++ b/batch-autoscale-formula.txt
@@ -17,4 +17,4 @@ cappedPoolSize = 120;
 $TargetDedicatedNodes = max(0, min($targetVMs, cappedPoolSize));
 
 // Set node deallocation mode - keep nodes active only until tasks finish
-$NodeDeallocationOption = taskcompletion;
\ No newline at end of file
+$NodeDeallocationOption = taskcompletion;

From a0c33994bf504f407803df7d7f10e92cf6f806b2 Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Thu, 26 Sep 2024 02:43:42 +0000
Subject: [PATCH 32/53] newline syntax fix for bash batch cli code

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index db79589..caa2b81 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -1,5 +1,5 @@
 name: Run Epinow2 Pipeline
-run-name: (2) Epinow2 Pipeline - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
+run-name: (2) Pipeline - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
 
 # This GitHub Actions workflow builds a Docker image for the
 # cfa-epinow2-pipeline-docker project. It consists of two jobs:
@@ -102,7 +102,7 @@ jobs:
           --id "${{ env.POOL_ID }}" \
           --image "${{ env.VM_IMAGE_TAG }}" \
           --node-agent-sku-id "${{ env.NODE_AGENT_SKU_ID }}" \
-          --auto-scale-formula "$(cat './batch-autoscale-formula.txt')"
+          --auto-scale-formula "$(cat './batch-autoscale-formula.txt')" \
           --vm-size "${{ env.VM_SIZE }}" \
           --account-name "${{ env.BATCH_ACCOUNT }}" \
           --account-endpoint "${{ env.BATCH_ENDPOINT }}"

From 62fb4169dd124cda62567793087faffae91e1a26 Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Thu, 26 Sep 2024 02:45:55 +0000
Subject: [PATCH 33/53] fixed endpoint uri

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index caa2b81..f9ec3fc 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -91,7 +91,7 @@ jobs:
         env:
           POOL_ID:           "cfa-epinow2-${{ github.sha }}" # version the pools with the github sha
           BATCH_ACCOUNT:     "cfaprdba"
-          BATCH_ENDPOINT:    "https://cfaprdba.eastus.batch.azure.com/pools/cfa-epinow2"
+          BATCH_ENDPOINT:    "https://cfaprdba.eastus.batch.azure.com/"
           VM_IMAGE_TAG:      "canonical:0001-com-ubuntu-server-focal:20_04-lts"
           NODE_AGENT_SKU_ID: "batch.node.ubuntu 20.04"
           VM_SIZE:           "standard_a4m_v2"

From 73360c774bad4550b539731174ce650d63048ced Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Thu, 26 Sep 2024 02:50:59 +0000
Subject: [PATCH 34/53] autoscale enablement?;

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index f9ec3fc..966c11e 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -102,6 +102,7 @@ jobs:
           --id "${{ env.POOL_ID }}" \
           --image "${{ env.VM_IMAGE_TAG }}" \
           --node-agent-sku-id "${{ env.NODE_AGENT_SKU_ID }}" \
+          --auto-scale \
           --auto-scale-formula "$(cat './batch-autoscale-formula.txt')" \
           --vm-size "${{ env.VM_SIZE }}" \
           --account-name "${{ env.BATCH_ACCOUNT }}" \

From 914395b2af66ad63dc586dbb32fc55ac96cc381e Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Thu, 26 Sep 2024 03:05:15 +0000
Subject: [PATCH 35/53] autoscale as a separate step

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 966c11e..4c2f829 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -95,20 +95,27 @@ jobs:
           VM_IMAGE_TAG:      "canonical:0001-com-ubuntu-server-focal:20_04-lts"
           NODE_AGENT_SKU_ID: "batch.node.ubuntu 20.04"
           VM_SIZE:           "standard_a4m_v2"
+          RESOURCE_GROUP:    ${{ secrets.PRD_RESOURCE_GROUP }}
 
         # The call to the az cli that actually generates the pool
         run: |
+          az batch account login \
+          --resource-group ${{ secrets.PRD_RESOURCE_GROUP }} \
+          --name "${{ env.BATCH_ACCOUNT }}" \
+          
           az batch pool create \
+          --account-endpoint "${{ env.BATCH_ENDPOINT }}" \
           --id "${{ env.POOL_ID }}" \
           --image "${{ env.VM_IMAGE_TAG }}" \
           --node-agent-sku-id "${{ env.NODE_AGENT_SKU_ID }}" \
-          --auto-scale \
-          --auto-scale-formula "$(cat './batch-autoscale-formula.txt')" \
           --vm-size "${{ env.VM_SIZE }}" \
-          --account-name "${{ env.BATCH_ACCOUNT }}" \
-          --account-endpoint "${{ env.BATCH_ENDPOINT }}"
 
-          # Let's get this var for the next step too
+
+          az batch pool autoscale enable \
+          --pool-id ${{ env.POOL_ID }} \
+          --auto-scale-formula "$(cat './batch-autoscale-formula.txt')"
+
+          # Let's get this POOL_ID var for the next step too
           echo "POOL_ID=${{ env.POOL_ID}}" >> $GITHUB_ENV
 
       # - name: legacy_test

From 2b30029a2e5ccdb2525de2d081ff2349b6a6797c Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 26 Sep 2024 03:06:59 +0000
Subject: [PATCH 36/53] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 4c2f829..f4ee6e2 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -102,7 +102,7 @@ jobs:
           az batch account login \
           --resource-group ${{ secrets.PRD_RESOURCE_GROUP }} \
           --name "${{ env.BATCH_ACCOUNT }}" \
-          
+
           az batch pool create \
           --account-endpoint "${{ env.BATCH_ENDPOINT }}" \
           --id "${{ env.POOL_ID }}" \

From 921105850d83c6dd14cf362d374fd6d234ea13f8 Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Thu, 26 Sep 2024 03:08:09 +0000
Subject: [PATCH 37/53] pool id var needs fixing in the last step

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 4c2f829..423120d 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -126,4 +126,4 @@ jobs:
       - name: Submit azure batch job
         id: submit_batch_job
         run: |
-          az batch job create --pool-id ${{ github.env.POOL_ID }}
+          az batch job create --pool-id $POOL_ID 

From 25d42c54c6acb9923d84673c05807cbb7eff0c2e Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 26 Sep 2024 03:08:50 +0000
Subject: [PATCH 38/53] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 7fc3249..ff5acd3 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -126,4 +126,4 @@ jobs:
       - name: Submit azure batch job
         id: submit_batch_job
         run: |
-          az batch job create --pool-id $POOL_ID 
+          az batch job create --pool-id $POOL_ID

From aaf2a1ace724c927383c689558671068c8f2491a Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Thu, 26 Sep 2024 03:21:44 +0000
Subject: [PATCH 39/53] environment variables

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 7fc3249..18a6f66 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -83,13 +83,19 @@ jobs:
         # managed by EDAV. Contact Amit Mantri or Jon Kislin if you have issues.
           creds: ${{ secrets.EDAV_CFA_PREDICT_NNHT_SP }}
 
+
+      - name: Get Github Short sha
+        run: |
+          shortSHA=$(git rev-parse --short ${{ github.sha }})
+          echo "SHORT_SHA=$shortSHA" >> $GITHUB_ENV
+
       - name: Create cfa-epinow2-pipeline Pool
         id: create_batch_pool
 
         # Every Azure Batch Pool parameter can simply go here,
         # no python module or config toml necessary
         env:
-          POOL_ID:           "cfa-epinow2-${{ github.sha }}" # version the pools with the github sha
+          POOL_ID:           "cfa-epinow2-${{ env.SHORT_SHA }}" # version the pools with the github sha
           BATCH_ACCOUNT:     "cfaprdba"
           BATCH_ENDPOINT:    "https://cfaprdba.eastus.batch.azure.com/"
           VM_IMAGE_TAG:      "canonical:0001-com-ubuntu-server-focal:20_04-lts"
@@ -97,6 +103,7 @@ jobs:
           VM_SIZE:           "standard_a4m_v2"
           RESOURCE_GROUP:    ${{ secrets.PRD_RESOURCE_GROUP }}
 
+
         # The call to the az cli that actually generates the pool
         run: |
           az batch account login \
@@ -110,7 +117,6 @@ jobs:
           --node-agent-sku-id "${{ env.NODE_AGENT_SKU_ID }}" \
           --vm-size "${{ env.VM_SIZE }}" \
 
-
           az batch pool autoscale enable \
           --pool-id ${{ env.POOL_ID }} \
           --auto-scale-formula "$(cat './batch-autoscale-formula.txt')"
@@ -125,5 +131,10 @@ jobs:
 
       - name: Submit azure batch job
         id: submit_batch_job
+        env:
+          POOL_ID: $POOL_ID
+          BATCH_JOB_ID: cfa-epinow2-pipeline-job-${{ env.SHORT_SHA }}
         run: |
-          az batch job create --pool-id $POOL_ID 
+          az batch job create \
+          --pool-id ${{ env.POOL_ID }} \
+          --id ${{ env.BATCH_JOB_ID }}

From aa7438f90f171be89a40fe7476092b1d9640cb80 Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Thu, 26 Sep 2024 03:25:01 +0000
Subject: [PATCH 40/53] quotes?

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 18a6f66..a31f1a7 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -133,8 +133,8 @@ jobs:
         id: submit_batch_job
         env:
           POOL_ID: $POOL_ID
-          BATCH_JOB_ID: cfa-epinow2-pipeline-job-${{ env.SHORT_SHA }}
+          BATCH_JOB_ID: "cfa-epinow2-pipeline-job-${{ env.SHORT_SHA }}"
         run: |
           az batch job create \
-          --pool-id ${{ env.POOL_ID }} \
-          --id ${{ env.BATCH_JOB_ID }}
+          --pool-id "${{ env.POOL_ID }}" \
+          --id "${{ env.BATCH_JOB_ID }}"

From 8c1bb0f291dada9929f3162b054dcc47f975ca4c Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Thu, 26 Sep 2024 03:28:35 +0000
Subject: [PATCH 41/53] comments

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 36 +++++++++----------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index a31f1a7..3fd4b84 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -52,15 +52,15 @@ jobs:
           password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
 
       # Comment out if you want to just test the pool creation without waiting for build and push
-      # - name: Build and push model pipeline image for Azure batch # second stage, the actual payload
-      #   id: build_and_push_model_image
-      #   uses: docker/build-push-action@v6
-      #   with:
-      #     push: true
-      #     tags: |
-      #       ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:latest
-      #       ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:${{ github.sha }}
-      #     file: ./Dockerfile # second stage Dockerfile
+      - name: Build and push model pipeline image for Azure batch # second stage, the actual payload
+        id: build_and_push_model_image
+        uses: docker/build-push-action@v6
+        with:
+          push: true
+          tags: |
+            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:latest
+            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:${{ github.sha }}
+          file: ./Dockerfile # second stage Dockerfile
 
   _02_create-batch-pool-and-submit-jobs:
     runs-on: cfa-cdcgov
@@ -129,12 +129,12 @@ jobs:
       #     az batch pool create --json-file ./cfa-epinow2-batch-pool-config.json
 
 
-      - name: Submit azure batch job
-        id: submit_batch_job
-        env:
-          POOL_ID: $POOL_ID
-          BATCH_JOB_ID: "cfa-epinow2-pipeline-job-${{ env.SHORT_SHA }}"
-        run: |
-          az batch job create \
-          --pool-id "${{ env.POOL_ID }}" \
-          --id "${{ env.BATCH_JOB_ID }}"
+      # - name: Submit azure batch job
+      #   id: submit_batch_job
+      #   env:
+      #     POOL_ID: "$POOL_ID"
+      #     BATCH_JOB_ID: "cfa-epinow2-pipeline-job-${{ env.SHORT_SHA }}"
+      #   run: |
+      #     az batch job create \
+      #     --pool-id "${{ env.POOL_ID }}" \
+      #     --id "${{ env.BATCH_JOB_ID }}"

From dde360ed015f5d5188b6e61302dfb3b42e588033 Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Thu, 26 Sep 2024 03:29:43 +0000
Subject: [PATCH 42/53] run name in quotes

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 3fd4b84..925d34f 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -1,5 +1,5 @@
 name: Run Epinow2 Pipeline
-run-name: (2) Pipeline - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
+run-name: (2) Pipeline - "${{ github.event.head_commit.message }}" - ${{ github.ref_name }}
 
 # This GitHub Actions workflow builds a Docker image for the
 # cfa-epinow2-pipeline-docker project. It consists of two jobs:

From 28195672862010d0e80417e3151d7b1efbea76c4 Mon Sep 17 00:00:00 2001
From: Jon Kislin <136386616+jkislin@users.noreply.github.com>
Date: Fri, 27 Sep 2024 00:46:23 +0000
Subject: [PATCH 43/53] Update .github/workflows/1-Build-Dependency-Image.yaml

Co-authored-by: Nate McIntosh <NMcIntosh@cdc.gov>
---
 .github/workflows/1-Build-Dependency-Image.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/1-Build-Dependency-Image.yaml b/.github/workflows/1-Build-Dependency-Image.yaml
index beb723e..a894ec5 100644
--- a/.github/workflows/1-Build-Dependency-Image.yaml
+++ b/.github/workflows/1-Build-Dependency-Image.yaml
@@ -6,7 +6,7 @@ run-name: (1) Dependencies Image Build - ${{ github.event.head_commit.message }}
 # build_image_dependencies and build_image.
 #
 # - The `build_image_dependencies` job carries out the first part of a
-#   multi-stage build. It downloads and install all the dependencies
+#   multi-stage build. It downloads and installs all the dependencies
 #   listed in the `DESCRIPTION` file. It uses the `Dockerfile-dependencies`
 #   file to build the image.
 #

From a64e632b064da69bd944601385912f6800e2b86f Mon Sep 17 00:00:00 2001
From: Jon Kislin <136386616+jkislin@users.noreply.github.com>
Date: Fri, 27 Sep 2024 00:49:41 +0000
Subject: [PATCH 44/53] Update .gitignore

Co-authored-by: Zachary Susswein <46581799+zsusswein@users.noreply.github.com>
---
 .gitignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 6073966..31f5dcf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -382,5 +382,5 @@ docs/site/
 docs
 
 # cfa-epinow2-pool-config.json
-# for now... will have to gpg encry[pt
+# for now... will have to gpg encrypt
 cfa-epinow2-batch-pool-config.json

From ea4573709d40c7562f68172616762bb8b2f22308 Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Fri, 27 Sep 2024 01:06:10 +0000
Subject: [PATCH 45/53] added cron trigger and removed jk-azure-readiness push
 trigger

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 925d34f..b7522f1 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -30,8 +30,12 @@ on:
   push:
     branches:
       - main
-      - jk-azure-readiness # temporary
+  pull_request:
+    branches:
+      - main
   workflow_dispatch:
+  schedule:
+    - cron: "0 12 * * 3"
 
 env:
   # Together, these form: cfaprdbatchcr.azurecr.io/cfa-epinow2-pipeline

From f82110a49d24eca11344978c7dc32503a07b9a00 Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Fri, 27 Sep 2024 01:29:34 +0000
Subject: [PATCH 46/53] documentation edits; file renames; revived "cacheing"
 for testing

---
 ...e.yaml => 1-Build-Dependencies-Image.yaml} | 36 +++++++---------
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 17 +-------
 .../2_pre-Test-Model-Image-Build.yaml         | 42 +++++++++++++++++++
 3 files changed, 58 insertions(+), 37 deletions(-)
 rename .github/workflows/{1-Build-Dependency-Image.yaml => 1-Build-Dependencies-Image.yaml} (68%)
 create mode 100644 .github/workflows/2_pre-Test-Model-Image-Build.yaml

diff --git a/.github/workflows/1-Build-Dependency-Image.yaml b/.github/workflows/1-Build-Dependencies-Image.yaml
similarity index 68%
rename from .github/workflows/1-Build-Dependency-Image.yaml
rename to .github/workflows/1-Build-Dependencies-Image.yaml
index a894ec5..24c6c6c 100644
--- a/.github/workflows/1-Build-Dependency-Image.yaml
+++ b/.github/workflows/1-Build-Dependencies-Image.yaml
@@ -1,9 +1,8 @@
 name: Build Dependencies Container Image
 run-name: (1) Dependencies Image Build - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
 
-# This GitHub Actions workflow builds a Docker image for the
-# cfa-epinow2-pipeline-docker project. It consists of two jobs:
-# build_image_dependencies and build_image.
+# This GitHub Actions workflow builds a Dependencies Docker image for the
+# cfa-epinow2-pipeline-docker project. 
 #
 # - The `build_image_dependencies` job carries out the first part of a
 #   multi-stage build. It downloads and installs all the dependencies
@@ -15,21 +14,15 @@ run-name: (1) Dependencies Image Build - ${{ github.event.head_commit.message }}
 #   The process is cached to avoid rebuilding the image if the dependencies
 #   have not changed. This is by hashing the `DESCRIPTION` file and the
 #   `Dockerfile-dependencies` file.
-#
-# - The build_image job builds the final image using the `Dockerfile` file.
-#   It uses the image built in the previous job as a base image.
-#
-#   During the build process, the package is installed and built. Furthermore
-#   the package is checked using `R CMD check` to ensure that it is working
-#   correctly.
-#
-#   Once the image is built, it is pushed to the corresponding registry.
+
+# - This built docker image is then referenced by the simply named 'Dockerfile', built in: 
+#     - "2_pre-Test-Model-Image-Build" and 
+#     - "2-Run-Epinow2-Pipeline.yaml"
 
 on:
   push:
     branches:
       - main
-      - jk-azure-readiness # temporary
     paths:
       - "./Dockerfile-dependencies" # the dockerfile this workflow builds from
       - "./.github/workflows/build-dependency-image.yaml" # this workflow
@@ -55,13 +48,13 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      # - name: Check cache
-      #   uses: actions/cache@v3
-      #   id: cache
-      #   with:
-      #     key: docker-dependencies-${{ runner.os }}-${{ hashFiles('./DESCRIPTION', './Dockerfile-dependencies') }}
-      #     path:
-      #       ./DESCRIPTION
+      - name: Check cache
+        uses: actions/cache@v3
+        id: cache
+        with:
+          key: docker-dependencies-${{ runner.os }}-${{ hashFiles('./DESCRIPTION', './Dockerfile-dependencies') }}
+          path:
+            ./DESCRIPTION
 
       - name: Login to the Container Registry
         uses: docker/login-action@v3
@@ -70,9 +63,8 @@ jobs:
           username: "cfaprdbatchcr"
           password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
 
-      # if: steps.cache.outputs.cache-hit != 'true'
-
       - name: Build and push
+        if: steps.cache.outputs.cache-hit != 'true'
         uses: docker/build-push-action@v6
         with:
           push: true
diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index b7522f1..c1d33f7 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -2,22 +2,10 @@ name: Run Epinow2 Pipeline
 run-name: (2) Pipeline - "${{ github.event.head_commit.message }}" - ${{ github.ref_name }}
 
 # This GitHub Actions workflow builds a Docker image for the
-# cfa-epinow2-pipeline-docker project. It consists of two jobs:
-# build_image_dependencies and build_image.
-#
-# - The `build_image_dependencies` job carries out the first part of a
-#   multi-stage build. It downloads and install all the dependencies
-#   listed in the `DESCRIPTION` file. It uses the `Dockerfile-dependencies`
-#   file to build the image.
-#
-#   The built image is then pushed to the corresponding registry.
-#
-#   The process is cached to avoid rebuilding the image if the dependencies
-#   have not changed. This is by hashing the `DESCRIPTION` file and the
-#   `Dockerfile-dependencies` file.
+# cfa-epinow2-pipeline-docker project. 
 #
 # - The build_image job builds the final image using the `Dockerfile` file.
-#   It uses the image built in the previous job as a base image.
+#   It uses the image built in "1-Build-Dependencies-Image.yaml" as a base image.
 #
 #   During the build process, the package is installed and built. Furthermore
 #   the package is checked using `R CMD check` to ensure that it is working
@@ -25,7 +13,6 @@ run-name: (2) Pipeline - "${{ github.event.head_commit.message }}" - ${{ github.
 #
 #   Once the image is built, it is pushed to the corresponding registry.
 
-
 on:
   push:
     branches:
diff --git a/.github/workflows/2_pre-Test-Model-Image-Build.yaml b/.github/workflows/2_pre-Test-Model-Image-Build.yaml
new file mode 100644
index 0000000..d92e57b
--- /dev/null
+++ b/.github/workflows/2_pre-Test-Model-Image-Build.yaml
@@ -0,0 +1,42 @@
+name: Test Model Image Build
+run-name: (2_pre) Test Model Image - "${{ github.event.head_commit.message }}" - ${{ github.ref_name }}
+
+# This GitHub Actions workflow builds a Docker image for the
+# cfa-epinow2-pipeline-docker project. In-container tests can be added here.
+
+on:
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+  schedule:
+    # Tuesdays at noon GMT time, 24 hours before the prod container is built and deployed
+    - cron: "0 12 * * 2" 
+
+env:
+  # Together, these form: cfaprdbatchcr.azurecr.io/cfa-epinow2-pipeline
+  REGISTRY: cfaprdbatchcr.azurecr.io
+  IMAGE_NAME: cfa-epinow2-pipeline
+
+jobs:
+
+  _01_build-model-image:
+    runs-on: cfa-cdcgov 
+    steps:
+
+      - name: Login to the Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: "cfaprdbatchcr.azurecr.io"
+          username: "cfaprdbatchcr"
+          password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }}
+
+      - name: Build and push model pipeline image for Azure batch
+        id: build_and_push_model_image
+        uses: docker/build-push-action@v6
+        with:
+          push: false # This can be toggled manually for tweaking.
+          tags: |
+            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:buildtest-latest
+            ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:buildtest-${{ github.sha }}
+          file: ./Dockerfile 

From ed939ad658b23cdf5af353d46f1b6f10f484c06b Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 27 Sep 2024 01:30:06 +0000
Subject: [PATCH 47/53] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .github/workflows/1-Build-Dependencies-Image.yaml   | 6 +++---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml       | 2 +-
 .github/workflows/2_pre-Test-Model-Image-Build.yaml | 6 +++---
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/1-Build-Dependencies-Image.yaml b/.github/workflows/1-Build-Dependencies-Image.yaml
index 24c6c6c..4052605 100644
--- a/.github/workflows/1-Build-Dependencies-Image.yaml
+++ b/.github/workflows/1-Build-Dependencies-Image.yaml
@@ -2,7 +2,7 @@ name: Build Dependencies Container Image
 run-name: (1) Dependencies Image Build - ${{ github.event.head_commit.message }} - ${{ github.ref_name }}
 
 # This GitHub Actions workflow builds a Dependencies Docker image for the
-# cfa-epinow2-pipeline-docker project. 
+# cfa-epinow2-pipeline-docker project.
 #
 # - The `build_image_dependencies` job carries out the first part of a
 #   multi-stage build. It downloads and installs all the dependencies
@@ -15,8 +15,8 @@ run-name: (1) Dependencies Image Build - ${{ github.event.head_commit.message }}
 #   have not changed. This is by hashing the `DESCRIPTION` file and the
 #   `Dockerfile-dependencies` file.
 
-# - This built docker image is then referenced by the simply named 'Dockerfile', built in: 
-#     - "2_pre-Test-Model-Image-Build" and 
+# - This built docker image is then referenced by the simply named 'Dockerfile', built in:
+#     - "2_pre-Test-Model-Image-Build" and
 #     - "2-Run-Epinow2-Pipeline.yaml"
 
 on:
diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index c1d33f7..eb938c5 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -2,7 +2,7 @@ name: Run Epinow2 Pipeline
 run-name: (2) Pipeline - "${{ github.event.head_commit.message }}" - ${{ github.ref_name }}
 
 # This GitHub Actions workflow builds a Docker image for the
-# cfa-epinow2-pipeline-docker project. 
+# cfa-epinow2-pipeline-docker project.
 #
 # - The build_image job builds the final image using the `Dockerfile` file.
 #   It uses the image built in "1-Build-Dependencies-Image.yaml" as a base image.
diff --git a/.github/workflows/2_pre-Test-Model-Image-Build.yaml b/.github/workflows/2_pre-Test-Model-Image-Build.yaml
index d92e57b..29d4c40 100644
--- a/.github/workflows/2_pre-Test-Model-Image-Build.yaml
+++ b/.github/workflows/2_pre-Test-Model-Image-Build.yaml
@@ -11,7 +11,7 @@ on:
   workflow_dispatch:
   schedule:
     # Tuesdays at noon GMT time, 24 hours before the prod container is built and deployed
-    - cron: "0 12 * * 2" 
+    - cron: "0 12 * * 2"
 
 env:
   # Together, these form: cfaprdbatchcr.azurecr.io/cfa-epinow2-pipeline
@@ -21,7 +21,7 @@ env:
 jobs:
 
   _01_build-model-image:
-    runs-on: cfa-cdcgov 
+    runs-on: cfa-cdcgov
     steps:
 
       - name: Login to the Container Registry
@@ -39,4 +39,4 @@ jobs:
           tags: |
             ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:buildtest-latest
             ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:buildtest-${{ github.sha }}
-          file: ./Dockerfile 
+          file: ./Dockerfile

From 22cc94678192e57fed5506652e7d814238429b50 Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Fri, 27 Sep 2024 01:38:23 +0000
Subject: [PATCH 48/53] ubuntu image update?

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 19 ++-----------------
 1 file changed, 2 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index c1d33f7..f7e7f13 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -89,7 +89,7 @@ jobs:
           POOL_ID:           "cfa-epinow2-${{ env.SHORT_SHA }}" # version the pools with the github sha
           BATCH_ACCOUNT:     "cfaprdba"
           BATCH_ENDPOINT:    "https://cfaprdba.eastus.batch.azure.com/"
-          VM_IMAGE_TAG:      "canonical:0001-com-ubuntu-server-focal:20_04-lts"
+          VM_IMAGE_TAG:      "canonical:0001-com-ubuntu-server-focal:24_04-lts"
           NODE_AGENT_SKU_ID: "batch.node.ubuntu 20.04"
           VM_SIZE:           "standard_a4m_v2"
           RESOURCE_GROUP:    ${{ secrets.PRD_RESOURCE_GROUP }}
@@ -113,19 +113,4 @@ jobs:
           --auto-scale-formula "$(cat './batch-autoscale-formula.txt')"
 
           # Let's get this POOL_ID var for the next step too
-          echo "POOL_ID=${{ env.POOL_ID}}" >> $GITHUB_ENV
-
-      # - name: legacy_test
-      #   run: |
-      #     az batch pool create --json-file ./cfa-epinow2-batch-pool-config.json
-
-
-      # - name: Submit azure batch job
-      #   id: submit_batch_job
-      #   env:
-      #     POOL_ID: "$POOL_ID"
-      #     BATCH_JOB_ID: "cfa-epinow2-pipeline-job-${{ env.SHORT_SHA }}"
-      #   run: |
-      #     az batch job create \
-      #     --pool-id "${{ env.POOL_ID }}" \
-      #     --id "${{ env.BATCH_JOB_ID }}"
+          echo "POOL_ID=${{ env.POOL_ID}}" >> $GITHUB_ENV
\ No newline at end of file

From f25af3ea77601d09aaebbf00cf04b516c81e3e7c Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 27 Sep 2024 01:39:01 +0000
Subject: [PATCH 49/53] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 16b7632..38e6c39 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -113,4 +113,4 @@ jobs:
           --auto-scale-formula "$(cat './batch-autoscale-formula.txt')"
 
           # Let's get this POOL_ID var for the next step too
-          echo "POOL_ID=${{ env.POOL_ID}}" >> $GITHUB_ENV
\ No newline at end of file
+          echo "POOL_ID=${{ env.POOL_ID}}" >> $GITHUB_ENV

From da7040cb3d19463294247c75eaee560e59ffee55 Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Fri, 27 Sep 2024 01:44:03 +0000
Subject: [PATCH 50/53] node agent sku also...

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 16b7632..d1ae49e 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -90,7 +90,7 @@ jobs:
           BATCH_ACCOUNT:     "cfaprdba"
           BATCH_ENDPOINT:    "https://cfaprdba.eastus.batch.azure.com/"
           VM_IMAGE_TAG:      "canonical:0001-com-ubuntu-server-focal:24_04-lts"
-          NODE_AGENT_SKU_ID: "batch.node.ubuntu 20.04"
+          NODE_AGENT_SKU_ID: "batch.node.ubuntu 24.04"
           VM_SIZE:           "standard_a4m_v2"
           RESOURCE_GROUP:    ${{ secrets.PRD_RESOURCE_GROUP }}
 

From 3cd0126d4f92c82c8de0d3e9eef3761cd6dc59ef Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Fri, 27 Sep 2024 01:46:37 +0000
Subject: [PATCH 51/53] attempting ubuntu 22 as 24 not yet supported

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index ef32e29..279f804 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -89,8 +89,8 @@ jobs:
           POOL_ID:           "cfa-epinow2-${{ env.SHORT_SHA }}" # version the pools with the github sha
           BATCH_ACCOUNT:     "cfaprdba"
           BATCH_ENDPOINT:    "https://cfaprdba.eastus.batch.azure.com/"
-          VM_IMAGE_TAG:      "canonical:0001-com-ubuntu-server-focal:24_04-lts"
-          NODE_AGENT_SKU_ID: "batch.node.ubuntu 24.04"
+          VM_IMAGE_TAG:      "canonical:0001-com-ubuntu-server-focal:22_04-lts"
+          NODE_AGENT_SKU_ID: "batch.node.ubuntu 22.04"
           VM_SIZE:           "standard_a4m_v2"
           RESOURCE_GROUP:    ${{ secrets.PRD_RESOURCE_GROUP }}
 

From 835596572aad77347bf25316b9afd6a386399970 Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Fri, 27 Sep 2024 01:51:29 +0000
Subject: [PATCH 52/53] simplified commit message display in workflow gui

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml       | 2 +-
 .github/workflows/2_pre-Test-Model-Image-Build.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 279f804..8fd9fd9 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -1,5 +1,5 @@
 name: Run Epinow2 Pipeline
-run-name: (2) Pipeline - "${{ github.event.head_commit.message }}" - ${{ github.ref_name }}
+run-name: (2) Pipeline - "${{ github.event.head_commit.message }}"
 
 # This GitHub Actions workflow builds a Docker image for the
 # cfa-epinow2-pipeline-docker project.
diff --git a/.github/workflows/2_pre-Test-Model-Image-Build.yaml b/.github/workflows/2_pre-Test-Model-Image-Build.yaml
index 29d4c40..9a79e3d 100644
--- a/.github/workflows/2_pre-Test-Model-Image-Build.yaml
+++ b/.github/workflows/2_pre-Test-Model-Image-Build.yaml
@@ -1,5 +1,5 @@
 name: Test Model Image Build
-run-name: (2_pre) Test Model Image - "${{ github.event.head_commit.message }}" - ${{ github.ref_name }}
+run-name: (2_pre) Test Model Image - "${{ github.event.head_commit.message }}"
 
 # This GitHub Actions workflow builds a Docker image for the
 # cfa-epinow2-pipeline-docker project. In-container tests can be added here.

From b5ebbf7d647c0486b86d340cf039e03c0aad02e5 Mon Sep 17 00:00:00 2001
From: jkislin <qxk3@cdc.gov>
Date: Fri, 27 Sep 2024 01:52:33 +0000
Subject: [PATCH 53/53] reverted to ubuntu 20. will have to investigate

---
 .github/workflows/2-Run-Epinow2-Pipeline.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
index 8fd9fd9..d580194 100644
--- a/.github/workflows/2-Run-Epinow2-Pipeline.yaml
+++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml
@@ -89,8 +89,8 @@ jobs:
           POOL_ID:           "cfa-epinow2-${{ env.SHORT_SHA }}" # version the pools with the github sha
           BATCH_ACCOUNT:     "cfaprdba"
           BATCH_ENDPOINT:    "https://cfaprdba.eastus.batch.azure.com/"
-          VM_IMAGE_TAG:      "canonical:0001-com-ubuntu-server-focal:22_04-lts"
-          NODE_AGENT_SKU_ID: "batch.node.ubuntu 22.04"
+          VM_IMAGE_TAG:      "canonical:0001-com-ubuntu-server-focal:20_04-lts"
+          NODE_AGENT_SKU_ID: "batch.node.ubuntu 20.04"
           VM_SIZE:           "standard_a4m_v2"
           RESOURCE_GROUP:    ${{ secrets.PRD_RESOURCE_GROUP }}