diff --git a/.github/workflows/1-Build-Dependencies-Image.yaml b/.github/workflows/1-Build-Dependencies-Image.yaml new file mode 100644 index 0000000..4052605 --- /dev/null +++ b/.github/workflows/1-Build-Dependencies-Image.yaml @@ -0,0 +1,74 @@ +name: Build Dependencies Container Image +run-name: (1) Dependencies Image Build - ${{ github.event.head_commit.message }} - ${{ github.ref_name }} + +# This GitHub Actions workflow builds a Dependencies Docker image for the +# cfa-epinow2-pipeline-docker project. +# +# - The `build_image_dependencies` job carries out the first part of a +# multi-stage build. It downloads and installs all the dependencies +# listed in the `DESCRIPTION` file. It uses the `Dockerfile-dependencies` +# file to build the image. +# +# The built image is then pushed to the corresponding registry. +# +# The process is cached to avoid rebuilding the image if the dependencies +# have not changed. This is by hashing the `DESCRIPTION` file and the +# `Dockerfile-dependencies` file. + +# - This built docker image is then referenced by the simply named 'Dockerfile', built in: +# - "2_pre-Test-Model-Image-Build" and +# - "2-Run-Epinow2-Pipeline.yaml" + +on: + push: + branches: + - main + paths: + - "./Dockerfile-dependencies" # the dockerfile this workflow builds from + - "./.github/workflows/build-dependency-image.yaml" # this workflow + + pull_request: + branches: + - main + paths: + - "./Dockerfile-dependencies" # the dockerfile this workflow builds from + - "./.github/workflows/build-dependency-image.yaml" # this workflow + + workflow_dispatch: + +env: + # Together, these form: cfaprdbatchcr.azurecr.io/cfa-epinow2-pipeline + REGISTRY: cfaprdbatchcr.azurecr.io + IMAGE_NAME: cfa-epinow2-pipeline + +jobs: + Job01-build_image_dependencies: + runs-on: cfa-cdcgov # VM based runner serving CFA's cdcgov repos (as opposed to cdcent) + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Check cache + uses: actions/cache@v3 + id: cache + with: + key: docker-dependencies-${{ runner.os }}-${{ hashFiles('./DESCRIPTION', './Dockerfile-dependencies') }} + path: + ./DESCRIPTION + + - name: Login to the Container Registry + uses: docker/login-action@v3 + with: + registry: "cfaprdbatchcr.azurecr.io" + username: "cfaprdbatchcr" + password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }} + + - name: Build and push + if: steps.cache.outputs.cache-hit != 'true' + uses: docker/build-push-action@v6 + with: + push: true + tags: | + ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:latest + ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:${{ github.sha }} + file: ./Dockerfile-dependencies diff --git a/.github/workflows/2-Run-Epinow2-Pipeline.yaml b/.github/workflows/2-Run-Epinow2-Pipeline.yaml new file mode 100644 index 0000000..d580194 --- /dev/null +++ b/.github/workflows/2-Run-Epinow2-Pipeline.yaml @@ -0,0 +1,116 @@ +name: Run Epinow2 Pipeline +run-name: (2) Pipeline - "${{ github.event.head_commit.message }}" + +# This GitHub Actions workflow builds a Docker image for the +# cfa-epinow2-pipeline-docker project. +# +# - The build_image job builds the final image using the `Dockerfile` file. +# It uses the image built in "1-Build-Dependencies-Image.yaml" as a base image. +# +# During the build process, the package is installed and built. Furthermore +# the package is checked using `R CMD check` to ensure that it is working +# correctly. +# +# Once the image is built, it is pushed to the corresponding registry. + +on: + push: + branches: + - main + pull_request: + branches: + - main + workflow_dispatch: + schedule: + - cron: "0 12 * * 3" + +env: + # Together, these form: cfaprdbatchcr.azurecr.io/cfa-epinow2-pipeline + REGISTRY: cfaprdbatchcr.azurecr.io + IMAGE_NAME: cfa-epinow2-pipeline + +jobs: + + _01_build-model-image: + runs-on: cfa-cdcgov # + steps: + + - name: Login to the Container Registry + uses: docker/login-action@v3 + with: + registry: "cfaprdbatchcr.azurecr.io" + username: "cfaprdbatchcr" + password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }} + + # Comment out if you want to just test the pool creation without waiting for build and push + - name: Build and push model pipeline image for Azure batch # second stage, the actual payload + id: build_and_push_model_image + uses: docker/build-push-action@v6 + with: + push: true + tags: | + ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:latest + ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:${{ github.sha }} + file: ./Dockerfile # second stage Dockerfile + + _02_create-batch-pool-and-submit-jobs: + runs-on: cfa-cdcgov + needs: + - _01_build-model-image + + permissions: + contents: read + packages: write + + steps: + - name: Checkout Repo + id: checkout_repo + uses: actions/checkout@v4 + + - name: Login to Azure with NNH Service Principal + id: azure_login_2 + uses: azure/login@v2 + with: + # managed by EDAV. Contact Amit Mantri or Jon Kislin if you have issues. + creds: ${{ secrets.EDAV_CFA_PREDICT_NNHT_SP }} + + + - name: Get Github Short sha + run: | + shortSHA=$(git rev-parse --short ${{ github.sha }}) + echo "SHORT_SHA=$shortSHA" >> $GITHUB_ENV + + - name: Create cfa-epinow2-pipeline Pool + id: create_batch_pool + + # Every Azure Batch Pool parameter can simply go here, + # no python module or config toml necessary + env: + POOL_ID: "cfa-epinow2-${{ env.SHORT_SHA }}" # version the pools with the github sha + BATCH_ACCOUNT: "cfaprdba" + BATCH_ENDPOINT: "https://cfaprdba.eastus.batch.azure.com/" + VM_IMAGE_TAG: "canonical:0001-com-ubuntu-server-focal:20_04-lts" + NODE_AGENT_SKU_ID: "batch.node.ubuntu 20.04" + VM_SIZE: "standard_a4m_v2" + RESOURCE_GROUP: ${{ secrets.PRD_RESOURCE_GROUP }} + + + # The call to the az cli that actually generates the pool + run: | + az batch account login \ + --resource-group ${{ secrets.PRD_RESOURCE_GROUP }} \ + --name "${{ env.BATCH_ACCOUNT }}" \ + + az batch pool create \ + --account-endpoint "${{ env.BATCH_ENDPOINT }}" \ + --id "${{ env.POOL_ID }}" \ + --image "${{ env.VM_IMAGE_TAG }}" \ + --node-agent-sku-id "${{ env.NODE_AGENT_SKU_ID }}" \ + --vm-size "${{ env.VM_SIZE }}" \ + + az batch pool autoscale enable \ + --pool-id ${{ env.POOL_ID }} \ + --auto-scale-formula "$(cat './batch-autoscale-formula.txt')" + + # Let's get this POOL_ID var for the next step too + echo "POOL_ID=${{ env.POOL_ID}}" >> $GITHUB_ENV diff --git a/.github/workflows/2_pre-Test-Model-Image-Build.yaml b/.github/workflows/2_pre-Test-Model-Image-Build.yaml new file mode 100644 index 0000000..9a79e3d --- /dev/null +++ b/.github/workflows/2_pre-Test-Model-Image-Build.yaml @@ -0,0 +1,42 @@ +name: Test Model Image Build +run-name: (2_pre) Test Model Image - "${{ github.event.head_commit.message }}" + +# This GitHub Actions workflow builds a Docker image for the +# cfa-epinow2-pipeline-docker project. In-container tests can be added here. + +on: + pull_request: + branches: + - main + workflow_dispatch: + schedule: + # Tuesdays at noon GMT time, 24 hours before the prod container is built and deployed + - cron: "0 12 * * 2" + +env: + # Together, these form: cfaprdbatchcr.azurecr.io/cfa-epinow2-pipeline + REGISTRY: cfaprdbatchcr.azurecr.io + IMAGE_NAME: cfa-epinow2-pipeline + +jobs: + + _01_build-model-image: + runs-on: cfa-cdcgov + steps: + + - name: Login to the Container Registry + uses: docker/login-action@v3 + with: + registry: "cfaprdbatchcr.azurecr.io" + username: "cfaprdbatchcr" + password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }} + + - name: Build and push model pipeline image for Azure batch + id: build_and_push_model_image + uses: docker/build-push-action@v6 + with: + push: false # This can be toggled manually for tweaking. + tags: | + ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:buildtest-latest + ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:buildtest-${{ github.sha }} + file: ./Dockerfile diff --git a/.github/workflows/check-news-md.yaml b/.github/workflows/3-check-news-md.yaml similarity index 77% rename from .github/workflows/check-news-md.yaml rename to .github/workflows/3-check-news-md.yaml index 07ae7de..83028a5 100644 --- a/.github/workflows/check-news-md.yaml +++ b/.github/workflows/3-check-news-md.yaml @@ -1,4 +1,8 @@ +# All PRs into main MUST be deliberately labelled in the NEWS.md with a succint but informative entry +# describing the changes made - this workflow checks to make sure that this has been done. + name: Check NEWS.md Update +run-name: (3) Check News.md for Compliance - ${{ github.event.head_commit.message }} - ${{ github.ref_name }} on: pull_request: diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/4-R-CMD-check.yaml similarity index 93% rename from .github/workflows/R-CMD-check.yaml rename to .github/workflows/4-R-CMD-check.yaml index 0489bb7..0aac7d1 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/4-R-CMD-check.yaml @@ -1,5 +1,9 @@ # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help + +name: R-CMD-check +run-name: (4) R-CMD-check - ${{ github.event.head_commit.message }} - ${{ github.ref_name }} + on: pull_request: branches: [main] @@ -7,8 +11,6 @@ on: branches: - main -name: R-CMD-check - jobs: R-CMD-check: runs-on: ${{ matrix.config.os }} diff --git a/.github/workflows/block-fixup.yaml b/.github/workflows/5-block-fixup.yaml similarity index 59% rename from .github/workflows/block-fixup.yaml rename to .github/workflows/5-block-fixup.yaml index b42aeb4..c88a4f5 100644 --- a/.github/workflows/block-fixup.yaml +++ b/.github/workflows/5-block-fixup.yaml @@ -1,4 +1,6 @@ -name: Git Checks +name: Block Fix-up (Git Check) +run-name: (5) Block Fix-up / Git Check - ${{ github.event.head_commit.message }} - ${{ github.ref_name }} + on: [pull_request] diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/6-pkgdown.yaml similarity index 93% rename from .github/workflows/pkgdown.yaml rename to .github/workflows/6-pkgdown.yaml index a7276e8..86e18ca 100644 --- a/.github/workflows/pkgdown.yaml +++ b/.github/workflows/6-pkgdown.yaml @@ -10,6 +10,7 @@ on: workflow_dispatch: name: pkgdown +run-name: (6) pkgdown - ${{ github.event.head_commit.message }} - ${{ github.ref_name }} jobs: pkgdown: diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/7-test-coverage.yaml similarity index 93% rename from .github/workflows/test-coverage.yaml rename to .github/workflows/7-test-coverage.yaml index e8a8471..713acd7 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/7-test-coverage.yaml @@ -7,6 +7,7 @@ on: branches: [main] name: test-coverage +run-name: (7) Test Coverage - ${{ github.event.head_commit.message }} - ${{ github.ref_name }} jobs: test-coverage: @@ -27,6 +28,7 @@ jobs: needs: coverage - name: Test coverage + shell: Rscript {0} run: | covr::codecov( quiet = FALSE, @@ -34,7 +36,6 @@ jobs: install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package"), token = "${{ secrets.CODECOV_TOKEN }}" ) - shell: Rscript {0} - name: Show testthat output if: always() diff --git a/.github/workflows/pr-commands.yaml b/.github/workflows/8-pr-commands.yaml similarity index 96% rename from .github/workflows/pr-commands.yaml rename to .github/workflows/8-pr-commands.yaml index c9914e1..cb11c80 100644 --- a/.github/workflows/pr-commands.yaml +++ b/.github/workflows/8-pr-commands.yaml @@ -5,7 +5,9 @@ on: types: [created] workflow_dispatch: -name: Commands +name: PR Commands +run-name: (8) PR Commands - ${{ github.event.head_commit.message }} - ${{ github.ref_name }} + jobs: document: diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml deleted file mode 100644 index 2cd6653..0000000 --- a/.github/workflows/build-docker.yaml +++ /dev/null @@ -1,73 +0,0 @@ -# This GitHub Actions workflow builds a Docker image for the -# cfa-epinow2-pipeline-docker project. It consists of two jobs: -# build_image_dependencies and build_image. -# -# - The `build_image_dependencies` job carries out the first part of a -# multi-stage build. It downloads and install all the dependencies -# listed in the `DESCRIPTION` file. It uses the `Dockerfile-dependencies` -# file to build the image. -# -# The built image is then pushed to the corresponding registry. -# -# The process is cached to avoid rebuilding the image if the dependencies -# have not changed. This is by hashing the `DESCRIPTION` file and the -# `Dockerfile-dependencies` file. -# -# - The build_image job builds the final image using the `Dockerfile` file. -# It uses the image built in the previous job as a base image. -# -# During the build process, the package is installed and built. Furthermore -# the package is checked using `R CMD check` to ensure that it is working -# correctly. -# -# Once the image is built, it is pushed to the corresponding registry. - -name: Builds Docker image - -on: - push: - branches: - - main - workflow_dispatch: - -env: - IMAGE_NAME: gvegayon/cfa-epinow2-pipeline - REGISTRY: docker.io - -jobs: - build_image_dependencies: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Check cache - uses: actions/cache@v3 - id: cache - with: - key: docker-dependencies-${{ runner.os }}-${{ hashFiles('./DESCRIPTION', './Dockerfile-dependencies') }} - path: - ./DESCRIPTION - - - name: Build and push - uses: docker/build-push-action@v6 - if: steps.cache.outputs.cache-hit != 'true' - with: - push: false - tags: ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:latest - file: ./Dockerfile-dependencies - - outputs: - IMAGE_NAME: ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}:latest - - build_image: - needs: build_image_dependencies - runs-on: ubuntu-latest - - steps: - - name: Build and push - uses: docker/build-push-action@v6 - with: - push: false - tags: ${{ needs.build_image_dependencies.outputs.IMAGE_NAME }} - file: ./Dockerfile diff --git a/.github/workflows/gh-act/2-dry.sh b/.github/workflows/gh-act/2-dry.sh new file mode 100644 index 0000000..f147536 --- /dev/null +++ b/.github/workflows/gh-act/2-dry.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# Requires that you have first run 'gh extension install nektos/gh-act' +# as well as having installed the docker engine and added your user to the docker group + +# This checks syntax before you push to Github Actions, helping with debug hell +# To run the entire pipeline locally, see 2-full.sh + +gh act -P cfa-cdcgov=... -n -W '.github/workflows/2-Run-Epinow2-Pipeline.yaml' diff --git a/.github/workflows/gh-act/2-full.sh b/.github/workflows/gh-act/2-full.sh new file mode 100644 index 0000000..cfafa9a --- /dev/null +++ b/.github/workflows/gh-act/2-full.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Requires that you have first run 'gh extension install nektos/gh-act' +# as well as having installed the docker engine and added your user to the docker group + +# This runs the github actions workflow locally + +gh act -P cfa-cdcgov=catthehacker/ubuntu:full-20.04 -W '.github/workflows/2-Run-Epinow2-Pipeline.yaml' diff --git a/.gitignore b/.gitignore index 2998db1..31f5dcf 100644 --- a/.gitignore +++ b/.gitignore @@ -380,3 +380,7 @@ docs/site/ # applications that require a static environment. #Manifest.toml docs + +# cfa-epinow2-pool-config.json +# for now... will have to gpg encrypt +cfa-epinow2-batch-pool-config.json diff --git a/.secrets.baseline b/.secrets.baseline index 63f42f9..4fc48f2 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1,5 +1,5 @@ { - "version": "1.4.0", + "version": "1.5.0", "plugins_used": [ { "name": "ArtifactoryDetector" @@ -26,6 +26,9 @@ { "name": "GitHubTokenDetector" }, + { + "name": "GitLabTokenDetector" + }, { "name": "HexHighEntropyString", "limit": 3.0 @@ -36,6 +39,9 @@ { "name": "IbmCosHmacDetector" }, + { + "name": "IPPublicDetector" + }, { "name": "JwtTokenDetector" }, @@ -49,9 +55,15 @@ { "name": "NpmDetector" }, + { + "name": "OpenAIDetector" + }, { "name": "PrivateKeyDetector" }, + { + "name": "PypiTokenDetector" + }, { "name": "SendGridDetector" }, @@ -67,6 +79,9 @@ { "name": "StripeDetector" }, + { + "name": "TelegramBotTokenDetector" + }, { "name": "TwilioKeyDetector" } @@ -108,5 +123,5 @@ } ], "results": {}, - "generated_at": "2023-09-24T19:52:08Z" + "generated_at": "2024-09-20T17:50:20Z" } diff --git a/Dockerfile b/Dockerfile index 22adfd2..ac300ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM gvegayon/cfa-epinow2-pipeline-dependencies:latest +# This requires access to the Azure Container Registry +FROM cfaprdbatchcr.azurecr.io/cfa-epinow2-pipeline-dependencies:latest # Will copy the package to the container preserving the directory structure COPY . pkg/ diff --git a/NEWS.md b/NEWS.md index 821858a..5a69e5a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,5 @@ # CFAEpiNow2Pipeline (development version) - +* Now uses CFA Azure ACR and images in the workflows and Dockerfiles, etc. * Added Docker image with all the requirements to build the package. * Bump pre-commit hooks * Fix bug in warning message for incomplete data read (h/t @damonbayer) diff --git a/batch-autoscale-formula.txt b/batch-autoscale-formula.txt new file mode 100644 index 0000000..c98a8ee --- /dev/null +++ b/batch-autoscale-formula.txt @@ -0,0 +1,20 @@ +// In this example, the pool size is adjusted based on the number of tasks in the queue. +// Note that both comments and line breaks are acceptable in formula strings. + +// Get pending tasks for the past 15 minutes. +$samples = $ActiveTasks.GetSamplePercent(TimeInterval_Minute * 15); + +// If we have fewer than 70 percent data points, we use the last sample point, +// otherwise we use the maximum of last sample point and the history average. +$tasks = $samples < 70 ? max(0, $ActiveTasks.GetSample(1)) : +max( $ActiveTasks.GetSample(1), avg($ActiveTasks.GetSample(TimeInterval_Minute * 15))); + +// If number of pending tasks is not 0, set targetVM to pending tasks, otherwise half of current dedicated. +$targetVMs = $tasks > 0 ? $tasks : max(0, $TargetDedicatedNodes / 2); + +// The pool size is capped at 120, if target VM value is more than that, set it to 120. +cappedPoolSize = 120; +$TargetDedicatedNodes = max(0, min($targetVMs, cappedPoolSize)); + +// Set node deallocation mode - keep nodes active only until tasks finish +$NodeDeallocationOption = taskcompletion;