From c3baf17cd1e3f8f24ebb1971403d1dd4bf905567 Mon Sep 17 00:00:00 2001 From: austincanada <162146803+austincanada@users.noreply.github.com> Date: Tue, 18 Jun 2024 13:10:00 -0400 Subject: [PATCH] Add opt-out import and cclf-import workflows (#957) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 🎫 Ticket https://jira.cms.gov/browse/BCDA-7899 ## 🛠 Changes Added cclf-import workflows. Added Sean's opt-out-import workflows. Added unit testing workflow for cclf and opt-out workflow. ## ℹ️ Context for reviewers Creating a merge PR in order to test run the workflows. Some things will probably have to change within the workflows themselves. ## ✅ Acceptance Validation Workflows completed: Screenshot 2024-06-18 at 12 23 13 PM Screenshot 2024-06-18 at 12 23 38 PM Lambdas updated: Screenshot 2024-06-18 at 12 29 49 PM ## 🔒 Security Implications - [ ] This PR adds a new software dependency or dependencies. - [ ] This PR modifies or invalidates one or more of our security controls. - [ ] This PR stores or transmits data that was not stored or transmitted before. - [ ] This PR requires additional review of its security implications for other reasons. If any security implications apply, add Jason Ashbaugh (GitHub username: StewGoin) as a reviewer and do not merge this PR without his approval. --------- Co-authored-by: Sean Fern --- .github/workflows/cclf-import-dev-deploy.yml | 38 ++++++++++++++ .github/workflows/cclf-import-prod-deploy.yml | 25 +++++++++ .github/workflows/cclf-import-test-deploy.yml | 35 +++++++++++++ .../cclf-import-test-integration.yml | 51 +++++++++++++++++++ .github/workflows/ci-workflow.yml | 6 +++ .../workflows/opt-out-import-dev-deploy.yml | 38 ++++++++++++++ .../workflows/opt-out-import-prod-deploy.yml | 24 +++++++++ .../workflows/opt-out-import-test-deploy.yml | 35 +++++++++++++ .../opt-out-import-test-integration.yml | 51 +++++++++++++++++++ 9 files changed, 303 insertions(+) create mode 100644 .github/workflows/cclf-import-dev-deploy.yml create mode 100644 .github/workflows/cclf-import-prod-deploy.yml create mode 100644 .github/workflows/cclf-import-test-deploy.yml create mode 100644 .github/workflows/cclf-import-test-integration.yml create mode 100644 .github/workflows/opt-out-import-dev-deploy.yml create mode 100644 .github/workflows/opt-out-import-prod-deploy.yml create mode 100644 .github/workflows/opt-out-import-test-deploy.yml create mode 100644 .github/workflows/opt-out-import-test-integration.yml diff --git a/.github/workflows/cclf-import-dev-deploy.yml b/.github/workflows/cclf-import-dev-deploy.yml new file mode 100644 index 000000000..3d6371f8b --- /dev/null +++ b/.github/workflows/cclf-import-dev-deploy.yml @@ -0,0 +1,38 @@ +name: cclf-import dev deploy + +on: + push: + branches: + - main + paths: + - cclf-import/** + - .github/workflows/cclf-import-dev-deploy.yml + workflow_dispatch: + +jobs: + test: + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + defaults: + run: + working-directory: bcda + environment: dev + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + - name: Build cclf-import zip file + run: | + go build -o bin/cclf-import ./lambda/cclf/main.go + zip function.zip bin/cclf-import + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.ACCOUNT_ID }}:role/delegatedadmin/developer/bcda-dev-github-actions + - name: Upload and reload + run: | + aws s3 cp --no-progress function.zip \ + s3://bcda-dev-cclf-import-function/function-${{ github.sha }}.zip + aws lambda update-function-code --function-name bcda-dev-cclf-import \ + --s3-bucket bcda-dev-cclf-import-function --s3-key function-${{ github.sha }}.zip diff --git a/.github/workflows/cclf-import-prod-deploy.yml b/.github/workflows/cclf-import-prod-deploy.yml new file mode 100644 index 000000000..2d3a27334 --- /dev/null +++ b/.github/workflows/cclf-import-prod-deploy.yml @@ -0,0 +1,25 @@ +name: cclf-import prod deploy + +on: + workflow_dispatch: + +jobs: + deploy: + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + environment: prod + steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.ACCOUNT_ID }}:role/delegatedadmin/developer/bcda-prod-github-actions + - name: Promote lambda code from test to prod + run: | + aws s3 cp --no-progress \ + s3://bcda-test-cclf-import-function/function-${{ github.sha }}.zip \ + s3://bcda-prod-cclf-import-function/function-${{ github.sha }}.zip + aws lambda update-function-code --function-name bcda-prod-cclf-import \ + --s3-bucket bcda-prod-cclf-import-function --s3-key function-${{ github.sha }}.zip + diff --git a/.github/workflows/cclf-import-test-deploy.yml b/.github/workflows/cclf-import-test-deploy.yml new file mode 100644 index 000000000..df2d4be76 --- /dev/null +++ b/.github/workflows/cclf-import-test-deploy.yml @@ -0,0 +1,35 @@ +name: cclf-import test deploy + +on: + workflow_call: + workflow_dispatch: + +jobs: + test: + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + defaults: + run: + working-directory: bcda + environment: test + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + - name: Build cclf-import zip file + env: + CGO_ENABLED: 0 + run: | + go build -o bin/cclf-import ./lambda/cclf/main.go + zip function.zip bin/cclf-import + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.ACCOUNT_ID }}:role/delegatedadmin/developer/bcda-test-github-actions + - name: Upload and reload + run: | + aws s3 cp --no-progress function.zip \ + s3://bcda-test-cclf-import-function/function-${{ github.sha }}.zip + aws lambda update-function-code --function-name bcda-test-cclf-import \ + --s3-bucket bcda-test-cclf-import-function --s3-key function-${{ github.sha }}.zip diff --git a/.github/workflows/cclf-import-test-integration.yml b/.github/workflows/cclf-import-test-integration.yml new file mode 100644 index 000000000..537044fd8 --- /dev/null +++ b/.github/workflows/cclf-import-test-integration.yml @@ -0,0 +1,51 @@ +name: cclf-import test integration + +on: + pull_request: + paths: + - .github/workflows/cclf-import-test-integration.yml + - .github/workflows/cclf-import-test-deploy.yml + - cclf/** + workflow_dispatch: + +# Ensure we have only one integration test running at a time +concurrency: + group: cclf-import-test-integration + +jobs: + # Deploy first if triggered by pull_request + deploy: + if: ${{ github.event_name == 'pull_request' }} + uses: ./.github/workflows/cclf-import-test-deploy.yml + secrets: inherit + + trigger: + if: ${{ always() }} + needs: deploy + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + defaults: + run: + working-directory: bcda + steps: + - uses: actions/checkout@v4 + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.ACCOUNT_ID }}:role/delegatedadmin/developer/bcda-test-cclf-import-function + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ vars.AWS_REGION }} + # Note that we use the BFD role with access to the bucket + role-to-assume: arn:aws:iam::${{ secrets.BFD_ACCOUNT_ID }}:role/bfd-test-eft-bcda-bucket-role + role-chaining: true + role-skip-session-tagging: true + - name: Upload test file to the BFD bucket to trigger lambda function via SNS message + run: | + aws s3 cp --no-progress ../shared_files/cclf/files/synthetic/test/small/ZC0 \ + s3://bfd-test-eft/bfdeft01/bcda/in/T.NGD.DPC.RSP.D$(date +'%y%m%d').T$(date +'%H%M%S')1.IN + + # TODO Check bucket for response file + # TODO Run another job to check database for update diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index 95786bb65..58b9a5e65 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -4,7 +4,13 @@ on: push: branches: - main + paths-ignore: + - .github/workflows/opt-out-import-* + - optout/** pull_request: + paths-ignore: + - .github/workflows/opt-out-import-* + - optout/** env: COMPOSE_INTERACTIVE_NO_CLI: 1 diff --git a/.github/workflows/opt-out-import-dev-deploy.yml b/.github/workflows/opt-out-import-dev-deploy.yml new file mode 100644 index 000000000..e28ada44a --- /dev/null +++ b/.github/workflows/opt-out-import-dev-deploy.yml @@ -0,0 +1,38 @@ +name: opt-out-import dev deploy + +on: + push: + branches: + - main + paths: + - optout/** + - .github/workflows/opt-out-import-dev-deploy.yml + workflow_dispatch: + +jobs: + test: + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + defaults: + run: + working-directory: bcda + environment: dev + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + - name: Build opt-out-import zip file + run: | + go build -o bin/opt-out-import ./lambda/optout/main.go + zip function.zip bin/opt-out-import + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.ACCOUNT_ID }}:role/delegatedadmin/developer/bcda-dev-github-actions + - name: Upload and reload + run: | + aws s3 cp --no-progress function.zip \ + s3://bcda-dev-opt-out-import-function/function-${{ github.sha }}.zip + aws lambda update-function-code --function-name bcda-dev-opt-out-import \ + --s3-bucket bcda-dev-opt-out-import-function --s3-key function-${{ github.sha }}.zip diff --git a/.github/workflows/opt-out-import-prod-deploy.yml b/.github/workflows/opt-out-import-prod-deploy.yml new file mode 100644 index 000000000..d208cb6d0 --- /dev/null +++ b/.github/workflows/opt-out-import-prod-deploy.yml @@ -0,0 +1,24 @@ +name: opt-out-import prod deploy + +on: + workflow_dispatch: + +jobs: + deploy: + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + environment: prod + steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.ACCOUNT_ID }}:role/delegatedadmin/developer/bcda-prod-github-actions + - name: Promote lambda code from test to prod + run: | + aws s3 cp --no-progress \ + s3://bcda-test-opt-out-import-function/function-${{ github.sha }}.zip \ + s3://bcda-prod-opt-out-import-function/function-${{ github.sha }}.zip + aws lambda update-function-code --function-name bcda-prod-opt-out-import \ + --s3-bucket bcda-prod-opt-out-import-function --s3-key function-${{ github.sha }}.zip diff --git a/.github/workflows/opt-out-import-test-deploy.yml b/.github/workflows/opt-out-import-test-deploy.yml new file mode 100644 index 000000000..aa35caec5 --- /dev/null +++ b/.github/workflows/opt-out-import-test-deploy.yml @@ -0,0 +1,35 @@ +name: opt-out-import test deploy + +on: + workflow_call: + workflow_dispatch: + +jobs: + test: + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + defaults: + run: + working-directory: bcda + environment: test + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + - name: Build opt-out-import zip file + env: + CGO_ENABLED: 0 + run: | + go build -o bin/opt-out-import ./lambda/optout/main.go + zip function.zip bin/opt-out-import + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.ACCOUNT_ID }}:role/delegatedadmin/developer/bcda-test-github-actions + - name: Upload and reload + run: | + aws s3 cp --no-progress function.zip \ + s3://bcda-test-opt-out-import-function/function-${{ github.sha }}.zip + aws lambda update-function-code --function-name bcda-test-opt-out-import \ + --s3-bucket bcda-test-opt-out-import-function --s3-key function-${{ github.sha }}.zip diff --git a/.github/workflows/opt-out-import-test-integration.yml b/.github/workflows/opt-out-import-test-integration.yml new file mode 100644 index 000000000..96aced9a3 --- /dev/null +++ b/.github/workflows/opt-out-import-test-integration.yml @@ -0,0 +1,51 @@ +name: opt-out-import test integration + +on: + pull_request: + paths: + - .github/workflows/opt-out-import-test-integration.yml + - .github/workflows/opt-out-import-test-deploy.yml + - optout/** + workflow_dispatch: + +# Ensure we have only one integration test running at a time +concurrency: + group: opt-out-import-test-integration + +jobs: + # Deploy first if triggered by pull_request + deploy: + if: ${{ github.event_name == 'pull_request' }} + uses: ./.github/workflows/opt-out-import-test-deploy.yml + secrets: inherit + + trigger: + if: ${{ always() }} + needs: deploy + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./optout + steps: + - uses: actions/checkout@v4 + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.ACCOUNT_ID }}:role/delegatedadmin/developer/bcda-test-opt-out-import-function + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ vars.AWS_REGION }} + # Note that we use the BFD role with access to the bucket + role-to-assume: arn:aws:iam::${{ secrets.BFD_ACCOUNT_ID }}:role/bfd-test-eft-bcda-bucket-role + role-chaining: true + role-skip-session-tagging: true + - name: Upload test file to the BFD bucket to trigger lambda function via SNS message + run: | + aws s3 cp --no-progress ../shared_files/synthetic1800MedicareFiles/test/T\#EFT.ON.ACO.NGD1800.DPRF.D181120.T1000009 \ + s3://bfd-test-eft/bfdeft01/bcda/in/T.NGD.DPC.RSP.D$(date +'%y%m%d').T$(date +'%H%M%S')1.IN + + # TODO Check bucket for response file + # TODO Run another job to check database for update