Add integration tests for prod/staging

.github/workflows/upload-and-deploy.yaml

@@ -10,6 +10,7 @@ env:
   PYTHON_VERSION: 3.9
   DEV_INPUT_BUCKET: recover-dev-input-data
   DEV_PROCESSED_BUCKET: recover-dev-processed-data
+  PROD_INPUT_BUCKET: recover-input-data
 
 jobs:
 
@@ -313,3 +314,41 @@ jobs:
 
       - name: Deploy sceptre stacks to staging on prod
         run: pipenv run sceptre --var "namespace=staging" launch prod --yes
+
+
+  integration-test-staging:
+    name: Triggers staging workflow with production data
+    runs-on: ubuntu-latest
+    needs: sceptre-deploy-staging
+    environment: prod
+    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Setup code, pipenv, aws
+        uses: Sage-Bionetworks/action-pipenv-aws-setup@v3
+        with:
+          role_to_assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }}
+          role_session_name: integration-test-${{ github.run_id }}
+          python_version: ${{ env.PYTHON_VERSION }}
+
+      - name: generate test events
+        run: >
+          pipenv run python src/lambda_function/s3_to_glue/events/generate_test_event.py
+          --input-bucket $PROD_INPUT_BUCKET
+          --input-key-prefix $NAMESPACE
+          --output-directory ./src/lambda_function/s3_to_glue/events/
+
+      - name: Setup sam
+        uses: aws-actions/setup-sam@v2
+
+      - name: sam build lambda
+        run: >
+          sam build
+          -t src/lambda_function/s3_to_glue/template.yaml
+
+      - name: Invoke Lambda
+        run: |
+          cd src/lambda_function/s3_to_glue/
+          sam local invoke -e events/records.json --parameter-overrides "PrimaryWorkflowName=staging-PrimaryWorkflow"