daily-db-publisher.yaml

name: 'Daily DB Publisher'
on:
  # allow for kicking off DB builds manually
  workflow_dispatch:
    inputs:
      publish-databases:
        description: "build new databases and upload to S3"
        type: boolean
        required: true
        default: true
      publish-listing:
        description: "use S3 state to update and publish listing file"
        type: boolean
        required: true
        default: true

  # run 4 AM (UTC) daily
  schedule:
    - cron:  '0 4 * * *'

env:
  CGO_ENABLED: "0"
  AWS_BUCKET: toolbox-data.anchore.io
  AWS_BUCKET_PATH: grype/databases
  AWS_DEFAULT_REGION: us-west-2
  SLACK_NOTIFICATIONS: true

jobs:
  discover-schema-versions:
    # note about workflow dispatch inputs and booleans:
    # a) booleans come across as string types :(
    # b) if not using workflow_dispatch the default values are empty, which means we want these to effectively evaluate to true (so only check the negative case)
    if: ${{ github.event.inputs.publish-databases != 'false' }}
    name: "Pull vulnerability data"
    runs-on: ubuntu-20.04
    outputs:
      schema-versions: ${{ steps.read-schema-versions.outputs.schema-versions }}
      pull-date: ${{ steps.timestamp.outputs.date }}
    # set the permissions granted to the github token to read the pull cache from ghcr.io
    permissions:
      contents: read
      packages: read
    steps:

      - uses: actions/checkout@v3

      - name: Read supported schema versions
        id: read-schema-versions
        run: |
          content=`cat grype-schema-version-mapping.json | jq -c 'keys'`
          echo "schema-versions=$content" >> $GITHUB_OUTPUT

  generate-and-publish-dbs:
    # note about workflow dispatch inputs and booleans:
    # a) booleans come across as string types :(
    # b) if not using workflow_dispatch the default values are empty, which means we want these to effectively evaluate to true (so only check the negative case)
    if: ${{ github.event.inputs.publish-databases != 'false' }}
    name: "Generate and publish DBs"
    needs: discover-schema-versions
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        schema-version: ${{fromJson(needs.discover-schema-versions.outputs.schema-versions)}}
    # set the permissions granted to the github token to read the pull cache from ghcr.io
    permissions:
      contents: read
      packages: read
    steps:

      - uses: actions/checkout@v3
        with:
          # this downloads and initializes LFS, but does not pull the objects
          lfs: true

      - name: Checkout LFS objects
        # lfs pull does a lfs fetch and lfs checkout, this is NOT the same as "git pull"
        run: git lfs pull

      - name: Bootstrap environment
        uses: ./.github/actions/bootstrap

      - name: Install dependencies and package
        run: |
          cd publish && poetry install

      - name: Login to ghcr.io
        run: |
          echo ${{ secrets.GITHUB_TOKEN }} | oras login ghcr.io --username ${{ github.actor }} --password-stdin

      - name: Pull vulnerability data
        run: make download-all-provider-cache

      - name: Generate DB (schema ${{ matrix.schema-version }})
        run: |
          cd publish && \
          poetry run publisher generate --schema-version ${{ matrix.schema-version }}

      - name: Upload DB (schema ${{ matrix.schema-version }})
        run: publish/upload-dbs.sh ${{ env.AWS_BUCKET }} ${{ env.AWS_BUCKET_PATH }}
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.TOOLBOX_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLBOX_AWS_SECRET_ACCESS_KEY }}

      - uses: 8398a7/action-slack@v3
        with:
          status: ${{ job.status }}
          fields: workflow,eventName
          text: Publishing the Grype DB has failed (schema ${{ matrix.schema-version }})
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }}
        if: ${{ failure() && env.SLACK_NOTIFICATIONS == 'true' }}

  publish-listing-file:
    # fun! https://github.com/actions/runner/issues/491#issuecomment-850884422
    # essentially even if the workflow dispatch job is skipping steps, we still want to run this step.
    # however, if not running from a workflow dispatch then we want the job ordering to be honored.
    # also...
    # note about workflow dispatch inputs and booleans:
    # a) booleans come across as string types :(
    # b) if not using workflow_dispatch the default values are empty, which means we want these to effectively evaluate to true (so only check the negative case)
    if: |
      always() &&
      (needs.generate-and-publish-dbs.result == 'success' || needs.generate-and-publish-dbs.result == 'skipped') &&
      github.event.inputs.publish-listing != 'false'

    name: "Publish listing file"
    needs: generate-and-publish-dbs
    runs-on: ubuntu-20.04
    steps:

      - uses: actions/checkout@v3
        with:
          # this downloads and initializes LFS, but does not pull the objects
          lfs: true

      - name: Checkout LFS objects
        # lfs pull does a lfs fetch and lfs checkout, this is NOT the same as "git pull"
        run: git lfs pull

      - name: Bootstrap environment
        uses: ./.github/actions/bootstrap

      - name: Install dependencies and package
        run: |
          cd publish && poetry install

      - name: Publish listing file
        run: |
          cd publish && \
          poetry run publisher upload-listing --s3-bucket ${{ env.AWS_BUCKET }} --s3-path ${{ env.AWS_BUCKET_PATH }}
        env:
            AWS_ACCESS_KEY_ID: ${{ secrets.TOOLBOX_AWS_ACCESS_KEY_ID }}
            AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLBOX_AWS_SECRET_ACCESS_KEY }}

      - uses: 8398a7/action-slack@v3
        with:
          status: ${{ job.status }}
          fields: workflow,eventName
          text: Publishing the Grype DB listing file has failed
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }}
        if: ${{ failure() && env.SLACK_NOTIFICATIONS == 'true' }}