docker-unified.yml

name: Docker Build, Scan, Test
on:
  push:
    branches:
      - master
    paths-ignore:
      - "docs/**"
      - "**.md"
  pull_request:
    branches:
      - "**"
    paths-ignore:
      - "docs/**"
      - "**.md"
  release:
    types: [published]

concurrency:
  # Using `github.run_id` (unique val) instead of `github.ref` here
  # because we don't want to cancel this workflow on master only for PRs
  #   as that makes reproducing issues easier
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true

env:
  DATAHUB_GMS_IMAGE: "linkedin/datahub-gms"
  DATAHUB_FRONTEND_IMAGE: "linkedin/datahub-frontend-react"
  DATAHUB_MAE_CONSUMER_IMAGE: "linkedin/datahub-mae-consumer"
  DATAHUB_MCE_CONSUMER_IMAGE: "linkedin/datahub-mce-consumer"
  DATAHUB_KAFKA_SETUP_IMAGE: "linkedin/datahub-kafka-setup"
  DATAHUB_ELASTIC_SETUP_IMAGE: "linkedin/datahub-elasticsearch-setup"
  DATAHUB_MYSQL_SETUP_IMAGE: "acryldata/datahub-mysql-setup"
  DATAHUB_UPGRADE_IMAGE: "acryldata/datahub-upgrade"
  DATAHUB_INGESTION_BASE_IMAGE: "acryldata/datahub-ingestion-base"
  DATAHUB_INGESTION_IMAGE: "acryldata/datahub-ingestion"

jobs:
  setup:
    runs-on: ubuntu-latest
    outputs:
      tag: ${{ steps.tag.outputs.tag }}
      slim_tag: ${{ steps.tag.outputs.slim_tag }}
      full_tag: ${{ steps.tag.outputs.full_tag }}
      unique_tag: ${{ steps.tag.outputs.unique_tag }}
      unique_slim_tag: ${{ steps.tag.outputs.unique_slim_tag }}
      unique_full_tag: ${{ steps.tag.outputs.unique_full_tag }}
      publish: ${{ steps.publish.outputs.publish }}
      python_release_version: ${{ steps.tag.outputs.python_release_version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Compute Tag
        id: tag
        run: |
          source .github/scripts/docker_helpers.sh
          echo "tag=$(get_tag)" >> $GITHUB_OUTPUT
          echo "slim_tag=$(get_tag_slim)" >> $GITHUB_OUTPUT
          echo "full_tag=$(get_tag_full)" >> $GITHUB_OUTPUT
          echo "unique_tag=$(get_unique_tag)" >> $GITHUB_OUTPUT
          echo "unique_slim_tag=$(get_unique_tag_slim)" >> $GITHUB_OUTPUT
          echo "unique_full_tag=$(get_unique_tag_full)" >> $GITHUB_OUTPUT
          echo "python_release_version=$(get_python_docker_release_v)" >> $GITHUB_OUTPUT
      - name: Check whether publishing enabled
        id: publish
        env:
          ENABLE_PUBLISH: ${{ secrets.DOCKER_PASSWORD != '' && secrets.ACRYL_DOCKER_PASSWORD != '' }}
        run: |
          echo "Enable publish: ${{ env.ENABLE_PUBLISH }}"
          echo "publish=${{ env.ENABLE_PUBLISH }}" >> $GITHUB_OUTPUT

  gms_build:
    name: Build and Push DataHub GMS Docker Image
    runs-on: ubuntu-latest
    needs: setup
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 800
      - name: Pre-build artifacts for docker image
        run: |
          ./gradlew :metadata-service:war:build -x test --parallel
          mv ./metadata-service/war/build/libs/war.war .
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_GMS_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-gms/Dockerfile
          platforms: linux/amd64,linux/arm64/v8
  gms_scan:
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    name: "[Monitoring] Scan GMS images for vulnerabilities"
    runs-on: ubuntu-latest
    needs: [setup, gms_build]
    steps:
      - name: Checkout # adding checkout step just to make trivy upload happy
        uses: actions/checkout@v3
      - name: Download image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@0.8.0
        env:
          TRIVY_OFFLINE_SCAN: true
        with:
          image-ref: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
          format: "template"
          template: "@/contrib/sarif.tpl"
          output: "trivy-results.sarif"
          severity: "CRITICAL,HIGH"
          ignore-unfixed: true
          vuln-type: "os,library"
      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: "trivy-results.sarif"

  mae_consumer_build:
    name: Build and Push DataHub MAE Consumer Docker Image
    runs-on: ubuntu-latest
    needs: setup
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 800
      - name: Pre-build artifacts for docker image
        run: |
          ./gradlew :metadata-jobs:mae-consumer-job:build -x test --parallel
          mv ./metadata-jobs/mae-consumer-job/build/libs/mae-consumer-job.jar .
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-mae-consumer/Dockerfile
          platforms: linux/amd64,linux/arm64/v8
  mae_consumer_scan:
    name: "[Monitoring] Scan MAE consumer images for vulnerabilities"
    runs-on: ubuntu-latest
    needs: [setup, mae_consumer_build]
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    steps:
      - name: Checkout # adding checkout step just to make trivy upload happy
        uses: actions/checkout@v3
      - name: Download image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@0.8.0
        env:
          TRIVY_OFFLINE_SCAN: true
        with:
          image-ref: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
          format: "template"
          template: "@/contrib/sarif.tpl"
          output: "trivy-results.sarif"
          severity: "CRITICAL,HIGH"
          ignore-unfixed: true
          vuln-type: "os,library"
      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: "trivy-results.sarif"

  mce_consumer_build:
    name: Build and Push DataHub MCE Consumer Docker Image
    runs-on: ubuntu-latest
    needs: setup
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 800
      - name: Pre-build artifacts for docker image
        run: |
          ./gradlew :metadata-jobs:mce-consumer-job:build -x test --parallel
          mv ./metadata-jobs/mce-consumer-job/build/libs/mce-consumer-job.jar .
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-mce-consumer/Dockerfile
          platforms: linux/amd64,linux/arm64/v8
  mce_consumer_scan:
    name: "[Monitoring] Scan MCE consumer images for vulnerabilities"
    runs-on: ubuntu-latest
    needs: [setup, mce_consumer_build]
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    steps:
      - name: Checkout # adding checkout step just to make trivy upload happy
        uses: actions/checkout@v3
      - name: Download image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@0.8.0
        env:
          TRIVY_OFFLINE_SCAN: true
        with:
          image-ref: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
          format: "template"
          template: "@/contrib/sarif.tpl"
          output: "trivy-results.sarif"
          severity: "CRITICAL,HIGH"
          ignore-unfixed: true
          vuln-type: "os,library"
      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: "trivy-results.sarif"

  datahub_upgrade_build:
    name: Build and Push DataHub Upgrade Docker Image
    runs-on: ubuntu-latest
    needs: setup
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 800
      - name: Pre-build artifacts for docker image
        run: |
          ./gradlew :datahub-upgrade:build -x test --parallel
          mv ./datahub-upgrade/build/libs/datahub-upgrade.jar .
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_UPGRADE_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-upgrade/Dockerfile
          platforms: linux/amd64,linux/arm64/v8
  datahub_upgrade_scan:
    name: "[Monitoring] Scan DataHub Upgrade images for vulnerabilities"
    runs-on: ubuntu-latest
    needs: [setup, datahub_upgrade_build]
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    steps:
      - name: Checkout # adding checkout step just to make trivy upload happy
        uses: actions/checkout@v3
      - name: Download image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@0.8.0
        env:
          TRIVY_OFFLINE_SCAN: true
        with:
          image-ref: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
          format: "template"
          template: "@/contrib/sarif.tpl"
          output: "trivy-results.sarif"
          severity: "CRITICAL,HIGH"
          ignore-unfixed: true
          vuln-type: "os,library"
      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: "trivy-results.sarif"

  frontend_build:
    name: Build and Push DataHub Frontend Docker Image
    runs-on: ubuntu-latest
    needs: setup
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 800
      - name: Pre-build artifacts for docker image
        run: |
          ./gradlew :datahub-frontend:dist -x test -x yarnTest -x yarnLint --parallel
          mv ./datahub-frontend/build/distributions/datahub-frontend-*.zip datahub-frontend.zip
        env:
          NODE_OPTIONS: "--max-old-space-size=3072"
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_FRONTEND_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-frontend/Dockerfile
          platforms: linux/amd64,linux/arm64/v8
  frontend_scan:
    name: "[Monitoring] Scan Frontend images for vulnerabilities"
    runs-on: ubuntu-latest
    needs: [setup, frontend_build]
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    steps:
      - name: Checkout # adding checkout step just to make trivy upload happy
        uses: actions/checkout@v3
      - name: Download image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@0.8.0
        env:
          TRIVY_OFFLINE_SCAN: true
        with:
          image-ref: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
          format: "template"
          template: "@/contrib/sarif.tpl"
          output: "trivy-results.sarif"
          severity: "CRITICAL,HIGH"
          ignore-unfixed: true
          vuln-type: "os,library"
      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: "trivy-results.sarif"

  kafka_setup_build:
    name: Build and Push DataHub Kafka Setup Docker Image
    runs-on: ubuntu-latest
    needs: setup
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 800
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/kafka-setup/Dockerfile
          platforms: linux/amd64,linux/arm64/v8

  mysql_setup_build:
    name: Build and Push DataHub MySQL Setup Docker Image
    runs-on: ubuntu-latest
    needs: setup
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 800
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/mysql-setup/Dockerfile
          platforms: linux/amd64,linux/arm64/v8

  elasticsearch_setup_build:
    name: Build and Push DataHub Elasticsearch Setup Docker Image
    runs-on: ubuntu-latest
    needs: setup
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 800
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/elasticsearch-setup/Dockerfile
          platforms: linux/amd64,linux/arm64/v8

  datahub_ingestion_base_build:
    name: Build and Push DataHub Ingestion (Base) Docker Image
    runs-on: ubuntu-latest
    outputs:
      tag: ${{ steps.tag.outputs.tag }}
    needs: setup
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 800
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            datahub-ingestion-base:
              - 'docker/datahub-ingestion-base/**'
      - name: Build and push Base Image
        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' }}
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          target: base
          images: |
            ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-ingestion-base/Dockerfile
          platforms: linux/amd64,linux/arm64/v8
      - name: Compute DataHub Ingestion (Base) Tag
        id: tag
        run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }}" >> $GITHUB_OUTPUT
  datahub_ingestion_base_slim_build:
    name: Build and Push DataHub Ingestion (Base-Slim) Docker Image
    runs-on: ubuntu-latest
    outputs:
      tag: ${{ steps.tag.outputs.tag }}
    needs: [setup, datahub_ingestion_base_build]
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 800
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            datahub-ingestion-base:
              - 'docker/datahub-ingestion-base/**'
      - name: Download Base Image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' &&  steps.filter.outputs.datahub-ingestion-base == 'true' }}
        with:
          image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }}
      - name: Build and push Base-Slim Image
        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' }}
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          target: slim-install
          images: |
            ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}
          tags: ${{ needs.setup.outputs.slim_tag }}
          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
          build-args: |
            APP_ENV=slim
            BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-ingestion-base/Dockerfile
          platforms: linux/amd64,linux/arm64/v8
      - name: Compute DataHub Ingestion (Base-Slim) Tag
        id: tag
        run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT
  datahub_ingestion_base_full_build:
    name: Build and Push DataHub Ingestion (Base-Full) Docker Image
    runs-on: ubuntu-latest
    outputs:
      tag: ${{ steps.tag.outputs.tag }}
    needs: [setup, datahub_ingestion_base_build]
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 800
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            datahub-ingestion-base:
              - 'docker/datahub-ingestion-base/**'
      - name: Download Base Image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' &&  steps.filter.outputs.datahub-ingestion-base == 'true' }}
        with:
          image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }}
      - name: Build and push (Base-Full) Image
        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' }}
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          target: full-install
          images: |
            ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}
          tags: ${{ needs.setup.outputs.full_tag }}
          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
          build-args: |
            APP_ENV=full
            BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-ingestion-base/Dockerfile
          platforms: linux/amd64,linux/arm64/v8
      - name: Compute DataHub Ingestion (Base-Full) Tag
        id: tag
        run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_full_tag || 'head' }}" >> $GITHUB_OUTPUT

  datahub_ingestion_slim_build:
    name: Build and Push DataHub Ingestion Docker Images
    runs-on: ubuntu-latest
    outputs:
      tag: ${{ steps.tag.outputs.tag }}
      needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.publish != 'true' }}
    needs: [setup, datahub_ingestion_base_slim_build]
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 800
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            datahub-ingestion-base:
              - 'docker/datahub-ingestion-base/**'
            datahub-ingestion:
              - 'docker/datahub-ingestion/**'
      - name: Build codegen
        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }}
        run: ./gradlew :metadata-ingestion:codegen
      - name: Download Base Image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' && steps.filter.outputs.datahub-ingestion-base == 'true' }}
        with:
          image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}
      - name: Build and push Slim Image
        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }}
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          target: final
          images: |
            ${{ env.DATAHUB_INGESTION_IMAGE }}
          build-args: |
            BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }}
            DOCKER_VERSION=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}
            RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }}
            APP_ENV=slim
          tags: ${{ needs.setup.outputs.slim_tag }}
          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-ingestion/Dockerfile
          platforms: linux/amd64,linux/arm64/v8
      - name: Compute Tag
        id: tag
        run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT
  datahub_ingestion_slim_scan:
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    name: "[Monitoring] Scan Datahub Ingestion Slim images for vulnerabilities"
    runs-on: ubuntu-latest
    needs: [setup, datahub_ingestion_slim_build]
    steps:
      - name: Checkout # adding checkout step just to make trivy upload happy
        uses: actions/checkout@v3
      - name: Download image Slim Image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.datahub_ingestion_slim_build.outputs.needs_artifact_download == 'true' }}
        with:
          image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}
      - name: Run Trivy vulnerability scanner Slim Image
        uses: aquasecurity/trivy-action@0.8.0
        env:
          TRIVY_OFFLINE_SCAN: true
        with:
          image-ref: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}
          format: "template"
          template: "@/contrib/sarif.tpl"
          output: "trivy-results.sarif"
          severity: "CRITICAL,HIGH"
          ignore-unfixed: true
          vuln-type: "os,library"
      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: "trivy-results.sarif"

  datahub_ingestion_full_build:
    name: Build and Push DataHub Ingestion (Full) Docker Images
    runs-on: ubuntu-latest
    outputs:
      tag: ${{ steps.tag.outputs.tag }}
      needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.publish != 'true' }}
    needs: [setup, datahub_ingestion_base_full_build]
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 800
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            datahub-ingestion-base:
              - 'docker/datahub-ingestion-base/**'
            datahub-ingestion:
              - 'docker/datahub-ingestion/**'
      - name: Build codegen
        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }}
        run: ./gradlew :metadata-ingestion:codegen
      - name: Download Base Image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' && steps.filter.outputs.datahub-ingestion-base == 'true' }}
        with:
          image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }}
      - name: Build and push Full Image
        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }}
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          target: final
          images: |
            ${{ env.DATAHUB_INGESTION_IMAGE }}
          build-args: |
            BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }}
            DOCKER_VERSION=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }}
            RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-ingestion/Dockerfile
          platforms: linux/amd64,linux/arm64/v8
      - name: Compute Tag (Full)
        id: tag
        run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.unique_tag || 'head' }}" >> $GITHUB_OUTPUT
  datahub_ingestion_full_scan:
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    name: "[Monitoring] Scan Datahub Ingestion images for vulnerabilities"
    runs-on: ubuntu-latest
    needs: [setup, datahub_ingestion_full_build]
    steps:
      - name: Checkout # adding checkout step just to make trivy upload happy
        uses: actions/checkout@v3
      - name: Download image Full Image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.datahub_ingestion_full_build.outputs.needs_artifact_download == 'true' }}
        with:
          image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }}
      - name: Run Trivy vulnerability scanner Full Image
        uses: aquasecurity/trivy-action@0.8.0
        env:
          TRIVY_OFFLINE_SCAN: true
        with:
          image-ref: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }}
          format: "template"
          template: "@/contrib/sarif.tpl"
          output: "trivy-results.sarif"
          severity: "CRITICAL,HIGH"
          ignore-unfixed: true
          vuln-type: "os,library"
      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: "trivy-results.sarif"

  smoke_test:
    name: Run Smoke Tests
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        test_strategy: ["no_cypress", "cypress_suite1", "cypress_rest"]
    needs:
      [
        setup,
        gms_build,
        frontend_build,
        kafka_setup_build,
        mysql_setup_build,
        elasticsearch_setup_build,
        mae_consumer_build,
        mce_consumer_build,
        datahub_upgrade_build,
        datahub_ingestion_slim_build,
      ]
    steps:
      - name: Disk Check
        run: df -h . && docker images
      - name: Check out the repo
        uses: actions/checkout@v3
      - name: Set up JDK 11
        uses: actions/setup-java@v3
        with:
          distribution: "zulu"
          java-version: 11
      - uses: actions/setup-python@v4
        with:
          python-version: "3.10"
          cache: "pip"
      - name: Install dependencies
        run: ./metadata-ingestion/scripts/install_deps.sh
      - name: Build datahub cli
        run: |
          ./gradlew :metadata-ingestion:install
      - name: Disk Check
        run: df -h . && docker images
      - name: Remove images
        run: docker image prune -a -f || true
      - name: Disk Check
        run: df -h . && docker images
      - name: Download GMS image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Download Frontend image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Download Kafka Setup image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Download Mysql Setup image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Download Elastic Setup image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Download MCE Consumer image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Download MAE Consumer image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Download upgrade image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Download datahub-ingestion-slim image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.datahub_ingestion_slim_build.outputs.needs_artifact_download == 'true' }}
        with:
          image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}
      - name: Disk Check
        run: df -h . && docker images
      - name: run quickstart
        env:
          DATAHUB_TELEMETRY_ENABLED: false
          DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
          DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_INGESTION_IMAGE }}
          ACTIONS_VERSION: ${{ needs.datahub_ingestion_slim_build.outputs.tag }}
          ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor]==0.0.13 acryl-datahub-actions==0.0.13 acryl-datahub==0.10.5"
          ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml"
        run: |
          ./smoke-test/run-quickstart.sh
      - name: sleep 60s
        run: |
          # we are doing this because gms takes time to get ready
          # and we don't have a better readiness check when bootstrap is done
          sleep 60s
      - name: Disk Check
        run: df -h . && docker images
      - name: Disable ES Disk Threshold
        run: |
          curl -XPUT "http://localhost:9200/_cluster/settings" \
          -H 'Content-Type: application/json' -d'{
            "persistent": {
              "cluster": {
                "routing": {
                  "allocation.disk.threshold_enabled": false
                }
              }
            }
          }'
      - name: Remove Source Code
        run: find ./*/* ! -path "./metadata-ingestion*" ! -path "./smoke-test*" ! -path "./gradle*" -delete
      - name: Disk Check
        run: df -h . && docker images
      - name: Smoke test
        env:
          RUN_QUICKSTART: false
          DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
          CLEANUP_DATA: "false"
          TEST_STRATEGY: ${{ matrix.test_strategy }}
        run: |
          echo "$DATAHUB_VERSION"
          ./smoke-test/smoke.sh
      - name: Disk Check
        run: df -h . && docker images
      - name: store logs
        if: failure()
        run: |
          docker ps -a
          docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log || true
          docker logs datahub-actions >& actions-${{ matrix.test_strategy }}.log || true
          docker logs datahub-mae-consumer >& mae-${{ matrix.test_strategy }}.log || true
          docker logs datahub-mce-consumer >& mce-${{ matrix.test_strategy }}.log || true
          docker logs broker >& broker-${{ matrix.test_strategy }}.log || true
          docker logs mysql >& mysql-${{ matrix.test_strategy }}.log || true
          docker logs elasticsearch >& elasticsearch-${{ matrix.test_strategy }}.log || true
          docker logs datahub-frontend-react >& frontend-${{ matrix.test_strategy }}.log || true
      - name: Upload logs
        uses: actions/upload-artifact@v3
        if: failure()
        with:
          name: docker logs
          path: "*.log"
      - name: Upload screenshots
        uses: actions/upload-artifact@v3
        if: failure()
        with:
          name: cypress-snapshots-${{ matrix.test_strategy }}
          path: smoke-test/tests/cypress/cypress/screenshots/
      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: Test Results (smoke tests) ${{ matrix.test_strategy }}
          path: |
            **/build/reports/tests/test/**
            **/build/test-results/test/**
            **/junit.*.xml
      - name: Slack failure notification
        if: failure() && github.event_name == 'push'
        uses: kpritam/slack-job-status-action@v1
        with:
          job-status: ${{ job.status }}
          slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
          channel: github-activities