test-build-deploy.yml

name: ci
permissions: read-all
on:
  push:
    branches:
      - '**'
jobs:
  lint:
    runs-on: ubuntu-20.04
    container:
      image: quay.io/cortexproject/build-image:upgrade-to-go-1.20.4-6025f83e5
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
      - name: Setup Git safe.directory
        run: |
          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
          git config --system --add safe.directory $GITHUB_WORKSPACE
        # Commands in the Makefile are hardcoded with an assumed file structure of the CI container
        # Symlink ensures paths specified in previous commands don’t break
      - name: Sym Link Expected Path to Workspace
        run: |
          mkdir -p /go/src/github.com/cortexproject/cortex
          ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
      - name: cherry pick
        run: git apply .github/workflows/thanos-engine.patch
      - name: Lint
        run: make BUILD_IN_CONTAINER=false lint
      - name: Check Protos
        run: make BUILD_IN_CONTAINER=false check-protos
      - name: Check White Noise.
        run: make BUILD_IN_CONTAINER=false check-white-noise

  test:
    runs-on: ubuntu-20.04
    container:
      image: quay.io/cortexproject/build-image:upgrade-to-go-1.20.4-6025f83e5
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
      - name: Setup Git safe.directory
        run: |
          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
          git config --system --add safe.directory $GITHUB_WORKSPACE
      - name: Sym Link Expected Path to Workspace
        run: |
          mkdir -p /go/src/github.com/cortexproject/cortex
          ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
      - name: cherry pick
        run: git apply .github/workflows/thanos-engine.patch
      - name: Run Tests
        run: make BUILD_IN_CONTAINER=false test

  build:
    runs-on: ubuntu-20.04
    container:
      image: quay.io/cortexproject/build-image:upgrade-to-go-1.20.4-6025f83e5
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
      - name: Setup Git safe.directory
        run: |
          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
          git config --system --add safe.directory $GITHUB_WORKSPACE
      - name: Install Docker Client
        run: ./.github/workflows/scripts/install-docker.sh
      - name: Sym Link Expected Path to Workspace
        run: |
          mkdir -p /go/src/github.com/cortexproject/cortex
          ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
      - name: cherry pick
        run: git apply .github/workflows/thanos-engine.patch
      - name: Build Image
        run: |
          touch build-image/.uptodate
          make BUILD_IN_CONTAINER=false
      - name: Build Website
        run: |
          touch build-image/.uptodate
          make BUILD_IN_CONTAINER=false web-build
      - name: Upload Website Artifact
        uses: actions/upload-artifact@v2
        with:
          name: website public
          path: website/public/
      - name: Save Images
        run: |
          mkdir /tmp/images
          ln -s /tmp/images ./docker-images
          make BUILD_IN_CONTAINER=false save-images
      - name: Create Docker Images Archive
        run: tar -cvf images.tar /tmp/images
      - name: Upload Docker Images Artifact
        uses: actions/upload-artifact@v2
        with:
          name: Docker Images
          path: ./images.tar

  integration:
    needs: build
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        tags:
          - requires_docker
          - integration_alertmanager
          - integration_backward_compatibility
          - integration_memberlist
          - integration_querier
          - integration_ruler
          - integration_query_fuzz
    steps:
      - name: Upgrade golang
        uses: actions/setup-go@v2
        with:
          go-version: 1.20.1
      - name: Checkout Repo
        uses: actions/checkout@v2
      - name: Install Docker Client
        run: sudo ./.github/workflows/scripts/install-docker.sh
      - name: Sym Link Expected Path to Workspace
        run: |
          sudo mkdir -p /go/src/github.com/cortexproject/cortex
          sudo ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
      - name: cherry pick
        run: git apply .github/workflows/thanos-engine.patch
      - name: Download Docker Images Artifacts
        uses: actions/download-artifact@v2
        with:
          name: Docker Images
      - name: Extract Docker Images Archive
        run: tar -xvf images.tar -C /
      - name: Load Docker Images
        run: |
          ln -s /tmp/images ./docker-images
          make BUILD_IN_CONTAINER=false load-images
      - name: Preload Images
        # We download docker images used by integration tests so that all images are available
        # locally and the download time doesn't account in the test execution time, which is subject
        # to a timeout
        run: |
          docker pull minio/minio:RELEASE.2021-10-13T00-23-17Z
          docker pull consul:1.8.4
          docker pull gcr.io/etcd-development/etcd:v3.4.7
          if [ "$TEST_TAGS" = "integration_backward_compatibility" ]; then
            docker pull quay.io/cortexproject/cortex:v1.6.0
            docker pull quay.io/cortexproject/cortex:v1.7.0
            docker pull quay.io/cortexproject/cortex:v1.8.0
            docker pull quay.io/cortexproject/cortex:v1.9.0
            docker pull quay.io/cortexproject/cortex:v1.10.0
            docker pull quay.io/cortexproject/cortex:v1.11.1
            docker pull quay.io/cortexproject/cortex:v1.13.1
            docker pull quay.io/cortexproject/cortex:v1.13.2
            docker pull quay.io/cortexproject/cortex:v1.14.0
            docker pull quay.io/cortexproject/cortex:v1.14.1
            docker pull quay.io/cortexproject/cortex:v1.15.0
            docker pull quay.io/cortexproject/cortex:v1.15.1
          fi
          docker pull memcached:1.6.1
          docker pull redis:7.0.4-alpine
        env:
          TEST_TAGS: ${{ matrix.tags }}
      - name: Integration Tests
        run: |
          export CORTEX_IMAGE_PREFIX="${IMAGE_PREFIX:-quay.io/cortexproject/}"
          export IMAGE_TAG=$(make image-tag)
          export CORTEX_IMAGE="${CORTEX_IMAGE_PREFIX}cortex:$IMAGE_TAG-amd64"
          export CORTEX_CHECKOUT_DIR="/go/src/github.com/cortexproject/cortex"
          echo "Running integration tests with image: $CORTEX_IMAGE"
          go test -tags=integration,${{ matrix.tags }} -timeout 2400s -v -count=1 ./integration/...
        env:
          IMAGE_PREFIX: ${{ secrets.IMAGE_PREFIX }}

  integration-configs-db:
    needs: build
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
      - name: Install Docker Client
        run: sudo ./.github/workflows/scripts/install-docker.sh
      - name: Download Docker Images Artifact
        uses: actions/download-artifact@v2
        with:
          name: Docker Images
      - name: Extract Docker Images Archive
        run: tar -xvf images.tar -C /
      - name: Run Integration Configs Tests
        # Github Actions does not support TTY in their default runners yet
        run: |
          touch build-image/.uptodate
          MIGRATIONS_DIR=$(pwd)/cmd/cortex/migrations
          git apply .github/workflows/thanos-engine.patch
          make BUILD_IMAGE=quay.io/cortexproject/build-image:upgrade-to-go-1.20.4-6025f83e5 TTY='' configs-integration-test

  deploy_website:
    needs: [build, test]
    if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
    runs-on: ubuntu-20.04
    container:
      image: quay.io/cortexproject/build-image:upgrade-to-go-1.20.4-6025f83e5
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
        with:
          # web-deploy script expects repo to be cloned with ssh for some commands to work
          ssh-key: ${{ secrets.WEBSITE_DEPLOY_SSH_PRIVATE_KEY }}
      - name: Setup Git safe.directory
        run: |
          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
          git config --system --add safe.directory $GITHUB_WORKSPACE
      - name: Sym Link Expected Path to Workspace
        run: |
          mkdir -p /go/src/github.com/cortexproject/cortex
          ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
      - name: Download Website Artifact
        uses: actions/download-artifact@v2
        with:
          name: website public
          path: website/public
      - name: Setup SSH Keys and known_hosts for Github Authentication to Deploy Website
        run: |
          mkdir -p ~/.ssh
          ssh-keyscan github.com >> ~/.ssh/known_hosts
          ssh-agent -a $SSH_AUTH_SOCK > /dev/null
          ssh-add - <<< "${{ secrets.WEBSITE_DEPLOY_SSH_PRIVATE_KEY }}"
        env:
          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
        shell: bash
      - name: Deploy Website
        # SSH is used to authentricate with Github because web-deploy script uses git to checkout and push to gh-pages
        run: make BUILD_IN_CONTAINER=false web-deploy
        env:
          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
          GIT_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no"

  deploy:
    needs: [build, test, lint, integration, integration-configs-db]
    if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
    runs-on: ubuntu-20.04
    container:
      image: quay.io/cortexproject/build-image:upgrade-to-go-1.20.4-6025f83e5
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
      - name: Setup Git safe.directory
        run: |
          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
          git config --system --add safe.directory $GITHUB_WORKSPACE
      - name: Install Docker Client
        run: ./.github/workflows/scripts/install-docker.sh
      - name: Sym link Expected Path to Workspace
        run: |
          mkdir -p /go/src/github.com/cortexproject/cortex
          ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
      - name: Download Docker Images Artifact
        uses: actions/download-artifact@v2
        with:
          name: Docker Images
      - name: Extract Docker Images Archive
        run: tar -xvf images.tar -C /
      - name: Load Images
        run: |
          ln -s /tmp/images ./docker-images
          make BUILD_IN_CONTAINER=false load-images
      - name: Deploy
        run: |
          if [ -n "$DOCKER_REGISTRY_PASSWORD" ]; then
            docker login -u "$DOCKER_REGISTRY_USER" -p "$DOCKER_REGISTRY_PASSWORD"
          fi
          if [ -n "$QUAY_REGISTRY_PASSWORD" ]; then
            docker login -u "$QUAY_REGISTRY_USER" -p "$QUAY_REGISTRY_PASSWORD" quay.io;
          fi
          export IMAGE_TAG=$(make image-tag)
          ./push-images $NOQUAY
        env:
          DOCKER_REGISTRY_USER: ${{secrets.DOCKER_REGISTRY_USER}}
          DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
          QUAY_REGISTRY_USER: ${{secrets.QUAY_REGISTRY_USER}}
          QUAY_REGISTRY_PASSWORD: ${{secrets.QUAY_REGISTRY_PASSWORD}}
          NOQUAY: ${{secrets.NOQUAY}}