benchmark.yaml

name: Benchmark

on:
  pull_request:
    paths:
      - "Cargo.toml"
      - "Cargo.lock"
      - "rust-toolchain"
      - "crates/**"
      - "!crates/ruff_dev"
      - "!crates/ruff_shrinking"

  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

jobs:
  run-benchmark:
    if: github.event_name == 'pull_request'
    name: "Run | ${{ matrix.os }}"
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest]
    runs-on: ${{ matrix.os }}

    steps:
      - name: "PR - Checkout Branch"
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: "PR - Install Rust toolchain"
        run: rustup show

      - uses: Swatinem/rust-cache@v2

      - name: "PR - Build benchmarks"
        run: cargo bench -p ruff_benchmark --no-run

      - name: "PR - Run benchmarks"
        run: cargo benchmark --save-baseline=pr

      - name: "Main - Checkout Branch"
        uses: actions/checkout@v3
        with:
          clean: false
          ref: main

      - name: "Main - Install Rust toolchain"
        run: rustup show

      - name: "Main - Build benchmarks"
        run: cargo bench -p ruff_benchmark --no-run

      - name: "Main - Run benchmarks"
        run: cargo benchmark --save-baseline=main

      - name: "Upload benchmark results"
        uses: actions/upload-artifact@v3
        with:
          name: benchmark-results-${{ matrix.os }}
          path: ./target/criterion

      # Cleanup
      - name: Remove Criterion Artifact
        uses: JesseTG/rm@v1.0.3
        with:
          path: ./target/criterion

  benchmark-compare:
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    name: Compare
    needs:
      - run-benchmark

    steps:
      - name: "Install Rust toolchain"
        run: rustup show

      - name: "Install critcmp"
        uses: taiki-e/install-action@v2
        with:
          tool: critcmp

      - name: "Linux | Download PR benchmark results"
        uses: actions/download-artifact@v3
        with:
          name: benchmark-results-ubuntu-latest
          path: ./target/criterion

      - name: "Linux | Compare benchmark results"
        shell: bash
        run: |
          echo "### Benchmark" >> summary.md
          echo "#### Linux" >> summary.md
          echo "\`\`\`" >> summary.md
          critcmp main pr >> summary.md
          echo "\`\`\`" >> summary.md
          echo "" >> summary.md

      - name: "Linux | Cleanup benchmark results"
        run: rm -rf ./target/criterion

      - name: "Windows | Download PR benchmark results"
        uses: actions/download-artifact@v3
        with:
          name: benchmark-results-windows-latest
          path: ./target/criterion

      - name: "Windows | Compare benchmark results"
        shell: bash
        run: |
          echo "#### Windows" >> summary.md
          echo "\`\`\`" >> summary.md
          critcmp main pr >> summary.md
          echo "\`\`\`" >> summary.md
          echo "" >> summary.md

          echo ${{ github.event.pull_request.number }} > pr-number

          cat summary.md > $GITHUB_STEP_SUMMARY

      - uses: actions/upload-artifact@v3
        name: Upload PR Number
        with:
          name: pr-number
          path: pr-number

      - uses: actions/upload-artifact@v3
        name: Upload Summary
        with:
          name: summary
          path: summary.md