bench_cli.yml

# CLI benchmark, compares main and PR branch with Hyperfine.
# Comment with text containing `!bench_cli`, a new result will be commented at the bottom of this PR.

name: CLI Benchmark

on:
  issue_comment:
    types: [ created ]

env:
  RUST_LOG: info

jobs:
  bench:
    permissions:
      pull-requests: write
    name: Bench
    if: github.event.issue.pull_request && contains(github.event.comment.body, '!bench_cli')
    runs-on: ubuntu-latest

    steps:
      - name: Get PR SHA
        id: sha
        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
        with:
          result-encoding: string
          script: |
            const response = await github.request(context.payload.issue.pull_request.url);
            return response.data.head.sha;

      - name: Checkout PR Branch
        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
        with:
          ref: ${{ steps.sha.outputs.result }}

      - name: Install toolchain
        uses: moonrepo/setup-rust@e013866c4215f77c925f42f60257dec7dd18836e # v1.2.1
        with:
          channel: stable
          cache-target: release
          cache-base: main
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Install hyperfine
        run: cargo install hyperfine

      - name: Compile on PR Branch
        run: |
          cargo build --release --bin biome
          mkdir -p benchmark/target
          cp target/release/biome benchmark/target/biome_pr

      - name: Checkout Main Branch
        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
        with:
          clean: false
          ref: main

      - name: Compile on Main Branch
        run: |
          cargo build --release --bin biome
          cp target/release/biome benchmark/target/biome_main

      - name: Checkout webpack
        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
        with:
          repository: webpack/webpack
          path: benchmark/target/webpack
      - name: Checkout prettier
        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
        with:
          repository: prettier/prettier
          path: benchmark/target/prettier
      - name: Checkout eslint
        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
        with:
          repository: eslint/eslint
          path: benchmark/target/eslint

      - name: Run Benchmarks
        id: benchmarks
        working-directory: benchmark/target
        env:
          FORMAT_BENCH_COMMAND: "format webpack/lib webpack/examples webpack/declarations webpack/benchmark prettier/src prettier/scripts --write"
          CHECK_BENCH_COMMAND: "--max-diagnostics=0 eslint/lib eslint/messages eslint/tests/lib eslint/tests/performance eslint/tools webpack/lib --no-errors-on-unmatched"
        run: |
          hyperfine -w 2 --export-markdown benchmark_format.md \
            -n "biome format (main)" "./biome_main $FORMAT_BENCH_COMMAND" \
            -n "biome format (pr)" "./biome_pr $FORMAT_BENCH_COMMAND"
          hyperfine -w 2 --export-markdown --ignore-failure benchmark_check.md \
            -n "biome check (main)" "./biome_main check $CHECK_BENCH_COMMAND" \
            -n "biome check (pr)" "./biome_pr check $CHECK_BENCH_COMMAND"
          cat benchmark_format.md >> benchmark.md
          echo $'\n' >> benchmark.md
          cat benchmark_check.md >> benchmark.md

      - name: Write a new comment
        # Check if the event is not triggered by a fork
        if: github.event.pull_request.head.repo.full_name == github.repository
        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
        continue-on-error: true
        with:
          issue-number: ${{ github.event.issue.number }}
          body-file: benchmark/target/benchmark.md