Smoke Tests #7072
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Tests a broad set of Quarto functionality that users are likely to encounter. | |
| # A failure indicates some signficant portion of functionality is likely to be broken. | |
| name: Smoke Tests | |
| on: | |
| workflow_call: | |
| inputs: | |
| buckets: | |
| description: "JSON string for buckets of tests to run in loop. Array of grouped tests." | |
| required: true | |
| type: string | |
| time-test: | |
| description: "Should we run tests to produce test file" | |
| required: false | |
| type: boolean | |
| default: false | |
| extra-r-packages: | |
| description: "extra R package to install for the runs (like a dev version of one of the deps) - comma separated, passed to renv::install" | |
| required: false | |
| type: string | |
| default: "" | |
| workflow_dispatch: | |
| inputs: | |
| buckets: | |
| description: "JSON string for buckets of tests to run in loop. Array of grouped tests." | |
| required: false | |
| type: string | |
| default: "" | |
| time-test: | |
| description: "Should we run tests to produce test file" | |
| required: false | |
| type: boolean | |
| default: false | |
| extra-r-packages: | |
| description: "extra R package to install for the runs (like a dev version of one of the deps) - comma separated, passed to renv::install" | |
| required: false | |
| type: string | |
| default: "" | |
| schedule: | |
| - cron: "0 6 * * *" | |
| jobs: | |
| run-smokes: | |
| name: Run smoke (${{ matrix.os }})${{ matrix.time-test && ' with timed file' || ''}}${{ inputs.extra-r-packages && ' - with some extra R packages'|| ''}} | |
| if: github.event_name != 'schedule' || (github.event_name == 'schedule' && github.repository == 'quarto-dev/quarto-cli') | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest] | |
| time-test: | |
| - ${{ inputs.time-test }} | |
| exclude: | |
| # only run timed tests on Linux | |
| - os: windows-latest | |
| time-test: true | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout Repo | |
| uses: actions/checkout@v4 | |
| - name: Fix temp dir to use runner one (windows) | |
| if: runner.os == 'Windows' | |
| run: | | |
| echo "TMPDIR=${{ runner.temp }}" >> $GITHUB_ENV | |
| echo "TMP=${{ runner.temp }}" >> $GITHUB_ENV | |
| echo "TEMP=${{ runner.temp }}" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Set up R | |
| uses: r-lib/actions/setup-r@v2 | |
| with: | |
| r-version: "4.3.2" | |
| use-public-rspm: true | |
| # required to avoid rtools bin in path | |
| windows-path-include-rtools: false | |
| - name: Install node (for Playwright, MECA) | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Install node dependencies | |
| if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} | |
| run: yarn | |
| working-directory: ./tests/integration/playwright | |
| shell: bash | |
| - name: Install Playwright Browsers | |
| if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} | |
| run: npx playwright install --with-deps | |
| working-directory: ./tests/integration/playwright | |
| - name: Install MECA validator | |
| if: ${{ runner.os != 'Windows' }} | |
| run: npm install -g meca | |
| - name: Set RENV_PATHS_ROOT | |
| shell: bash | |
| run: | | |
| echo "RENV_PATHS_ROOT=${{ runner.temp }}/renv" >> $GITHUB_ENV | |
| - name: Get R and OS version | |
| id: get-version | |
| run: | | |
| cat("os-version=", sessionInfo()$running, "\n", file = Sys.getenv("GITHUB_OUTPUT"), sep = "", append = TRUE) | |
| cat("r-version=", R.Version()$version.string, "\n", file = Sys.getenv("GITHUB_OUTPUT"), sep = "", append = TRUE) | |
| shell: Rscript {0} | |
| - name: Cache R packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.RENV_PATHS_ROOT }} | |
| key: ${{ steps.get-version.outputs.os-version }}-${{ steps.get-version.outputs.r-version }}-renv-2-${{ hashFiles('tests/renv.lock') }} | |
| restore-keys: | | |
| ${{ steps.get-version.outputs.os-version }}-${{ steps.get-version.outputs.r-version }}-renv-2- | |
| save-always: ${{ inputs.extra-r-packages == '' }} # don't save cache if we have extra R packages | |
| - name: Install missing system deps | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo apt-get update -y | |
| sudo apt-get install -y libcurl4-openssl-dev | |
| sudo apt-get install -y libxml2-utils | |
| sudo apt-get install -y libharfbuzz-dev libfribidi-dev | |
| sudo apt-get install -y poppler-utils | |
| - name: Restore R packages | |
| working-directory: tests | |
| run: | | |
| if (!requireNamespace('renv', quietly = TRUE)) install.packages('renv') | |
| renv::restore() | |
| # Install dev versions for our testing | |
| # Use r-universe to avoid github api calls | |
| try(install.packages('knitr', repos = 'https://yihui.r-universe.dev')) | |
| try(install.packages('rmarkdown', repos = 'https://rstudio.r-universe.dev')) | |
| if ('${{ inputs.extra-r-packages }}' != '') { | |
| cat(sprintf("::notice::Running with the following extra R packages for pak: %s\n", "${{ inputs.extra-r-packages }}")) | |
| renv::install(strsplit("${{ inputs.extra-r-packages }}", split = ",")[[1]]) | |
| } | |
| shell: Rscript {0} | |
| env: | |
| GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache: "pipenv" | |
| cache-dependency-path: "./tests/Pipfile.lock" | |
| - name: Restore Python Dependencies | |
| working-directory: tests | |
| run: | | |
| python -m pip install pipenv==2024.3.0 | |
| pipenv install | |
| - uses: ./.github/workflows/actions/quarto-dev | |
| - name: Install Tinytex | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| quarto install tinytex | |
| - name: Cache Typst packages | |
| uses: ./.github/actions/cache-typst | |
| - name: Install Chrome | |
| uses: browser-actions/setup-chrome@latest | |
| - name: Setup Julia | |
| uses: julia-actions/setup-julia@v2 | |
| with: | |
| version: "1.10" | |
| - name: Cache Julia Packages | |
| uses: julia-actions/cache@v2 | |
| - name: Restore Julia Packages | |
| working-directory: tests | |
| shell: bash | |
| run: | | |
| # Setup IJulia with the jupyter from the Python environment | |
| # https://julialang.github.io/IJulia.jl/stable/manual/installation/ | |
| export JUPYTER=$(find $(dirname $(pipenv run which jupyter))/ -type f -name "jupyter.exe" -o -name "jupyter") | |
| pipenv run julia --color=yes --project=. -e "import Pkg; Pkg.instantiate(); Pkg.build(\"IJulia\"); Pkg.precompile()" | |
| echo "Julia Jupyter:" | |
| julia --project=. -e "import IJulia;println(IJulia.JUPYTER);println(IJulia.find_jupyter_subcommand(\"notebook\"))" | |
| - name: Setup timing file for timed test | |
| if: ${{ matrix.time-test == true }} | |
| run: | | |
| echo "QUARTO_TEST_TIMING=timing-for-ci.txt" >> "$GITHUB_ENV" | |
| # - name: Setup tmate session | |
| # uses: mxschmitt/action-tmate@v3 | |
| - name: Run all Smoke Tests Windows | |
| if: ${{ runner.os == 'Windows' && format('{0}', inputs.buckets) == '' && matrix.time-test == false }} | |
| env: | |
| # Useful as TinyTeX latest release is checked in run-test.sh | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: ./run-tests.ps1 | |
| working-directory: tests | |
| shell: pwsh | |
| - name: Run all Smoke Tests Linux | |
| if: ${{ runner.os != 'Windows' && format('{0}', inputs.buckets) == '' }} | |
| env: | |
| # Useful as TinyTeX latest release is checked in run-test.sh | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: ./run-tests.sh | |
| working-directory: tests | |
| shell: bash | |
| - name: Run Smoke Tests as a bucket on Linux | |
| if: ${{ runner.os != 'Windows' && format('{0}', inputs.buckets) != '' }} | |
| env: | |
| # Useful as TinyTeX latest release is checked in run-test.sh | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| QUARTO_LOG_LEVEL: DEBUG | |
| run: | | |
| haserror=0 | |
| readarray -t my_array < <(echo '${{ inputs.buckets }}' | jq -rc '.[]') | |
| for file in "${my_array[@]}"; do | |
| echo ">>> ./run-tests.sh ${file}" | |
| ./run-tests.sh $file | |
| status=$? | |
| [ $status -eq 0 ] && echo ">>> No error in this test file" || haserror=1 | |
| done | |
| [ $haserror -eq 0 ] && echo ">>> All tests passed" || exit 1 | |
| working-directory: tests | |
| shell: bash | |
| - name: Run Smoke Tests as a bucket on Windows | |
| if: ${{ runner.os == 'Windows' && format('{0}', inputs.buckets) != '' }} | |
| env: | |
| # Useful as TinyTeX latest release is checked in run-test.sh | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| $haserror=$false | |
| foreach ($file in ('${{ inputs.buckets }}' | ConvertFrom-Json)) { | |
| Write-Host ">>> ./run-tests.ps1 ${file}" | |
| ./run-tests.ps1 $file | |
| $status=$LASTEXITCODE | |
| if ($status -eq 1) { | |
| Write-Host ">>> Error found in test file" | |
| $haserror=$true | |
| } else { | |
| Write-Host ">>> No error in this test file" | |
| } | |
| } | |
| if ($haserror) { | |
| Exit 1 | |
| } else { | |
| Write-Host ">>> All tests have passed" | |
| } | |
| working-directory: tests | |
| shell: pwsh | |
| - name: Create Pull Request for new tests timing | |
| if: matrix.time-test | |
| id: cpr | |
| uses: peter-evans/create-pull-request@v6 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| add-paths: tests/timing-for-ci.txt | |
| commit-message: | | |
| timing for new tests [auto PR] | |
| branch: updates/timing-for-ci | |
| delete-branch: true | |
| assignees: cderv | |
| reviewers: cderv | |
| title: | | |
| [CI] Update timing file | |
| body: | | |
| This PR was created automatically with new timing test file for our parallel smoke tests. | |
| Please review and merge to trigger a new build of the website. | |
| - name: Check auto PR outputs | |
| if: steps.cpr.outcome == 'success' | |
| run: | | |
| echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" | |
| echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" | |
| echo "Pull Request Action Performed - ${{ steps.cpr.outputs.pull-request-operation }}" | |
| - name: Upload test timing file | |
| uses: actions/upload-artifact@v4 | |
| if: matrix.time-test && ( failure() || cancelled()) | |
| with: | |
| name: timed test file | |
| path: tests/timing-for-ci.txt | |
| - uses: actions/upload-artifact@v4 | |
| # PLaywright test only runs on Linux for now | |
| if: ${{ !cancelled() && runner.os != 'Windows' }} | |
| with: | |
| name: playwright-report | |
| path: ./tests/integration/playwright/playwright-report/ | |
| retention-days: 30 |