From 4dc68ecaabed4e51bb1871717f62ef397ed7c188 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 21 Oct 2023 16:10:56 -0700 Subject: [PATCH 01/93] .ci/docker-exec-script.sh: New --- .ci/docker-exec-script.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100755 .ci/docker-exec-script.sh diff --git a/.ci/docker-exec-script.sh b/.ci/docker-exec-script.sh new file mode 100755 index 00000000000..60bbd70aee7 --- /dev/null +++ b/.ci/docker-exec-script.sh @@ -0,0 +1,14 @@ +#!/bin/sh -x +if [ $# -lt 3 ]; then + echo >&2 "usage: docker-exec-script.sh CONTAINER WORKDIR [VAR=VALUE...] SCRIPT" + exit 1 +fi +CONTAINER=$1 +WORKDIR=$2 +shift 2 +(echo "cd \"$WORKDIR\""; + while [ $# -gt 1 ]; do + echo "export \"$1\"" + shift + done; + cat "$1") | docker exec -i $CONTAINER bash -ex From 8c845bc960c1de04d1d2f43434ab61d3c9ea3cee Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 22 Oct 2023 16:22:38 -0700 Subject: [PATCH 02/93] .github/workflows/doc-build.yml: Run container explicitly --- .github/workflows/doc-build.yml | 63 +++++++++------------------------ 1 file changed, 17 insertions(+), 46 deletions(-) diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 5eb3998feee..882d5e5b948 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -16,39 +16,29 @@ concurrency: cancel-in-progress: true jobs: - get_ci_fixes: + build-docs: runs-on: ubuntu-latest steps: - name: Checkout - id: checkout uses: actions/checkout@v4 + - name: Merge CI fixes from sagemath/sage run: | .ci/merge-fixes.sh env: GH_TOKEN: ${{ github.token }} - - name: Store CI fixes in upstream artifact - run: | - mkdir -p upstream - if git format-patch --stdout test_base > ci_fixes.patch; then - cp ci_fixes.patch upstream/ - fi - - uses: actions/upload-artifact@v3 - with: - path: upstream - name: upstream - build-docs: - runs-on: ubuntu-latest - container: ghcr.io/sagemath/sage/sage-ubuntu-focal-standard-with-targets:dev - needs: [get_ci_fixes] - steps: - - name: Checkout - uses: actions/checkout@v4 + - name: Start container + run: | + docker run --name BUILD -dit \ + --mount type=bind,src=$(pwd),dst=$(pwd) \ + --workdir $(pwd) \ + ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}} /bin/sh - name: Update system packages run: | apt-get update && apt-get install -y git zip + shell: sh .ci/docker-exec-script.sh BUILD . {0} - name: Add prebuilt tree as a worktree id: worktree @@ -73,40 +63,23 @@ jobs: (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && \ (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; \ git add -A && git commit --quiet -m "old") - - - name: Download upstream artifact - uses: actions/download-artifact@v3 - with: - path: upstream - name: upstream - - - name: Apply CI fixes from sagemath/sage - # After applying the fixes, make sure all changes are marked as uncommitted changes. - run: | - if [ -r upstream/ci_fixes.patch ]; then - (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch - fi + shell: sh .ci/docker-exec-script.sh BUILD . {0} - name: Incremental build id: incremental run: | + export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps. ./bootstrap && make sagemath_doc_html-build-deps - working-directory: ./worktree-image - env: - MAKE: make -j2 --output-sync=recurse - SAGE_NUM_THREADS: 2 + shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - name: Build (fallback to non-incremental) id: build if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success' run: | - set -ex + export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 make sagelib-clean && git clean -fx src/sage && ./config.status && make sagemath_doc_html-build-deps - working-directory: ./worktree-image - env: - MAKE: make -j2 --output-sync=recurse - SAGE_NUM_THREADS: 2 + shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - name: Build docs id: docbuild @@ -114,16 +87,13 @@ jobs: # Always non-incremental because of the concern that # incremental docbuild may introduce broken links (inter-file references) though build succeeds run: | - set -ex + export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 export SAGE_USE_CDNS=yes mv /sage/local/share/doc/sage/html/en/.git /sage/.git-doc make doc-clean doc-uninstall mkdir -p /sage/local/share/doc/sage/html/en/ && mv /sage/.git-doc /sage/local/share/doc/sage/html/en/.git ./config.status && make sagemath_doc_html-no-deps - working-directory: ./worktree-image - env: - MAKE: make -j2 --output-sync=recurse - SAGE_NUM_THREADS: 2 + shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - name: Copy docs id: copy @@ -189,6 +159,7 @@ jobs: cp -r -L /sage/local/share/doc/sage/html/en/* ./docs # Zip everything for increased performance zip -r docs.zip docs + shell: sh .ci/docker-exec-script.sh BUILD . {0} - name: Upload docs if: always() && steps.copy.outcome == 'success' From e1ce9fe44aed3f354b8ab893111c74e067e9b16f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 1 Nov 2023 12:24:54 -0700 Subject: [PATCH 03/93] .github/workflows/doc-build-pdf.yml: Run container explicitly --- .github/workflows/doc-build-pdf.yml | 62 ++++++++--------------------- 1 file changed, 16 insertions(+), 46 deletions(-) diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index aedee3a54a7..7fead7ca270 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -21,35 +21,24 @@ concurrency: cancel-in-progress: true jobs: - get_ci_fixes: + build-docs-pdf: runs-on: ubuntu-latest steps: - name: Checkout - id: checkout uses: actions/checkout@v4 + - name: Merge CI fixes from sagemath/sage run: | .ci/merge-fixes.sh env: GH_TOKEN: ${{ github.token }} - - name: Store CI fixes in upstream artifact - run: | - mkdir -p upstream - if git format-patch --stdout test_base > ci_fixes.patch; then - cp ci_fixes.patch upstream/ - fi - - uses: actions/upload-artifact@v3 - with: - path: upstream - name: upstream - build-docs-pdf: - runs-on: ubuntu-latest - container: ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}} - needs: [get_ci_fixes] - steps: - - name: Checkout - uses: actions/checkout@v4 + - name: Start container + run: | + docker run --name BUILD -dit \ + --mount type=bind,src=$(pwd),dst=$(pwd) \ + --workdir $(pwd) \ + ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}} /bin/sh - name: Update system packages run: | @@ -57,7 +46,7 @@ jobs: eval $(sage-print-system-package-command auto update) eval $(sage-print-system-package-command auto --yes --no-install-recommends install zip) eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git texlive) - + shell: sh .ci/docker-exec-script.sh BUILD . {0} - name: Add prebuilt tree as a worktree id: worktree @@ -67,49 +56,29 @@ jobs: git config --global user.name "Build & Test workflow" .ci/retrofit-worktree.sh worktree-image /sage - - name: Download upstream artifact - uses: actions/download-artifact@v3 - with: - path: upstream - name: upstream - - - name: Apply CI fixes from sagemath/sage - # After applying the fixes, make sure all changes are marked as uncommitted changes. - run: | - if [ -r upstream/ci_fixes.patch ]; then - (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch - fi - - name: Incremental build id: incremental run: | + export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps. - ./bootstrap && make build - working-directory: ./worktree-image - env: - MAKE: make -j2 --output-sync=recurse - SAGE_NUM_THREADS: 2 + ./bootstrap && make sagemath_doc_html-build-deps + shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - name: Build (fallback to non-incremental) id: build if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success' run: | - set -ex + export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 make sagelib-clean && git clean -fx src/sage && ./config.status && make build - working-directory: ./worktree-image - env: - MAKE: make -j2 --output-sync=recurse - SAGE_NUM_THREADS: 2 + shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - name: Build docs (PDF) id: docbuild if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success') run: | + export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 make doc-clean doc-uninstall; make sagemath_doc_html-build-deps sagemath_doc_pdf-no-deps working-directory: ./worktree-image - env: - MAKE: make -j2 --output-sync=recurse - SAGE_NUM_THREADS: 2 - name: Copy docs id: copy @@ -121,6 +90,7 @@ jobs: cp -r -L /sage/local/share/doc/sage/pdf/en/* ./docs # Zip everything for increased performance zip -r docs-pdf.zip docs + shell: sh .ci/docker-exec-script.sh BUILD . {0} - name: Upload docs if: always() && steps.copy.outcome == 'success' From be7dacf79052f65ec574b0670a0ef9eec5f6899a Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 1 Nov 2023 13:35:18 -0700 Subject: [PATCH 04/93] .github/workflows/build.yml: Run containers explicitly with docker run, docker exec --- .github/workflows/build.yml | 123 +++++++++--------------------------- 1 file changed, 31 insertions(+), 92 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 53a281ca5b4..00b6f2b19b0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,7 @@ concurrency: cancel-in-progress: true jobs: - get_ci_fixes: + build: runs-on: ubuntu-latest steps: - name: Checkout @@ -38,25 +38,13 @@ jobs: .ci/merge-fixes.sh env: GH_TOKEN: ${{ github.token }} - - name: Store CI fixes in upstream artifact - run: | - mkdir -p upstream - if git format-patch --stdout test_base > ci_fixes.patch; then - cp ci_fixes.patch upstream/ - fi - - uses: actions/upload-artifact@v3 - with: - path: upstream - name: upstream - build: - runs-on: ubuntu-latest - container: ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}} - needs: [get_ci_fixes] - steps: - - name: Checkout - id: checkout - uses: actions/checkout@v4 + - name: Start container + run: | + docker run --name BUILD -dit \ + --mount type=bind,src=$(pwd),dst=$(pwd) \ + --workdir $(pwd) \ + ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}} /bin/sh - name: Update system packages id: prepare @@ -64,80 +52,38 @@ jobs: export PATH="build/bin:$PATH" eval $(sage-print-system-package-command auto update) eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git) + shell: sh .ci/docker-exec-script.sh BUILD . {0} - name: Add prebuilt tree as a worktree id: worktree run: | - set -ex git config --global --add safe.directory $(pwd) .ci/retrofit-worktree.sh worktree-image /sage - - - name: Download upstream artifact - uses: actions/download-artifact@v3 - with: - path: upstream - name: upstream - - - name: Apply CI fixes from sagemath/sage - # After applying the fixes, make sure all changes are marked as uncommitted changes. - run: | - if [ -r upstream/ci_fixes.patch ]; then - (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch - fi + shell: sh .ci/docker-exec-script.sh BUILD . {0} - name: Incremental build id: incremental run: | # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps. - ./bootstrap && make build - working-directory: ./worktree-image - env: - MAKE: make -j2 --output-sync=recurse - SAGE_NUM_THREADS: 2 + ./bootstrap + MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 make build + shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - name: Build modularized distributions if: (success() || failure()) && steps.worktree.outcome == 'success' - run: make V=0 tox && make SAGE_CHECK=no pypi-wheels - working-directory: ./worktree-image - env: - MAKE: make -j2 --output-sync=recurse - SAGE_NUM_THREADS: 2 - - - name: Static code check with pyright - if: (success() || failure()) && steps.worktree.outcome == 'success' - uses: jakebailey/pyright-action@v1 - with: - version: 1.1.332 - # Many warnings issued by pyright are not yet helpful because there is not yet enough type information. - no-comments: true - working-directory: ./worktree-image - env: - # To avoid out of memory errors - NODE_OPTIONS: --max-old-space-size=8192 - - - name: Static code check with pyright (annotated) - if: (success() || failure()) && steps.worktree.outcome == 'success' - uses: jakebailey/pyright-action@v1 - with: - version: 1.1.332 - # Issue errors - no-comments: false - level: error - working-directory: ./worktree-image - env: - # To avoid out of memory errors - NODE_OPTIONS: --max-old-space-size=8192 + run: | + export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 + make V=0 tox + make SAGE_CHECK=no pypi-wheels + shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - name: Clean (fallback to non-incremental) id: clean if: (success() || failure()) && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success' run: | - set -ex + export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 ./bootstrap && make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status - working-directory: ./worktree-image - env: - MAKE: make -j2 - SAGE_NUM_THREADS: 2 + shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - name: Build # This step is needed because building the modularized distributions installs some optional packages, @@ -145,48 +91,41 @@ jobs: id: build if: (success() || failure()) && (steps.incremental.outcome == 'success' || steps.clean.outcome == 'success') run: | + export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 make build - working-directory: ./worktree-image - env: - MAKE: make -j2 --output-sync=recurse - SAGE_NUM_THREADS: 2 + shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} # Testing - name: Test changed files (sage -t --new) if: (success() || failure()) && steps.build.outcome == 'success' run: | + export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 # We run tests with "sage -t --new"; this only tests the uncommitted changes. ./sage -t --new -p2 - working-directory: ./worktree-image - env: - MAKE: make -j2 --output-sync=recurse - SAGE_NUM_THREADS: 2 + shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - name: Test modularized distributions if: (success() || failure()) && steps.build.outcome == 'success' - run: make V=0 tox && make pypi-wheels-check - working-directory: ./worktree-image - env: - MAKE: make -j2 --output-sync=recurse - SAGE_NUM_THREADS: 2 + run: | + export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 + make V=0 tox && make pypi-wheels-check + shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - name: Pytest if: contains(github.ref, 'pytest') run: | ../sage -python -m pip install coverage pytest-xdist ../sage -python -m coverage run -m pytest -c tox.ini --doctest-modules || true - working-directory: ./worktree-image/src - env: - # Increase the length of the lines in the "short summary" - COLUMNS: 120 + # Increase the length of the lines in the "short summary" + shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image COLUMNS=120 {0} - name: Test all files (sage -t --all --long) if: (success() || failure()) && steps.build.outcome == 'success' run: | ../sage -python -m pip install coverage ../sage -python -m coverage run ./bin/sage-runtests --all --long -p2 --random-seed=286735480429121101562228604801325644303 - working-directory: ./worktree-image/src + shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image/src {0} - name: Prepare coverage results if: (success() || failure()) && steps.build.outcome == 'success' @@ -195,7 +134,7 @@ jobs: ./venv/bin/python3 -m coverage xml mkdir -p coverage-report mv coverage.xml coverage-report/ - working-directory: ./worktree-image + shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - name: Upload coverage to codecov if: (success() || failure()) && steps.build.outcome == 'success' From 4a3c700cfd3ccd720cc048eb83fc9853578ace89 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 22 Oct 2023 09:18:47 -0700 Subject: [PATCH 05/93] .github/workflows/build.yml: Run tests from SAGE_ROOT as current dir --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 00b6f2b19b0..e66963d9072 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -115,17 +115,17 @@ jobs: - name: Pytest if: contains(github.ref, 'pytest') run: | - ../sage -python -m pip install coverage pytest-xdist - ../sage -python -m coverage run -m pytest -c tox.ini --doctest-modules || true + ./sage -python -m pip install coverage pytest-xdist + ./sage -python -m coverage run -m pytest -c tox.ini --doctest-modules || true # Increase the length of the lines in the "short summary" shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image COLUMNS=120 {0} - name: Test all files (sage -t --all --long) if: (success() || failure()) && steps.build.outcome == 'success' run: | - ../sage -python -m pip install coverage - ../sage -python -m coverage run ./bin/sage-runtests --all --long -p2 --random-seed=286735480429121101562228604801325644303 - shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image/src {0} + ./sage -python -m pip install coverage + ./sage -python -m coverage run ./bin/sage-runtests --all --long -p2 --random-seed=286735480429121101562228604801325644303 + shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - name: Prepare coverage results if: (success() || failure()) && steps.build.outcome == 'success' From c7fa34f43d7a559cfaca5795f6f217e13f62691c Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 1 Nov 2023 13:21:21 -0700 Subject: [PATCH 06/93] .github/workflows/build.yml: Create coverage.xml in workspace, not container --- .github/workflows/build.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e66963d9072..ceee019aa6a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -124,20 +124,19 @@ jobs: if: (success() || failure()) && steps.build.outcome == 'success' run: | ./sage -python -m pip install coverage - ./sage -python -m coverage run ./bin/sage-runtests --all --long -p2 --random-seed=286735480429121101562228604801325644303 + ./sage -python -m coverage run src/bin/sage-runtests --all --long -p2 --random-seed=286735480429121101562228604801325644303 shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - name: Prepare coverage results if: (success() || failure()) && steps.build.outcome == 'success' run: | - ./venv/bin/python3 -m coverage combine src/.coverage/ - ./venv/bin/python3 -m coverage xml - mkdir -p coverage-report - mv coverage.xml coverage-report/ + venv/bin/python3 -m coverage combine src/.coverage/ + mkdir -p ${{ github.workspace }}/coverage-report + venv/bin/python3 -m coverage xml -o ${{ github.workspace }}/coverage-report/coverage.xml shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - name: Upload coverage to codecov if: (success() || failure()) && steps.build.outcome == 'success' uses: codecov/codecov-action@v3 with: - directory: ./worktree-image/coverage-report + directory: ./coverage-report From c62b12106dc2c2105e15b10430b7af4269d2667b Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 1 Nov 2023 13:14:20 -0700 Subject: [PATCH 07/93] .github/workflows/pyright.yml: New --- .github/workflows/pyright.yml | 84 +++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 .github/workflows/pyright.yml diff --git a/.github/workflows/pyright.yml b/.github/workflows/pyright.yml new file mode 100644 index 00000000000..1b608561b15 --- /dev/null +++ b/.github/workflows/pyright.yml @@ -0,0 +1,84 @@ +name: Static check with Pyright + +on: + pull_request: + merge_group: + push: + branches: + - master + - develop + workflow_dispatch: + # Allow to run manually + +concurrency: + # Cancel previous runs of this workflow for the same branch + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + pyright: + runs-on: ubuntu-latest + container: ghcr.io/sagemath/sage/sage-ubuntu-focal-standard-with-targets:dev + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v4 + + - name: Update system packages + id: prepare + run: | + export PATH="build/bin:$PATH" + eval $(sage-print-system-package-command auto update) + eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git) + + - name: Install GH CLI + uses: dev-hanz-ops/install-gh-cli-action@v0.1.0 + with: + gh-cli-version: 2.32.0 + + - name: Merge CI fixes from sagemath/sage + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + .ci/merge-fixes.sh + + - name: Add prebuilt tree as a worktree + id: worktree + run: | + set -ex + .ci/retrofit-worktree.sh worktree-image /sage + + - name: Incremental build (sagelib deps) + id: incremental + run: | + # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps. + # pyright does not need a built sagelib; it only needs + # the libraries from which sagelib imports. + ./bootstrap && make sagelib-build-deps + working-directory: ./worktree-image + env: + MAKE: make -j2 --output-sync=recurse + SAGE_NUM_THREADS: 2 + + - name: Static code check with pyright + uses: jakebailey/pyright-action@v1 + with: + version: 1.1.332 + # Many warnings issued by pyright are not yet helpful because there is not yet enough type information. + no-comments: true + working-directory: ./worktree-image + env: + # To avoid out of memory errors + NODE_OPTIONS: --max-old-space-size=8192 + + - name: Static code check with pyright (annotated) + if: (success() || failure()) && steps.incremental.outcome == 'success' + uses: jakebailey/pyright-action@v1 + with: + version: 1.1.332 + # Issue errors + no-comments: false + level: error + working-directory: ./worktree-image + env: + # To avoid out of memory errors + NODE_OPTIONS: --max-old-space-size=8192 From caed8e33238caeec7741613c93fc7d7233705f8b Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 1 Nov 2023 14:44:38 -0700 Subject: [PATCH 08/93] Makefile (ci-build-with-fallback): New --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index ae90db4e9e2..e99dfadf967 100644 --- a/Makefile +++ b/Makefile @@ -80,6 +80,14 @@ download: dist: build/make/Makefile ./sage --sdist +ci-build-with-fallback: + $(MAKE) build && $(MAKE) SAGE_CHECK=no pypi-wheels; \ + if [ $$? != 0 ]; then \ + echo "Incremental build failed, falling back"; \ + $(MAKE) doc-clean doc-uninstall sagelib-clean; \ + $(MAKE) build && $(MAKE) SAGE_CHECK=no pypi-wheels; \ + fi + ############################################################################### # Cleaning up ############################################################################### From ba92f3f24d3ea4abe9be009931a6b43458eef406 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 1 Nov 2023 15:01:52 -0700 Subject: [PATCH 09/93] .github/workflows/build.yml: Separate jobs 'modularized', 'test-long' --- .github/workflows/build.yml | 159 +++++++++++++++++++++++------------- 1 file changed, 100 insertions(+), 59 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ceee019aa6a..1312e6c8b67 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,6 +26,18 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + # Adapted from docker.yml + TOX_ENV: "docker-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-incremental" + LOGS_ARTIFACT_NAME: "logs-commit-${{ github.sha }}-tox-docker-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}" + DOCKER_TARGETS: "with-targets" + TARGETS_PRE: "build/make/Makefile" + TARGETS: "ci-build-with-fallback" + FROM_DOCKER_REPOSITORY: "ghcr.io/sagemath/sage/" + FROM_DOCKER_TARGET: "with-targets" + FROM_DOCKER_TAG: ${{ github.event.inputs.docker_tag || 'dev'}} + EXTRA_CONFIGURE_ARGS: --enable-fat-binary + EXTRA_DOCKER_TAGS: ci jobs: build: runs-on: ubuntu-latest @@ -33,110 +45,139 @@ jobs: - name: Checkout id: checkout uses: actions/checkout@v4 + - name: Install test prerequisites + # From docker.yml + run: | + sudo DEBIAN_FRONTEND=noninteractive apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install tox + sudo apt-get clean + df -h - name: Merge CI fixes from sagemath/sage + # From docker.yml + # This step needs to happen after the commit sha is put in DOCKER_TAG + # so that multi-stage builds can work correctly. run: | .ci/merge-fixes.sh env: GH_TOKEN: ${{ github.token }} + - name: Configure and build Sage distribution within a Docker container + # From docker.yml + run: | + set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - name: Start container run: | docker run --name BUILD -dit \ --mount type=bind,src=$(pwd),dst=$(pwd) \ --workdir $(pwd) \ - ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}} /bin/sh + sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:ci /bin/sh - - name: Update system packages - id: prepare - run: | - export PATH="build/bin:$PATH" - eval $(sage-print-system-package-command auto update) - eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git) - shell: sh .ci/docker-exec-script.sh BUILD . {0} + # Testing - - name: Add prebuilt tree as a worktree - id: worktree + - name: Test changed files (sage -t --new) run: | - git config --global --add safe.directory $(pwd) - .ci/retrofit-worktree.sh worktree-image /sage - shell: sh .ci/docker-exec-script.sh BUILD . {0} + export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 + # We run tests with "sage -t --new"; this only tests the uncommitted changes. + ./sage -t --new -p2 + shell: sh .ci/docker-exec-script.sh BUILD /sage {0} - - name: Incremental build - id: incremental + modularized: + runs-on: ubuntu-latest + needs: [build] + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v4 + - name: Install test prerequisites + # From docker.yml run: | - # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps. - ./bootstrap - MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 make build - shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - - - name: Build modularized distributions - if: (success() || failure()) && steps.worktree.outcome == 'success' + sudo DEBIAN_FRONTEND=noninteractive apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install tox + sudo apt-get clean + df -h + - name: Merge CI fixes from sagemath/sage + # From docker.yml + # This step needs to happen after the commit sha is put in DOCKER_TAG + # so that multi-stage builds can work correctly. run: | - export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 - make V=0 tox - make SAGE_CHECK=no pypi-wheels - shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} + .ci/merge-fixes.sh + env: + GH_TOKEN: ${{ github.token }} + - name: Configure and build Sage distribution within a Docker container + # From docker.yml + run: | + set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - - name: Clean (fallback to non-incremental) - id: clean - if: (success() || failure()) && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success' + - name: Start container run: | - export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 - ./bootstrap && make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status - shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} + docker run --name BUILD -dit \ + --mount type=bind,src=$(pwd),dst=$(pwd) \ + --workdir $(pwd) \ + sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:ci /bin/sh - - name: Build - # This step is needed because building the modularized distributions installs some optional packages, - # so the editable install of sagelib needs to build the corresponding optional extension modules. - id: build - if: (success() || failure()) && (steps.incremental.outcome == 'success' || steps.clean.outcome == 'success') + - name: Test modularized distributions run: | export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 - make build - shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - - # Testing + make V=0 tox && make pypi-wheels-check + shell: sh .ci/docker-exec-script.sh BUILD /sage {0} - - name: Test changed files (sage -t --new) - if: (success() || failure()) && steps.build.outcome == 'success' + test-long: + runs-on: ubuntu-latest + needs: [build] + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v4 + - name: Install test prerequisites + # From docker.yml run: | - export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 - # We run tests with "sage -t --new"; this only tests the uncommitted changes. - ./sage -t --new -p2 - shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} + sudo DEBIAN_FRONTEND=noninteractive apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install tox + sudo apt-get clean + df -h + - name: Merge CI fixes from sagemath/sage + # From docker.yml + # This step needs to happen after the commit sha is put in DOCKER_TAG + # so that multi-stage builds can work correctly. + run: | + .ci/merge-fixes.sh + env: + GH_TOKEN: ${{ github.token }} + - name: Configure and build Sage distribution within a Docker container + # From docker.yml + run: | + set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - - name: Test modularized distributions - if: (success() || failure()) && steps.build.outcome == 'success' + - name: Start container run: | - export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 - make V=0 tox && make pypi-wheels-check - shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} + docker run --name BUILD -dit \ + --mount type=bind,src=$(pwd),dst=$(pwd) \ + --workdir $(pwd) \ + sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:ci /bin/sh - name: Pytest - if: contains(github.ref, 'pytest') + if: (success() || failure()) && contains(github.ref, 'pytest') run: | ./sage -python -m pip install coverage pytest-xdist ./sage -python -m coverage run -m pytest -c tox.ini --doctest-modules || true # Increase the length of the lines in the "short summary" - shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image COLUMNS=120 {0} + shell: sh .ci/docker-exec-script.sh BUILD /sage COLUMNS=120 {0} - name: Test all files (sage -t --all --long) - if: (success() || failure()) && steps.build.outcome == 'success' run: | ./sage -python -m pip install coverage ./sage -python -m coverage run src/bin/sage-runtests --all --long -p2 --random-seed=286735480429121101562228604801325644303 - shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} + shell: sh .ci/docker-exec-script.sh BUILD /sage {0} - name: Prepare coverage results - if: (success() || failure()) && steps.build.outcome == 'success' + if: success() || failure() run: | venv/bin/python3 -m coverage combine src/.coverage/ mkdir -p ${{ github.workspace }}/coverage-report venv/bin/python3 -m coverage xml -o ${{ github.workspace }}/coverage-report/coverage.xml - shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} + shell: sh .ci/docker-exec-script.sh BUILD /sage {0} - name: Upload coverage to codecov - if: (success() || failure()) && steps.build.outcome == 'success' uses: codecov/codecov-action@v3 with: directory: ./coverage-report From 13ca5ffc8537100fab2d7cc14784f0bc3fa529d8 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 12 Oct 2023 22:35:22 -0700 Subject: [PATCH 10/93] build/bin/write-dockerfile.sh: Only unminimize once --- build/bin/write-dockerfile.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build/bin/write-dockerfile.sh b/build/bin/write-dockerfile.sh index bfc4dd751a4..0cd648d9ded 100755 --- a/build/bin/write-dockerfile.sh +++ b/build/bin/write-dockerfile.sh @@ -52,7 +52,10 @@ case $SYSTEM in ;; *) cat < /dev/null; then \ + (yes | unminimize) || echo "(ignored)"; \ + rm -f "$(command -v unminimize)"; \ + fi EOF if [ -n "$DIST_UPGRADE" ]; then cat < Date: Tue, 10 Oct 2023 17:55:16 -0700 Subject: [PATCH 11/93] build/bin/write-dockerfile.sh, tox.ini (docker-incremental): Make incremental build more incremental via git, via git, .ci/retrofit-worktree.sh --- build/bin/write-dockerfile.sh | 49 +++++++++++++++++++++++++++-------- tox.ini | 6 +++-- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/build/bin/write-dockerfile.sh b/build/bin/write-dockerfile.sh index 0cd648d9ded..307e5cea6c5 100755 --- a/build/bin/write-dockerfile.sh +++ b/build/bin/write-dockerfile.sh @@ -219,17 +219,34 @@ cat <> .gitignore && \ + ./.ci/retrofit-worktree.sh worktree-image /sage); \ + else \ + for a in local logs; do \ + if [ -d /sage/\$a ]; then mv /sage/\$a /new/; fi; \ + done; \ + rm -rf /sage; \ + mv /new /sage; \ + fi; \ + else \ + mv /new /sage; \ + fi WORKDIR /sage -$ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap bootstrap-conda configure.ac sage .homebrew-build-env tox.ini Pipfile.m4 ./ -$ADD config/config.rpath config/config.rpath -$ADD src/doc/bootstrap src/doc/bootstrap -$ADD src/bin src/bin -$ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/setup.cfg.m4 src/VERSION.txt src/ -$ADD m4 ./m4 -$ADD pkgs pkgs -$ADD build ./build -$ADD .upstream.d ./.upstream.d + ARG BOOTSTRAP=./bootstrap $RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN @@ -275,7 +292,17 @@ ENV MAKE="make -j\${NUMPROC}" ARG USE_MAKEFLAGS="-k V=0" ENV SAGE_CHECK=warn ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,!cmake,!rpy2,!sage_sws2rst" -$ADD src src +$ADD .gitignore /new +$ADD src /new/src +RUN if command -v git; then \ + rm -f /sage/.git && \ + cd /new && \ + ./.ci/retrofit-worktree.sh worktree-pre /sage; \ + else \ + rm -rf /sage/src; \ + mv /new/src /sage/src; \ + fi + ARG TARGETS="build" $RUN make SAGE_SPKG="sage-spkg -y -o" \${USE_MAKEFLAGS} \${TARGETS} $ENDRUN diff --git a/tox.ini b/tox.ini index 6293831c44f..a73b92258b9 100644 --- a/tox.ini +++ b/tox.ini @@ -185,6 +185,7 @@ setenv = # What system packages should be installed. Default: All standard packages with spkg-configure. SAGE_PACKAGE_LIST_ARGS=--has-file=spkg-configure.m4 :standard: recommended: EXTRA_SAGE_PACKAGES_3=_recommended $(head -n 1 build/pkgs/_recommended/dependencies) + incremental: EXTRA_SAGE_PACKAGES_4=git develop: EXTRA_SAGE_PACKAGES_4=_develop $(head -n 1 build/pkgs/_develop/dependencies) minimal: SAGE_PACKAGE_LIST_ARGS=_prereq maximal: SAGE_PACKAGE_LIST_ARGS=:standard: :optional: @@ -490,8 +491,9 @@ setenv = # docker: FULL_BASE_IMAGE_AND_TAG={env:ARCH_IMAGE_PREFIX:}{env:BASE_IMAGE}{env:ARCH_IMAGE_SUFFIX:}:{env:ARCH_TAG_PREFIX:}{env:BASE_TAG}{env:ARCH_TAG_SUFFIX:} docker-incremental: FULL_BASE_IMAGE_AND_TAG={env:FROM_DOCKER_REPOSITORY:ghcr.io/sagemath/sage/}sage-$(echo {envname} | sed -E "s/(docker-|-incremental|-sitepackages)//g")-{env:FROM_DOCKER_TARGET:with-targets}:{env:FROM_DOCKER_TAG:dev} - docker-incremental: SKIP_SYSTEM_PKG_INSTALL=yes - docker-incremental-sitepackages: SKIP_SYSTEM_PKG_INSTALL=no + # Can SKIP_SYSTEM_PKG_INSTALL if the base image already has git + docker-incremental-{develop,recommended,maximal}: SKIP_SYSTEM_PKG_INSTALL=yes + docker-incremental-sitepackages: SKIP_SYSTEM_PKG_INSTALL=no # docker-nobootstrap: BOOTSTRAP=./bootstrap -D ### From 0db6b1d07e529272724710c0fc8aa5aa5afe76cc Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 1 Nov 2023 17:02:52 -0700 Subject: [PATCH 12/93] Use build kit, --cache-{from,to}=type=gha --- .github/workflows/build.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1312e6c8b67..ade53022fee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,6 +38,7 @@ env: FROM_DOCKER_TAG: ${{ github.event.inputs.docker_tag || 'dev'}} EXTRA_CONFIGURE_ARGS: --enable-fat-binary EXTRA_DOCKER_TAGS: ci + DOCKER_BUILDKIT: 1 jobs: build: runs-on: ubuntu-latest @@ -63,7 +64,7 @@ jobs: - name: Configure and build Sage distribution within a Docker container # From docker.yml run: | - set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" + set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--cache-from=type=gha --cache-to=type=gha --build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - name: Start container run: | @@ -106,7 +107,7 @@ jobs: - name: Configure and build Sage distribution within a Docker container # From docker.yml run: | - set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" + set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--cache-from=type=gha --cache-to=type=gha --build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - name: Start container run: | @@ -146,7 +147,7 @@ jobs: - name: Configure and build Sage distribution within a Docker container # From docker.yml run: | - set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" + set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--cache-from=type=gha --cache-to=type=gha --build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - name: Start container run: | From 7f352c81db359ed1a80e480bc7b00d1a16b91533 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 1 Nov 2023 18:15:00 -0700 Subject: [PATCH 13/93] Use --cache-to=type=gha,mode=max --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ade53022fee..b321c851337 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,7 +64,7 @@ jobs: - name: Configure and build Sage distribution within a Docker container # From docker.yml run: | - set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--cache-from=type=gha --cache-to=type=gha --build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" + set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--cache-from=type=gha --cache-to=type=gha,mode=max --build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - name: Start container run: | @@ -107,7 +107,7 @@ jobs: - name: Configure and build Sage distribution within a Docker container # From docker.yml run: | - set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--cache-from=type=gha --cache-to=type=gha --build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" + set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--cache-from=type=gha --cache-to=type=gha,mode=max --build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - name: Start container run: | @@ -147,7 +147,7 @@ jobs: - name: Configure and build Sage distribution within a Docker container # From docker.yml run: | - set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--cache-from=type=gha --cache-to=type=gha --build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" + set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--cache-from=type=gha --cache-to=type=gha,mode=max --build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - name: Start container run: | From 35997330dd9e5e9aa16a81959fd8cdef11880e92 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 1 Nov 2023 19:00:09 -0700 Subject: [PATCH 14/93] tox.ini (docker): Pass ACTIONS_... variables --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index a73b92258b9..2a28aedff3e 100644 --- a/tox.ini +++ b/tox.ini @@ -156,6 +156,9 @@ passenv = docker-incremental: FROM_DOCKER_REPOSITORY docker-incremental: FROM_DOCKER_TARGET docker-incremental: FROM_DOCKER_TAG + docker: ACTIONS_CACHE_URL + docker: ACTIONS_RUNTIME_URL + docker: ACTIONS_RUNTIME_TOKEN local: MAKE local: PREFIX local: SAGE_NUM_THREADS From a158d3966dcacc94cd83ef32c2058a25cd79bddc Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 1 Nov 2023 20:00:19 -0700 Subject: [PATCH 15/93] .github/workflows/build.yml: Make 'modularized' a matrix job --- .github/workflows/build.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b321c851337..1708a282551 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -82,9 +82,14 @@ jobs: ./sage -t --new -p2 shell: sh .ci/docker-exec-script.sh BUILD /sage {0} - modularized: + mod: runs-on: ubuntu-latest needs: [build] + strategy: + fail-fast: false + matrix: + targets: + - sagemath_categories-check steps: - name: Checkout id: checkout @@ -119,7 +124,7 @@ jobs: - name: Test modularized distributions run: | export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 - make V=0 tox && make pypi-wheels-check + make V=0 tox-ensure && make ${{ matrix.targets }} shell: sh .ci/docker-exec-script.sh BUILD /sage {0} test-long: From 7541781214982d5ef682184329f8ba24a6025a4a Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 Nov 2023 13:46:43 -0700 Subject: [PATCH 16/93] .github/workflows/build.yml: Use docker/setup-buildx-action, docker/build-push-action --- .github/workflows/build.yml | 97 ++++++++++++++++++++++++++++++++----- 1 file changed, 86 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1708a282551..af2199a454d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,15 +29,14 @@ concurrency: env: # Adapted from docker.yml TOX_ENV: "docker-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-incremental" + IMAGE: "sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:ci" LOGS_ARTIFACT_NAME: "logs-commit-${{ github.sha }}-tox-docker-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}" - DOCKER_TARGETS: "with-targets" TARGETS_PRE: "build/make/Makefile" TARGETS: "ci-build-with-fallback" FROM_DOCKER_REPOSITORY: "ghcr.io/sagemath/sage/" FROM_DOCKER_TARGET: "with-targets" FROM_DOCKER_TAG: ${{ github.event.inputs.docker_tag || 'dev'}} EXTRA_CONFIGURE_ARGS: --enable-fat-binary - EXTRA_DOCKER_TAGS: ci DOCKER_BUILDKIT: 1 jobs: build: @@ -61,17 +60,41 @@ jobs: .ci/merge-fixes.sh env: GH_TOKEN: ${{ github.token }} - - name: Configure and build Sage distribution within a Docker container + + # Building + + - name: Generate Dockerfile # From docker.yml run: | - set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--cache-from=type=gha --cache-to=type=gha,mode=max --build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" + tox -e ${{ env.TOX_ENV }} + cp .tox/${{ env.TOX_ENV }}/Dockerfile . + env: + # Only generate the Dockerfile, do not run 'docker build' here + DOCKER_TARGETS: "" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build Docker image + uses: docker/build-push-action@v5 + with: + push: false + load: true + context: . + tags: ${{ env.IMAGE }} + target: with-targets + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + NUMPROC=4 + USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse - name: Start container run: | docker run --name BUILD -dit \ --mount type=bind,src=$(pwd),dst=$(pwd) \ --workdir $(pwd) \ - sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:ci /bin/sh + ${{ env.IMAGE }} /bin/sh # Testing @@ -109,17 +132,43 @@ jobs: .ci/merge-fixes.sh env: GH_TOKEN: ${{ github.token }} - - name: Configure and build Sage distribution within a Docker container + + # Building + + - name: Generate Dockerfile # From docker.yml run: | - set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--cache-from=type=gha --cache-to=type=gha,mode=max --build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" + tox -e ${{ env.TOX_ENV }} + cp .tox/${{ env.TOX_ENV }}/Dockerfile . + env: + # Only generate the Dockerfile, do not run 'docker build' here + DOCKER_TARGETS: "" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build Docker image + uses: docker/build-push-action@v5 + with: + push: false + load: true + context: . + tags: ${{ env.IMAGE }} + target: with-targets + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + NUMPROC=4 + USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse - name: Start container run: | docker run --name BUILD -dit \ --mount type=bind,src=$(pwd),dst=$(pwd) \ --workdir $(pwd) \ - sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:ci /bin/sh + ${{ env.IMAGE }} /bin/sh + + # Testing - name: Test modularized distributions run: | @@ -149,17 +198,43 @@ jobs: .ci/merge-fixes.sh env: GH_TOKEN: ${{ github.token }} - - name: Configure and build Sage distribution within a Docker container + + # Building + + - name: Generate Dockerfile # From docker.yml run: | - set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--cache-from=type=gha --cache-to=type=gha,mode=max --build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" + tox -e ${{ env.TOX_ENV }} + cp .tox/${{ env.TOX_ENV }}/Dockerfile . + env: + # Only generate the Dockerfile, do not run 'docker build' here + DOCKER_TARGETS: "" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build Docker image + uses: docker/build-push-action@v5 + with: + push: false + load: true + context: . + tags: ${{ env.IMAGE }} + target: with-targets + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + NUMPROC=4 + USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse - name: Start container run: | docker run --name BUILD -dit \ --mount type=bind,src=$(pwd),dst=$(pwd) \ --workdir $(pwd) \ - sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:ci /bin/sh + ${{ env.IMAGE }} /bin/sh + + # Testing - name: Pytest if: (success() || failure()) && contains(github.ref, 'pytest') From 46e01110198d5509edcf849250993dcd36153d9f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 Nov 2023 14:13:26 -0700 Subject: [PATCH 17/93] tox.ini: Fill some build-args in build/bin/write-dockerfile.sh --- build/bin/write-dockerfile.sh | 8 ++++---- tox.ini | 4 ---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/build/bin/write-dockerfile.sh b/build/bin/write-dockerfile.sh index 307e5cea6c5..8c8f8a8299f 100755 --- a/build/bin/write-dockerfile.sh +++ b/build/bin/write-dockerfile.sh @@ -30,7 +30,7 @@ echo "# to simplify writing scripts that customize this file" ADD="ADD $__CHOWN" RUN=RUN cat < Date: Thu, 2 Nov 2023 14:13:40 -0700 Subject: [PATCH 18/93] .github/workflows/build.yml: Use docker/setup-buildx-action, docker/build-push-action (fixup) --- .github/workflows/build.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index af2199a454d..24ff40de285 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,8 +31,6 @@ env: TOX_ENV: "docker-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-incremental" IMAGE: "sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:ci" LOGS_ARTIFACT_NAME: "logs-commit-${{ github.sha }}-tox-docker-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}" - TARGETS_PRE: "build/make/Makefile" - TARGETS: "ci-build-with-fallback" FROM_DOCKER_REPOSITORY: "ghcr.io/sagemath/sage/" FROM_DOCKER_TARGET: "with-targets" FROM_DOCKER_TAG: ${{ github.event.inputs.docker_tag || 'dev'}} @@ -88,6 +86,8 @@ jobs: build-args: | NUMPROC=4 USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse + TARGETS_PRE=build/make/Makefile + TARGETS=ci-build-with-fallback - name: Start container run: | @@ -160,6 +160,8 @@ jobs: build-args: | NUMPROC=4 USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse + TARGETS_PRE=build/make/Makefile + TARGETS=ci-build-with-fallback - name: Start container run: | @@ -226,6 +228,8 @@ jobs: build-args: | NUMPROC=4 USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse + TARGETS_PRE=build/make/Makefile + TARGETS=ci-build-with-fallback - name: Start container run: | From be4cc8c565cbbdc60688de30f0587f2bc8e18b4a Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 Nov 2023 14:22:13 -0700 Subject: [PATCH 19/93] build/bin/write-dockerfile.sh: eval FULL_BASE_IMAGE_AND_TAG to expand shell logic --- build/bin/write-dockerfile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/bin/write-dockerfile.sh b/build/bin/write-dockerfile.sh index 8c8f8a8299f..8c60aca20a3 100755 --- a/build/bin/write-dockerfile.sh +++ b/build/bin/write-dockerfile.sh @@ -30,7 +30,7 @@ echo "# to simplify writing scripts that customize this file" ADD="ADD $__CHOWN" RUN=RUN cat < Date: Thu, 2 Nov 2023 14:45:53 -0700 Subject: [PATCH 20/93] .github/workflows/build.yml: Include ghcr.io... in image name --- .github/workflows/build.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 24ff40de285..e892188d6f3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,7 +29,7 @@ concurrency: env: # Adapted from docker.yml TOX_ENV: "docker-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-incremental" - IMAGE: "sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:ci" + BUILD_IMAGE: "ghcr.io/${{ github.repository }}/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:ci" LOGS_ARTIFACT_NAME: "logs-commit-${{ github.sha }}-tox-docker-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}" FROM_DOCKER_REPOSITORY: "ghcr.io/sagemath/sage/" FROM_DOCKER_TARGET: "with-targets" @@ -79,7 +79,7 @@ jobs: push: false load: true context: . - tags: ${{ env.IMAGE }} + tags: ${{ env.BUILD_IMAGE }} target: with-targets cache-from: type=gha cache-to: type=gha,mode=max @@ -94,7 +94,7 @@ jobs: docker run --name BUILD -dit \ --mount type=bind,src=$(pwd),dst=$(pwd) \ --workdir $(pwd) \ - ${{ env.IMAGE }} /bin/sh + ${{ env.BUILD_IMAGE }} /bin/sh # Testing @@ -153,7 +153,7 @@ jobs: push: false load: true context: . - tags: ${{ env.IMAGE }} + tags: ${{ env.BUILD_IMAGE }} target: with-targets cache-from: type=gha cache-to: type=gha,mode=max @@ -168,7 +168,7 @@ jobs: docker run --name BUILD -dit \ --mount type=bind,src=$(pwd),dst=$(pwd) \ --workdir $(pwd) \ - ${{ env.IMAGE }} /bin/sh + ${{ env.BUILD_IMAGE }} /bin/sh # Testing @@ -221,7 +221,7 @@ jobs: push: false load: true context: . - tags: ${{ env.IMAGE }} + tags: ${{ env.BUILD_IMAGE }} target: with-targets cache-from: type=gha cache-to: type=gha,mode=max @@ -236,7 +236,7 @@ jobs: docker run --name BUILD -dit \ --mount type=bind,src=$(pwd),dst=$(pwd) \ --workdir $(pwd) \ - ${{ env.IMAGE }} /bin/sh + ${{ env.BUILD_IMAGE }} /bin/sh # Testing From 430aaaa94b065b3cabe244f46f8f4ded8951367b Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 Nov 2023 15:02:38 -0700 Subject: [PATCH 21/93] .github/workflows/doc-build-pdf: Fix up merge --- .github/workflows/doc-build-pdf.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index 7fead7ca270..904854cecb1 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -55,6 +55,7 @@ jobs: git config --global user.email "ci-sage@example.com" git config --global user.name "Build & Test workflow" .ci/retrofit-worktree.sh worktree-image /sage + shell: sh .ci/docker-exec-script.sh BUILD . {0} - name: Incremental build id: incremental @@ -69,7 +70,7 @@ jobs: if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success' run: | export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 - make sagelib-clean && git clean -fx src/sage && ./config.status && make build + make sagelib-clean && git clean -fx src/sage && ./config.status && make sagemath_doc_html-build-deps shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - name: Build docs (PDF) @@ -77,8 +78,8 @@ jobs: if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success') run: | export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 - make doc-clean doc-uninstall; make sagemath_doc_html-build-deps sagemath_doc_pdf-no-deps - working-directory: ./worktree-image + make doc-clean doc-uninstall; make sagemath_doc_pdf-no-deps + shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - name: Copy docs id: copy From d6eccdaaed2be0b7a795ea2472563a700d21e6c5 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 Nov 2023 15:33:33 -0700 Subject: [PATCH 22/93] .github/workflows/build.yml: Remove unnecessary env vars --- .github/workflows/build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e892188d6f3..6fabfb4afd4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -30,12 +30,11 @@ env: # Adapted from docker.yml TOX_ENV: "docker-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-incremental" BUILD_IMAGE: "ghcr.io/${{ github.repository }}/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:ci" - LOGS_ARTIFACT_NAME: "logs-commit-${{ github.sha }}-tox-docker-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}" FROM_DOCKER_REPOSITORY: "ghcr.io/sagemath/sage/" FROM_DOCKER_TARGET: "with-targets" FROM_DOCKER_TAG: ${{ github.event.inputs.docker_tag || 'dev'}} EXTRA_CONFIGURE_ARGS: --enable-fat-binary - DOCKER_BUILDKIT: 1 + jobs: build: runs-on: ubuntu-latest From f82da4a705a07a4e2480aa1608ff1cc1f4c9606d Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 Nov 2023 15:35:05 -0700 Subject: [PATCH 23/93] .github/workflows/doc-build.yml: Build container with cache, as in build.yml --- .github/workflows/doc-build.yml | 64 +++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 6864b92f385..ae21eb3168f 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -15,6 +15,15 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + # Same as in build.yml + TOX_ENV: "docker-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-incremental" + BUILD_IMAGE: "ghcr.io/${{ github.repository }}/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:ci" + FROM_DOCKER_REPOSITORY: "ghcr.io/sagemath/sage/" + FROM_DOCKER_TARGET: "with-targets" + FROM_DOCKER_TAG: ${{ github.event.inputs.docker_tag || 'dev'}} + EXTRA_CONFIGURE_ARGS: --enable-fat-binary + jobs: build-docs: runs-on: ubuntu-latest @@ -28,19 +37,44 @@ jobs: env: GH_TOKEN: ${{ github.token }} + # Building + + - name: Generate Dockerfile + # From docker.yml + run: | + tox -e ${{ env.TOX_ENV }} + cp .tox/${{ env.TOX_ENV }}/Dockerfile . + env: + # Only generate the Dockerfile, do not run 'docker build' here + DOCKER_TARGETS: "" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build Docker image + uses: docker/build-push-action@v5 + with: + push: false + load: true + context: . + tags: ${{ env.BUILD_IMAGE }} + target: with-targets + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + NUMPROC=4 + USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse + TARGETS_PRE=build/make/Makefile + TARGETS=ci-build-with-fallback + - name: Start container run: | docker run --name BUILD -dit \ --mount type=bind,src=$(pwd),dst=$(pwd) \ --workdir $(pwd) \ - ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}} /bin/sh + ${{ env.BUILD_IMAGE }} /bin/sh - - name: Update system packages - run: | - apt-get update && apt-get install -y git zip - shell: sh .ci/docker-exec-script.sh BUILD . {0} - - - name: Add prebuilt tree as a worktree + - name: Store old docs id: worktree run: | git config --global --add safe.directory $(pwd) @@ -65,22 +99,6 @@ jobs: git add -A && git commit --quiet -m "old") shell: sh .ci/docker-exec-script.sh BUILD . {0} - - name: Incremental build - id: incremental - run: | - export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 - # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps. - ./bootstrap && make sagemath_doc_html-build-deps - shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - - - name: Build (fallback to non-incremental) - id: build - if: (success() || failure()) && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success' - run: | - export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 - make sagelib-clean && git clean -fx src/sage && ./config.status && make sagemath_doc_html-build-deps - shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - - name: Build docs id: docbuild if: (success() || failure()) && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success') From e0cd8914add50025402f4adcbe2d70151c6cc9dd Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 Nov 2023 16:23:59 -0700 Subject: [PATCH 24/93] .github/workflows/doc-build.yml: Build container with cache, as in build.yml (fixup) --- .github/workflows/doc-build.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index ae21eb3168f..fb841bb106f 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -30,7 +30,13 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - + - name: Install test prerequisites + # From docker.yml + run: | + sudo DEBIAN_FRONTEND=noninteractive apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install tox + sudo apt-get clean + df -h - name: Merge CI fixes from sagemath/sage run: | .ci/merge-fixes.sh @@ -101,7 +107,6 @@ jobs: - name: Build docs id: docbuild - if: (success() || failure()) && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success') # Always non-incremental because of the concern that # incremental docbuild may introduce broken links (inter-file references) though build succeeds run: | @@ -110,7 +115,7 @@ jobs: mv /sage/local/share/doc/sage/html/en/.git /sage/.git-doc make doc-clean doc-uninstall mkdir -p /sage/local/share/doc/sage/html/en/ && mv /sage/.git-doc /sage/local/share/doc/sage/html/en/.git - ./config.status && make sagemath_doc_html-no-deps + ./config.status && make sagemath_doc_html shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - name: Copy docs From 26bdc1f1108ec88a2f144a45fd6c14f1b57747e6 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 Nov 2023 16:37:13 -0700 Subject: [PATCH 25/93] .github/workflows/doc-build-pdf.yml: Build container with cache, as in build.yml --- .github/workflows/doc-build-pdf.yml | 85 +++++++++++++++++------------ .github/workflows/doc-build.yml | 11 ++++ 2 files changed, 60 insertions(+), 36 deletions(-) diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index 9f427bfa34a..6a24188e276 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -20,65 +20,78 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + # Same as in build.yml + TOX_ENV: "docker-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-incremental" + BUILD_IMAGE: "ghcr.io/${{ github.repository }}/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:ci" + FROM_DOCKER_REPOSITORY: "ghcr.io/sagemath/sage/" + FROM_DOCKER_TARGET: "with-targets" + FROM_DOCKER_TAG: ${{ github.event.inputs.docker_tag || 'dev'}} + EXTRA_CONFIGURE_ARGS: --enable-fat-binary + jobs: build-docs-pdf: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - + - name: Install test prerequisites + # From docker.yml + run: | + sudo DEBIAN_FRONTEND=noninteractive apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install tox + sudo apt-get clean + df -h - name: Merge CI fixes from sagemath/sage run: | .ci/merge-fixes.sh env: GH_TOKEN: ${{ github.token }} - - name: Start container - run: | - docker run --name BUILD -dit \ - --mount type=bind,src=$(pwd),dst=$(pwd) \ - --workdir $(pwd) \ - ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}} /bin/sh + # Building - - name: Update system packages + - name: Generate Dockerfile + # From docker.yml run: | - export PATH="build/bin:$PATH" - eval $(sage-print-system-package-command auto update) - eval $(sage-print-system-package-command auto --yes --no-install-recommends install zip) - eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git texlive) - shell: sh .ci/docker-exec-script.sh BUILD . {0} + tox -e ${{ env.TOX_ENV }} + cp .tox/${{ env.TOX_ENV }}/Dockerfile . + env: + # Only generate the Dockerfile, do not run 'docker build' here + DOCKER_TARGETS: "" - - name: Add prebuilt tree as a worktree - id: worktree - run: | - git config --global --add safe.directory $(pwd) - git config --global user.email "ci-sage@example.com" - git config --global user.name "Build & Test workflow" - .ci/retrofit-worktree.sh worktree-image /sage - shell: sh .ci/docker-exec-script.sh BUILD . {0} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - - name: Incremental build - id: incremental - run: | - export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 - # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps. - ./bootstrap && make sagemath_doc_html-build-deps - shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} + - name: Build Docker image + uses: docker/build-push-action@v5 + with: + push: false + load: true + context: . + tags: ${{ env.BUILD_IMAGE }} + target: with-targets + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + NUMPROC=4 + USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse + TARGETS_PRE=build/make/Makefile + TARGETS=ci-build-with-fallback - - name: Build (fallback to non-incremental) - id: build - if: (success() || failure()) && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success' + - name: Start container run: | - export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 - make sagelib-clean && git clean -fx src/sage && ./config.status && make sagemath_doc_html-build-deps - shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} + docker run --name BUILD -dit \ + --mount type=bind,src=$(pwd),dst=$(pwd) \ + --workdir $(pwd) \ + ${{ env.BUILD_IMAGE }} /bin/sh + + # Docs - name: Build docs (PDF) id: docbuild - if: (success() || failure()) && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success') run: | export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 - make doc-clean doc-uninstall; make sagemath_doc_pdf-no-deps + make doc-clean doc-uninstall; make sagemath_doc_html-build-deps sagemath_doc_pdf-no-deps shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - name: Copy docs diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index fb841bb106f..b152d11b200 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -9,6 +9,15 @@ on: - develop workflow_dispatch: # Allow to run manually + inputs: + platform: + description: 'Platform' + required: true + default: 'ubuntu-focal-standard' + docker_tag: + description: 'Docker tag' + required: true + default: 'dev' concurrency: # Cancel previous runs of this workflow for the same branch @@ -80,6 +89,8 @@ jobs: --workdir $(pwd) \ ${{ env.BUILD_IMAGE }} /bin/sh + # Docs + - name: Store old docs id: worktree run: | From a2a15242e5292ef8d3d69f4b77c1c3a865a3ab28 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 Nov 2023 16:50:09 -0700 Subject: [PATCH 26/93] .github/workflows/doc-build[-pdf].yml: Use /sage in container --- .github/workflows/doc-build-pdf.yml | 4 ++-- .github/workflows/doc-build.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index 6a24188e276..ce57637e184 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -92,7 +92,7 @@ jobs: run: | export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 make doc-clean doc-uninstall; make sagemath_doc_html-build-deps sagemath_doc_pdf-no-deps - shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} + shell: sh .ci/docker-exec-script.sh BUILD /sage {0} - name: Copy docs id: copy @@ -104,7 +104,7 @@ jobs: cp -r -L /sage/local/share/doc/sage/pdf/en/* ./docs # Zip everything for increased performance zip -r docs-pdf.zip docs - shell: sh .ci/docker-exec-script.sh BUILD . {0} + shell: sh .ci/docker-exec-script.sh BUILD /sage {0} - name: Upload docs if: (success() || failure()) && steps.copy.outcome == 'success' diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index b152d11b200..fab72733951 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -114,7 +114,7 @@ jobs: (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && \ (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; \ git add -A && git commit --quiet -m "old") - shell: sh .ci/docker-exec-script.sh BUILD . {0} + shell: sh .ci/docker-exec-script.sh BUILD /sage {0} - name: Build docs id: docbuild @@ -127,7 +127,7 @@ jobs: make doc-clean doc-uninstall mkdir -p /sage/local/share/doc/sage/html/en/ && mv /sage/.git-doc /sage/local/share/doc/sage/html/en/.git ./config.status && make sagemath_doc_html - shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} + shell: sh .ci/docker-exec-script.sh BUILD /sage {0} - name: Copy docs id: copy @@ -193,7 +193,7 @@ jobs: cp -r -L /sage/local/share/doc/sage/html/en/* ./docs # Zip everything for increased performance zip -r docs.zip docs - shell: sh .ci/docker-exec-script.sh BUILD . {0} + shell: sh .ci/docker-exec-script.sh BUILD /sage {0} - name: Upload docs if: (success() || failure()) && steps.copy.outcome == 'success' From 6e8ec0fa774171329bf3bdaaf7e5dc18046730b0 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 Nov 2023 17:38:16 -0700 Subject: [PATCH 27/93] .github/workflows/doc-build.yml: Run HTML diffing outside of the container --- .github/workflows/doc-build.yml | 39 +++++++++++++++------------------ 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 652f35285ed..b808b9a33bc 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -96,25 +96,25 @@ jobs: run: | git config --global --add safe.directory $(pwd) git config --global user.email "ci-sage@example.com" - git config --global user.name "Build & Test workflow" - # mathjax path in old doc - mathjax_path_from=$(SAGE_USE_CDNS=no /sage/sage -python -c "from sage_docbuild.conf import mathjax_path; print(mathjax_path)") - .ci/retrofit-worktree.sh worktree-image /sage + git config --global user.name "Build documentation workflow" + # mathjax path in old doc (regex) + mathjax_path_from="[-./A-Za-z_]*/tex-chtml[.]js" # mathjax path in new doc - mathjax_path_to=$(SAGE_USE_CDNS=yes /sage/sage -python -c "from sage_docbuild.conf import mathjax_path; print(mathjax_path)") - new_version=$(cat src/VERSION.txt) + mathjax_path_to=$(docker exec -e SAGE_USE_CDNS=yes BUILD /sage/sage -python -c "from sage_docbuild.conf import mathjax_path; print(mathjax_path)") + new_version=$(docker exec BUILD cat src/VERSION.txt) + mkdir -p docs/html + docker cp BUILD:/sage/local/share/doc/sage/html docs/html # Wipe out chronic diffs between old doc and new doc - (cd /sage/local/share/doc/sage/html && \ + (cd docs && \ find . -name "*.html" | xargs sed -i -e '/class="sidebar-brand-text"/ s/Sage [0-9a-z.]* /Sage '"$new_version"' /' \ -e 's;'"$mathjax_path_from"';'"$mathjax_path_to"';' \ -e '\;; d') # Create git repo from old doc - (cd /sage/local/share/doc/sage/html && \ + (cd docs && \ git init && \ (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && \ (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; \ git add -A && git commit --quiet -m "old") - shell: sh .ci/docker-exec-script.sh BUILD /sage {0} - name: Build docs id: docbuild @@ -123,9 +123,7 @@ jobs: run: | export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 export SAGE_USE_CDNS=yes - mv /sage/local/share/doc/sage/html/.git /sage/.git-doc make doc-clean doc-uninstall - mkdir -p /sage/local/share/doc/sage/html/ && mv /sage/.git-doc /sage/local/share/doc/sage/html/.git ./config.status && make sagemath_doc_html shell: sh .ci/docker-exec-script.sh BUILD /sage {0} @@ -134,10 +132,11 @@ jobs: if: (success() || failure()) && steps.docbuild.outcome == 'success' run: | set -ex - mkdir -p ./docs - (cd /sage/local/share/doc/sage/html && git commit -a -m 'new') + docker cp BUILD:/sage/local/share/doc/sage/html docs/html + docker cp BUILD:/sage/local/share/doc/sage/index.html docs/index.html + (cd html && git commit -a -m 'new') # Wipe out chronic diffs between old doc and new doc - (cd /sage/local/share/doc/sage/html && \ + (cd html && \ find . -name "*.html" | xargs sed -i -e '\;; d') # Create CHANGES.html echo '' > ./docs/CHANGES.html @@ -166,8 +165,8 @@ jobs: EOF echo '' >> ./docs/CHANGES.html echo '' >> ./docs/CHANGES.html - (cd /sage/local/share/doc/sage/html && git diff HEAD^ -- *.html; rm -rf .git) > ./docs/diff.txt - /sage/sage -python - << EOF + (cd docs && git diff HEAD^ -- *.html; rm -rf .git) > ./docs/diff.txt + python3 - << EOF import re, html with open('./docs/diff.txt', 'r') as f: diff_text = f.read() @@ -186,12 +185,10 @@ jobs: echo '' >> ./docs/CHANGES.html echo '' >>./docs/CHANGES.html rm ./docs/diff.txt ./docs/diff.html - (cd /sage/local/share/doc/sage/html && git reset --hard HEAD) - # For some reason the deploy step below cannot find /sage/... - # So copy everything from there to local folder + (cd docs && git reset --hard HEAD) # We also need to replace the symlinks because netlify is not following them - cp -r -L /sage/local/share/doc/sage/html ./docs - cp /sage/local/share/doc/sage/index.html ./docs + # CHECK IF STILL NEEDED + ##### cp -r -L /sage/local/share/doc/sage/html ./docs # Zip everything for increased performance zip -r docs.zip docs shell: sh .ci/docker-exec-script.sh BUILD /sage {0} From 12e11d813456e578020bdbb88e617025a1612886 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 Nov 2023 18:16:05 -0700 Subject: [PATCH 28/93] .github/workflows/[doc-]build[-pdf].yml: Use 'docker run --pull never' for the built container --- .github/workflows/build.yml | 6 +++--- .github/workflows/doc-build-pdf.yml | 2 +- .github/workflows/doc-build.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6fabfb4afd4..6069b8d1f2c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -90,7 +90,7 @@ jobs: - name: Start container run: | - docker run --name BUILD -dit \ + docker run --name BUILD --pull never -dit \ --mount type=bind,src=$(pwd),dst=$(pwd) \ --workdir $(pwd) \ ${{ env.BUILD_IMAGE }} /bin/sh @@ -164,7 +164,7 @@ jobs: - name: Start container run: | - docker run --name BUILD -dit \ + docker run --name BUILD --pull never -dit \ --mount type=bind,src=$(pwd),dst=$(pwd) \ --workdir $(pwd) \ ${{ env.BUILD_IMAGE }} /bin/sh @@ -232,7 +232,7 @@ jobs: - name: Start container run: | - docker run --name BUILD -dit \ + docker run --name BUILD --pull never -dit \ --mount type=bind,src=$(pwd),dst=$(pwd) \ --workdir $(pwd) \ ${{ env.BUILD_IMAGE }} /bin/sh diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index cc3a152cc92..81a4e9b7101 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -80,7 +80,7 @@ jobs: - name: Start container run: | - docker run --name BUILD -dit \ + docker run --name BUILD --pull never -dit \ --mount type=bind,src=$(pwd),dst=$(pwd) \ --workdir $(pwd) \ ${{ env.BUILD_IMAGE }} /bin/sh diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index b808b9a33bc..a54fb0afa90 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -84,7 +84,7 @@ jobs: - name: Start container run: | - docker run --name BUILD -dit \ + docker run --name BUILD --pull never -dit \ --mount type=bind,src=$(pwd),dst=$(pwd) \ --workdir $(pwd) \ ${{ env.BUILD_IMAGE }} /bin/sh From 70fe088aea2c5e9b479a72fad2d55ba08f0640ed Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 Nov 2023 18:43:18 -0700 Subject: [PATCH 29/93] .github/workflows/[doc-]build[-pdf].yml: Retry if the built container is not available immediately --- .github/workflows/build.yml | 33 ++++++++++++++++++----------- .github/workflows/doc-build-pdf.yml | 11 ++++++---- .github/workflows/doc-build.yml | 11 ++++++---- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6069b8d1f2c..b47386cadcf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -90,10 +90,13 @@ jobs: - name: Start container run: | - docker run --name BUILD --pull never -dit \ - --mount type=bind,src=$(pwd),dst=$(pwd) \ - --workdir $(pwd) \ - ${{ env.BUILD_IMAGE }} /bin/sh + while ! docker run --name BUILD --pull never -dit \ + --mount type=bind,src=$(pwd),dst=$(pwd) \ + --workdir $(pwd) \ + ${{ env.BUILD_IMAGE }} /bin/sh; \ + do \ + sleep 30; \ + done # Testing @@ -164,10 +167,13 @@ jobs: - name: Start container run: | - docker run --name BUILD --pull never -dit \ - --mount type=bind,src=$(pwd),dst=$(pwd) \ - --workdir $(pwd) \ - ${{ env.BUILD_IMAGE }} /bin/sh + while ! docker run --name BUILD --pull never -dit \ + --mount type=bind,src=$(pwd),dst=$(pwd) \ + --workdir $(pwd) \ + ${{ env.BUILD_IMAGE }} /bin/sh; \ + do \ + sleep 30; \ + done # Testing @@ -232,10 +238,13 @@ jobs: - name: Start container run: | - docker run --name BUILD --pull never -dit \ - --mount type=bind,src=$(pwd),dst=$(pwd) \ - --workdir $(pwd) \ - ${{ env.BUILD_IMAGE }} /bin/sh + while ! docker run --name BUILD --pull never -dit \ + --mount type=bind,src=$(pwd),dst=$(pwd) \ + --workdir $(pwd) \ + ${{ env.BUILD_IMAGE }} /bin/sh; \ + do \ + sleep 30; \ + done # Testing diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index 81a4e9b7101..53f7bc806c2 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -80,10 +80,13 @@ jobs: - name: Start container run: | - docker run --name BUILD --pull never -dit \ - --mount type=bind,src=$(pwd),dst=$(pwd) \ - --workdir $(pwd) \ - ${{ env.BUILD_IMAGE }} /bin/sh + while ! docker run --name BUILD --pull never -dit \ + --mount type=bind,src=$(pwd),dst=$(pwd) \ + --workdir $(pwd) \ + ${{ env.BUILD_IMAGE }} /bin/sh; \ + do \ + sleep 30; \ + done # Docs diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index a54fb0afa90..7a9d89d3f42 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -84,10 +84,13 @@ jobs: - name: Start container run: | - docker run --name BUILD --pull never -dit \ - --mount type=bind,src=$(pwd),dst=$(pwd) \ - --workdir $(pwd) \ - ${{ env.BUILD_IMAGE }} /bin/sh + while ! docker run --name BUILD --pull never -dit \ + --mount type=bind,src=$(pwd),dst=$(pwd) \ + --workdir $(pwd) \ + ${{ env.BUILD_IMAGE }} /bin/sh; \ + do \ + sleep 30; \ + done # Docs From 261fbd9df763c4cc8e02230df8a04bc1b14c7b35 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 Nov 2023 19:24:10 -0700 Subject: [PATCH 30/93] .github/workflows/doc-build.yml: Run HTML diffing outside of the container (fixup) --- .github/workflows/doc-build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 7a9d89d3f42..de5c3fe8c78 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -194,7 +194,6 @@ jobs: ##### cp -r -L /sage/local/share/doc/sage/html ./docs # Zip everything for increased performance zip -r docs.zip docs - shell: sh .ci/docker-exec-script.sh BUILD /sage {0} - name: Upload docs if: (success() || failure()) && steps.copy.outcome == 'success' From 87e40a3ffc821c58a4f2846f46c21dbd73600745 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 Nov 2023 19:55:29 -0700 Subject: [PATCH 31/93] src/doc/Makefile (doc-pdf-other): Depend on doc-pdf-reference, not ...html... --- src/doc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/doc/Makefile b/src/doc/Makefile index 2f76dfb9cb4..08e0a17e19d 100644 --- a/src/doc/Makefile +++ b/src/doc/Makefile @@ -79,7 +79,7 @@ doc-pdf-reference: doc-inventory-reference $(MAKE) SAGE_DOCBUILD_OPTS="$(SAGE_DOCBUILD_OPTS) --no-prune-empty-dirs" doc-pdf--reference_top # other documentation, pdf -doc-pdf-other: doc-html-reference +doc-pdf-other: doc-pdf-reference $(eval DOCS = $(shell sage --docbuild --all-documents all)) @if [ -z "$(DOCS)" ]; then echo "Error: 'sage --docbuild --all-documents' failed"; exit 1; fi $(MAKE) SAGE_DOCBUILD_OPTS="$(SAGE_DOCBUILD_OPTS) --no-prune-empty-dirs" $(foreach doc, $(wordlist 2, 100, $(DOCS)), doc-pdf--$(subst /,-,$(doc))) From 292cc89e100332baaaf0746a046542e2c03e9e9f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 Nov 2023 22:02:27 -0700 Subject: [PATCH 32/93] .github/workflows/doc-build.yml: Run HTML diffing outside of the container (fixup) --- .github/workflows/doc-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index de5c3fe8c78..6a8c5384e8c 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -137,9 +137,9 @@ jobs: set -ex docker cp BUILD:/sage/local/share/doc/sage/html docs/html docker cp BUILD:/sage/local/share/doc/sage/index.html docs/index.html - (cd html && git commit -a -m 'new') + (cd docs && git commit -a -m 'new') # Wipe out chronic diffs between old doc and new doc - (cd html && \ + (cd docs && \ find . -name "*.html" | xargs sed -i -e '\;; d') # Create CHANGES.html echo '' > ./docs/CHANGES.html From 192e6b5a6f5e10c97db203d31e7debcf756af402 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 Nov 2023 22:40:35 -0700 Subject: [PATCH 33/93] .github/workflows/[doc-]build[-pdf].yml: Refer to the built container by an output --- .github/workflows/build.yml | 36 ++++++++++++----------------- .github/workflows/doc-build-pdf.yml | 12 ++++------ .github/workflows/doc-build.yml | 12 ++++------ 3 files changed, 25 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b47386cadcf..3d9139ebefe 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -73,6 +73,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Build Docker image + id: image uses: docker/build-push-action@v5 with: push: false @@ -90,13 +91,10 @@ jobs: - name: Start container run: | - while ! docker run --name BUILD --pull never -dit \ - --mount type=bind,src=$(pwd),dst=$(pwd) \ - --workdir $(pwd) \ - ${{ env.BUILD_IMAGE }} /bin/sh; \ - do \ - sleep 30; \ - done + docker run --name BUILD --pull never -dit \ + --mount type=bind,src=$(pwd),dst=$(pwd) \ + --workdir $(pwd) \ + ${{ steps.image.outputs.imageid }} /bin/sh # Testing @@ -150,6 +148,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Build Docker image + id: image uses: docker/build-push-action@v5 with: push: false @@ -167,13 +166,10 @@ jobs: - name: Start container run: | - while ! docker run --name BUILD --pull never -dit \ - --mount type=bind,src=$(pwd),dst=$(pwd) \ - --workdir $(pwd) \ - ${{ env.BUILD_IMAGE }} /bin/sh; \ - do \ - sleep 30; \ - done + docker run --name BUILD --pull never -dit \ + --mount type=bind,src=$(pwd),dst=$(pwd) \ + --workdir $(pwd) \ + ${{ steps.image.outputs.imageid }} /bin/sh # Testing @@ -221,6 +217,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Build Docker image + id: image uses: docker/build-push-action@v5 with: push: false @@ -238,13 +235,10 @@ jobs: - name: Start container run: | - while ! docker run --name BUILD --pull never -dit \ - --mount type=bind,src=$(pwd),dst=$(pwd) \ - --workdir $(pwd) \ - ${{ env.BUILD_IMAGE }} /bin/sh; \ - do \ - sleep 30; \ - done + docker run --name BUILD --pull never -dit \ + --mount type=bind,src=$(pwd),dst=$(pwd) \ + --workdir $(pwd) \ + ${{ steps.image.outputs.imageid }} /bin/sh # Testing diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index 53f7bc806c2..18bb019d6c0 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -63,6 +63,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Build Docker image + id: image uses: docker/build-push-action@v5 with: push: false @@ -80,13 +81,10 @@ jobs: - name: Start container run: | - while ! docker run --name BUILD --pull never -dit \ - --mount type=bind,src=$(pwd),dst=$(pwd) \ - --workdir $(pwd) \ - ${{ env.BUILD_IMAGE }} /bin/sh; \ - do \ - sleep 30; \ - done + docker run --name BUILD --pull never -dit \ + --mount type=bind,src=$(pwd),dst=$(pwd) \ + --workdir $(pwd) \ + ${{ steps.image.outputs.imageid }} /bin/sh # Docs diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 6a8c5384e8c..0e6e5c6a780 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -67,6 +67,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Build Docker image + id: image uses: docker/build-push-action@v5 with: push: false @@ -84,13 +85,10 @@ jobs: - name: Start container run: | - while ! docker run --name BUILD --pull never -dit \ - --mount type=bind,src=$(pwd),dst=$(pwd) \ - --workdir $(pwd) \ - ${{ env.BUILD_IMAGE }} /bin/sh; \ - do \ - sleep 30; \ - done + docker run --name BUILD --pull never -dit \ + --mount type=bind,src=$(pwd),dst=$(pwd) \ + --workdir $(pwd) \ + ${{ steps.image.outputs.imageid }} /bin/sh # Docs From 12709e9d3638cf04037319f70d1aef3622d07935 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 Nov 2023 23:08:28 -0700 Subject: [PATCH 34/93] Revert "tox.ini (docker): Pass ACTIONS_... variables" This reverts commit 35997330dd9e5e9aa16a81959fd8cdef11880e92. --- tox.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/tox.ini b/tox.ini index c54a6cb2356..8047c24433f 100644 --- a/tox.ini +++ b/tox.ini @@ -156,9 +156,6 @@ passenv = docker-incremental: FROM_DOCKER_REPOSITORY docker-incremental: FROM_DOCKER_TARGET docker-incremental: FROM_DOCKER_TAG - docker: ACTIONS_CACHE_URL - docker: ACTIONS_RUNTIME_URL - docker: ACTIONS_RUNTIME_TOKEN local: MAKE local: PREFIX local: SAGE_NUM_THREADS From 54a54bc459ca5b80afaf2b1b49bf63e277008095 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 Nov 2023 23:13:16 -0700 Subject: [PATCH 35/93] .github/workflows/doc-build-pdf.yml: Restore install of texlive --- .github/workflows/doc-build-pdf.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index 18bb019d6c0..f27fdc20978 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -88,6 +88,14 @@ jobs: # Docs + - name: Update system packages + run: | + export PATH="build/bin:$PATH" + eval $(sage-print-system-package-command auto update) + eval $(sage-print-system-package-command auto --yes --no-install-recommends install zip) + eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git texlive) + shell: sh .ci/docker-exec-script.sh BUILD /sage {0} + - name: Build docs (PDF) id: docbuild run: | From 9670c8658206be4a515f078cb1382f390373af6f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 Nov 2023 23:27:48 -0700 Subject: [PATCH 36/93] .github/workflows/doc-build.yml: Fix output path --- .github/workflows/doc-build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 0e6e5c6a780..40427195a4b 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -103,8 +103,8 @@ jobs: # mathjax path in new doc mathjax_path_to=$(docker exec -e SAGE_USE_CDNS=yes BUILD /sage/sage -python -c "from sage_docbuild.conf import mathjax_path; print(mathjax_path)") new_version=$(docker exec BUILD cat src/VERSION.txt) - mkdir -p docs/html - docker cp BUILD:/sage/local/share/doc/sage/html docs/html + mkdir -p docs/ + docker cp BUILD:/sage/local/share/doc/sage/html docs/ # Wipe out chronic diffs between old doc and new doc (cd docs && \ find . -name "*.html" | xargs sed -i -e '/class="sidebar-brand-text"/ s/Sage [0-9a-z.]* /Sage '"$new_version"' /' \ @@ -133,8 +133,8 @@ jobs: if: (success() || failure()) && steps.docbuild.outcome == 'success' run: | set -ex - docker cp BUILD:/sage/local/share/doc/sage/html docs/html - docker cp BUILD:/sage/local/share/doc/sage/index.html docs/index.html + docker cp BUILD:/sage/local/share/doc/sage/html docs + docker cp BUILD:/sage/local/share/doc/sage/index.html docs (cd docs && git commit -a -m 'new') # Wipe out chronic diffs between old doc and new doc (cd docs && \ From 38eb74af70f3249962ac045ad255f68de73108c9 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 9 Nov 2023 19:50:01 -0800 Subject: [PATCH 37/93] .github/workflows/build.yml: Use a local registry --- .github/workflows/build.yml | 51 +++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3d9139ebefe..bd08501310b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,7 +29,7 @@ concurrency: env: # Adapted from docker.yml TOX_ENV: "docker-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-incremental" - BUILD_IMAGE: "ghcr.io/${{ github.repository }}/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:ci" + BUILD_IMAGE: "localhost:5000/${{ github.repository }}/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:ci" FROM_DOCKER_REPOSITORY: "ghcr.io/sagemath/sage/" FROM_DOCKER_TARGET: "with-targets" FROM_DOCKER_TAG: ${{ github.event.inputs.docker_tag || 'dev'}} @@ -38,6 +38,12 @@ env: jobs: build: runs-on: ubuntu-latest + services: + # https://docs.docker.com/build/ci/github-actions/local-registry/ + registry: + image: registry:2 + ports: + - 5000:5000 steps: - name: Checkout id: checkout @@ -71,13 +77,16 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + driver-opts: network=host - name: Build Docker image id: image uses: docker/build-push-action@v5 with: - push: false - load: true + # push and load may not be set together at the moment + push: true + load: false context: . tags: ${{ env.BUILD_IMAGE }} target: with-targets @@ -91,10 +100,10 @@ jobs: - name: Start container run: | - docker run --name BUILD --pull never -dit \ + docker run --name BUILD -dit \ --mount type=bind,src=$(pwd),dst=$(pwd) \ --workdir $(pwd) \ - ${{ steps.image.outputs.imageid }} /bin/sh + ${{ env.BUILD_IMAGE }} /bin/sh # Testing @@ -108,6 +117,12 @@ jobs: mod: runs-on: ubuntu-latest needs: [build] + services: + # https://docs.docker.com/build/ci/github-actions/local-registry/ + registry: + image: registry:2 + ports: + - 5000:5000 strategy: fail-fast: false matrix: @@ -146,13 +161,15 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + driver-opts: network=host - name: Build Docker image id: image uses: docker/build-push-action@v5 with: - push: false - load: true + push: true + load: false context: . tags: ${{ env.BUILD_IMAGE }} target: with-targets @@ -166,10 +183,10 @@ jobs: - name: Start container run: | - docker run --name BUILD --pull never -dit \ + docker run --name BUILD -dit \ --mount type=bind,src=$(pwd),dst=$(pwd) \ --workdir $(pwd) \ - ${{ steps.image.outputs.imageid }} /bin/sh + ${{ env.BUILD_IMAGE }} /bin/sh # Testing @@ -182,6 +199,12 @@ jobs: test-long: runs-on: ubuntu-latest needs: [build] + services: + # https://docs.docker.com/build/ci/github-actions/local-registry/ + registry: + image: registry:2 + ports: + - 5000:5000 steps: - name: Checkout id: checkout @@ -215,13 +238,15 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + driver-opts: network=host - name: Build Docker image id: image uses: docker/build-push-action@v5 with: - push: false - load: true + push: true + load: false context: . tags: ${{ env.BUILD_IMAGE }} target: with-targets @@ -235,10 +260,10 @@ jobs: - name: Start container run: | - docker run --name BUILD --pull never -dit \ + docker run --name BUILD -dit \ --mount type=bind,src=$(pwd),dst=$(pwd) \ --workdir $(pwd) \ - ${{ steps.image.outputs.imageid }} /bin/sh + ${{ env.BUILD_IMAGE }} /bin/sh # Testing From dd14ffd5e0b65a9198f1026d464bce93805ba002 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 9 Nov 2023 21:33:47 -0800 Subject: [PATCH 38/93] .github/workflows/doc-build[-pdf].yml: Use a local registry --- .github/workflows/doc-build-pdf.yml | 19 ++++++++++++++----- .github/workflows/doc-build.yml | 19 ++++++++++++++----- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index f27fdc20978..6630acf2570 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -23,7 +23,7 @@ concurrency: env: # Same as in build.yml TOX_ENV: "docker-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-incremental" - BUILD_IMAGE: "ghcr.io/${{ github.repository }}/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:ci" + BUILD_IMAGE: "localhost:5000/${{ github.repository }}/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:ci" FROM_DOCKER_REPOSITORY: "ghcr.io/sagemath/sage/" FROM_DOCKER_TARGET: "with-targets" FROM_DOCKER_TAG: ${{ github.event.inputs.docker_tag || 'dev'}} @@ -32,6 +32,12 @@ env: jobs: build-docs-pdf: runs-on: ubuntu-latest + services: + # https://docs.docker.com/build/ci/github-actions/local-registry/ + registry: + image: registry:2 + ports: + - 5000:5000 steps: - name: Checkout uses: actions/checkout@v4 @@ -61,13 +67,16 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + driver-opts: network=host - name: Build Docker image id: image uses: docker/build-push-action@v5 with: - push: false - load: true + # push and load may not be set together at the moment + push: true + load: false context: . tags: ${{ env.BUILD_IMAGE }} target: with-targets @@ -81,10 +90,10 @@ jobs: - name: Start container run: | - docker run --name BUILD --pull never -dit \ + docker run --name BUILD -dit \ --mount type=bind,src=$(pwd),dst=$(pwd) \ --workdir $(pwd) \ - ${{ steps.image.outputs.imageid }} /bin/sh + ${{ env.BUILD_IMAGE }} /bin/sh # Docs diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index d776b305979..ed193864016 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -27,7 +27,7 @@ concurrency: env: # Same as in build.yml TOX_ENV: "docker-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-incremental" - BUILD_IMAGE: "ghcr.io/${{ github.repository }}/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:ci" + BUILD_IMAGE: "localhost:5000/${{ github.repository }}/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:ci" FROM_DOCKER_REPOSITORY: "ghcr.io/sagemath/sage/" FROM_DOCKER_TARGET: "with-targets" FROM_DOCKER_TAG: ${{ github.event.inputs.docker_tag || 'dev'}} @@ -36,6 +36,12 @@ env: jobs: build-docs: runs-on: ubuntu-latest + services: + # https://docs.docker.com/build/ci/github-actions/local-registry/ + registry: + image: registry:2 + ports: + - 5000:5000 steps: - name: Checkout uses: actions/checkout@v4 @@ -65,13 +71,16 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + driver-opts: network=host - name: Build Docker image id: image uses: docker/build-push-action@v5 with: - push: false - load: true + # push and load may not be set together at the moment + push: true + load: false context: . tags: ${{ env.BUILD_IMAGE }} target: with-targets @@ -85,10 +94,10 @@ jobs: - name: Start container run: | - docker run --name BUILD --pull never -dit \ + docker run --name BUILD -dit \ --mount type=bind,src=$(pwd),dst=$(pwd) \ --workdir $(pwd) \ - ${{ steps.image.outputs.imageid }} /bin/sh + ${{ env.BUILD_IMAGE }} /bin/sh # Docs From dcf6d6d9f7c7565bc862db3213353ea380b63b14 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 13 Nov 2023 14:05:34 -0800 Subject: [PATCH 39/93] .github/workflows/doc-build.yml: Use explicit docker exec also for livedoc --- .github/workflows/doc-build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 73844be9f4b..864f4768e03 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -172,26 +172,26 @@ jobs: id: buildlivedoc if: (success() || failure()) && steps.copy.outcome == 'success' && github.repository == 'sagemath/sage' && github.ref == 'refs/heads/develop' run: | - set -ex + export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 export SAGE_USE_CDNS=yes export SAGE_LIVE_DOC=yes export SAGE_JUPYTER_SERVER=binder:sagemath/sage-binder-env/dev make doc-clean doc-uninstall ./config.status && make sagemath_doc_html-no-deps - working-directory: ./worktree-image env: MAKE: make -j2 --output-sync=recurse SAGE_NUM_THREADS: 2 + shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0} - name: Copy live doc id: copylivedoc if: (success() || failure()) && steps.buildlivedoc.outcome == 'success' run: | - set -ex mkdir -p ./livedoc cp -r -L /sage/local/share/doc/sage/html ./livedoc cp /sage/local/share/doc/sage/index.html ./livedoc zip -r livedoc.zip livedoc + shell: sh .ci/docker-exec-script.sh BUILD . {0} - name: Upload live doc if: (success() || failure()) && steps.copylivedoc.outcome == 'success' From d9479b76a823c4208d831e7ee7acadf20c043a4c Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 10 Nov 2023 16:06:46 -0800 Subject: [PATCH 40/93] .github/workflows/[doc-]build[-pdf].yml: Free disk space --- .github/workflows/build.yml | 30 +++++++++++++++++++++++++++++ .github/workflows/doc-build-pdf.yml | 10 ++++++++++ .github/workflows/doc-build.yml | 10 ++++++++++ 3 files changed, 50 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd08501310b..0b160ab9633 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,6 +45,16 @@ jobs: ports: - 5000:5000 steps: + - name: Maximize build disk space + uses: easimon/maximize-build-space@v8 + with: + # need space in /var for Docker images + root-reserve-mb: 40000 + remove-dotnet: true + remove-android: true + remove-haskell: true + remove-codeql: true + remove-docker-images: true - name: Checkout id: checkout uses: actions/checkout@v4 @@ -129,6 +139,16 @@ jobs: targets: - sagemath_categories-check steps: + - name: Maximize build disk space + uses: easimon/maximize-build-space@v8 + with: + # need space in /var for Docker images + root-reserve-mb: 40000 + remove-dotnet: true + remove-android: true + remove-haskell: true + remove-codeql: true + remove-docker-images: true - name: Checkout id: checkout uses: actions/checkout@v4 @@ -206,6 +226,16 @@ jobs: ports: - 5000:5000 steps: + - name: Maximize build disk space + uses: easimon/maximize-build-space@v8 + with: + # need space in /var for Docker images + root-reserve-mb: 40000 + remove-dotnet: true + remove-android: true + remove-haskell: true + remove-codeql: true + remove-docker-images: true - name: Checkout id: checkout uses: actions/checkout@v4 diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index 6630acf2570..26c948ee018 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -39,6 +39,16 @@ jobs: ports: - 5000:5000 steps: + - name: Maximize build disk space + uses: easimon/maximize-build-space@v8 + with: + # need space in /var for Docker images + root-reserve-mb: 40000 + remove-dotnet: true + remove-android: true + remove-haskell: true + remove-codeql: true + remove-docker-images: true - name: Checkout uses: actions/checkout@v4 - name: Install test prerequisites diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 75c3f430c27..2f4d2c740ac 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -43,6 +43,16 @@ jobs: ports: - 5000:5000 steps: + - name: Maximize build disk space + uses: easimon/maximize-build-space@v8 + with: + # need space in /var for Docker images + root-reserve-mb: 40000 + remove-dotnet: true + remove-android: true + remove-haskell: true + remove-codeql: true + remove-docker-images: true - name: Checkout uses: actions/checkout@v4 - name: Install test prerequisites From 1fd1c5f162da199c39333b0b2693b1ede304fea6 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 13 Nov 2023 20:54:29 -0800 Subject: [PATCH 41/93] .github/workflows/build.yml: Fix location of .coverage --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0b160ab9633..2c1d3d6fe84 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -314,7 +314,7 @@ jobs: - name: Prepare coverage results if: success() || failure() run: | - venv/bin/python3 -m coverage combine src/.coverage/ + venv/bin/python3 -m coverage combine .coverage/ mkdir -p ${{ github.workspace }}/coverage-report venv/bin/python3 -m coverage xml -o ${{ github.workspace }}/coverage-report/coverage.xml shell: sh .ci/docker-exec-script.sh BUILD /sage {0} From 8253cdfdfc063fc64f76f5004014b9f846215288 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 15 Nov 2023 21:20:46 -0800 Subject: [PATCH 42/93] src/tox.ini (coverage.py-xml): New --- src/tox.ini | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/tox.ini b/src/tox.ini index 00f7a153f35..dabc52162fa 100644 --- a/src/tox.ini +++ b/src/tox.ini @@ -86,6 +86,24 @@ commands_post = {env:SAGE} --python -m coverage report {env:SAGE} --python -m coverage html -d "{envdir}" +[testenv:coverage.py-xml] +# https://coverage.readthedocs.io/en/latest/index.html +description = + run the Sage doctester with Coverage.py, generate XML report +## This toxenv bypasses the virtual environment set up by tox. +passenv = {[sagedirect]passenv} +setenv = {[sagedirect]setenv} +envdir = {[sagedirect]envdir} +allowlist_externals = {[sagedirect]allowlist_externals} +commands_pre = + {env:SAGE} -pip install -U coverage +commands = + {env:SAGE} --python -m coverage run "{toxinidir}/../venv/bin/sage-runtests" -p 0 {posargs:--all} +commands_post = + {env:SAGE} --python -m coverage combine + {env:SAGE} --python -m coverage report + {env:SAGE} --python -m coverage xml -o "{envdir}/coverage.xml" + [testenv:coverage] description = give information about doctest coverage of files From 30975487e5cad4af147570acf78d1ef6665aeca7 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 15 Nov 2023 21:21:12 -0800 Subject: [PATCH 43/93] .github/workflows/build.yml: Run tests with coverage via tox --- .github/workflows/build.yml | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2c1d3d6fe84..f6c665d560e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -297,26 +297,11 @@ jobs: # Testing - - name: Pytest - if: (success() || failure()) && contains(github.ref, 'pytest') - run: | - ./sage -python -m pip install coverage pytest-xdist - ./sage -python -m coverage run -m pytest -c tox.ini --doctest-modules || true - # Increase the length of the lines in the "short summary" - shell: sh .ci/docker-exec-script.sh BUILD /sage COLUMNS=120 {0} - - name: Test all files (sage -t --all --long) run: | - ./sage -python -m pip install coverage - ./sage -python -m coverage run src/bin/sage-runtests --all --long -p2 --random-seed=286735480429121101562228604801325644303 - shell: sh .ci/docker-exec-script.sh BUILD /sage {0} - - - name: Prepare coverage results - if: success() || failure() - run: | - venv/bin/python3 -m coverage combine .coverage/ + ./sage -tox -e coverage.py-xml -- --all --long -p2 --random-seed=286735480429121101562228604801325644303 mkdir -p ${{ github.workspace }}/coverage-report - venv/bin/python3 -m coverage xml -o ${{ github.workspace }}/coverage-report/coverage.xml + cp src/.tox/sagedirect/coverage.xml ${{ github.workspace }}/coverage-report/ shell: sh .ci/docker-exec-script.sh BUILD /sage {0} - name: Upload coverage to codecov From 309d0a585431bcc39fde7c0116d723e06a1017a8 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 15 Nov 2023 22:03:41 -0800 Subject: [PATCH 44/93] Add redirects to SAGE_ROOT/tox.ini --- tox.ini | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index bb0cc3e4af0..6dc97c67b14 100644 --- a/tox.ini +++ b/tox.ini @@ -794,7 +794,22 @@ allowlist_externals = {[sage_src]allowlist_externals} [testenv:coverage.py] description = run the Sage doctester with Coverage.py - (https://coverage.readthedocs.io/en/latest/index.html) +passenv = {[sage_src]passenv} +envdir = {[sage_src]envdir} +commands = {[sage_src]commands} +allowlist_externals = {[sage_src]allowlist_externals} + +[testenv:coverage.py-html] +description = + run the Sage doctester with Coverage.py, generate HTML report +passenv = {[sage_src]passenv} +envdir = {[sage_src]envdir} +commands = {[sage_src]commands} +allowlist_externals = {[sage_src]allowlist_externals} + +[testenv:coverage.py-xml] +description = + run the Sage doctester with Coverage.py, generate XML report passenv = {[sage_src]passenv} envdir = {[sage_src]envdir} commands = {[sage_src]commands} From b4c35a8da3f31424cea771ac15f2bc44a149a2ef Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 15 Nov 2023 22:27:22 -0800 Subject: [PATCH 45/93] build/bin/write-dockerfile.sh (incremental): Fix retrofitting of src/ --- build/bin/write-dockerfile.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/build/bin/write-dockerfile.sh b/build/bin/write-dockerfile.sh index 8c60aca20a3..18715f17331 100755 --- a/build/bin/write-dockerfile.sh +++ b/build/bin/write-dockerfile.sh @@ -292,11 +292,10 @@ ENV MAKE="make -j\${NUMPROC}" ARG USE_MAKEFLAGS="-k V=0" ENV SAGE_CHECK=warn ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,!cmake,!rpy2,!sage_sws2rst" -$ADD .gitignore /new +$ADD .gitignore /new/.gitignore $ADD src /new/src RUN if command -v git; then \ - rm -f /sage/.git && \ - cd /new && \ + cd /new && rm -rf .git && \ ./.ci/retrofit-worktree.sh worktree-pre /sage; \ else \ rm -rf /sage/src; \ From 041c3dd8af4d909e9670c1edff139f702f44b5b6 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 16 Nov 2023 12:20:20 -0800 Subject: [PATCH 46/93] .github/workflows/build.yml: Upload to codecov also on failed tests --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f6c665d560e..d1bb8fcaf43 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -289,6 +289,7 @@ jobs: TARGETS=ci-build-with-fallback - name: Start container + id: container run: | docker run --name BUILD -dit \ --mount type=bind,src=$(pwd),dst=$(pwd) \ @@ -305,6 +306,7 @@ jobs: shell: sh .ci/docker-exec-script.sh BUILD /sage {0} - name: Upload coverage to codecov + if: (success() || failure()) && steps.container.outcome == 'success' uses: codecov/codecov-action@v3 with: directory: ./coverage-report From a25a9813b8183f8ceb451736b5b35ed50f9b4797 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 16 Nov 2023 21:05:07 -0800 Subject: [PATCH 47/93] .github/workflows/build.yml: Copy coverage results from container in a separate step --- .github/workflows/build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d1bb8fcaf43..7f5917c4bdc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -301,6 +301,11 @@ jobs: - name: Test all files (sage -t --all --long) run: | ./sage -tox -e coverage.py-xml -- --all --long -p2 --random-seed=286735480429121101562228604801325644303 + shell: sh .ci/docker-exec-script.sh BUILD /sage {0} + + - name: Copy coverage results + if: (success() || failure()) && steps.container.outcome == 'success' + run: | mkdir -p ${{ github.workspace }}/coverage-report cp src/.tox/sagedirect/coverage.xml ${{ github.workspace }}/coverage-report/ shell: sh .ci/docker-exec-script.sh BUILD /sage {0} From faccef1b9795d55cec0ca616e32d75b27bfdc595 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 10 Dec 2023 20:54:15 -0800 Subject: [PATCH 48/93] src/sage/misc/package.py: Update # needs from #35095 --- src/sage/misc/package.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sage/misc/package.py b/src/sage/misc/package.py index dcae7d0c0fc..8287286eb51 100644 --- a/src/sage/misc/package.py +++ b/src/sage/misc/package.py @@ -179,16 +179,16 @@ def pip_installed_packages(normalization=None): sage: # optional - sage_spkg sage: from sage.misc.package import pip_installed_packages sage: d = pip_installed_packages() - sage: 'scipy' in d or 'SciPy' in d + sage: 'scipy' in d or 'SciPy' in d # needs scipy True - sage: d['beautifulsoup4'] # optional - beautifulsoup4 + sage: d['beautifulsoup4'] # needs beautifulsoup4 '...' sage: d['prompt-toolkit'] '...' sage: d = pip_installed_packages(normalization='spkg') sage: d['prompt_toolkit'] '...' - sage: d['scipy'] + sage: d['scipy'] # needs scipy '...' """ with open(os.devnull, 'w') as devnull: From deffac275dae946b1c6bdb9b39e4cb1c1b43c340 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 10 Dec 2023 23:47:22 -0800 Subject: [PATCH 49/93] .ci/create-changes-html.sh: Quoting fix --- .ci/create-changes-html.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/create-changes-html.sh b/.ci/create-changes-html.sh index 1a6fbeef27b..021beb8e5ab 100755 --- a/.ci/create-changes-html.sh +++ b/.ci/create-changes-html.sh @@ -51,7 +51,7 @@ diffParagraphs.forEach(paragraph => { EOF echo '' >> CHANGES.html echo '' >> CHANGES.html -(cd $DOC_REPOSITORY && git diff $BASE_DOC_COMMIT -- *.html) > diff.txt +(cd $DOC_REPOSITORY && git diff $BASE_DOC_COMMIT -- "*.html") > diff.txt python3 - << EOF import os, re, html with open('diff.txt', 'r') as f: From 7f982bb0591ea5f34e72ce8893f319fc0df0fead Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 20 Dec 2023 14:07:05 -0800 Subject: [PATCH 50/93] .github/workflows/ci-conda-known-test-failures.json: Add random failure https://github.com/sagemath/sage/issues/36832 --- .github/workflows/ci-conda-known-test-failures.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-conda-known-test-failures.json b/.github/workflows/ci-conda-known-test-failures.json index 2d828ac98fb..4680c5e7c81 100644 --- a/.github/workflows/ci-conda-known-test-failures.json +++ b/.github/workflows/ci-conda-known-test-failures.json @@ -47,6 +47,9 @@ "sage.rings.polynomial.skew_polynomial_finite_field": { "failed": true }, + "sage.schemes.elliptic_curves.ell_field": { + "failed": true + }, "sage.tests.gap_packages": { "failed": true } From ee0cb6b033f3d31be8c09cf8705a307c21e2b84c Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 20 Dec 2023 23:03:28 -0800 Subject: [PATCH 51/93] .github/workflows/ci-conda-known-test-failures.json: Add https://github.com/sagemath/sage/issues/36939 --- .github/workflows/ci-conda-known-test-failures.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-conda-known-test-failures.json b/.github/workflows/ci-conda-known-test-failures.json index 4680c5e7c81..18b898c6685 100644 --- a/.github/workflows/ci-conda-known-test-failures.json +++ b/.github/workflows/ci-conda-known-test-failures.json @@ -35,6 +35,9 @@ "sage.modular.modform.l_series_gross_zagier": { "failed": true }, + "sage.parallel.map_reduce": { + "failed": true + }, "sage.rings.function_field.drinfeld_modules.morphism": { "failed": true }, From fed6570ef3dffebaa58689ddae78c514addcac16 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 20 Dec 2023 23:15:09 -0800 Subject: [PATCH 52/93] .github/workflows/ci-conda-known-test-failures.json: Add https://github.com/sagemath/sage/issues/35973 --- .github/workflows/ci-conda-known-test-failures.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-conda-known-test-failures.json b/.github/workflows/ci-conda-known-test-failures.json index 18b898c6685..8010bfc3727 100644 --- a/.github/workflows/ci-conda-known-test-failures.json +++ b/.github/workflows/ci-conda-known-test-failures.json @@ -53,6 +53,9 @@ "sage.schemes.elliptic_curves.ell_field": { "failed": true }, + "sage.structure.coerce_actions": { + "failed": true + }, "sage.tests.gap_packages": { "failed": true } From 22ef48640af4094cfad0f999b94645fdc5617b64 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 20 Dec 2023 23:20:05 -0800 Subject: [PATCH 53/93] .github/workflows/ci-conda-known-test-failures.json: Add some failures from https://github.com/sagemath/sage/pull/36372 --- .github/workflows/ci-conda-known-test-failures.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci-conda-known-test-failures.json b/.github/workflows/ci-conda-known-test-failures.json index 8010bfc3727..12ae507bedc 100644 --- a/.github/workflows/ci-conda-known-test-failures.json +++ b/.github/workflows/ci-conda-known-test-failures.json @@ -2,6 +2,9 @@ "sage_setup.clean": { "failed": true }, + "sage.algebras.fusion_rings.fusion_ring": { + "failed": true + }, "sage.combinat.cluster_algebra_quiver.quiver": { "failed": true }, @@ -47,12 +50,18 @@ "sage.rings.polynomial.multi_polynomial_libsingular": { "failed": true }, + "sage.rings.polynomial.polynomial_element": { + "failed": true + }, "sage.rings.polynomial.skew_polynomial_finite_field": { "failed": true }, "sage.schemes.elliptic_curves.ell_field": { "failed": true }, + "sage.sets.recursively_enumerated_set": { + "failed": true + }, "sage.structure.coerce_actions": { "failed": true }, From 179ebb4d4f2519fd069cbccfed240251fb48e41e Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 21 Dec 2023 12:11:15 -0800 Subject: [PATCH 54/93] .github/workflows/ci-conda-known-test-failures.json: Add references from commit messages --- .../ci-conda-known-test-failures.json | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci-conda-known-test-failures.json b/.github/workflows/ci-conda-known-test-failures.json index 12ae507bedc..348815e5720 100644 --- a/.github/workflows/ci-conda-known-test-failures.json +++ b/.github/workflows/ci-conda-known-test-failures.json @@ -3,13 +3,13 @@ "failed": true }, "sage.algebras.fusion_rings.fusion_ring": { - "failed": true + "failed": "see https://github.com/sagemath/sage/pull/36372" }, "sage.combinat.cluster_algebra_quiver.quiver": { - "failed": true + "failed": "failure seen in https://github.com/sagemath/sage/actions/runs/6836592771/job/18591690058#step:11:10059" }, "sage.geometry.cone": { - "failed": true + "failed": "failure seen in https://github.com/sagemath/sage/actions/runs/6827937663/job/18571052628#step:11:12362" }, "sage.groups.matrix_gps.finitely_generated_gap": { "failed": true @@ -18,31 +18,31 @@ "failed": true }, "sage.libs.gap.element": { - "failed": true + "failed": "failure seen in https://github.com/sagemath/sage/actions/runs/6840579851/job/18600012965#step:11:13016" }, "sage.libs.singular.singular": { "failed": true }, "sage.matrix.matrix2": { - "failed": true + "failed": "failure seen in https://github.com/sagemath/sage/actions/runs/6835143781/job/18588599649#step:11:16939" }, "sage.matrix.matrix_integer_sparse": { - "failed": true + "failed": "failure seen in https://github.com/sagemath/sage/actions/runs/6827937663/job/18571052628#step:11:12362" }, "sage.misc.lazy_import": { "failed": true }, "sage.misc.weak_dict": { - "failed": true + "failed": "failure seen in https://github.com/sagemath/sage/actions/runs/6870325919/job/18684964234#step:11:10059" }, "sage.modular.modform.l_series_gross_zagier": { - "failed": true + "failed": "failure seen in https://github.com/sagemath/sage/actions/runs/6859244281/job/18651257775#step:11:10102" }, "sage.parallel.map_reduce": { - "failed": true + "failed": "random failure https://github.com/sagemath/sage/issues/36939" }, "sage.rings.function_field.drinfeld_modules.morphism": { - "failed": true + "failed": "failure seen in https://github.com/sagemath/sage/actions/runs/6840502530/job/18599835766#step:11:10107" }, "sage.rings.polynomial.multi_polynomial_ideal": { "failed": true @@ -51,19 +51,19 @@ "failed": true }, "sage.rings.polynomial.polynomial_element": { - "failed": true + "failed": "see https://github.com/sagemath/sage/pull/36372" }, "sage.rings.polynomial.skew_polynomial_finite_field": { "failed": true }, "sage.schemes.elliptic_curves.ell_field": { - "failed": true + "failed": "random failure https://github.com/sagemath/sage/issues/36832" }, "sage.sets.recursively_enumerated_set": { - "failed": true + "failed": "see https://github.com/sagemath/sage/pull/36372" }, "sage.structure.coerce_actions": { - "failed": true + "failed": "random failure https://github.com/sagemath/sage/issues/35973" }, "sage.tests.gap_packages": { "failed": true From b41999bf330d591480cbe88d0b4893feb63a26a4 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 21 Dec 2023 12:40:00 -0800 Subject: [PATCH 55/93] .github/workflows/ci-conda-known-test-failures.json: Add details from https://github.com/sagemath/sage/pull/36372 --- .../ci-conda-known-test-failures.json | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-conda-known-test-failures.json b/.github/workflows/ci-conda-known-test-failures.json index 348815e5720..47d0adcd8e9 100644 --- a/.github/workflows/ci-conda-known-test-failures.json +++ b/.github/workflows/ci-conda-known-test-failures.json @@ -1,15 +1,18 @@ { + "doc.en.constructions.calculus": { + "failed": "unreported random failures in plotting" + }, "sage_setup.clean": { - "failed": true + "failed": "_find_stale_files finds some stale files under sage/tests when executed under conda" }, "sage.algebras.fusion_rings.fusion_ring": { - "failed": "see https://github.com/sagemath/sage/pull/36372" + "failed": "unreported random timeouts" }, "sage.combinat.cluster_algebra_quiver.quiver": { "failed": "failure seen in https://github.com/sagemath/sage/actions/runs/6836592771/job/18591690058#step:11:10059" }, "sage.geometry.cone": { - "failed": "failure seen in https://github.com/sagemath/sage/actions/runs/6827937663/job/18571052628#step:11:12362" + "failed": "unreported random timeouts seen in https://github.com/sagemath/sage/actions/runs/6827937663/job/18571052628#step:11:12362" }, "sage.groups.matrix_gps.finitely_generated_gap": { "failed": true @@ -36,13 +39,16 @@ "failed": "failure seen in https://github.com/sagemath/sage/actions/runs/6870325919/job/18684964234#step:11:10059" }, "sage.modular.modform.l_series_gross_zagier": { - "failed": "failure seen in https://github.com/sagemath/sage/actions/runs/6859244281/job/18651257775#step:11:10102" + "failed": "unreported failure seen in https://github.com/sagemath/sage/actions/runs/6859244281/job/18651257775#step:11:10102" }, "sage.parallel.map_reduce": { "failed": "random failure https://github.com/sagemath/sage/issues/36939" }, + "sage.plot.plot": { + "failed": "unreported random failure (macOS)" + }, "sage.rings.function_field.drinfeld_modules.morphism": { - "failed": "failure seen in https://github.com/sagemath/sage/actions/runs/6840502530/job/18599835766#step:11:10107" + "failed": "unreported random failure seen in https://github.com/sagemath/sage/actions/runs/6840502530/job/18599835766#step:11:10107" }, "sage.rings.polynomial.multi_polynomial_ideal": { "failed": true @@ -51,7 +57,7 @@ "failed": true }, "sage.rings.polynomial.polynomial_element": { - "failed": "see https://github.com/sagemath/sage/pull/36372" + "failed": "unreported random failure in symbolic 'roots' (macOS)" }, "sage.rings.polynomial.skew_polynomial_finite_field": { "failed": true @@ -60,7 +66,7 @@ "failed": "random failure https://github.com/sagemath/sage/issues/36832" }, "sage.sets.recursively_enumerated_set": { - "failed": "see https://github.com/sagemath/sage/pull/36372" + "failed": "random failures related to AlarmInterrupt (macOS)" }, "sage.structure.coerce_actions": { "failed": "random failure https://github.com/sagemath/sage/issues/35973" From 63b8b108fee7e18a8c51d7e1460cea7ad9a02804 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 21 Dec 2023 12:40:58 -0800 Subject: [PATCH 56/93] src/sage/doctest/reporting.py: Include message in '[failed in baseline]' if provided --- src/sage/doctest/reporting.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/sage/doctest/reporting.py b/src/sage/doctest/reporting.py index 42cc3b6d85e..a1e85af1a57 100644 --- a/src/sage/doctest/reporting.py +++ b/src/sage/doctest/reporting.py @@ -213,10 +213,13 @@ def report_head(self, source, fail_msg=None): baseline = self.controller.source_baseline(source) if fail_msg: cmd += " # " + fail_msg - if baseline.get('failed', False): + if failed := baseline.get('failed', False): if not fail_msg: cmd += " #" - cmd += " [failed in baseline]" + if failed is True: + cmd += " [failed in baseline]" + else: + cmd += f" [failed in baseline: {failed}]" return cmd def report(self, source, timeout, return_code, results, output, pid=None): From 2bc41439aaf016cd672f61d8c9aae037d6ce6e81 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 21 Mar 2024 22:36:32 -0700 Subject: [PATCH 57/93] .github/workflows/*build*.yml: Update use of easimon/maximize-build-space --- .github/workflows/build.yml | 12 ++++++------ .github/workflows/doc-build-pdf.yml | 2 +- .github/workflows/doc-build.yml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9aef34f7586..a5c96663c66 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,10 +46,10 @@ jobs: - 5000:5000 steps: - name: Maximize build disk space - uses: easimon/maximize-build-space@v8 + uses: easimon/maximize-build-space@v10 with: # need space in /var for Docker images - root-reserve-mb: 40000 + root-reserve-mb: 30000 remove-dotnet: true remove-android: true remove-haskell: true @@ -152,10 +152,10 @@ jobs: - sagemath_categories-check steps: - name: Maximize build disk space - uses: easimon/maximize-build-space@v8 + uses: easimon/maximize-build-space@v10 with: # need space in /var for Docker images - root-reserve-mb: 40000 + root-reserve-mb: 30000 remove-dotnet: true remove-android: true remove-haskell: true @@ -239,10 +239,10 @@ jobs: - 5000:5000 steps: - name: Maximize build disk space - uses: easimon/maximize-build-space@v8 + uses: easimon/maximize-build-space@v10 with: # need space in /var for Docker images - root-reserve-mb: 40000 + root-reserve-mb: 30000 remove-dotnet: true remove-android: true remove-haskell: true diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index e44c81f3096..5d4e05082ec 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -43,7 +43,7 @@ jobs: uses: easimon/maximize-build-space@v8 with: # need space in /var for Docker images - root-reserve-mb: 40000 + root-reserve-mb: 30000 remove-dotnet: true remove-android: true remove-haskell: true diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 450dbe68d2f..416dcf7f684 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -47,7 +47,7 @@ jobs: uses: easimon/maximize-build-space@v8 with: # need space in /var for Docker images - root-reserve-mb: 40000 + root-reserve-mb: 30000 remove-dotnet: true remove-android: true remove-haskell: true From bc2585c070d791c59d5328b16fd5cf0e3bf8d558 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 21 Mar 2024 22:44:55 -0700 Subject: [PATCH 58/93] .github/workflows/*build*.yml: Update NUMPROC, SAGE_NUMTHREADS etc. for updated Linux runners --- .github/workflows/build.yml | 20 ++++++++++---------- .github/workflows/doc-build-pdf.yml | 6 +++--- .github/workflows/doc-build.yml | 8 ++++---- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a5c96663c66..51bfe6c10c5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -104,8 +104,8 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max build-args: | - NUMPROC=4 - USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse + NUMPROC=6 + USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse TARGETS_PRE=build/make/Makefile TARGETS=ci-build-with-fallback @@ -131,9 +131,9 @@ jobs: - name: Test changed files (sage -t --new) run: | - export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 + export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=4 # We run tests with "sage -t --new"; this only tests the uncommitted changes. - ./sage -t --new -p2 + ./sage -t --new -p4 shell: sh .ci/docker-exec-script.sh BUILD /sage {0} mod: @@ -208,8 +208,8 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max build-args: | - NUMPROC=4 - USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse + NUMPROC=6 + USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse TARGETS_PRE=build/make/Makefile TARGETS=ci-build-with-fallback @@ -224,7 +224,7 @@ jobs: - name: Test modularized distributions run: | - export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 + export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=4 make V=0 tox-ensure && make ${{ matrix.targets }} shell: sh .ci/docker-exec-script.sh BUILD /sage {0} @@ -295,8 +295,8 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max build-args: | - NUMPROC=4 - USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse + NUMPROC=6 + USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse TARGETS_PRE=build/make/Makefile TARGETS=ci-build-with-fallback @@ -313,7 +313,7 @@ jobs: - name: Test all files (sage -t --all --long) run: | ./sage -python -m pip install coverage - ./sage -python -m coverage run --rcfile=src/tox.ini src/bin/sage-runtests --all --long -p2 --format github --random-seed=286735480429121101562228604801325644303 + ./sage -python -m coverage run --rcfile=src/tox.ini src/bin/sage-runtests --all --long -p4 --format github --random-seed=286735480429121101562228604801325644303 shell: sh .ci/docker-exec-script.sh BUILD /sage {0} - name: Copy coverage results diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index 5d4e05082ec..167293a5898 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -94,8 +94,8 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max build-args: | - NUMPROC=4 - USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse + NUMPROC=6 + USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse TARGETS_PRE=build/make/Makefile TARGETS=ci-build-with-fallback @@ -119,7 +119,7 @@ jobs: - name: Build docs (PDF) id: docbuild run: | - export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 + export MAKE="make -j5 --output-sync=recurse" SAGE_NUM_THREADS=5 make doc-clean doc-uninstall; make sagemath_doc_html-build-deps sagemath_doc_pdf-no-deps shell: sh .ci/docker-exec-script.sh BUILD /sage {0} diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 416dcf7f684..8803b6104a1 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -98,8 +98,8 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max build-args: | - NUMPROC=4 - USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=3 --output-sync=recurse + NUMPROC=6 + USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse TARGETS_PRE=build/make/Makefile TARGETS=ci-build-with-fallback @@ -142,7 +142,7 @@ jobs: # Always non-incremental because of the concern that # incremental docbuild may introduce broken links (inter-file references) though build succeeds run: | - export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 + export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=5 make doc-clean doc-uninstall export SAGE_USE_CDNS=yes ./config.status && make sagemath_doc_html-no-deps @@ -177,7 +177,7 @@ jobs: id: buildlivedoc if: (success() || failure()) && steps.copy.outcome == 'success' && github.repository == 'sagemath/sage' && github.ref == 'refs/heads/develop' run: | - export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2 + export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=5 export PATH="build/bin:$PATH" eval $(sage-print-system-package-command auto update) eval $(sage-print-system-package-command auto --yes --no-install-recommends install zip) From 6888dc70446faad849a0159a2e7b2df1e162be08 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 21 Mar 2024 23:32:21 -0700 Subject: [PATCH 59/93] .github/workflows/doc-build.yml: fixup --- .github/workflows/doc-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 8803b6104a1..28aa6bee0fb 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -142,7 +142,7 @@ jobs: # Always non-incremental because of the concern that # incremental docbuild may introduce broken links (inter-file references) though build succeeds run: | - export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=5 + export MAKE="make -j5 --output-sync=recurse" SAGE_NUM_THREADS=5 make doc-clean doc-uninstall export SAGE_USE_CDNS=yes ./config.status && make sagemath_doc_html-no-deps @@ -177,7 +177,7 @@ jobs: id: buildlivedoc if: (success() || failure()) && steps.copy.outcome == 'success' && github.repository == 'sagemath/sage' && github.ref == 'refs/heads/develop' run: | - export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=5 + export MAKE="make -j5 --output-sync=recurse" SAGE_NUM_THREADS=5 export PATH="build/bin:$PATH" eval $(sage-print-system-package-command auto update) eval $(sage-print-system-package-command auto --yes --no-install-recommends install zip) From 793ae732269a9e9f5385fc568210a3c8daa8bdd2 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 4 Apr 2024 23:38:08 -0700 Subject: [PATCH 60/93] build/pkgs/sagemath_*/spkg-check: Replace by spkg-check.in from #35095 --- build/pkgs/sagemath_categories/spkg-check | 1 - build/pkgs/sagemath_categories/spkg-check.in | 1 + build/pkgs/sagemath_environment/spkg-check | 1 - build/pkgs/sagemath_environment/spkg-check.in | 1 + build/pkgs/sagemath_objects/spkg-check | 27 ---------- build/pkgs/sagemath_objects/spkg-check.in | 49 +++++++++++++++++++ build/pkgs/sagemath_repl/spkg-check | 1 - build/pkgs/sagemath_repl/spkg-check.in | 1 + 8 files changed, 52 insertions(+), 30 deletions(-) delete mode 120000 build/pkgs/sagemath_categories/spkg-check create mode 120000 build/pkgs/sagemath_categories/spkg-check.in delete mode 120000 build/pkgs/sagemath_environment/spkg-check create mode 120000 build/pkgs/sagemath_environment/spkg-check.in delete mode 100755 build/pkgs/sagemath_objects/spkg-check create mode 100644 build/pkgs/sagemath_objects/spkg-check.in delete mode 120000 build/pkgs/sagemath_repl/spkg-check create mode 120000 build/pkgs/sagemath_repl/spkg-check.in diff --git a/build/pkgs/sagemath_categories/spkg-check b/build/pkgs/sagemath_categories/spkg-check deleted file mode 120000 index 91c6b1835fc..00000000000 --- a/build/pkgs/sagemath_categories/spkg-check +++ /dev/null @@ -1 +0,0 @@ -../sagemath_objects/spkg-check \ No newline at end of file diff --git a/build/pkgs/sagemath_categories/spkg-check.in b/build/pkgs/sagemath_categories/spkg-check.in new file mode 120000 index 00000000000..24eb3322489 --- /dev/null +++ b/build/pkgs/sagemath_categories/spkg-check.in @@ -0,0 +1 @@ +../sagemath_objects/spkg-check.in \ No newline at end of file diff --git a/build/pkgs/sagemath_environment/spkg-check b/build/pkgs/sagemath_environment/spkg-check deleted file mode 120000 index 91c6b1835fc..00000000000 --- a/build/pkgs/sagemath_environment/spkg-check +++ /dev/null @@ -1 +0,0 @@ -../sagemath_objects/spkg-check \ No newline at end of file diff --git a/build/pkgs/sagemath_environment/spkg-check.in b/build/pkgs/sagemath_environment/spkg-check.in new file mode 120000 index 00000000000..24eb3322489 --- /dev/null +++ b/build/pkgs/sagemath_environment/spkg-check.in @@ -0,0 +1 @@ +../sagemath_objects/spkg-check.in \ No newline at end of file diff --git a/build/pkgs/sagemath_objects/spkg-check b/build/pkgs/sagemath_objects/spkg-check deleted file mode 100755 index 87cea0fc38f..00000000000 --- a/build/pkgs/sagemath_objects/spkg-check +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -cd src - -export PIP_NO_INDEX=true -export PIP_FIND_LINKS="file://$SAGE_SPKG_WHEELS" - -export TOX_PARALLEL_NO_SPINNER=1 -wheel="$(sed -n '1s,.*@ file://,,p' $SAGE_SPKG_SCRIPTS/$PKG_BASE/spkg-requirements.txt)" -echo Running "tox -r -p auto -v --installpkg $wheel" -tox -r -p auto -v --installpkg "$wheel" -status=$? -case $status:$SAGE_CHECK:$([ -r known-test-failures.json ]; echo $?) in - 0:*:0) echo "Passed the test suite (modulo baseline known-test-failures*.json)";; - 0:*:*) echo "Passed the test suite";; - *:warn:0) echo "Warning: New failures (not in baseline known-test-failures*.json (ignored)"; status=0;; - *:warn:*) echo "Warning: Failures testing the package (ignored)"; status=0;; - *:yes:0) echo "New failures, not in baseline known-test-failures*.json";; - *:yes:*) echo "Failures testing the package";; -esac -# Show summaries of failures (suppress lines ending with '[failed in baseline]') -for f in $(pwd)/.tox/sagepython-sagewheels-nopypi-norequirements*/log/*-command*.log; do - if [ -r "$f" ]; then - echo "$f" - grep '^sage -t.*#[^]]*$' "$f" - fi -done -exit $status diff --git a/build/pkgs/sagemath_objects/spkg-check.in b/build/pkgs/sagemath_objects/spkg-check.in new file mode 100644 index 00000000000..9ff603e1b46 --- /dev/null +++ b/build/pkgs/sagemath_objects/spkg-check.in @@ -0,0 +1,49 @@ +cd src + +if [ ! -r tox.ini ]; then + echo "Not testing the package because there is no tox.ini" + exit 0 +fi + +export PIP_NO_INDEX=true +export PIP_FIND_LINKS="file://$SAGE_SPKG_WHEELS" + +unset tox_args + +wheel="$(sed -n '1s,.*@ file://,,p' $SAGE_SPKG_SCRIPTS/$PKG_BASE/spkg-requirements.txt)" +if [ -n "$wheel" ]; then + tox_envs=$(tox -l) + tox_args="-r -p auto -v --installpkg $wheel" +elif [ "$SAGE_EDITABLE" = yes ]; then + tox_envs=$(tox -l | sed s/norequirements/editable/) + # FIXME: Should use -r if sage_setup or another build requirement changes + tox_args="-r -v -v -v -v -e $(echo $tox_envs | sed 's/ /,/g')" +else + echo "Not testing the package because SAGE_WHEELS=$SAGE_WHEELS and SAGE_EDITABLE=$SAGE_EDITABLE" + exit 0 +fi + +export TOX_PARALLEL_NO_SPINNER=1 + +echo Running "tox $tox_args" +tox $tox_args +status=$? +case $status:$SAGE_CHECK:$([ -r known-test-failures.json ]; echo $?) in + 0:no:*) echo "Not testing the package because SAGE_CHECK=no";; + 0:*:0) echo "Passed the test suite (modulo baseline known-test-failures*.json)";; + 0:*:*) echo "Passed the test suite";; + *:warn:0) echo "Warning: New failures (not in baseline known-test-failures*.json (ignored)"; status=0;; + *:warn:*) echo "Warning: Failures testing the package (ignored)"; status=0;; + *:yes:0) echo "New failures, not in baseline known-test-failures*.json";; + *:yes:*) echo "Failures testing the package";; +esac +# Show summaries of failures (suppress lines ending with '[failed in baseline]') +for e in $tox_envs; do + for f in $(pwd)/.tox/$e/log/*-command*.log; do + if [ -r "$f" ]; then + echo "$f" + grep '^sage -t.*#[^]]*$' "$f" + fi + done +done +exit $status diff --git a/build/pkgs/sagemath_repl/spkg-check b/build/pkgs/sagemath_repl/spkg-check deleted file mode 120000 index 91c6b1835fc..00000000000 --- a/build/pkgs/sagemath_repl/spkg-check +++ /dev/null @@ -1 +0,0 @@ -../sagemath_objects/spkg-check \ No newline at end of file diff --git a/build/pkgs/sagemath_repl/spkg-check.in b/build/pkgs/sagemath_repl/spkg-check.in new file mode 120000 index 00000000000..24eb3322489 --- /dev/null +++ b/build/pkgs/sagemath_repl/spkg-check.in @@ -0,0 +1 @@ +../sagemath_objects/spkg-check.in \ No newline at end of file From b163a9bb6f3c117b879915853e2c03c88624e195 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 00:00:55 -0700 Subject: [PATCH 61/93] .github/workflows/ci-linux-incremental.yml: Do not run on changes to our script packages --- .github/workflows/ci-linux-incremental.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-linux-incremental.yml b/.github/workflows/ci-linux-incremental.yml index 82b14472c55..4861ef1d1d0 100644 --- a/.github/workflows/ci-linux-incremental.yml +++ b/.github/workflows/ci-linux-incremental.yml @@ -19,7 +19,12 @@ on: pull_request: paths: - 'build/pkgs/**' - - 'pkgs/**' + - '!build/pkgs/sage_conf/**' + - '!build/pkgs/sage_docbuild/**' + - '!build/pkgs/sage_setup/**' + - '!build/pkgs/sage_sws2rst/**' + - '!build/pkgs/sagelib/**' + - '!build/pkgs/sagemath_*/**' workflow_dispatch: concurrency: From 95641d378d23d31b826d97e674fcfc10e1151ed8 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 11:29:40 -0700 Subject: [PATCH 62/93] pkgs/sagemath-{objects,categories,environment,repl}/tox.ini: Update from #35095 --- pkgs/sagemath-categories/tox.ini | 27 ++++++++++++++++++++++----- pkgs/sagemath-environment/tox.ini | 16 ++++++++++++++++ pkgs/sagemath-objects/tox.ini | 16 ++++++++++++++++ pkgs/sagemath-repl/tox.ini | 27 ++++++++++++++++++++++----- 4 files changed, 76 insertions(+), 10 deletions(-) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index b4bb49d132a..29b6b15f30b 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -49,6 +49,9 @@ passenv = {[pkgenv]passenv} setenv = {[pkgenv]setenv} # Sage scripts such as sage-runtests like to use $HOME/.sage HOME={envdir} + # Stop 'sage -t --installed' from picking up doc installed in SAGE_LOCAL + SAGE_DOC=/doesnotexist + KNOWN_TEST_FAILURES={toxinidir}/known-test-failures.json allowlist_externals = bash @@ -61,7 +64,7 @@ commands = {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.categories.all import *; SimplicialComplexes(); FunctionFields()' bash -c 'cd $(python -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") \ - && sage-runtests -p --initial --environment=sage.all__sagemath_categories --probe all --baseline-stats-path={toxinidir}/known-test-failures.json --optional=sage --installed' + && sage-runtests -p --force-lib --initial --environment=sage.all__sagemath_categories --probe all --baseline-stats-path=$KNOWN_TEST_FAILURES --optional=sage --installed' [testenv:.tox] # Allow access to PyPI for auto-provisioning a suitable tox version @@ -84,6 +87,11 @@ setenv = {[pkgenv]setenv} basepython = {env:SAGE_VENV}/bin/python3 +[testenv:.pkg-sagepython-sagewheels-nopypi-editable] +config_settings_build_editable = + editable_mode = strict + + [testenv:sagepython] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython @@ -92,10 +100,6 @@ package_env = .pkg-sagepython basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython-sagewheels-nopypi -[testenv:sagepython-sagewheels-nopypi-norequirements] -basepython = {env:SAGE_VENV}/bin/python3 -package_env = .pkg-sagepython-sagewheels-nopypi - [testenv:sagepython-sagewheels] basepython = {env:SAGE_VENV}/bin/python package_env = .pkg-sagepython @@ -103,3 +107,16 @@ package_env = .pkg-sagepython [testenv:sagepython-norequirements] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython + + +[testenv:sagepython-sagewheels-nopypi-norequirements] +basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi + +[testenv:sagepython-sagewheels-nopypi-editable] +basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi-editable +package = editable +deps = -r requirements-editable.txt +config_settings_build_editable = + editable_mode = strict diff --git a/pkgs/sagemath-environment/tox.ini b/pkgs/sagemath-environment/tox.ini index 6bf1f2a6ebb..96bcbdf4397 100644 --- a/pkgs/sagemath-environment/tox.ini +++ b/pkgs/sagemath-environment/tox.ini @@ -49,6 +49,9 @@ passenv = {[pkgenv]passenv} setenv = {[pkgenv]setenv} # Sage scripts such as sage-runtests like to use $HOME/.sage HOME={envdir} + # Stop 'sage -t --installed' from picking up doc installed in SAGE_LOCAL + SAGE_DOC=/doesnotexist + KNOWN_TEST_FAILURES={toxinidir}/known-test-failures.json allowlist_externals = bash @@ -78,6 +81,11 @@ setenv = {[pkgenv]setenv} basepython = {env:SAGE_VENV}/bin/python3 +[testenv:.pkg-sagepython-sagewheels-nopypi-editable] +config_settings_build_editable = + editable_mode = strict + + [testenv:sagepython] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython @@ -98,3 +106,11 @@ package_env = .pkg-sagepython [testenv:sagepython-sagewheels-nopypi-norequirements] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython-sagewheels-nopypi + +[testenv:sagepython-sagewheels-nopypi-editable] +basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi-editable +package = editable +deps = -r requirements-editable.txt +config_settings_build_editable = + editable_mode = strict diff --git a/pkgs/sagemath-objects/tox.ini b/pkgs/sagemath-objects/tox.ini index a8f5a6d6a76..d51d473d84f 100644 --- a/pkgs/sagemath-objects/tox.ini +++ b/pkgs/sagemath-objects/tox.ini @@ -49,6 +49,9 @@ passenv = {[pkgenv]passenv} setenv = {[pkgenv]setenv} # Sage scripts such as sage-runtests like to use $HOME/.sage HOME={envdir} + # Stop 'sage -t --installed' from picking up doc installed in SAGE_LOCAL + SAGE_DOC=/doesnotexist + KNOWN_TEST_FAILURES={toxinidir}/known-test-failures.json allowlist_externals = bash @@ -82,6 +85,11 @@ setenv = {[pkgenv]setenv} basepython = {env:SAGE_VENV}/bin/python3 +[testenv:.pkg-sagepython-sagewheels-nopypi-editable] +config_settings_build_editable = + editable_mode = strict + + [testenv:sagepython] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython @@ -102,3 +110,11 @@ package_env = .pkg-sagepython [testenv:sagepython-sagewheels-nopypi-norequirements] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython-sagewheels-nopypi + +[testenv:sagepython-sagewheels-nopypi-editable] +basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi-editable +package = editable +deps = -r requirements-editable.txt +config_settings_build_editable = + editable_mode = strict diff --git a/pkgs/sagemath-repl/tox.ini b/pkgs/sagemath-repl/tox.ini index 679153a2947..ffdf5b97b1c 100644 --- a/pkgs/sagemath-repl/tox.ini +++ b/pkgs/sagemath-repl/tox.ini @@ -49,6 +49,9 @@ passenv = {[pkgenv]passenv} setenv = {[pkgenv]setenv} # Sage scripts such as sage-runtests like to use $HOME/.sage HOME={envdir} + # Stop 'sage -t --installed' from picking up doc installed in SAGE_LOCAL + SAGE_DOC=/doesnotexist + KNOWN_TEST_FAILURES={toxinidir}/known-test-failures.json allowlist_externals = bash @@ -57,7 +60,7 @@ commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.repl.all; import sage.doctest.all' - bash -c 'cd $({envpython} -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") && sage-runtests -p --environment=sage.all__sagemath_repl --baseline-stats-path={toxinidir}/known-test-failures.json --initial --optional=sage sage/repl sage/doctest sage/misc/sage_input.py sage/misc/sage_eval.py' + bash -c 'cd $({envpython} -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") && sage-runtests -p --environment=sage.all__sagemath_repl --baseline-stats-path=$KNOWN_TEST_FAILURES --initial --optional=sage sage/repl sage/doctest sage/misc/sage_input.py sage/misc/sage_eval.py' [testenv:.tox] # Allow access to PyPI for auto-provisioning a suitable tox version @@ -80,6 +83,11 @@ setenv = {[pkgenv]setenv} basepython = {env:SAGE_VENV}/bin/python3 +[testenv:.pkg-sagepython-sagewheels-nopypi-editable] +config_settings_build_editable = + editable_mode = strict + + [testenv:sagepython] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython @@ -88,10 +96,6 @@ package_env = .pkg-sagepython basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython-sagewheels-nopypi -[testenv:sagepython-sagewheels-nopypi-norequirements] -basepython = {env:SAGE_VENV}/bin/python3 -package_env = .pkg-sagepython-sagewheels-nopypi - [testenv:sagepython-sagewheels] basepython = {env:SAGE_VENV}/bin/python package_env = .pkg-sagepython @@ -99,3 +103,16 @@ package_env = .pkg-sagepython [testenv:sagepython-norequirements] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython + + +[testenv:sagepython-sagewheels-nopypi-norequirements] +basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi + +[testenv:sagepython-sagewheels-nopypi-editable] +basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi-editable +package = editable +deps = -r requirements-editable.txt +config_settings_build_editable = + editable_mode = strict From cca2b8dd1cdd54f902b85d5adea57e0edbb481bd Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 11:30:08 -0700 Subject: [PATCH 63/93] pkgs/sagemath-{objects,categories,environment,repl}/tox.ini: Replace use of {envpython} by just python3 --- pkgs/sagemath-categories/tox.ini | 4 ++-- pkgs/sagemath-environment/tox.ini | 2 +- pkgs/sagemath-objects/tox.ini | 4 ++-- pkgs/sagemath-repl/tox.ini | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index 29b6b15f30b..3f237f7feef 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -58,10 +58,10 @@ allowlist_externals = commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. - {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.cpython.builtin_types, sage.cpython.cython_metaclass, sage.cpython.debug, sage.structure.all, sage.categories.all' + python3 -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.cpython.builtin_types, sage.cpython.cython_metaclass, sage.cpython.debug, sage.structure.all, sage.categories.all' # Test that importing sage.categories.all initializes categories - {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.categories.all import *; SimplicialComplexes(); FunctionFields()' + python3 -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.categories.all import *; SimplicialComplexes(); FunctionFields()' bash -c 'cd $(python -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") \ && sage-runtests -p --force-lib --initial --environment=sage.all__sagemath_categories --probe all --baseline-stats-path=$KNOWN_TEST_FAILURES --optional=sage --installed' diff --git a/pkgs/sagemath-environment/tox.ini b/pkgs/sagemath-environment/tox.ini index 96bcbdf4397..d3dd3dcaef9 100644 --- a/pkgs/sagemath-environment/tox.ini +++ b/pkgs/sagemath-environment/tox.ini @@ -58,7 +58,7 @@ allowlist_externals = commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. - {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.features.all import all_features; print(sorted(all_features(), key=lambda x: x.name)); import sage.misc.package' + python3 -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.features.all import all_features; print(sorted(all_features(), key=lambda x: x.name)); import sage.misc.package' [testenv:.tox] # Allow access to PyPI for auto-provisioning a suitable tox version diff --git a/pkgs/sagemath-objects/tox.ini b/pkgs/sagemath-objects/tox.ini index d51d473d84f..91a0d0bfe62 100644 --- a/pkgs/sagemath-objects/tox.ini +++ b/pkgs/sagemath-objects/tox.ini @@ -58,9 +58,9 @@ allowlist_externals = commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. - {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.structure.all, sage.categories.sets_cat' + python3 -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.structure.all, sage.categories.sets_cat' - {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.all__sagemath_objects import *' + python3 -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.all__sagemath_objects import *' #bash -c 'cd {temp_dir} && SAGE_SRC=$(python -c "from sage.env import SAGE_SRC; print(SAGE_SRC)") && sage-runtests --environment=sage.all__sagemath_objects --initial --optional=sage $SAGE_SRC/sage/structure || echo "(lots of doctest failures are expected)"' diff --git a/pkgs/sagemath-repl/tox.ini b/pkgs/sagemath-repl/tox.ini index ffdf5b97b1c..9ac2e942948 100644 --- a/pkgs/sagemath-repl/tox.ini +++ b/pkgs/sagemath-repl/tox.ini @@ -58,9 +58,9 @@ allowlist_externals = commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. - {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.repl.all; import sage.doctest.all' + python3 -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.repl.all; import sage.doctest.all' - bash -c 'cd $({envpython} -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") && sage-runtests -p --environment=sage.all__sagemath_repl --baseline-stats-path=$KNOWN_TEST_FAILURES --initial --optional=sage sage/repl sage/doctest sage/misc/sage_input.py sage/misc/sage_eval.py' + bash -c 'cd $(python3 -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") && sage-runtests -p --environment=sage.all__sagemath_repl --baseline-stats-path=$KNOWN_TEST_FAILURES --initial --optional=sage sage/repl sage/doctest sage/misc/sage_input.py sage/misc/sage_eval.py' [testenv:.tox] # Allow access to PyPI for auto-provisioning a suitable tox version From a8bffd3726d8f3f06cbb732e3852a1c38826dbfa Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 19:51:07 -0700 Subject: [PATCH 64/93] build/pkgs/sagemath_{objects,categories,environment,repl}: Back to using spkg-check, source sage-env etc. --- build/pkgs/sagemath_categories/spkg-check | 1 + build/pkgs/sagemath_categories/spkg-check.in | 1 - build/pkgs/sagemath_environment/spkg-check | 1 + build/pkgs/sagemath_environment/spkg-check.in | 1 - .../sagemath_objects/{spkg-check.in => spkg-check} | 10 ++++++++++ build/pkgs/sagemath_repl/spkg-check | 1 + build/pkgs/sagemath_repl/spkg-check.in | 1 - 7 files changed, 13 insertions(+), 3 deletions(-) create mode 120000 build/pkgs/sagemath_categories/spkg-check delete mode 120000 build/pkgs/sagemath_categories/spkg-check.in create mode 120000 build/pkgs/sagemath_environment/spkg-check delete mode 120000 build/pkgs/sagemath_environment/spkg-check.in rename build/pkgs/sagemath_objects/{spkg-check.in => spkg-check} (81%) mode change 100644 => 100755 create mode 120000 build/pkgs/sagemath_repl/spkg-check delete mode 120000 build/pkgs/sagemath_repl/spkg-check.in diff --git a/build/pkgs/sagemath_categories/spkg-check b/build/pkgs/sagemath_categories/spkg-check new file mode 120000 index 00000000000..91c6b1835fc --- /dev/null +++ b/build/pkgs/sagemath_categories/spkg-check @@ -0,0 +1 @@ +../sagemath_objects/spkg-check \ No newline at end of file diff --git a/build/pkgs/sagemath_categories/spkg-check.in b/build/pkgs/sagemath_categories/spkg-check.in deleted file mode 120000 index 24eb3322489..00000000000 --- a/build/pkgs/sagemath_categories/spkg-check.in +++ /dev/null @@ -1 +0,0 @@ -../sagemath_objects/spkg-check.in \ No newline at end of file diff --git a/build/pkgs/sagemath_environment/spkg-check b/build/pkgs/sagemath_environment/spkg-check new file mode 120000 index 00000000000..91c6b1835fc --- /dev/null +++ b/build/pkgs/sagemath_environment/spkg-check @@ -0,0 +1 @@ +../sagemath_objects/spkg-check \ No newline at end of file diff --git a/build/pkgs/sagemath_environment/spkg-check.in b/build/pkgs/sagemath_environment/spkg-check.in deleted file mode 120000 index 24eb3322489..00000000000 --- a/build/pkgs/sagemath_environment/spkg-check.in +++ /dev/null @@ -1 +0,0 @@ -../sagemath_objects/spkg-check.in \ No newline at end of file diff --git a/build/pkgs/sagemath_objects/spkg-check.in b/build/pkgs/sagemath_objects/spkg-check old mode 100644 new mode 100755 similarity index 81% rename from build/pkgs/sagemath_objects/spkg-check.in rename to build/pkgs/sagemath_objects/spkg-check index 9ff603e1b46..99a8528e0da --- a/build/pkgs/sagemath_objects/spkg-check.in +++ b/build/pkgs/sagemath_objects/spkg-check @@ -1,3 +1,4 @@ +#!/usr/bin/env bash cd src if [ ! -r tox.ini ]; then @@ -5,6 +6,15 @@ if [ ! -r tox.ini ]; then exit 0 fi +for lib in "$SAGE_SRC/bin/sage-src-env-config" "$SAGE_SRC/bin/sage-env-config" "$SAGE_SRC/bin/sage-env" "$SAGE_ROOT/build/bin/sage-build-env-config" "$SAGE_ROOT/build/bin/sage-build-env"; do + source "$lib" + if [ $? -ne 0 ]; then + echo >&2 "Error: failed to source $lib" + echo >&2 "Is $SAGE_ROOT the correct SAGE_ROOT?" + exit 1 + fi +done + export PIP_NO_INDEX=true export PIP_FIND_LINKS="file://$SAGE_SPKG_WHEELS" diff --git a/build/pkgs/sagemath_repl/spkg-check b/build/pkgs/sagemath_repl/spkg-check new file mode 120000 index 00000000000..91c6b1835fc --- /dev/null +++ b/build/pkgs/sagemath_repl/spkg-check @@ -0,0 +1 @@ +../sagemath_objects/spkg-check \ No newline at end of file diff --git a/build/pkgs/sagemath_repl/spkg-check.in b/build/pkgs/sagemath_repl/spkg-check.in deleted file mode 120000 index 24eb3322489..00000000000 --- a/build/pkgs/sagemath_repl/spkg-check.in +++ /dev/null @@ -1 +0,0 @@ -../sagemath_objects/spkg-check.in \ No newline at end of file From 77fa23b571e758711feb5718975e2a4beeb19468 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 20:21:45 -0700 Subject: [PATCH 65/93] CythonFeature: Do not fail on Python 3.12 without setuptools --- src/sage/features/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sage/features/__init__.py b/src/sage/features/__init__.py index a8b11ef7db5..ebeb55527a8 100644 --- a/src/sage/features/__init__.py +++ b/src/sage/features/__init__.py @@ -948,7 +948,10 @@ def _is_present(self): # Available since https://setuptools.pypa.io/en/latest/history.html#v59-0-0 from setuptools.errors import CCompilerError except ImportError: - from distutils.errors import CCompilerError + try: + from distutils.errors import CCompilerError + except ImportError: + CCompilerError = () with open(tmp_filename(ext=".pyx"), 'w') as pyx: pyx.write(self.test_code) try: From 05aec74ce6c1c4c2364c52ba1a9622d343c8da92 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 21:24:30 -0700 Subject: [PATCH 66/93] pkgs/sagemath-repl/pyproject.toml.m4: Add dependencies ipykernel, jupyter-client --- pkgs/sagemath-repl/pyproject.toml.m4 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/sagemath-repl/pyproject.toml.m4 b/pkgs/sagemath-repl/pyproject.toml.m4 index 2bc276158a4..1f8ad82cccf 100644 --- a/pkgs/sagemath-repl/pyproject.toml.m4 +++ b/pkgs/sagemath-repl/pyproject.toml.m4 @@ -13,8 +13,10 @@ description = "Sage: Open Source Mathematics Software: IPython kernel, Sage prep dependencies = [ SPKG_INSTALL_REQUIRES_sagemath_objects SPKG_INSTALL_REQUIRES_sagemath_environment + SPKG_INSTALL_REQUIRES_ipykernel SPKG_INSTALL_REQUIRES_ipython SPKG_INSTALL_REQUIRES_ipywidgets + SPKG_INSTALL_REQUIRES_jupyter_client ] dynamic = ["version"] include(`pyproject_toml_metadata.m4')dnl' From 9255a388771657c9777b03dcbd5dd46891f14860 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 21:31:22 -0700 Subject: [PATCH 67/93] pkgs/sagemath-{repl,categories}/tox.ini: Set PYDEVD_DISABLE_FILE_VALIDATION=1 --- pkgs/sagemath-categories/tox.ini | 2 ++ pkgs/sagemath-repl/tox.ini | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index 3f237f7feef..843b0ab242f 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -52,6 +52,8 @@ setenv = {[pkgenv]setenv} # Stop 'sage -t --installed' from picking up doc installed in SAGE_LOCAL SAGE_DOC=/doesnotexist KNOWN_TEST_FAILURES={toxinidir}/known-test-failures.json + # See src/bin/sage-env + PYDEVD_DISABLE_FILE_VALIDATION=1 allowlist_externals = bash diff --git a/pkgs/sagemath-repl/tox.ini b/pkgs/sagemath-repl/tox.ini index 9ac2e942948..9386ca22ae9 100644 --- a/pkgs/sagemath-repl/tox.ini +++ b/pkgs/sagemath-repl/tox.ini @@ -52,6 +52,8 @@ setenv = {[pkgenv]setenv} # Stop 'sage -t --installed' from picking up doc installed in SAGE_LOCAL SAGE_DOC=/doesnotexist KNOWN_TEST_FAILURES={toxinidir}/known-test-failures.json + # See src/bin/sage-env + PYDEVD_DISABLE_FILE_VALIDATION=1 allowlist_externals = bash From bdb342cd731e80ee7f59431efa4021f374bc53e0 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 21:41:07 -0700 Subject: [PATCH 68/93] pkgs/sagemath-categories/known-test-failures.json: Update --- .../known-test-failures.json | 101 ++++++++++-------- 1 file changed, 55 insertions(+), 46 deletions(-) diff --git a/pkgs/sagemath-categories/known-test-failures.json b/pkgs/sagemath-categories/known-test-failures.json index ddae185a7d9..cf70863646e 100644 --- a/pkgs/sagemath-categories/known-test-failures.json +++ b/pkgs/sagemath-categories/known-test-failures.json @@ -26,7 +26,7 @@ "ntests": 28 }, "sage.categories.affine_weyl_groups": { - "ntests": 14 + "ntests": 15 }, "sage.categories.algebra_ideals": { "failed": true, @@ -38,7 +38,7 @@ }, "sage.categories.algebras": { "failed": true, - "ntests": 20 + "ntests": 25 }, "sage.categories.algebras_with_basis": { "failed": true, @@ -126,7 +126,7 @@ }, "sage.categories.commutative_rings": { "failed": true, - "ntests": 39 + "ntests": 41 }, "sage.categories.complete_discrete_valuation": { "failed": true, @@ -148,15 +148,19 @@ }, "sage.categories.coxeter_groups": { "failed": true, - "ntests": 362 + "ntests": 392 }, "sage.categories.cw_complexes": { "failed": true, "ntests": 36 }, + "sage.categories.dedekind_domains": { + "failed": true, + "ntests": 30 + }, "sage.categories.discrete_valuation": { "failed": true, - "ntests": 23 + "ntests": 28 }, "sage.categories.distributive_magmas_and_additive_magmas": { "failed": true, @@ -169,9 +173,6 @@ "failed": true, "ntests": 7 }, - "sage.categories.drinfeld_modules": { - "ntests": 2 - }, "sage.categories.dual": { "failed": true, "ntests": 1 @@ -297,18 +298,18 @@ }, "sage.categories.filtered_modules_with_basis": { "failed": true, - "ntests": 65 + "ntests": 74 }, "sage.categories.finite_complex_reflection_groups": { "failed": true, - "ntests": 178 + "ntests": 189 }, "sage.categories.finite_coxeter_groups": { "ntests": 6 }, "sage.categories.finite_dimensional_algebras_with_basis": { "failed": true, - "ntests": 87 + "ntests": 91 }, "sage.categories.finite_dimensional_bialgebras_with_basis": { "failed": true, @@ -328,7 +329,7 @@ }, "sage.categories.finite_dimensional_lie_algebras_with_basis": { "failed": true, - "ntests": 34 + "ntests": 44 }, "sage.categories.finite_dimensional_modules_with_basis": { "failed": true, @@ -493,10 +494,6 @@ "failed": true, "ntests": 13 }, - "sage.categories.inner_product_spaces": { - "failed": true, - "ntests": 9 - }, "sage.categories.integral_domains": { "failed": true, "ntests": 22 @@ -530,7 +527,7 @@ }, "sage.categories.lie_algebras": { "failed": true, - "ntests": 125 + "ntests": 135 }, "sage.categories.lie_algebras_with_basis": { "ntests": 19 @@ -586,7 +583,7 @@ }, "sage.categories.modules_with_basis": { "failed": true, - "ntests": 221 + "ntests": 232 }, "sage.categories.monoid_algebras": { "failed": true, @@ -668,14 +665,14 @@ }, "sage.categories.rings": { "failed": true, - "ntests": 141 + "ntests": 146 }, "sage.categories.rngs": { "ntests": 6 }, "sage.categories.schemes": { "failed": true, - "ntests": 23 + "ntests": 33 }, "sage.categories.semigroups": { "failed": true, @@ -711,7 +708,7 @@ }, "sage.categories.simplicial_sets": { "failed": true, - "ntests": 57 + "ntests": 98 }, "sage.categories.subobjects": { "ntests": 2 @@ -765,7 +762,7 @@ }, "sage.categories.unital_algebras": { "failed": true, - "ntests": 39 + "ntests": 37 }, "sage.categories.vector_spaces": { "failed": true, @@ -783,7 +780,7 @@ }, "sage.cpython.debug": { "failed": true, - "ntests": 14 + "ntests": 13 }, "sage.cpython.dict_del_by_value": { "failed": true, @@ -808,26 +805,24 @@ }, "sage.doctest.control": { "failed": true, - "ntests": 0 + "ntests": 232 }, "sage.doctest.external": { "ntests": 42 }, "sage.doctest.fixtures": { - "failed": true, "ntests": 59 }, "sage.doctest.forker": { "failed": true, - "ntests": 433 + "ntests": 434 }, "sage.doctest.parsing": { "failed": true, "ntests": 321 }, "sage.doctest.reporting": { - "failed": true, - "ntests": 124 + "ntests": 125 }, "sage.doctest.sources": { "failed": true, @@ -846,7 +841,7 @@ "ntests": 41 }, "sage.features": { - "ntests": 145 + "ntests": 143 }, "sage.features.all": { "ntests": 14 @@ -865,17 +860,23 @@ }, "sage.features.databases": { "failed": true, - "ntests": 26 + "ntests": 38 }, "sage.features.dvipng": { "ntests": 4 }, + "sage.features.ecm": { + "ntests": 4 + }, "sage.features.ffmpeg": { "ntests": 4 }, "sage.features.four_ti_2": { "ntests": 6 }, + "sage.features.fricas": { + "ntests": 6 + }, "sage.features.gap": { "ntests": 6 }, @@ -901,7 +902,11 @@ "sage.features.internet": { "ntests": 5 }, + "sage.features.jmol": { + "ntests": 4 + }, "sage.features.join_feature": { + "failed": true, "ntests": 25 }, "sage.features.kenzo": { @@ -960,7 +965,7 @@ }, "sage.features.sagemath": { "failed": true, - "ntests": 147 + "ntests": 151 }, "sage.features.singular": { "ntests": 4 @@ -968,9 +973,16 @@ "sage.features.sphinx": { "ntests": 4 }, + "sage.features.symengine_py": { + "ntests": 4 + }, "sage.features.tdlib": { "ntests": 2 }, + "sage.features.threejs": { + "failed": true, + "ntests": 6 + }, "sage.misc.abstract_method": { "failed": true, "ntests": 33 @@ -1001,7 +1013,7 @@ }, "sage.misc.decorators": { "failed": true, - "ntests": 126 + "ntests": 125 }, "sage.misc.fast_methods": { "failed": true, @@ -1059,7 +1071,7 @@ }, "sage.misc.package_dir": { "failed": true, - "ntests": 29 + "ntests": 35 }, "sage.misc.persist": { "failed": true, @@ -1091,7 +1103,7 @@ }, "sage.misc.sageinspect": { "failed": true, - "ntests": 329 + "ntests": 322 }, "sage.misc.superseded": { "failed": true, @@ -1157,7 +1169,7 @@ }, "sage.repl.interpreter": { "failed": true, - "ntests": 118 + "ntests": 114 }, "sage.repl.ipython_extension": { "failed": true, @@ -1165,7 +1177,7 @@ }, "sage.repl.ipython_kernel.install": { "failed": true, - "ntests": 40 + "ntests": 35 }, "sage.repl.ipython_kernel.interact": { "failed": true, @@ -1188,7 +1200,7 @@ }, "sage.repl.load": { "failed": true, - "ntests": 42 + "ntests": 38 }, "sage.repl.preparse": { "failed": true, @@ -1206,7 +1218,7 @@ }, "sage.repl.rich_output.backend_ipython": { "failed": true, - "ntests": 78 + "ntests": 75 }, "sage.repl.rich_output.buffer": { "failed": true, @@ -1214,7 +1226,7 @@ }, "sage.repl.rich_output.display_manager": { "failed": true, - "ntests": 88 + "ntests": 86 }, "sage.repl.rich_output.output_basic": { "ntests": 47 @@ -1251,7 +1263,7 @@ }, "sage.rings.ring": { "failed": true, - "ntests": 337 + "ntests": 298 }, "sage.sets.pythonclass": { "failed": true, @@ -1263,7 +1275,7 @@ }, "sage.structure.coerce": { "failed": true, - "ntests": 314 + "ntests": 310 }, "sage.structure.coerce_actions": { "failed": true, @@ -1298,7 +1310,7 @@ }, "sage.structure.factorization": { "failed": true, - "ntests": 203 + "ntests": 200 }, "sage.structure.factorization_integer": { "failed": true, @@ -1315,9 +1327,6 @@ "sage.structure.global_options": { "ntests": 145 }, - "sage.structure.graphics_file": { - "ntests": 8 - }, "sage.structure.indexed_generators": { "failed": true, "ntests": 90 @@ -1345,7 +1354,7 @@ }, "sage.structure.parent": { "failed": true, - "ntests": 300 + "ntests": 303 }, "sage.structure.parent_gens": { "failed": true, From 87a042a9609036a767af675a9bc2c2f6a80f64c8 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 22:55:06 -0700 Subject: [PATCH 69/93] **/tox.ini: Set upper bound tox < 4.14.1 because of https://github.com/tox-dev/tox/issues/3238 --- pkgs/sage-conf_pypi/tox.ini | 4 ++++ pkgs/sage-docbuild/tox.ini | 4 ++++ pkgs/sage-setup/tox.ini | 4 ++++ pkgs/sagemath-categories/tox.ini | 3 ++- pkgs/sagemath-environment/tox.ini | 3 ++- pkgs/sagemath-objects/tox.ini | 3 ++- pkgs/sagemath-repl/tox.ini | 3 ++- pkgs/sagemath-standard/tox.ini | 3 ++- src/tox.ini | 5 +++-- tox.ini | 5 +++-- 10 files changed, 28 insertions(+), 9 deletions(-) diff --git a/pkgs/sage-conf_pypi/tox.ini b/pkgs/sage-conf_pypi/tox.ini index fadf9a9cc6c..7160d4db678 100644 --- a/pkgs/sage-conf_pypi/tox.ini +++ b/pkgs/sage-conf_pypi/tox.ini @@ -1,6 +1,10 @@ [tox] envlist = py39, py310, py311, py39-user, py310-user, py311-user +requires = + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox<4.14.1 + [testenv:.pkg] basepython = py311 passenv = diff --git a/pkgs/sage-docbuild/tox.ini b/pkgs/sage-docbuild/tox.ini index 77da5a78ede..efa222028ff 100644 --- a/pkgs/sage-docbuild/tox.ini +++ b/pkgs/sage-docbuild/tox.ini @@ -8,6 +8,10 @@ # [tox] +requires = + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox<4.14.1 + [testenv] deps = -rrequirements.txt diff --git a/pkgs/sage-setup/tox.ini b/pkgs/sage-setup/tox.ini index fd935f2e978..e8a22f03101 100644 --- a/pkgs/sage-setup/tox.ini +++ b/pkgs/sage-setup/tox.ini @@ -12,6 +12,10 @@ # [tox] +requires = + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox<4.14.1 + [testenv] deps = -rrequirements.txt diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index 843b0ab242f..065f0790411 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -13,7 +13,8 @@ envlist = requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance - tox>=4.2 + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox>=4.2,<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-environment/tox.ini b/pkgs/sagemath-environment/tox.ini index d3dd3dcaef9..7973b0f46a2 100644 --- a/pkgs/sagemath-environment/tox.ini +++ b/pkgs/sagemath-environment/tox.ini @@ -13,7 +13,8 @@ envlist = requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance - tox>=4.2 + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox>=4.2,<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-objects/tox.ini b/pkgs/sagemath-objects/tox.ini index 91a0d0bfe62..52c5434e67e 100644 --- a/pkgs/sagemath-objects/tox.ini +++ b/pkgs/sagemath-objects/tox.ini @@ -13,7 +13,8 @@ envlist = requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance - tox>=4.2 + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox>=4.2,<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-repl/tox.ini b/pkgs/sagemath-repl/tox.ini index 9386ca22ae9..b65157e4f58 100644 --- a/pkgs/sagemath-repl/tox.ini +++ b/pkgs/sagemath-repl/tox.ini @@ -13,7 +13,8 @@ envlist = requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance - tox>=4.2 + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox>=4.2,<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-standard/tox.ini b/pkgs/sagemath-standard/tox.ini index 6aae1ef1bfa..c06afe8229d 100644 --- a/pkgs/sagemath-standard/tox.ini +++ b/pkgs/sagemath-standard/tox.ini @@ -62,7 +62,8 @@ envlist = requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance - tox>=4.2 + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox>=4.2,<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/src/tox.ini b/src/tox.ini index 8e562cced8e..e0dc1509f1f 100644 --- a/src/tox.ini +++ b/src/tox.ini @@ -26,8 +26,9 @@ envlist = doctest, coverage, startuptime, pycodestyle-minimal, relint, codespell skipsdist = true requires = - # For the renamed "allowlist_externals" keyword - tox>=3.18 + # For the renamed "allowlist_externals" keyword, need >= 3.18 + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox>=3.18,<4.14.1 [sagedirect] # Base for tox environments that bypass the virtual environment set up by tox, diff --git a/tox.ini b/tox.ini index 99d04aad388..c8d4a5a4ebc 100644 --- a/tox.ini +++ b/tox.ini @@ -129,8 +129,9 @@ envlist = # pycodestyle requires = - # For repaired numerical factors in tox 4: - tox>=4.2.7 + # For repaired numerical factors in tox 4, need >= 4.2.7. + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox>=4.2.7,<4.14.1 skipsdist = true From cd5a53ae9d48ac70d9d3346d149d6d0dbabd4552 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 23:22:32 -0700 Subject: [PATCH 70/93] Revert "pkgs/sagemath-{objects,categories,environment,repl}/tox.ini: Replace use of {envpython} by just python3" This reverts commit cca2b8dd1cdd54f902b85d5adea57e0edbb481bd. --- pkgs/sagemath-categories/tox.ini | 4 ++-- pkgs/sagemath-environment/tox.ini | 2 +- pkgs/sagemath-objects/tox.ini | 4 ++-- pkgs/sagemath-repl/tox.ini | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index 065f0790411..6eaa17fc20e 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -61,10 +61,10 @@ allowlist_externals = commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. - python3 -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.cpython.builtin_types, sage.cpython.cython_metaclass, sage.cpython.debug, sage.structure.all, sage.categories.all' + {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.cpython.builtin_types, sage.cpython.cython_metaclass, sage.cpython.debug, sage.structure.all, sage.categories.all' # Test that importing sage.categories.all initializes categories - python3 -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.categories.all import *; SimplicialComplexes(); FunctionFields()' + {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.categories.all import *; SimplicialComplexes(); FunctionFields()' bash -c 'cd $(python -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") \ && sage-runtests -p --force-lib --initial --environment=sage.all__sagemath_categories --probe all --baseline-stats-path=$KNOWN_TEST_FAILURES --optional=sage --installed' diff --git a/pkgs/sagemath-environment/tox.ini b/pkgs/sagemath-environment/tox.ini index 7973b0f46a2..fec1966d9a9 100644 --- a/pkgs/sagemath-environment/tox.ini +++ b/pkgs/sagemath-environment/tox.ini @@ -59,7 +59,7 @@ allowlist_externals = commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. - python3 -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.features.all import all_features; print(sorted(all_features(), key=lambda x: x.name)); import sage.misc.package' + {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.features.all import all_features; print(sorted(all_features(), key=lambda x: x.name)); import sage.misc.package' [testenv:.tox] # Allow access to PyPI for auto-provisioning a suitable tox version diff --git a/pkgs/sagemath-objects/tox.ini b/pkgs/sagemath-objects/tox.ini index 52c5434e67e..3e539e2e3b0 100644 --- a/pkgs/sagemath-objects/tox.ini +++ b/pkgs/sagemath-objects/tox.ini @@ -59,9 +59,9 @@ allowlist_externals = commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. - python3 -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.structure.all, sage.categories.sets_cat' + {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.structure.all, sage.categories.sets_cat' - python3 -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.all__sagemath_objects import *' + {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.all__sagemath_objects import *' #bash -c 'cd {temp_dir} && SAGE_SRC=$(python -c "from sage.env import SAGE_SRC; print(SAGE_SRC)") && sage-runtests --environment=sage.all__sagemath_objects --initial --optional=sage $SAGE_SRC/sage/structure || echo "(lots of doctest failures are expected)"' diff --git a/pkgs/sagemath-repl/tox.ini b/pkgs/sagemath-repl/tox.ini index b65157e4f58..d565aa7b2a4 100644 --- a/pkgs/sagemath-repl/tox.ini +++ b/pkgs/sagemath-repl/tox.ini @@ -61,9 +61,9 @@ allowlist_externals = commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. - python3 -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.repl.all; import sage.doctest.all' + {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.repl.all; import sage.doctest.all' - bash -c 'cd $(python3 -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") && sage-runtests -p --environment=sage.all__sagemath_repl --baseline-stats-path=$KNOWN_TEST_FAILURES --initial --optional=sage sage/repl sage/doctest sage/misc/sage_input.py sage/misc/sage_eval.py' + bash -c 'cd $({envpython} -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") && sage-runtests -p --environment=sage.all__sagemath_repl --baseline-stats-path=$KNOWN_TEST_FAILURES --initial --optional=sage sage/repl sage/doctest sage/misc/sage_input.py sage/misc/sage_eval.py' [testenv:.tox] # Allow access to PyPI for auto-provisioning a suitable tox version From 7db544816f1a02c28b3ff733771daa4c5e0ce816 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 23:25:45 -0700 Subject: [PATCH 71/93] pkgs/sagemath-{repl,categories,environment,objects}/requirements-editable.txt.m4: Copy from #35095 --- pkgs/sagemath-categories/requirements-editable.txt.m4 | 7 +++++++ .../sagemath-environment/requirements-editable.txt.m4 | 0 pkgs/sagemath-objects/requirements-editable.txt.m4 | 4 ++++ pkgs/sagemath-repl/requirements-editable.txt.m4 | 11 +++++++++++ 4 files changed, 22 insertions(+) create mode 100644 pkgs/sagemath-categories/requirements-editable.txt.m4 create mode 100644 pkgs/sagemath-environment/requirements-editable.txt.m4 create mode 100644 pkgs/sagemath-objects/requirements-editable.txt.m4 create mode 100644 pkgs/sagemath-repl/requirements-editable.txt.m4 diff --git a/pkgs/sagemath-categories/requirements-editable.txt.m4 b/pkgs/sagemath-categories/requirements-editable.txt.m4 new file mode 100644 index 00000000000..1c0abbe40ae --- /dev/null +++ b/pkgs/sagemath-categories/requirements-editable.txt.m4 @@ -0,0 +1,7 @@ +include(`sage_spkg_versions.m4')dnl +dnl Same as setup.cfg.m4 install_requires; FIXME: should pin to built wheels. +SPKG_INSTALL_REQUIRES_gmpy2 +SPKG_INSTALL_REQUIRES_cysignals +SPKG_INSTALL_REQUIRES_memory_allocator +-e ../sagemath-environment +-e ../sagemath-objects diff --git a/pkgs/sagemath-environment/requirements-editable.txt.m4 b/pkgs/sagemath-environment/requirements-editable.txt.m4 new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pkgs/sagemath-objects/requirements-editable.txt.m4 b/pkgs/sagemath-objects/requirements-editable.txt.m4 new file mode 100644 index 00000000000..df9b22b43aa --- /dev/null +++ b/pkgs/sagemath-objects/requirements-editable.txt.m4 @@ -0,0 +1,4 @@ +include(`sage_spkg_versions.m4')dnl +dnl Same as setup.cfg.m4 install_requires; FIXME: should pin to built wheels. +SPKG_INSTALL_REQUIRES_gmpy2 +SPKG_INSTALL_REQUIRES_cysignals diff --git a/pkgs/sagemath-repl/requirements-editable.txt.m4 b/pkgs/sagemath-repl/requirements-editable.txt.m4 new file mode 100644 index 00000000000..fa89ade7621 --- /dev/null +++ b/pkgs/sagemath-repl/requirements-editable.txt.m4 @@ -0,0 +1,11 @@ +include(`sage_spkg_versions.m4')dnl +dnl Same as setup.cfg.m4 install_requires (+ their install-requires) +dnl FIXME: should pin to built wheels. +SPKG_INSTALL_REQUIRES_gmpy2 +SPKG_INSTALL_REQUIRES_cysignals +SPKG_INSTALL_REQUIRES_memory_allocator +SPKG_INSTALL_REQUIRES_ipython +SPKG_INSTALL_REQUIRES_ipywidgets +dnl To be added when ready for editable: +-e ../sagemath-environment +-e ../sagemath-objects From ed2b0897e200d601b4dd8eabf22e57935a06bffd Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 23:32:52 -0700 Subject: [PATCH 72/93] **/tox.ini: Set upper bound tox < 4.14.1 because of https://github.com/tox-dev/tox/issues/3238 (fixup) --- pkgs/sagemath-categories/tox.ini | 2 +- pkgs/sagemath-environment/tox.ini | 2 +- pkgs/sagemath-objects/tox.ini | 2 +- pkgs/sagemath-repl/tox.ini | 2 +- pkgs/sagemath-standard/tox.ini | 2 +- src/tox.ini | 2 +- tox.ini | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index 6eaa17fc20e..2dff44b2f8d 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -14,7 +14,7 @@ requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=4.2,<4.14.1 + tox>=4.2, tox<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-environment/tox.ini b/pkgs/sagemath-environment/tox.ini index fec1966d9a9..502f6967798 100644 --- a/pkgs/sagemath-environment/tox.ini +++ b/pkgs/sagemath-environment/tox.ini @@ -14,7 +14,7 @@ requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=4.2,<4.14.1 + tox>=4.2, tox<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-objects/tox.ini b/pkgs/sagemath-objects/tox.ini index 3e539e2e3b0..24e7a2f107d 100644 --- a/pkgs/sagemath-objects/tox.ini +++ b/pkgs/sagemath-objects/tox.ini @@ -14,7 +14,7 @@ requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=4.2,<4.14.1 + tox>=4.2, tox<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-repl/tox.ini b/pkgs/sagemath-repl/tox.ini index d565aa7b2a4..f0874d4e22b 100644 --- a/pkgs/sagemath-repl/tox.ini +++ b/pkgs/sagemath-repl/tox.ini @@ -14,7 +14,7 @@ requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=4.2,<4.14.1 + tox>=4.2, tox<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-standard/tox.ini b/pkgs/sagemath-standard/tox.ini index c06afe8229d..6e38a8e0f23 100644 --- a/pkgs/sagemath-standard/tox.ini +++ b/pkgs/sagemath-standard/tox.ini @@ -63,7 +63,7 @@ requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=4.2,<4.14.1 + tox>=4.2, tox<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/src/tox.ini b/src/tox.ini index e0dc1509f1f..49172605b3e 100644 --- a/src/tox.ini +++ b/src/tox.ini @@ -28,7 +28,7 @@ skipsdist = true requires = # For the renamed "allowlist_externals" keyword, need >= 3.18 # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=3.18,<4.14.1 + tox>=3.18, tox<4.14.1 [sagedirect] # Base for tox environments that bypass the virtual environment set up by tox, diff --git a/tox.ini b/tox.ini index c8d4a5a4ebc..49652fd9934 100644 --- a/tox.ini +++ b/tox.ini @@ -131,7 +131,7 @@ envlist = requires = # For repaired numerical factors in tox 4, need >= 4.2.7. # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=4.2.7,<4.14.1 + tox>=4.2.7, tox<4.14.1 skipsdist = true From a4e925f6e94c8c8ea543e31d0750b428863b49da Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 6 Apr 2024 00:02:59 -0700 Subject: [PATCH 73/93] tox.ini: Fix 'requires' for autoprovisioning on tox 3 --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 49652fd9934..0e9b8c62b50 100644 --- a/tox.ini +++ b/tox.ini @@ -131,7 +131,8 @@ envlist = requires = # For repaired numerical factors in tox 4, need >= 4.2.7. # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=4.2.7, tox<4.14.1 + tox>=4.2.7 + tox<4.14.1 skipsdist = true From 6333308c08761a50b46cbcb6a1a1497405104152 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 9 Apr 2024 22:36:46 -0700 Subject: [PATCH 74/93] .github/workflows/build.yml: Rename jobs as test-new, test-mod, test-long as suggested --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a2f9da4671a..69804ce8676 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,7 +36,7 @@ env: EXTRA_CONFIGURE_ARGS: --enable-fat-binary jobs: - build: + test-new: runs-on: ubuntu-latest services: # https://docs.docker.com/build/ci/github-actions/local-registry/ @@ -137,9 +137,9 @@ jobs: ./sage -t --new -p4 shell: sh .ci/docker-exec-script.sh BUILD /sage {0} - mod: + test-mod: runs-on: ubuntu-latest - needs: [build] + needs: [test-new] services: # https://docs.docker.com/build/ci/github-actions/local-registry/ registry: @@ -231,7 +231,7 @@ jobs: test-long: runs-on: ubuntu-latest - needs: [build] + needs: [test-new] services: # https://docs.docker.com/build/ci/github-actions/local-registry/ registry: From afe72b2f85622d5f3d68d9a0972b65dd3120ff32 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 9 Apr 2024 22:37:35 -0700 Subject: [PATCH 75/93] .github/workflows/pyright.yml: Upgrade to ubuntu-jammy here too --- .github/workflows/pyright.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pyright.yml b/.github/workflows/pyright.yml index 1b608561b15..cafe75db728 100644 --- a/.github/workflows/pyright.yml +++ b/.github/workflows/pyright.yml @@ -18,7 +18,7 @@ concurrency: jobs: pyright: runs-on: ubuntu-latest - container: ghcr.io/sagemath/sage/sage-ubuntu-focal-standard-with-targets:dev + container: ghcr.io/sagemath/sage/sage-ubuntu-jammy-standard-with-targets:dev steps: - name: Checkout id: checkout From 17583ab633a255dfa4fa587db0198828e043f9d6 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 10 Apr 2024 18:11:05 -0700 Subject: [PATCH 76/93] .github/workflows/build.yml: Add documentation --- .github/workflows/build.yml | 45 +++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 69804ce8676..e9703d30bfa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,6 +26,46 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +# +# The three workflows: +# +# - build.yml (with jobs test-new, test-mod, test-long), +# - doc-build.yml, +# - doc-build-pdf.yml +# +# each build Sage: +# +# - incrementally starting from a Docker image that ci-linux.yml +# publishes on each development release to ghcr.io, +# - orchestrated using a tox-generated Dockerfile, +# - using https://github.com/marketplace/actions/build-and-push-docker-images, +# - pushing the image to a local registry, +# - then executing a container loaded from that registry, +# +# and then run various tests or build the documentation. +# +# Without the use of a cache, this runs the same incremental rebuild of Sage +# multiple times; there is no interaction between these workflow and jobs. +# +# This baseline is transparently improved by our use of the GH Actions cache, +# see https://docs.docker.com/build/ci/github-actions/cache/#cache-backend-api. +# +# Jobs test-mod and test-long are only started after test-new completed; +# but the workflows doc-build.yml and doc-build-pdf.yml are started independently. +# +# - When nothing is cached and the 3 workflows are launched in parallel, +# they may each run the same incremental rebuild. +# +# - When there's congestion that leads to the workflows to be run serially, +# the 2nd and 3rd workflow will be able to use the cache from the 1st workflow. +# +# This elasticity may be helpful in reducing congestion. +# +# There is a rather small per-project limit of 10 GB for this cache. +# Use https://github.com/sagemath/sage/actions/caches to monitor the utilization +# of the cache. +# + env: # Adapted from docker.yml TOX_ENV: "docker-${{ github.event.inputs.platform || 'ubuntu-jammy-standard' }}-incremental" @@ -97,6 +137,11 @@ jobs: uses: docker/build-push-action@v5 with: # push and load may not be set together at the moment + # + # We are using "push" (to the local registry) because it was + # more reliable than "load", for which we observed random failure + # conditions in which the built image could not be found. + # push: true load: false context: . From c18ba755eba1ceb0974965ecba6ff21ef617e271 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 10 Apr 2024 18:13:19 -0700 Subject: [PATCH 77/93] build/bin/write-dockerfile.sh: Add documentation for 'unminimize' --- build/bin/write-dockerfile.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/build/bin/write-dockerfile.sh b/build/bin/write-dockerfile.sh index 27681502370..9724264f0e4 100755 --- a/build/bin/write-dockerfile.sh +++ b/build/bin/write-dockerfile.sh @@ -54,6 +54,16 @@ case $SYSTEM in 1|y*|Y*) ;; *) + # + # The Ubuntu Docker images are "minimized", meaning that some large + # bits such as documentation has been removed. We have to unminimize + # once (which reinstalls the full versions of some minimized packages), + # or e.g. the maxima documentation (which we depend on for correct operation) + # will be missing. + # + # But we only have to do this once. To save time in incremental builds, + # we remove the unminimize binary here after it has done its job. + # cat < /dev/null; then \ (yes | unminimize) || echo "(ignored)"; \ From e0d5253d55d659b3715297d1bb148c4d63dbac81 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 4 Apr 2024 23:38:08 -0700 Subject: [PATCH 78/93] build/pkgs/sagemath_*/spkg-check: Replace by spkg-check.in from #35095 --- build/pkgs/sagemath_categories/spkg-check | 1 - build/pkgs/sagemath_categories/spkg-check.in | 1 + build/pkgs/sagemath_environment/spkg-check | 1 - build/pkgs/sagemath_environment/spkg-check.in | 1 + build/pkgs/sagemath_objects/spkg-check | 27 ---------- build/pkgs/sagemath_objects/spkg-check.in | 49 +++++++++++++++++++ build/pkgs/sagemath_repl/spkg-check | 1 - build/pkgs/sagemath_repl/spkg-check.in | 1 + 8 files changed, 52 insertions(+), 30 deletions(-) delete mode 120000 build/pkgs/sagemath_categories/spkg-check create mode 120000 build/pkgs/sagemath_categories/spkg-check.in delete mode 120000 build/pkgs/sagemath_environment/spkg-check create mode 120000 build/pkgs/sagemath_environment/spkg-check.in delete mode 100755 build/pkgs/sagemath_objects/spkg-check create mode 100644 build/pkgs/sagemath_objects/spkg-check.in delete mode 120000 build/pkgs/sagemath_repl/spkg-check create mode 120000 build/pkgs/sagemath_repl/spkg-check.in diff --git a/build/pkgs/sagemath_categories/spkg-check b/build/pkgs/sagemath_categories/spkg-check deleted file mode 120000 index 91c6b1835fc..00000000000 --- a/build/pkgs/sagemath_categories/spkg-check +++ /dev/null @@ -1 +0,0 @@ -../sagemath_objects/spkg-check \ No newline at end of file diff --git a/build/pkgs/sagemath_categories/spkg-check.in b/build/pkgs/sagemath_categories/spkg-check.in new file mode 120000 index 00000000000..24eb3322489 --- /dev/null +++ b/build/pkgs/sagemath_categories/spkg-check.in @@ -0,0 +1 @@ +../sagemath_objects/spkg-check.in \ No newline at end of file diff --git a/build/pkgs/sagemath_environment/spkg-check b/build/pkgs/sagemath_environment/spkg-check deleted file mode 120000 index 91c6b1835fc..00000000000 --- a/build/pkgs/sagemath_environment/spkg-check +++ /dev/null @@ -1 +0,0 @@ -../sagemath_objects/spkg-check \ No newline at end of file diff --git a/build/pkgs/sagemath_environment/spkg-check.in b/build/pkgs/sagemath_environment/spkg-check.in new file mode 120000 index 00000000000..24eb3322489 --- /dev/null +++ b/build/pkgs/sagemath_environment/spkg-check.in @@ -0,0 +1 @@ +../sagemath_objects/spkg-check.in \ No newline at end of file diff --git a/build/pkgs/sagemath_objects/spkg-check b/build/pkgs/sagemath_objects/spkg-check deleted file mode 100755 index 87cea0fc38f..00000000000 --- a/build/pkgs/sagemath_objects/spkg-check +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -cd src - -export PIP_NO_INDEX=true -export PIP_FIND_LINKS="file://$SAGE_SPKG_WHEELS" - -export TOX_PARALLEL_NO_SPINNER=1 -wheel="$(sed -n '1s,.*@ file://,,p' $SAGE_SPKG_SCRIPTS/$PKG_BASE/spkg-requirements.txt)" -echo Running "tox -r -p auto -v --installpkg $wheel" -tox -r -p auto -v --installpkg "$wheel" -status=$? -case $status:$SAGE_CHECK:$([ -r known-test-failures.json ]; echo $?) in - 0:*:0) echo "Passed the test suite (modulo baseline known-test-failures*.json)";; - 0:*:*) echo "Passed the test suite";; - *:warn:0) echo "Warning: New failures (not in baseline known-test-failures*.json (ignored)"; status=0;; - *:warn:*) echo "Warning: Failures testing the package (ignored)"; status=0;; - *:yes:0) echo "New failures, not in baseline known-test-failures*.json";; - *:yes:*) echo "Failures testing the package";; -esac -# Show summaries of failures (suppress lines ending with '[failed in baseline]') -for f in $(pwd)/.tox/sagepython-sagewheels-nopypi-norequirements*/log/*-command*.log; do - if [ -r "$f" ]; then - echo "$f" - grep '^sage -t.*#[^]]*$' "$f" - fi -done -exit $status diff --git a/build/pkgs/sagemath_objects/spkg-check.in b/build/pkgs/sagemath_objects/spkg-check.in new file mode 100644 index 00000000000..9ff603e1b46 --- /dev/null +++ b/build/pkgs/sagemath_objects/spkg-check.in @@ -0,0 +1,49 @@ +cd src + +if [ ! -r tox.ini ]; then + echo "Not testing the package because there is no tox.ini" + exit 0 +fi + +export PIP_NO_INDEX=true +export PIP_FIND_LINKS="file://$SAGE_SPKG_WHEELS" + +unset tox_args + +wheel="$(sed -n '1s,.*@ file://,,p' $SAGE_SPKG_SCRIPTS/$PKG_BASE/spkg-requirements.txt)" +if [ -n "$wheel" ]; then + tox_envs=$(tox -l) + tox_args="-r -p auto -v --installpkg $wheel" +elif [ "$SAGE_EDITABLE" = yes ]; then + tox_envs=$(tox -l | sed s/norequirements/editable/) + # FIXME: Should use -r if sage_setup or another build requirement changes + tox_args="-r -v -v -v -v -e $(echo $tox_envs | sed 's/ /,/g')" +else + echo "Not testing the package because SAGE_WHEELS=$SAGE_WHEELS and SAGE_EDITABLE=$SAGE_EDITABLE" + exit 0 +fi + +export TOX_PARALLEL_NO_SPINNER=1 + +echo Running "tox $tox_args" +tox $tox_args +status=$? +case $status:$SAGE_CHECK:$([ -r known-test-failures.json ]; echo $?) in + 0:no:*) echo "Not testing the package because SAGE_CHECK=no";; + 0:*:0) echo "Passed the test suite (modulo baseline known-test-failures*.json)";; + 0:*:*) echo "Passed the test suite";; + *:warn:0) echo "Warning: New failures (not in baseline known-test-failures*.json (ignored)"; status=0;; + *:warn:*) echo "Warning: Failures testing the package (ignored)"; status=0;; + *:yes:0) echo "New failures, not in baseline known-test-failures*.json";; + *:yes:*) echo "Failures testing the package";; +esac +# Show summaries of failures (suppress lines ending with '[failed in baseline]') +for e in $tox_envs; do + for f in $(pwd)/.tox/$e/log/*-command*.log; do + if [ -r "$f" ]; then + echo "$f" + grep '^sage -t.*#[^]]*$' "$f" + fi + done +done +exit $status diff --git a/build/pkgs/sagemath_repl/spkg-check b/build/pkgs/sagemath_repl/spkg-check deleted file mode 120000 index 91c6b1835fc..00000000000 --- a/build/pkgs/sagemath_repl/spkg-check +++ /dev/null @@ -1 +0,0 @@ -../sagemath_objects/spkg-check \ No newline at end of file diff --git a/build/pkgs/sagemath_repl/spkg-check.in b/build/pkgs/sagemath_repl/spkg-check.in new file mode 120000 index 00000000000..24eb3322489 --- /dev/null +++ b/build/pkgs/sagemath_repl/spkg-check.in @@ -0,0 +1 @@ +../sagemath_objects/spkg-check.in \ No newline at end of file From 3e7798f43996e8cc7782a5a0bb8ce381e42173fe Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 00:00:55 -0700 Subject: [PATCH 79/93] .github/workflows/ci-linux-incremental.yml: Do not run on changes to our script packages --- .github/workflows/ci-linux-incremental.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-linux-incremental.yml b/.github/workflows/ci-linux-incremental.yml index 82b14472c55..4861ef1d1d0 100644 --- a/.github/workflows/ci-linux-incremental.yml +++ b/.github/workflows/ci-linux-incremental.yml @@ -19,7 +19,12 @@ on: pull_request: paths: - 'build/pkgs/**' - - 'pkgs/**' + - '!build/pkgs/sage_conf/**' + - '!build/pkgs/sage_docbuild/**' + - '!build/pkgs/sage_setup/**' + - '!build/pkgs/sage_sws2rst/**' + - '!build/pkgs/sagelib/**' + - '!build/pkgs/sagemath_*/**' workflow_dispatch: concurrency: From 89432393c8ecf847ad93d14451a2102938b38717 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 11:29:40 -0700 Subject: [PATCH 80/93] pkgs/sagemath-{objects,categories,environment,repl}/tox.ini: Update from #35095 --- pkgs/sagemath-categories/tox.ini | 27 ++++++++++++++++++++++----- pkgs/sagemath-environment/tox.ini | 16 ++++++++++++++++ pkgs/sagemath-objects/tox.ini | 16 ++++++++++++++++ pkgs/sagemath-repl/tox.ini | 27 ++++++++++++++++++++++----- 4 files changed, 76 insertions(+), 10 deletions(-) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index b4bb49d132a..29b6b15f30b 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -49,6 +49,9 @@ passenv = {[pkgenv]passenv} setenv = {[pkgenv]setenv} # Sage scripts such as sage-runtests like to use $HOME/.sage HOME={envdir} + # Stop 'sage -t --installed' from picking up doc installed in SAGE_LOCAL + SAGE_DOC=/doesnotexist + KNOWN_TEST_FAILURES={toxinidir}/known-test-failures.json allowlist_externals = bash @@ -61,7 +64,7 @@ commands = {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.categories.all import *; SimplicialComplexes(); FunctionFields()' bash -c 'cd $(python -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") \ - && sage-runtests -p --initial --environment=sage.all__sagemath_categories --probe all --baseline-stats-path={toxinidir}/known-test-failures.json --optional=sage --installed' + && sage-runtests -p --force-lib --initial --environment=sage.all__sagemath_categories --probe all --baseline-stats-path=$KNOWN_TEST_FAILURES --optional=sage --installed' [testenv:.tox] # Allow access to PyPI for auto-provisioning a suitable tox version @@ -84,6 +87,11 @@ setenv = {[pkgenv]setenv} basepython = {env:SAGE_VENV}/bin/python3 +[testenv:.pkg-sagepython-sagewheels-nopypi-editable] +config_settings_build_editable = + editable_mode = strict + + [testenv:sagepython] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython @@ -92,10 +100,6 @@ package_env = .pkg-sagepython basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython-sagewheels-nopypi -[testenv:sagepython-sagewheels-nopypi-norequirements] -basepython = {env:SAGE_VENV}/bin/python3 -package_env = .pkg-sagepython-sagewheels-nopypi - [testenv:sagepython-sagewheels] basepython = {env:SAGE_VENV}/bin/python package_env = .pkg-sagepython @@ -103,3 +107,16 @@ package_env = .pkg-sagepython [testenv:sagepython-norequirements] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython + + +[testenv:sagepython-sagewheels-nopypi-norequirements] +basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi + +[testenv:sagepython-sagewheels-nopypi-editable] +basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi-editable +package = editable +deps = -r requirements-editable.txt +config_settings_build_editable = + editable_mode = strict diff --git a/pkgs/sagemath-environment/tox.ini b/pkgs/sagemath-environment/tox.ini index 6bf1f2a6ebb..96bcbdf4397 100644 --- a/pkgs/sagemath-environment/tox.ini +++ b/pkgs/sagemath-environment/tox.ini @@ -49,6 +49,9 @@ passenv = {[pkgenv]passenv} setenv = {[pkgenv]setenv} # Sage scripts such as sage-runtests like to use $HOME/.sage HOME={envdir} + # Stop 'sage -t --installed' from picking up doc installed in SAGE_LOCAL + SAGE_DOC=/doesnotexist + KNOWN_TEST_FAILURES={toxinidir}/known-test-failures.json allowlist_externals = bash @@ -78,6 +81,11 @@ setenv = {[pkgenv]setenv} basepython = {env:SAGE_VENV}/bin/python3 +[testenv:.pkg-sagepython-sagewheels-nopypi-editable] +config_settings_build_editable = + editable_mode = strict + + [testenv:sagepython] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython @@ -98,3 +106,11 @@ package_env = .pkg-sagepython [testenv:sagepython-sagewheels-nopypi-norequirements] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython-sagewheels-nopypi + +[testenv:sagepython-sagewheels-nopypi-editable] +basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi-editable +package = editable +deps = -r requirements-editable.txt +config_settings_build_editable = + editable_mode = strict diff --git a/pkgs/sagemath-objects/tox.ini b/pkgs/sagemath-objects/tox.ini index a8f5a6d6a76..d51d473d84f 100644 --- a/pkgs/sagemath-objects/tox.ini +++ b/pkgs/sagemath-objects/tox.ini @@ -49,6 +49,9 @@ passenv = {[pkgenv]passenv} setenv = {[pkgenv]setenv} # Sage scripts such as sage-runtests like to use $HOME/.sage HOME={envdir} + # Stop 'sage -t --installed' from picking up doc installed in SAGE_LOCAL + SAGE_DOC=/doesnotexist + KNOWN_TEST_FAILURES={toxinidir}/known-test-failures.json allowlist_externals = bash @@ -82,6 +85,11 @@ setenv = {[pkgenv]setenv} basepython = {env:SAGE_VENV}/bin/python3 +[testenv:.pkg-sagepython-sagewheels-nopypi-editable] +config_settings_build_editable = + editable_mode = strict + + [testenv:sagepython] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython @@ -102,3 +110,11 @@ package_env = .pkg-sagepython [testenv:sagepython-sagewheels-nopypi-norequirements] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython-sagewheels-nopypi + +[testenv:sagepython-sagewheels-nopypi-editable] +basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi-editable +package = editable +deps = -r requirements-editable.txt +config_settings_build_editable = + editable_mode = strict diff --git a/pkgs/sagemath-repl/tox.ini b/pkgs/sagemath-repl/tox.ini index 679153a2947..ffdf5b97b1c 100644 --- a/pkgs/sagemath-repl/tox.ini +++ b/pkgs/sagemath-repl/tox.ini @@ -49,6 +49,9 @@ passenv = {[pkgenv]passenv} setenv = {[pkgenv]setenv} # Sage scripts such as sage-runtests like to use $HOME/.sage HOME={envdir} + # Stop 'sage -t --installed' from picking up doc installed in SAGE_LOCAL + SAGE_DOC=/doesnotexist + KNOWN_TEST_FAILURES={toxinidir}/known-test-failures.json allowlist_externals = bash @@ -57,7 +60,7 @@ commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.repl.all; import sage.doctest.all' - bash -c 'cd $({envpython} -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") && sage-runtests -p --environment=sage.all__sagemath_repl --baseline-stats-path={toxinidir}/known-test-failures.json --initial --optional=sage sage/repl sage/doctest sage/misc/sage_input.py sage/misc/sage_eval.py' + bash -c 'cd $({envpython} -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") && sage-runtests -p --environment=sage.all__sagemath_repl --baseline-stats-path=$KNOWN_TEST_FAILURES --initial --optional=sage sage/repl sage/doctest sage/misc/sage_input.py sage/misc/sage_eval.py' [testenv:.tox] # Allow access to PyPI for auto-provisioning a suitable tox version @@ -80,6 +83,11 @@ setenv = {[pkgenv]setenv} basepython = {env:SAGE_VENV}/bin/python3 +[testenv:.pkg-sagepython-sagewheels-nopypi-editable] +config_settings_build_editable = + editable_mode = strict + + [testenv:sagepython] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython @@ -88,10 +96,6 @@ package_env = .pkg-sagepython basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython-sagewheels-nopypi -[testenv:sagepython-sagewheels-nopypi-norequirements] -basepython = {env:SAGE_VENV}/bin/python3 -package_env = .pkg-sagepython-sagewheels-nopypi - [testenv:sagepython-sagewheels] basepython = {env:SAGE_VENV}/bin/python package_env = .pkg-sagepython @@ -99,3 +103,16 @@ package_env = .pkg-sagepython [testenv:sagepython-norequirements] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython + + +[testenv:sagepython-sagewheels-nopypi-norequirements] +basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi + +[testenv:sagepython-sagewheels-nopypi-editable] +basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi-editable +package = editable +deps = -r requirements-editable.txt +config_settings_build_editable = + editable_mode = strict From 17dd1bc7311901b15b013eba9b5213fba659e796 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 11:30:08 -0700 Subject: [PATCH 81/93] pkgs/sagemath-{objects,categories,environment,repl}/tox.ini: Replace use of {envpython} by just python3 --- pkgs/sagemath-categories/tox.ini | 4 ++-- pkgs/sagemath-environment/tox.ini | 2 +- pkgs/sagemath-objects/tox.ini | 4 ++-- pkgs/sagemath-repl/tox.ini | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index 29b6b15f30b..3f237f7feef 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -58,10 +58,10 @@ allowlist_externals = commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. - {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.cpython.builtin_types, sage.cpython.cython_metaclass, sage.cpython.debug, sage.structure.all, sage.categories.all' + python3 -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.cpython.builtin_types, sage.cpython.cython_metaclass, sage.cpython.debug, sage.structure.all, sage.categories.all' # Test that importing sage.categories.all initializes categories - {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.categories.all import *; SimplicialComplexes(); FunctionFields()' + python3 -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.categories.all import *; SimplicialComplexes(); FunctionFields()' bash -c 'cd $(python -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") \ && sage-runtests -p --force-lib --initial --environment=sage.all__sagemath_categories --probe all --baseline-stats-path=$KNOWN_TEST_FAILURES --optional=sage --installed' diff --git a/pkgs/sagemath-environment/tox.ini b/pkgs/sagemath-environment/tox.ini index 96bcbdf4397..d3dd3dcaef9 100644 --- a/pkgs/sagemath-environment/tox.ini +++ b/pkgs/sagemath-environment/tox.ini @@ -58,7 +58,7 @@ allowlist_externals = commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. - {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.features.all import all_features; print(sorted(all_features(), key=lambda x: x.name)); import sage.misc.package' + python3 -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.features.all import all_features; print(sorted(all_features(), key=lambda x: x.name)); import sage.misc.package' [testenv:.tox] # Allow access to PyPI for auto-provisioning a suitable tox version diff --git a/pkgs/sagemath-objects/tox.ini b/pkgs/sagemath-objects/tox.ini index d51d473d84f..91a0d0bfe62 100644 --- a/pkgs/sagemath-objects/tox.ini +++ b/pkgs/sagemath-objects/tox.ini @@ -58,9 +58,9 @@ allowlist_externals = commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. - {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.structure.all, sage.categories.sets_cat' + python3 -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.structure.all, sage.categories.sets_cat' - {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.all__sagemath_objects import *' + python3 -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.all__sagemath_objects import *' #bash -c 'cd {temp_dir} && SAGE_SRC=$(python -c "from sage.env import SAGE_SRC; print(SAGE_SRC)") && sage-runtests --environment=sage.all__sagemath_objects --initial --optional=sage $SAGE_SRC/sage/structure || echo "(lots of doctest failures are expected)"' diff --git a/pkgs/sagemath-repl/tox.ini b/pkgs/sagemath-repl/tox.ini index ffdf5b97b1c..9ac2e942948 100644 --- a/pkgs/sagemath-repl/tox.ini +++ b/pkgs/sagemath-repl/tox.ini @@ -58,9 +58,9 @@ allowlist_externals = commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. - {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.repl.all; import sage.doctest.all' + python3 -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.repl.all; import sage.doctest.all' - bash -c 'cd $({envpython} -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") && sage-runtests -p --environment=sage.all__sagemath_repl --baseline-stats-path=$KNOWN_TEST_FAILURES --initial --optional=sage sage/repl sage/doctest sage/misc/sage_input.py sage/misc/sage_eval.py' + bash -c 'cd $(python3 -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") && sage-runtests -p --environment=sage.all__sagemath_repl --baseline-stats-path=$KNOWN_TEST_FAILURES --initial --optional=sage sage/repl sage/doctest sage/misc/sage_input.py sage/misc/sage_eval.py' [testenv:.tox] # Allow access to PyPI for auto-provisioning a suitable tox version From 190b657c6d142bf3da4d19fead9bf7016075e18f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 19:51:07 -0700 Subject: [PATCH 82/93] build/pkgs/sagemath_{objects,categories,environment,repl}: Back to using spkg-check, source sage-env etc. --- build/pkgs/sagemath_categories/spkg-check | 1 + build/pkgs/sagemath_categories/spkg-check.in | 1 - build/pkgs/sagemath_environment/spkg-check | 1 + build/pkgs/sagemath_environment/spkg-check.in | 1 - .../sagemath_objects/{spkg-check.in => spkg-check} | 10 ++++++++++ build/pkgs/sagemath_repl/spkg-check | 1 + build/pkgs/sagemath_repl/spkg-check.in | 1 - 7 files changed, 13 insertions(+), 3 deletions(-) create mode 120000 build/pkgs/sagemath_categories/spkg-check delete mode 120000 build/pkgs/sagemath_categories/spkg-check.in create mode 120000 build/pkgs/sagemath_environment/spkg-check delete mode 120000 build/pkgs/sagemath_environment/spkg-check.in rename build/pkgs/sagemath_objects/{spkg-check.in => spkg-check} (81%) mode change 100644 => 100755 create mode 120000 build/pkgs/sagemath_repl/spkg-check delete mode 120000 build/pkgs/sagemath_repl/spkg-check.in diff --git a/build/pkgs/sagemath_categories/spkg-check b/build/pkgs/sagemath_categories/spkg-check new file mode 120000 index 00000000000..91c6b1835fc --- /dev/null +++ b/build/pkgs/sagemath_categories/spkg-check @@ -0,0 +1 @@ +../sagemath_objects/spkg-check \ No newline at end of file diff --git a/build/pkgs/sagemath_categories/spkg-check.in b/build/pkgs/sagemath_categories/spkg-check.in deleted file mode 120000 index 24eb3322489..00000000000 --- a/build/pkgs/sagemath_categories/spkg-check.in +++ /dev/null @@ -1 +0,0 @@ -../sagemath_objects/spkg-check.in \ No newline at end of file diff --git a/build/pkgs/sagemath_environment/spkg-check b/build/pkgs/sagemath_environment/spkg-check new file mode 120000 index 00000000000..91c6b1835fc --- /dev/null +++ b/build/pkgs/sagemath_environment/spkg-check @@ -0,0 +1 @@ +../sagemath_objects/spkg-check \ No newline at end of file diff --git a/build/pkgs/sagemath_environment/spkg-check.in b/build/pkgs/sagemath_environment/spkg-check.in deleted file mode 120000 index 24eb3322489..00000000000 --- a/build/pkgs/sagemath_environment/spkg-check.in +++ /dev/null @@ -1 +0,0 @@ -../sagemath_objects/spkg-check.in \ No newline at end of file diff --git a/build/pkgs/sagemath_objects/spkg-check.in b/build/pkgs/sagemath_objects/spkg-check old mode 100644 new mode 100755 similarity index 81% rename from build/pkgs/sagemath_objects/spkg-check.in rename to build/pkgs/sagemath_objects/spkg-check index 9ff603e1b46..99a8528e0da --- a/build/pkgs/sagemath_objects/spkg-check.in +++ b/build/pkgs/sagemath_objects/spkg-check @@ -1,3 +1,4 @@ +#!/usr/bin/env bash cd src if [ ! -r tox.ini ]; then @@ -5,6 +6,15 @@ if [ ! -r tox.ini ]; then exit 0 fi +for lib in "$SAGE_SRC/bin/sage-src-env-config" "$SAGE_SRC/bin/sage-env-config" "$SAGE_SRC/bin/sage-env" "$SAGE_ROOT/build/bin/sage-build-env-config" "$SAGE_ROOT/build/bin/sage-build-env"; do + source "$lib" + if [ $? -ne 0 ]; then + echo >&2 "Error: failed to source $lib" + echo >&2 "Is $SAGE_ROOT the correct SAGE_ROOT?" + exit 1 + fi +done + export PIP_NO_INDEX=true export PIP_FIND_LINKS="file://$SAGE_SPKG_WHEELS" diff --git a/build/pkgs/sagemath_repl/spkg-check b/build/pkgs/sagemath_repl/spkg-check new file mode 120000 index 00000000000..91c6b1835fc --- /dev/null +++ b/build/pkgs/sagemath_repl/spkg-check @@ -0,0 +1 @@ +../sagemath_objects/spkg-check \ No newline at end of file diff --git a/build/pkgs/sagemath_repl/spkg-check.in b/build/pkgs/sagemath_repl/spkg-check.in deleted file mode 120000 index 24eb3322489..00000000000 --- a/build/pkgs/sagemath_repl/spkg-check.in +++ /dev/null @@ -1 +0,0 @@ -../sagemath_objects/spkg-check.in \ No newline at end of file From b144d8c5a98267a92b7230a520a58d5261e97f50 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 20:21:45 -0700 Subject: [PATCH 83/93] CythonFeature: Do not fail on Python 3.12 without setuptools --- src/sage/features/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sage/features/__init__.py b/src/sage/features/__init__.py index 5fdefceb7ee..3a57073434f 100644 --- a/src/sage/features/__init__.py +++ b/src/sage/features/__init__.py @@ -948,7 +948,10 @@ def _is_present(self): # Available since https://setuptools.pypa.io/en/latest/history.html#v59-0-0 from setuptools.errors import CCompilerError except ImportError: - from distutils.errors import CCompilerError + try: + from distutils.errors import CCompilerError + except ImportError: + CCompilerError = () with open(tmp_filename(ext=".pyx"), 'w') as pyx: pyx.write(self.test_code) try: From e7b66fedf2d5940d8f9a59aac9fe43253d15789b Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 21:24:30 -0700 Subject: [PATCH 84/93] pkgs/sagemath-repl/pyproject.toml.m4: Add dependencies ipykernel, jupyter-client --- pkgs/sagemath-repl/pyproject.toml.m4 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/sagemath-repl/pyproject.toml.m4 b/pkgs/sagemath-repl/pyproject.toml.m4 index 2d2f13008be..459a599f1fb 100644 --- a/pkgs/sagemath-repl/pyproject.toml.m4 +++ b/pkgs/sagemath-repl/pyproject.toml.m4 @@ -13,8 +13,10 @@ description = "Sage: Open Source Mathematics Software: IPython kernel, Sage prep dependencies = [ SPKG_INSTALL_REQUIRES_sagemath_objects SPKG_INSTALL_REQUIRES_sagemath_environment + SPKG_INSTALL_REQUIRES_ipykernel SPKG_INSTALL_REQUIRES_ipython SPKG_INSTALL_REQUIRES_ipywidgets + SPKG_INSTALL_REQUIRES_jupyter_client ] dynamic = ["version"] include(`pyproject_toml_metadata.m4')dnl' From 039bb06664a8321d4cccf4716e64b2bd28c747c6 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 21:31:22 -0700 Subject: [PATCH 85/93] pkgs/sagemath-{repl,categories}/tox.ini: Set PYDEVD_DISABLE_FILE_VALIDATION=1 --- pkgs/sagemath-categories/tox.ini | 2 ++ pkgs/sagemath-repl/tox.ini | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index 3f237f7feef..843b0ab242f 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -52,6 +52,8 @@ setenv = {[pkgenv]setenv} # Stop 'sage -t --installed' from picking up doc installed in SAGE_LOCAL SAGE_DOC=/doesnotexist KNOWN_TEST_FAILURES={toxinidir}/known-test-failures.json + # See src/bin/sage-env + PYDEVD_DISABLE_FILE_VALIDATION=1 allowlist_externals = bash diff --git a/pkgs/sagemath-repl/tox.ini b/pkgs/sagemath-repl/tox.ini index 9ac2e942948..9386ca22ae9 100644 --- a/pkgs/sagemath-repl/tox.ini +++ b/pkgs/sagemath-repl/tox.ini @@ -52,6 +52,8 @@ setenv = {[pkgenv]setenv} # Stop 'sage -t --installed' from picking up doc installed in SAGE_LOCAL SAGE_DOC=/doesnotexist KNOWN_TEST_FAILURES={toxinidir}/known-test-failures.json + # See src/bin/sage-env + PYDEVD_DISABLE_FILE_VALIDATION=1 allowlist_externals = bash From 650a743b963a68d4225f8b2009f4e8cc2ad591a9 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 12 Apr 2024 16:17:57 -0700 Subject: [PATCH 86/93] pkgs/sagemath-categories/known-test-failures.json: Update --- .../known-test-failures.json | 85 +++++++++++-------- 1 file changed, 51 insertions(+), 34 deletions(-) diff --git a/pkgs/sagemath-categories/known-test-failures.json b/pkgs/sagemath-categories/known-test-failures.json index 3f7973d7fee..3d460557ee4 100644 --- a/pkgs/sagemath-categories/known-test-failures.json +++ b/pkgs/sagemath-categories/known-test-failures.json @@ -38,7 +38,7 @@ }, "sage.categories.algebras": { "failed": true, - "ntests": 20 + "ntests": 25 }, "sage.categories.algebras_with_basis": { "failed": true, @@ -126,7 +126,7 @@ }, "sage.categories.commutative_rings": { "failed": true, - "ntests": 39 + "ntests": 41 }, "sage.categories.complete_discrete_valuation": { "failed": true, @@ -148,15 +148,19 @@ }, "sage.categories.coxeter_groups": { "failed": true, - "ntests": 363 + "ntests": 392 }, "sage.categories.cw_complexes": { "failed": true, "ntests": 36 }, + "sage.categories.dedekind_domains": { + "failed": true, + "ntests": 30 + }, "sage.categories.discrete_valuation": { "failed": true, - "ntests": 23 + "ntests": 28 }, "sage.categories.distributive_magmas_and_additive_magmas": { "failed": true, @@ -169,9 +173,6 @@ "failed": true, "ntests": 7 }, - "sage.categories.drinfeld_modules": { - "ntests": 2 - }, "sage.categories.dual": { "failed": true, "ntests": 1 @@ -297,18 +298,18 @@ }, "sage.categories.filtered_modules_with_basis": { "failed": true, - "ntests": 65 + "ntests": 74 }, "sage.categories.finite_complex_reflection_groups": { "failed": true, - "ntests": 199 + "ntests": 189 }, "sage.categories.finite_coxeter_groups": { "ntests": 6 }, "sage.categories.finite_dimensional_algebras_with_basis": { "failed": true, - "ntests": 87 + "ntests": 91 }, "sage.categories.finite_dimensional_bialgebras_with_basis": { "failed": true, @@ -328,7 +329,7 @@ }, "sage.categories.finite_dimensional_lie_algebras_with_basis": { "failed": true, - "ntests": 34 + "ntests": 44 }, "sage.categories.finite_dimensional_modules_with_basis": { "failed": true, @@ -526,7 +527,7 @@ }, "sage.categories.lie_algebras": { "failed": true, - "ntests": 125 + "ntests": 135 }, "sage.categories.lie_algebras_with_basis": { "ntests": 19 @@ -582,7 +583,7 @@ }, "sage.categories.modules_with_basis": { "failed": true, - "ntests": 221 + "ntests": 232 }, "sage.categories.monoid_algebras": { "failed": true, @@ -664,14 +665,14 @@ }, "sage.categories.rings": { "failed": true, - "ntests": 141 + "ntests": 146 }, "sage.categories.rngs": { "ntests": 6 }, "sage.categories.schemes": { "failed": true, - "ntests": 23 + "ntests": 33 }, "sage.categories.semigroups": { "failed": true, @@ -707,7 +708,7 @@ }, "sage.categories.simplicial_sets": { "failed": true, - "ntests": 57 + "ntests": 98 }, "sage.categories.subobjects": { "ntests": 2 @@ -761,7 +762,7 @@ }, "sage.categories.unital_algebras": { "failed": true, - "ntests": 39 + "ntests": 37 }, "sage.categories.vector_spaces": { "failed": true, @@ -804,25 +805,24 @@ }, "sage.doctest.control": { "failed": true, - "ntests": 0 + "ntests": 232 }, "sage.doctest.external": { "ntests": 42 }, "sage.doctest.fixtures": { - "failed": true, "ntests": 59 }, "sage.doctest.forker": { "failed": true, - "ntests": 433 + "ntests": 434 }, "sage.doctest.parsing": { "failed": true, "ntests": 321 }, "sage.doctest.reporting": { - "ntests": 124 + "ntests": 125 }, "sage.doctest.sources": { "failed": true, @@ -841,7 +841,7 @@ "ntests": 41 }, "sage.features": { - "ntests": 145 + "ntests": 143 }, "sage.features.all": { "ntests": 14 @@ -860,17 +860,23 @@ }, "sage.features.databases": { "failed": true, - "ntests": 26 + "ntests": 38 }, "sage.features.dvipng": { "ntests": 4 }, + "sage.features.ecm": { + "ntests": 4 + }, "sage.features.ffmpeg": { "ntests": 4 }, "sage.features.four_ti_2": { "ntests": 6 }, + "sage.features.fricas": { + "ntests": 6 + }, "sage.features.gap": { "ntests": 6 }, @@ -896,7 +902,11 @@ "sage.features.internet": { "ntests": 5 }, + "sage.features.jmol": { + "ntests": 4 + }, "sage.features.join_feature": { + "failed": true, "ntests": 25 }, "sage.features.kenzo": { @@ -955,7 +965,7 @@ }, "sage.features.sagemath": { "failed": true, - "ntests": 147 + "ntests": 151 }, "sage.features.singular": { "ntests": 4 @@ -963,9 +973,16 @@ "sage.features.sphinx": { "ntests": 4 }, + "sage.features.symengine_py": { + "ntests": 4 + }, "sage.features.tdlib": { "ntests": 2 }, + "sage.features.threejs": { + "failed": true, + "ntests": 6 + }, "sage.misc.abstract_method": { "failed": true, "ntests": 33 @@ -996,7 +1013,7 @@ }, "sage.misc.decorators": { "failed": true, - "ntests": 126 + "ntests": 125 }, "sage.misc.fast_methods": { "failed": true, @@ -1054,7 +1071,7 @@ }, "sage.misc.package_dir": { "failed": true, - "ntests": 29 + "ntests": 35 }, "sage.misc.persist": { "failed": true, @@ -1086,7 +1103,7 @@ }, "sage.misc.sageinspect": { "failed": true, - "ntests": 332 + "ntests": 322 }, "sage.misc.superseded": { "failed": true, @@ -1162,7 +1179,7 @@ }, "sage.repl.ipython_kernel.install": { "failed": true, - "ntests": 40 + "ntests": 35 }, "sage.repl.ipython_kernel.interact": { "failed": true, @@ -1203,7 +1220,7 @@ }, "sage.repl.rich_output.backend_ipython": { "failed": true, - "ntests": 78 + "ntests": 75 }, "sage.repl.rich_output.buffer": { "failed": true, @@ -1211,7 +1228,7 @@ }, "sage.repl.rich_output.display_manager": { "failed": true, - "ntests": 88 + "ntests": 86 }, "sage.repl.rich_output.output_basic": { "ntests": 47 @@ -1248,7 +1265,7 @@ }, "sage.rings.ring": { "failed": true, - "ntests": 337 + "ntests": 298 }, "sage.sets.pythonclass": { "failed": true, @@ -1260,7 +1277,7 @@ }, "sage.structure.coerce": { "failed": true, - "ntests": 314 + "ntests": 310 }, "sage.structure.coerce_actions": { "failed": true, @@ -1295,7 +1312,7 @@ }, "sage.structure.factorization": { "failed": true, - "ntests": 203 + "ntests": 200 }, "sage.structure.factorization_integer": { "failed": true, @@ -1339,7 +1356,7 @@ }, "sage.structure.parent": { "failed": true, - "ntests": 300 + "ntests": 303 }, "sage.structure.parent_gens": { "failed": true, From bb8e2581a1479e58a7e70fe2f8d488959d548229 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 22:55:06 -0700 Subject: [PATCH 87/93] **/tox.ini: Set upper bound tox < 4.14.1 because of https://github.com/tox-dev/tox/issues/3238 --- pkgs/sage-conf_pypi/tox.ini | 4 ++++ pkgs/sage-docbuild/tox.ini | 4 ++++ pkgs/sage-setup/tox.ini | 4 ++++ pkgs/sagemath-categories/tox.ini | 3 ++- pkgs/sagemath-environment/tox.ini | 3 ++- pkgs/sagemath-objects/tox.ini | 3 ++- pkgs/sagemath-repl/tox.ini | 3 ++- pkgs/sagemath-standard/tox.ini | 3 ++- src/tox.ini | 5 +++-- tox.ini | 5 +++-- 10 files changed, 28 insertions(+), 9 deletions(-) diff --git a/pkgs/sage-conf_pypi/tox.ini b/pkgs/sage-conf_pypi/tox.ini index fadf9a9cc6c..7160d4db678 100644 --- a/pkgs/sage-conf_pypi/tox.ini +++ b/pkgs/sage-conf_pypi/tox.ini @@ -1,6 +1,10 @@ [tox] envlist = py39, py310, py311, py39-user, py310-user, py311-user +requires = + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox<4.14.1 + [testenv:.pkg] basepython = py311 passenv = diff --git a/pkgs/sage-docbuild/tox.ini b/pkgs/sage-docbuild/tox.ini index 77da5a78ede..efa222028ff 100644 --- a/pkgs/sage-docbuild/tox.ini +++ b/pkgs/sage-docbuild/tox.ini @@ -8,6 +8,10 @@ # [tox] +requires = + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox<4.14.1 + [testenv] deps = -rrequirements.txt diff --git a/pkgs/sage-setup/tox.ini b/pkgs/sage-setup/tox.ini index fd935f2e978..e8a22f03101 100644 --- a/pkgs/sage-setup/tox.ini +++ b/pkgs/sage-setup/tox.ini @@ -12,6 +12,10 @@ # [tox] +requires = + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox<4.14.1 + [testenv] deps = -rrequirements.txt diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index 843b0ab242f..065f0790411 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -13,7 +13,8 @@ envlist = requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance - tox>=4.2 + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox>=4.2,<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-environment/tox.ini b/pkgs/sagemath-environment/tox.ini index d3dd3dcaef9..7973b0f46a2 100644 --- a/pkgs/sagemath-environment/tox.ini +++ b/pkgs/sagemath-environment/tox.ini @@ -13,7 +13,8 @@ envlist = requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance - tox>=4.2 + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox>=4.2,<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-objects/tox.ini b/pkgs/sagemath-objects/tox.ini index 91a0d0bfe62..52c5434e67e 100644 --- a/pkgs/sagemath-objects/tox.ini +++ b/pkgs/sagemath-objects/tox.ini @@ -13,7 +13,8 @@ envlist = requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance - tox>=4.2 + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox>=4.2,<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-repl/tox.ini b/pkgs/sagemath-repl/tox.ini index 9386ca22ae9..b65157e4f58 100644 --- a/pkgs/sagemath-repl/tox.ini +++ b/pkgs/sagemath-repl/tox.ini @@ -13,7 +13,8 @@ envlist = requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance - tox>=4.2 + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox>=4.2,<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-standard/tox.ini b/pkgs/sagemath-standard/tox.ini index fcd32d0764b..cfb4fb6d527 100644 --- a/pkgs/sagemath-standard/tox.ini +++ b/pkgs/sagemath-standard/tox.ini @@ -62,7 +62,8 @@ envlist = requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance - tox>=4.2 + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox>=4.2,<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/src/tox.ini b/src/tox.ini index aab671441a9..ab7c1b5f18e 100644 --- a/src/tox.ini +++ b/src/tox.ini @@ -26,8 +26,9 @@ envlist = doctest, coverage, startuptime, pycodestyle-minimal, relint, codespell skipsdist = true requires = - # For the renamed "allowlist_externals" keyword - tox>=3.18 + # For the renamed "allowlist_externals" keyword, need >= 3.18 + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox>=3.18,<4.14.1 [sagedirect] # Base for tox environments that bypass the virtual environment set up by tox, diff --git a/tox.ini b/tox.ini index 837764e6441..5167ead23b8 100644 --- a/tox.ini +++ b/tox.ini @@ -129,8 +129,9 @@ envlist = # pycodestyle requires = - # For repaired numerical factors in tox 4: - tox>=4.2.7 + # For repaired numerical factors in tox 4, need >= 4.2.7. + # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 + tox>=4.2.7,<4.14.1 skipsdist = true From f77bb619491473caccfd4e65a3582e030d136b2c Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 23:22:32 -0700 Subject: [PATCH 88/93] Revert "pkgs/sagemath-{objects,categories,environment,repl}/tox.ini: Replace use of {envpython} by just python3" This reverts commit cca2b8dd1cdd54f902b85d5adea57e0edbb481bd. --- pkgs/sagemath-categories/tox.ini | 4 ++-- pkgs/sagemath-environment/tox.ini | 2 +- pkgs/sagemath-objects/tox.ini | 4 ++-- pkgs/sagemath-repl/tox.ini | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index 065f0790411..6eaa17fc20e 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -61,10 +61,10 @@ allowlist_externals = commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. - python3 -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.cpython.builtin_types, sage.cpython.cython_metaclass, sage.cpython.debug, sage.structure.all, sage.categories.all' + {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.cpython.builtin_types, sage.cpython.cython_metaclass, sage.cpython.debug, sage.structure.all, sage.categories.all' # Test that importing sage.categories.all initializes categories - python3 -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.categories.all import *; SimplicialComplexes(); FunctionFields()' + {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.categories.all import *; SimplicialComplexes(); FunctionFields()' bash -c 'cd $(python -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") \ && sage-runtests -p --force-lib --initial --environment=sage.all__sagemath_categories --probe all --baseline-stats-path=$KNOWN_TEST_FAILURES --optional=sage --installed' diff --git a/pkgs/sagemath-environment/tox.ini b/pkgs/sagemath-environment/tox.ini index 7973b0f46a2..fec1966d9a9 100644 --- a/pkgs/sagemath-environment/tox.ini +++ b/pkgs/sagemath-environment/tox.ini @@ -59,7 +59,7 @@ allowlist_externals = commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. - python3 -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.features.all import all_features; print(sorted(all_features(), key=lambda x: x.name)); import sage.misc.package' + {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.features.all import all_features; print(sorted(all_features(), key=lambda x: x.name)); import sage.misc.package' [testenv:.tox] # Allow access to PyPI for auto-provisioning a suitable tox version diff --git a/pkgs/sagemath-objects/tox.ini b/pkgs/sagemath-objects/tox.ini index 52c5434e67e..3e539e2e3b0 100644 --- a/pkgs/sagemath-objects/tox.ini +++ b/pkgs/sagemath-objects/tox.ini @@ -59,9 +59,9 @@ allowlist_externals = commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. - python3 -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.structure.all, sage.categories.sets_cat' + {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.structure.all, sage.categories.sets_cat' - python3 -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.all__sagemath_objects import *' + {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.all__sagemath_objects import *' #bash -c 'cd {temp_dir} && SAGE_SRC=$(python -c "from sage.env import SAGE_SRC; print(SAGE_SRC)") && sage-runtests --environment=sage.all__sagemath_objects --initial --optional=sage $SAGE_SRC/sage/structure || echo "(lots of doctest failures are expected)"' diff --git a/pkgs/sagemath-repl/tox.ini b/pkgs/sagemath-repl/tox.ini index b65157e4f58..d565aa7b2a4 100644 --- a/pkgs/sagemath-repl/tox.ini +++ b/pkgs/sagemath-repl/tox.ini @@ -61,9 +61,9 @@ allowlist_externals = commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. - python3 -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.repl.all; import sage.doctest.all' + {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.repl.all; import sage.doctest.all' - bash -c 'cd $(python3 -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") && sage-runtests -p --environment=sage.all__sagemath_repl --baseline-stats-path=$KNOWN_TEST_FAILURES --initial --optional=sage sage/repl sage/doctest sage/misc/sage_input.py sage/misc/sage_eval.py' + bash -c 'cd $({envpython} -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") && sage-runtests -p --environment=sage.all__sagemath_repl --baseline-stats-path=$KNOWN_TEST_FAILURES --initial --optional=sage sage/repl sage/doctest sage/misc/sage_input.py sage/misc/sage_eval.py' [testenv:.tox] # Allow access to PyPI for auto-provisioning a suitable tox version From 7da30ed8bf0a32e0ee1385c543b6da60e08003ec Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 23:25:45 -0700 Subject: [PATCH 89/93] pkgs/sagemath-{repl,categories,environment,objects}/requirements-editable.txt.m4: Copy from #35095 --- pkgs/sagemath-categories/requirements-editable.txt.m4 | 7 +++++++ .../sagemath-environment/requirements-editable.txt.m4 | 0 pkgs/sagemath-objects/requirements-editable.txt.m4 | 4 ++++ pkgs/sagemath-repl/requirements-editable.txt.m4 | 11 +++++++++++ 4 files changed, 22 insertions(+) create mode 100644 pkgs/sagemath-categories/requirements-editable.txt.m4 create mode 100644 pkgs/sagemath-environment/requirements-editable.txt.m4 create mode 100644 pkgs/sagemath-objects/requirements-editable.txt.m4 create mode 100644 pkgs/sagemath-repl/requirements-editable.txt.m4 diff --git a/pkgs/sagemath-categories/requirements-editable.txt.m4 b/pkgs/sagemath-categories/requirements-editable.txt.m4 new file mode 100644 index 00000000000..1c0abbe40ae --- /dev/null +++ b/pkgs/sagemath-categories/requirements-editable.txt.m4 @@ -0,0 +1,7 @@ +include(`sage_spkg_versions.m4')dnl +dnl Same as setup.cfg.m4 install_requires; FIXME: should pin to built wheels. +SPKG_INSTALL_REQUIRES_gmpy2 +SPKG_INSTALL_REQUIRES_cysignals +SPKG_INSTALL_REQUIRES_memory_allocator +-e ../sagemath-environment +-e ../sagemath-objects diff --git a/pkgs/sagemath-environment/requirements-editable.txt.m4 b/pkgs/sagemath-environment/requirements-editable.txt.m4 new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pkgs/sagemath-objects/requirements-editable.txt.m4 b/pkgs/sagemath-objects/requirements-editable.txt.m4 new file mode 100644 index 00000000000..df9b22b43aa --- /dev/null +++ b/pkgs/sagemath-objects/requirements-editable.txt.m4 @@ -0,0 +1,4 @@ +include(`sage_spkg_versions.m4')dnl +dnl Same as setup.cfg.m4 install_requires; FIXME: should pin to built wheels. +SPKG_INSTALL_REQUIRES_gmpy2 +SPKG_INSTALL_REQUIRES_cysignals diff --git a/pkgs/sagemath-repl/requirements-editable.txt.m4 b/pkgs/sagemath-repl/requirements-editable.txt.m4 new file mode 100644 index 00000000000..fa89ade7621 --- /dev/null +++ b/pkgs/sagemath-repl/requirements-editable.txt.m4 @@ -0,0 +1,11 @@ +include(`sage_spkg_versions.m4')dnl +dnl Same as setup.cfg.m4 install_requires (+ their install-requires) +dnl FIXME: should pin to built wheels. +SPKG_INSTALL_REQUIRES_gmpy2 +SPKG_INSTALL_REQUIRES_cysignals +SPKG_INSTALL_REQUIRES_memory_allocator +SPKG_INSTALL_REQUIRES_ipython +SPKG_INSTALL_REQUIRES_ipywidgets +dnl To be added when ready for editable: +-e ../sagemath-environment +-e ../sagemath-objects From 59e2911305bc9642fd0566d620c44b25183dbc15 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 5 Apr 2024 23:32:52 -0700 Subject: [PATCH 90/93] **/tox.ini: Set upper bound tox < 4.14.1 because of https://github.com/tox-dev/tox/issues/3238 (fixup) --- pkgs/sagemath-categories/tox.ini | 2 +- pkgs/sagemath-environment/tox.ini | 2 +- pkgs/sagemath-objects/tox.ini | 2 +- pkgs/sagemath-repl/tox.ini | 2 +- pkgs/sagemath-standard/tox.ini | 2 +- src/tox.ini | 2 +- tox.ini | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index 6eaa17fc20e..2dff44b2f8d 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -14,7 +14,7 @@ requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=4.2,<4.14.1 + tox>=4.2, tox<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-environment/tox.ini b/pkgs/sagemath-environment/tox.ini index fec1966d9a9..502f6967798 100644 --- a/pkgs/sagemath-environment/tox.ini +++ b/pkgs/sagemath-environment/tox.ini @@ -14,7 +14,7 @@ requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=4.2,<4.14.1 + tox>=4.2, tox<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-objects/tox.ini b/pkgs/sagemath-objects/tox.ini index 3e539e2e3b0..24e7a2f107d 100644 --- a/pkgs/sagemath-objects/tox.ini +++ b/pkgs/sagemath-objects/tox.ini @@ -14,7 +14,7 @@ requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=4.2,<4.14.1 + tox>=4.2, tox<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-repl/tox.ini b/pkgs/sagemath-repl/tox.ini index d565aa7b2a4..f0874d4e22b 100644 --- a/pkgs/sagemath-repl/tox.ini +++ b/pkgs/sagemath-repl/tox.ini @@ -14,7 +14,7 @@ requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=4.2,<4.14.1 + tox>=4.2, tox<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-standard/tox.ini b/pkgs/sagemath-standard/tox.ini index cfb4fb6d527..d27d45db301 100644 --- a/pkgs/sagemath-standard/tox.ini +++ b/pkgs/sagemath-standard/tox.ini @@ -63,7 +63,7 @@ requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=4.2,<4.14.1 + tox>=4.2, tox<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/src/tox.ini b/src/tox.ini index ab7c1b5f18e..31ae15a8946 100644 --- a/src/tox.ini +++ b/src/tox.ini @@ -28,7 +28,7 @@ skipsdist = true requires = # For the renamed "allowlist_externals" keyword, need >= 3.18 # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=3.18,<4.14.1 + tox>=3.18, tox<4.14.1 [sagedirect] # Base for tox environments that bypass the virtual environment set up by tox, diff --git a/tox.ini b/tox.ini index 5167ead23b8..1ecfc8efd1b 100644 --- a/tox.ini +++ b/tox.ini @@ -131,7 +131,7 @@ envlist = requires = # For repaired numerical factors in tox 4, need >= 4.2.7. # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=4.2.7,<4.14.1 + tox>=4.2.7, tox<4.14.1 skipsdist = true From 6d4ee489b98bfe54760c170f06481bafd7adea75 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 6 Apr 2024 00:02:59 -0700 Subject: [PATCH 91/93] tox.ini: Fix 'requires' for autoprovisioning on tox 3 --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1ecfc8efd1b..1ed1352e187 100644 --- a/tox.ini +++ b/tox.ini @@ -131,7 +131,8 @@ envlist = requires = # For repaired numerical factors in tox 4, need >= 4.2.7. # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=4.2.7, tox<4.14.1 + tox>=4.2.7 + tox<4.14.1 skipsdist = true From b2c1131245c7f2c05df5de122c920211c3daae56 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 13 Apr 2024 13:16:57 -0700 Subject: [PATCH 92/93] src/sage/doctest/reporting.py: Add missing # needs SAGE_SRC --- src/sage/doctest/reporting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sage/doctest/reporting.py b/src/sage/doctest/reporting.py index c1ac8a6c7b8..31d4232fcda 100644 --- a/src/sage/doctest/reporting.py +++ b/src/sage/doctest/reporting.py @@ -145,7 +145,8 @@ def were_doctests_with_optional_tag_run(self, tag): When latex is available, doctests marked with optional tag ``latex`` are run by default since :issue:`32174`:: - sage: filename = os.path.join(SAGE_SRC,'sage','misc','latex.py') + sage: # needs SAGE_SRC + sage: filename = os.path.join(SAGE_SRC, 'sage', 'misc', 'latex.py') sage: DC = DocTestController(DocTestDefaults(), [filename]) sage: DTR = DocTestReporter(DC) sage: DTR.were_doctests_with_optional_tag_run('latex') # optional - latex From 2a8a95849ffd3b33145797c77ba6823c2d8dbb67 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 14 Apr 2024 22:50:35 -0700 Subject: [PATCH 93/93] **/tox.ini: Fix format of [tox] requires --- pkgs/sagemath-categories/tox.ini | 3 ++- pkgs/sagemath-environment/tox.ini | 3 ++- pkgs/sagemath-objects/tox.ini | 3 ++- pkgs/sagemath-repl/tox.ini | 3 ++- pkgs/sagemath-standard/tox.ini | 3 ++- src/tox.ini | 3 ++- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index 2dff44b2f8d..841209e88f1 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -14,7 +14,8 @@ requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=4.2, tox<4.14.1 + tox>=4.2 + tox<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-environment/tox.ini b/pkgs/sagemath-environment/tox.ini index 502f6967798..5d741bd0b48 100644 --- a/pkgs/sagemath-environment/tox.ini +++ b/pkgs/sagemath-environment/tox.ini @@ -14,7 +14,8 @@ requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=4.2, tox<4.14.1 + tox>=4.2 + tox<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-objects/tox.ini b/pkgs/sagemath-objects/tox.ini index 24e7a2f107d..a7b91f55990 100644 --- a/pkgs/sagemath-objects/tox.ini +++ b/pkgs/sagemath-objects/tox.ini @@ -14,7 +14,8 @@ requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=4.2, tox<4.14.1 + tox>=4.2 + tox<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-repl/tox.ini b/pkgs/sagemath-repl/tox.ini index f0874d4e22b..d7b557761ac 100644 --- a/pkgs/sagemath-repl/tox.ini +++ b/pkgs/sagemath-repl/tox.ini @@ -14,7 +14,8 @@ requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=4.2, tox<4.14.1 + tox>=4.2 + tox<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/pkgs/sagemath-standard/tox.ini b/pkgs/sagemath-standard/tox.ini index d27d45db301..c4d6d304155 100644 --- a/pkgs/sagemath-standard/tox.ini +++ b/pkgs/sagemath-standard/tox.ini @@ -63,7 +63,8 @@ requires = # Auto-provision a modern tox. # [pkgenv] added in 4.2 - https://tox.wiki/en/latest/upgrading.html#packaging-configuration-and-inheritance # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=4.2, tox<4.14.1 + tox>=4.2 + tox<4.14.1 [pkgenv] # Environment in which to build the sdist. diff --git a/src/tox.ini b/src/tox.ini index 31ae15a8946..57a7a11aaa9 100644 --- a/src/tox.ini +++ b/src/tox.ini @@ -28,7 +28,8 @@ skipsdist = true requires = # For the renamed "allowlist_externals" keyword, need >= 3.18 # Because of https://github.com/tox-dev/tox/issues/3238, need <4.14.1 - tox>=3.18, tox<4.14.1 + tox>=3.18 + tox<4.14.1 [sagedirect] # Base for tox environments that bypass the virtual environment set up by tox,