Trac #33788: GH Actions: Split ci-macos.yml out from tox.yml

When preparing a portability report such as https://groups.google.com/g
/sage-release/c/GOGWk66zaCQ/m/dBQww8WNAAAJ, I have found that the output
of `tox.yml` has gotten too big to be comfortably inspected in one go.
So we split out everything macOS into a separate workflow.

Preparation for #33062 and #32570

URL: https://trac.sagemath.org/33788
Reported by: mkoeppe
Ticket author(s): Matthias Koeppe
Reviewer(s): Dima Pasechnik

.github/workflows/ci-macos.yml

@@ -0,0 +1,226 @@
+name: CI macOS
+
+## This GitHub Actions workflow runs SAGE_ROOT/tox.ini with select environments,
+## whenever a GitHub pull request is opened or synchronized in a repository
+## where GitHub Actions are enabled.
+##
+## It builds and checks some sage spkgs as defined in TARGETS.
+##
+## A job succeeds if there is no error.
+##
+## The build is run with "make V=0", so the build logs of individual packages are suppressed.
+##
+## At the end, all package build logs that contain an error are printed out.
+##
+## After all jobs have finished (or are canceled) and a short delay,
+## tar files of all logs are made available as "build artifacts".
+
+#on: [push, pull_request]
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+  push:
+    tags:
+      - '*'
+  workflow_dispatch:
+    # Allow to run manually
+
+env:
+  TARGETS_PRE: all-sage-local
+  TARGETS: build doc-html
+  TARGETS_OPTIONAL: ptest
+
+jobs:
+  local-macos:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        stage: ["1", "2", "2-optional-0-o", "2-optional-p-z", "2-experimental-0-o", "2-experimental-p-z"]
+        # python3_xcode is only accepted if enough packages are available from the system
+        # --> to test "minimal", we would need https://trac.sagemath.org/ticket/30949
+        tox_env: [homebrew-macos-usrlocal-minimal, homebrew-macos-usrlocal-standard, homebrew-macos-usrlocal-maximal, homebrew-macos-usrlocal-python3_xcode-standard, conda-forge-macos-minimal, conda-forge-macos-standard, conda-forge-macos-maximal]
+        # As of 2021-12, default xcode
+        # - on macos-10.15:               12.4
+        # - on macos-latest (= macos-11): 13.1
+        # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md#xcode
+        xcode_version_factor: [default]
+        os: [ macos-10.15, macos-latest ]
+    env:
+      TOX_ENV: local-${{ matrix.tox_env }}
+      LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-tox-local-${{ matrix.tox_env }}-${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
+      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_env }}--${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Select Xcode version
+        run: |
+          if [ ${{ matrix.xcode_version_factor }} != default ]; then sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode_version_factor }}.app; fi
+      - name: Install test prerequisites
+        run: |
+          brew install tox
+      - uses: actions/download-artifact@v2
+        with:
+          path: sage-local-artifact
+          name: ${{ env.LOCAL_ARTIFACT_NAME }}
+        if: contains(matrix.stage, '2')
+      - name: Extract sage-local artifact
+        # This is macOS tar -- cannot use --listed-incremental
+        run: |
+          export SAGE_LOCAL=$(pwd)/.tox/$TOX_ENV/local
+          .github/workflows/extract-sage-local.sh sage-local-artifact/sage-local-*.tar
+        if: contains(matrix.stage, '2')
+      - name: Build and test with tox
+        # We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
+        # For doctesting, we use a lower parallelization to avoid timeouts.
+        run: |
+          case "${{ matrix.stage }}" in
+            1)               export TARGETS_PRE="all-sage-local" TARGETS="all-sage-local" TARGETS_OPTIONAL=""
+                             ;;
+            2)               export TARGETS_PRE="all-sage-local" TARGETS="build doc-html" TARGETS_OPTIONAL="ptest"
+                             ;;
+            2-optional*)     export TARGETS_PRE="build/make/Makefile" TARGETS="build/make/Makefile"
+                             targets_pattern="${{ matrix.stage }}"
+                             targets_pattern="${targets_pattern#2-optional-}"
+                             export TARGETS_OPTIONAL=$( echo $(export PATH=build/bin:$PATH && (for a in spkg-install.in spkg-install requirements.txt; do sage-package list :optional: --has-file $a --no-file huge --no-file has_nonfree_dependencies; done) | grep -v ^_ | grep -v sagemath_doc | grep "^[$targets_pattern]" ) )
+                             ;;
+            2-experimental*) export TARGETS_PRE="build/make/Makefile" TARGETS="build/make/Makefile"
+                             targets_pattern="${{ matrix.stage }}"
+                             targets_pattern="${targets_pattern#2-experimental-}"
+                             export TARGETS_OPTIONAL=$( echo $(export PATH=build/bin:$PATH && (for a in spkg-install.in spkg-install requirements.txt; do sage-package list :experimental: --has-file $a --no-file huge --no-file has_nonfree_dependencies; done) | grep -v ^_ | grep -v sagemath_doc | grep "^[$targets_pattern]" ) )
+                             ;;
+          esac
+          MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
+      - name: Prepare logs artifact
+        run: |
+          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
+        if: always()
+      - uses: actions/upload-artifact@v1
+        with:
+          path: artifacts
+          name: ${{ env.LOGS_ARTIFACT_NAME }}
+        if: always()
+      - name: Print out logs for immediate inspection
+        # and markup the output with GitHub Actions logging commands
+        run: |
+          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
+        if: always()
+      - name: Prepare sage-local artifact
+        # This also includes the copies of homebrew or conda installed in the tox environment.
+        # We use absolute pathnames in the tar file.
+        # This is macOS tar -- cannot use --remove-files.
+        # We remove the $SAGE_LOCAL/lib64 link, which will be recreated by the next stage.
+        run: |
+          mkdir -p sage-local-artifact && (cd .tox/$TOX_ENV && rm -f "local/lib64" && tar -cf - $(pwd)) > sage-local-artifact/sage-${{ env.TOX_ENV }}-${{ matrix.stage }}.tar
+        if: contains(matrix.stage, '1')
+      - uses: actions/upload-artifact@v2
+        with:
+          path: sage-local-artifact/sage-${{ env.TOX_ENV }}-${{ matrix.stage }}.tar
+          name: ${{ env.LOCAL_ARTIFACT_NAME }}
+        if: always()
+
+  dist:
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 500
+      - name: fetch tags
+        run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
+      - name: Install bootstrap prerequisites
+        run: |
+          sudo DEBIAN_FRONTEND=noninteractive apt-get update
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install $(build/bin/sage-get-system-packages debian _bootstrap)
+      - name: Bootstrap with sage-update-version
+        # We set SAGE_ROOT and SAGE_SRC by hand
+        # because 'sage -sh' does not work with an unconfigured tree,
+        # giving: Error: SAGE_SCRIPTS_DIR is set to a bad value
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "Sage GitHub CI"
+          SAGE_ROOT=. SAGE_SRC=./src src/bin/sage-update-version $(git describe) || echo "(ignoring error)"
+      - name: make dist
+        run: |
+          ./configure --enable-download-from-upstream-url && make dist
+      - uses: actions/upload-artifact@v2
+        with:
+          path: "dist/*.tar.gz"
+          name: dist
+
+  local-macos-nohomebrew:
+
+    needs: [dist]
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        os: [ macos-10.15, macos-11.0 ]
+        tox_system_factor: [macos-nobootstrap, macos-nobootstrap-python3_pythonorg]
+        tox_packages_factor: [minimal]
+        # As of 2021-03, default is 12.4
+        # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md#xcode
+        xcode_version_factor: [default]
+        include:
+          # Test xcode 11.7 only on macos-10.15
+          - tox_system_factor: macos-nobootstrap
+            tox_packages_factor: minimal
+            xcode_version_factor: 11.7
+            os: macos-10.15
+          - tox_system_factor: macos-nobootstrap-python3_pythonorg
+            tox_packages_factor: minimal
+            xcode_version_factor: 11.7
+            os: macos-10.15
+    env:
+      TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
+      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}-xcode_${{ matrix.xcode_version_factor }}
+    steps:
+      - uses: actions/checkout@v2
+        if: "!contains(matrix.tox_system_factor, 'nobootstrap')"
+      - uses: actions/download-artifact@v2
+        with:
+          path: .
+          name: dist
+        if: contains(matrix.tox_system_factor, 'nobootstrap')
+      - name: Unpack sage dist
+        run: |
+          tar xf sage*.tar.gz --strip-components=1
+        if: contains(matrix.tox_system_factor, 'nobootstrap')
+      - name: Move homebrew away
+        run: |
+          (cd /usr/local && for a in bin etc include lib opt sbin share; do sudo mv $a $a-moved; done)
+      - name: Select Xcode version
+        run: |
+          if [ ${{ matrix.xcode_version_factor }} != default ]; then sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode_version_factor }}.app; fi
+      - name: Install test prerequisites
+        run: |
+          sudo /usr/bin/python3 -m pip install tox
+      - name: Install python3 from python.org
+        # As of 2020-03-30 (https://github.com/actions/virtual-environments/blob/master/images/macos/macos-10.15-Readme.md),
+        # Python 3.7.7 is installed on GitHub Actions runners.  But we install our own copy from the python.org binary package.
+        run: |
+          curl -o python3.pkg https://www.python.org/ftp/python/3.7.7/python-3.7.7-macosx10.9.pkg
+          sudo installer -verbose -pkg python3.pkg -target /
+        if: contains(matrix.tox_system_factor, 'python3_pythonorg')
+      - name: Build and test with tox
+        # We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
+        # For doctesting, we use a lower parallelization to avoid timeouts.
+        run: |
+          MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
+      - name: Prepare logs artifact
+        run: |
+          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
+        if: always()
+      - uses: actions/upload-artifact@v1
+        with:
+          path: artifacts
+          name: ${{ env.LOGS_ARTIFACT_NAME }}
+        if: always()
+      - name: Print out logs for immediate inspection
+        # and markup the output with GitHub Actions logging commands
+        run: |
+          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
+        if: always()