Merge tag '9.3.beta3' into t/29850/install_sage_env_config_with_sage_…

…conf

SageMath version 9.3.beta3, Release Date: 2020-12-06

.github/workflows/ci-wsl.yml

@@ -1,11 +1,19 @@
 name: Build & Test WSL
 
-on: [push, pull_request]
+on:
+  pull_request:
+    types: [opened, synchronize]
+  push:
+    tags:
+      - '*'
+    branches:
+      - 'public/build/**wsl**'
+  workflow_dispatch:
 
 jobs:
   windows:
     runs-on: windows-latest
-    name: Windows (using WSL)
+    name: Ubuntu 20.04
     # Following https://trac.sagemath.org/ticket/25206#comment:63
     steps:
     - name: Configure git
@@ -18,22 +26,10 @@ jobs:
         Ubuntu\ubuntu2004.exe install --root
     - name: Install dependencies
       run: |
-        Function ExtractPackages
-        {
-           param($path)
-           @(Get-Content $path | Where-Object { !$_.StartsWith("#") })
-        }
-        $packages = Get-ChildItem build/pkgs -recurse -Include debian.txt, debian-bootstrap.txt | foreach-object { ExtractPackages $_.FullName } | sort -Unique
-        # Add tox which is use to build
-        $packages += 'tox'
-        echo "Install the following packages:" @packages
-        # Add deadsnakes so that libpython3.7-dev is found
-        & wsl sudo add-apt-repository ppa:deadsnakes/ppa
-        # Install
-        & wsl sudo apt-get update -y 
-        & wsl sudo apt-get install -y @packages
+        wsl sudo apt-get update -y 
+        wsl sudo apt-get install -y tox
     - name: Build
-      run: wsl tox -e local -- SAGE_NUM_THREADS=4 build
+      run: wsl tox -e local-sudo-ubuntu-standard -- SAGE_NUM_THREADS=4 build
       env:
         # WSL runs everything as root, so we have to enable build as root user
         EXTRA_CONFIGURE_ARGS: "--enable-build-as-root"
@@ -42,7 +38,7 @@ jobs:
         MAKE: "make -j2"
         WSLENV: EXTRA_CONFIGURE_ARGS:MAKE
     - name: Test
-      run: wsl tox -e local -- SAGE_NUM_THREADS=4 ptest
+      run: wsl tox -e local-sudo-ubuntu-standard -- SAGE_NUM_THREADS=4 ptest
     - name: Prepare logs artifact
       run: mkdir -p "artifacts/logs"; cp -r .tox/*/log "artifacts/logs"
       shell: bash

.github/workflows/tox-experimental.yml

@@ -99,20 +99,25 @@ jobs:
 
   local-macos:
 
-    runs-on: macos-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       max-parallel: 3
       matrix:
+        os: [ macos-10.15, macos-11.0 ]
         tox_system_factor: [homebrew-macos, homebrew-macos-python3_xcode, homebrew-macos-python3_xcode-nokegonly, homebrew-macos-python3_pythonorg, conda-forge-macos]
         tox_packages_factor: [maximal]
         targets_pattern: [0-g, h-o, p, q-z]
+        xcode_version_factor: [11.7, default, 12.3]
     env:
       TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
-      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
+      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}-${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
       TARGETS_OPTIONAL: "$( echo $(PATH=build/bin:$PATH build/bin/sage-package list :experimental: | grep -v  database_stein_watkins\\$ | grep -v polytopes_db_4d | grep '^[${{ matrix.targets_pattern }}]' ) )"
     steps:
       - uses: actions/checkout@v2
+      - name: Select Xcode version
+        run: |
+          if [ ${{ matrix.xcode_version_factor }} != default ]; then sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode_version_factor }}.app; fi
       - name: Install test prerequisites
         run: |
           brew install tox

.github/workflows/tox-optional.yml

@@ -101,22 +101,27 @@ jobs:
 
   local-macos:
 
-    runs-on: macos-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       max-parallel: 3
       matrix:
+        os: [ macos-10.15, macos-11.0 ]
         tox_system_factor: [homebrew-macos, homebrew-macos-python3_xcode, homebrew-macos-python3_xcode-nokegonly, homebrew-macos-python3_pythonorg, conda-forge-macos]
         tox_packages_factor: [maximal]
         targets_pattern: [0-g, h-o, p, q-z]
+        xcode_version_factor: [11.7, default, 12.3]
     env:
       TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
-      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
+      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}-${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
       # Test all optional packages, but do not test huge packages
       # and do not test packages that require external software
       TARGETS_OPTIONAL: "$( echo $(PATH=build/bin:$PATH build/bin/sage-package list :optional: | grep -v  database_stein_watkins\\$ | grep -v polytopes_db_4d | grep -v cplex | grep -v gurobi | grep '^[${{ matrix.targets_pattern }}]' ) )"
     steps:
       - uses: actions/checkout@v2
+      - name: Select Xcode version
+        run: |
+          if [ ${{ matrix.xcode_version_factor }} != default ]; then sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode_version_factor }}.app; fi
       - name: Install test prerequisites
         run: |
           brew install tox

.github/workflows/tox.yml

@@ -97,17 +97,18 @@ jobs:
 
   local-macos:
 
-    runs-on: macos-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       max-parallel: 4
       matrix:
+        os: [ macos-10.15, macos-11.0 ]
         tox_system_factor: [homebrew-macos, homebrew-macos-python3_xcode, homebrew-macos-python3_xcode-nokegonly, homebrew-macos-python3_pythonorg, homebrew-macos-python3_xcode-gcc_spkg, conda-forge-macos]
         tox_packages_factor: [minimal, standard]
-        xcode_version_factor: [default, 12]
+        xcode_version_factor: [11.7, default, 12.3]
     env:
       TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
-      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}-xcode_${{ matrix.xcode_version_factor }}
+      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}-${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
     steps:
       - uses: actions/checkout@v2
       - name: Select Xcode version
@@ -214,11 +215,12 @@ jobs:
 
     needs: [dist]
 
-    runs-on: macos-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       max-parallel: 4
       matrix:
+        os: [ macos-10.15, macos-11.0 ]
         tox_system_factor: [macos-nobootstrap, macos-nobootstrap-python3_pythonorg]
         tox_packages_factor: [minimal]
         xcode_version_factor: [default, 12]

.gitignore

@@ -18,6 +18,11 @@
 # no longer generated, but may still be in user worktrees
 /src/lib/pkgconfig
 
+/environment.yml
+/environment-optional.yml
+/src/environment.yml
+/src/environment-optional.yml
+
 # Various editors
 *~
 

.zenodo.json

@@ -1,10 +1,10 @@
 {
     "description": "Mirror of the Sage https://sagemath.org/ source tree", 
     "license": "other-open", 
-    "title": "sagemath/sage: 9.3.beta2", 
-    "version": "9.3.beta2", 
+    "title": "sagemath/sage: 9.3.beta3", 
+    "version": "9.3.beta3", 
     "upload_type": "software", 
-    "publication_date": "2020-11-24", 
+    "publication_date": "2020-12-06", 
     "creators": [
         {
             "affiliation": "SageMath.org", 
@@ -15,7 +15,7 @@
     "related_identifiers": [
         {
             "scheme": "url", 
-            "identifier": "https://github.com/sagemath/sage/tree/9.3.beta2", 
+            "identifier": "https://github.com/sagemath/sage/tree/9.3.beta3", 
             "relation": "isSupplementTo"
         }, 
         {

Makefile

@@ -64,11 +64,11 @@ build/make/Makefile: configure $(SPKG_COLLECT_FILES) $(CONFIG_FILES:%=%.in)
 buildbot-python3:
 	$(MAKE)
 
-# Preemptively download all standard upstream source tarballs.
+# Preemptively download all source tarballs of normal packages.
 download:
 	export SAGE_ROOT=$$(pwd) && \
-	export PATH=$$SAGE_ROOT/src/bin:$$PATH && \
-	./src/bin/sage-download-upstream
+	export PATH=$$SAGE_ROOT/build/bin:$$PATH && \
+	sage-package download :all:
 
 dist: build/make/Makefile
 	./sage --sdist
@@ -111,6 +111,10 @@ bootstrap-clean:
 	rm -f src/doc/en/installation/*.txt
 	rm -rf src/doc/en/reference/spkg/*.rst
 	rm -f src/doc/en/reference/repl/*.txt
+	rm -f environment.yml
+	rm -f src/environment.yml
+	rm -f environment-optional.yml
+	rm -f src/environment-optional.yml
 
 # Remove absolutely everything which isn't part of the git repo
 maintainer-clean: distclean bootstrap-clean
@@ -147,15 +151,15 @@ micro_release:
 
 # Leaves everything that is needed to make the next "make" fast but removes
 # all the cheap build artifacts that can be quickly regenerated.
+# Trac #30960: We no longer uninstall sagelib.
 fast-rebuild-clean: misc-clean
 	rm -rf upstream/
-	rm -rf src/build/temp.*
-	# Without site-packages/sage sage does not start but copying/compiling
-	# them from src/build is very fast.
-	rm -rf local/lib/python*/site-packages/sage
+	rm -rf build/pkgs/sagelib/src/build/temp.*
 	# The .py files in src/build are restored from src/sage without their
 	# mtimes changed.
-	find src/build -name '*.py' -exec rm \{\} \;
+	-find build/pkgs/sagelib/src/build -name '*.py' -exec rm \{\} \;
+	# Remove leftovers from ancient branches
+	rm -rf src/build
 
 TESTALL = ./sage -t --all
 PTESTALL = ./sage -t -p --all

VERSION.txt

@@ -1 +1 @@
-SageMath version 9.3.beta2, Release Date: 2020-11-24
+SageMath version 9.3.beta3, Release Date: 2020-12-06

bootstrap

@@ -190,7 +190,11 @@ save () {
 	build/make/Makefile-auto.in \
 	src/doc/en/installation/*.txt \
 	src/doc/en/reference/spkg/*.rst \
-	src/doc/en/reference/repl/*.txt
+	src/doc/en/reference/repl/*.txt \
+	environment.yml \
+	src/environment.yml \
+	environment-optional.yml \
+	src/environment-optional.yml
 
     # Update version
     echo "$NEWCONFVERSION" >$PKG/package-version.txt

build/bin/sage-build-num-threads

@@ -1,4 +1,4 @@
-#!/usr/bin/env sage-system-python
+#!/usr/bin/env sage-bootstrap-python
 #
 # Determine the number of threads to be used by Sage.
 #

build/bin/sage-print-system-package-command

@@ -5,9 +5,14 @@ shift
 IF_VERBOSE=:
 SUDO=
 PROMPT=
+COMMENT="# "
 while :
 do
     case "$1" in
+        --verbose=*)
+            IF_VERBOSE=
+            COMMENT="${1#--verbose=}"
+            ;;
         --verbose)
             IF_VERBOSE=
             ;;
@@ -16,7 +21,7 @@ do
             SUDO="sudo "
             ;;
         --prompt=*)
-            PROMPT=${1#--prompt=}
+            PROMPT="${1#--prompt=}"
             ;;
         --prompt)
             PROMPT='  $ '
@@ -47,11 +52,15 @@ options=
 shopt -s extglob
 case $system:$command in
     homebrew*:setup-build-env)
-        $IF_VERBOSE echo "# To automatically take care of homebrew messages regarding "
-        $IF_VERBOSE echo "# keg-only packages for the current shell session:"
+        $IF_VERBOSE echo "${COMMENT}"
+        $IF_VERBOSE echo "${COMMENT}Homebrew can issue suggestions regarding keg-only packages."
+        $IF_VERBOSE echo "${COMMENT}The following command is to automatically apply these suggestions"
+        $IF_VERBOSE echo "${COMMENT}for packages relevant for Sage to make them available for the build."
+        $IF_VERBOSE echo "${COMMENT}Run it once to apply the suggestions for the current session."
+        $IF_VERBOSE echo "${COMMENT}Add it to your shell profile to apply them for all future sessions."
+        $IF_VERBOSE echo "${COMMENT}"
         [ -n "$SAGE_ROOT" ] || SAGE_ROOT=.
         echo "${PROMPT}source $SAGE_ROOT/.homebrew-build-env"
-        $IF_VERBOSE echo "# Add this to your shell profile if you want it to persist between shell sessions."
         ;;
     *:setup-build-env)
         # Nothing needed

build/bin/write-dockerfile.sh

@@ -5,32 +5,26 @@
 set -e
 SYSTEM="${1:-debian}"
 shopt -s extglob
-TYPE_PATTERN="${2:-standard}"
+SAGE_PACKAGE_LIST_ARGS="${2:- --has-file=spkg-configure.m4 :standard:}"
 WITH_SYSTEM_SPKG="${3:-yes}"
 IGNORE_MISSING_SYSTEM_PACKAGES="${4:-no}"
 #
 STRIP_COMMENTS="sed s/#.*//;"
 SAGE_ROOT=.
-SYSTEM_PACKAGES=$(echo $(${STRIP_COMMENTS} $SAGE_ROOT/build/pkgs/$SYSTEM{,-bootstrap}.txt))
+export PATH="$SAGE_ROOT"/build/bin:$PATH
+SYSTEM_PACKAGES=$(echo $(${STRIP_COMMENTS} "$SAGE_ROOT"/build/pkgs/$SYSTEM{,-bootstrap}.txt))
 CONFIGURE_ARGS="--enable-option-checking "
-for PKG_SCRIPTS in build/pkgs/*; do
+for PKG_BASE in $($SAGE_ROOT/sage -package list --has-file=distros/$SYSTEM.txt $SAGE_PACKAGE_LIST_ARGS); do
+    PKG_SCRIPTS="$SAGE_ROOT"/build/pkgs/$PKG_BASE
     if [ -d $PKG_SCRIPTS ]; then
-        PKG_BASE=$(basename $PKG_SCRIPTS)
-        SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/distros/$SYSTEM.txt
-        if [ -f $PKG_SCRIPTS/type -a -f $SYSTEM_PACKAGES_FILE -a -f $PKG_SCRIPTS/spkg-configure.m4 ]; then
-           PKG_TYPE=$(cat $PKG_SCRIPTS/type)
-           PKG_SYSTEM_PACKAGES=$(echo $(${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE))
-           if [ -n "PKG_SYSTEM_PACKAGES" ]; then
-               case "$PKG_TYPE" in
-                   $TYPE_PATTERN)
-                       SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
-                       if [ -f $PKG_SCRIPTS/spkg-configure.m4 ]; then
-                           CONFIGURE_ARGS+="--with-system-$PKG_BASE=${WITH_SYSTEM_SPKG} "
-                       fi
-                       ;;
-               esac
+       SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/distros/$SYSTEM.txt
+       PKG_SYSTEM_PACKAGES=$(echo $(${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE))
+       if [ -n "PKG_SYSTEM_PACKAGES" ]; then
+           SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
+           if [ -f $PKG_SCRIPTS/spkg-configure.m4 ]; then
+               CONFIGURE_ARGS+="--with-system-$PKG_BASE=${WITH_SYSTEM_SPKG} "
            fi
-        fi
+       fi
     fi
 done
 echo "# Automatically generated by SAGE_ROOT/build/bin/write-dockerfile.sh"

build/pkgs/4ti2/distros/conda.txt

@@ -0,0 +1 @@
+4ti2

build/pkgs/_recommended/distros/debian.txt

@@ -1,7 +1,7 @@
 # From https://wiki.sagemath.org/prerequisitesUbuntu
 
 # to generate pdf documentation
-texlive-generic-extra
+texlive-latex-extra
 # to convert Jupyter notebooks to pdf
 texlive-xetex
 # to generate pdf documentation

build/pkgs/alabaster/distros/conda.txt

@@ -0,0 +1 @@
+alabaster

build/pkgs/arb/distros/homebrew.txt

@@ -1,3 +1 @@
-## This package depends on ntl, the homebrew package of which we cannot use
-## because it is built with NTL_THREADS.  See https://trac.sagemath.org/ticket/29339
-# sagemath/science/arb
+arb

build/pkgs/attrs/distros/conda.txt

@@ -0,0 +1 @@
+attrs

build/pkgs/babel/distros/conda.txt

@@ -0,0 +1 @@
+babel

build/pkgs/backcall/distros/conda.txt

@@ -0,0 +1 @@
+backcall

build/pkgs/beautifulsoup4/distros/conda.txt

@@ -0,0 +1 @@
+beautifulsoup4

build/pkgs/biopython/distros/conda.txt

@@ -0,0 +1 @@
+biopython

build/pkgs/bleach/distros/conda.txt

@@ -0,0 +1 @@
+bleach

build/pkgs/ccache/distros/conda.txt

@@ -0,0 +1 @@
+ccache

build/pkgs/cddlib/distros/conda.txt

@@ -0,0 +1 @@
+cddlib

build/pkgs/certifi/distros/conda.txt

@@ -0,0 +1 @@
+certifi

build/pkgs/cffi/distros/conda.txt

@@ -0,0 +1 @@
+cffi

build/pkgs/combinatorial_designs/distros/conda.txt

@@ -0,0 +1 @@
+sagemath-db-combinatorial-designs

build/pkgs/conda.txt

@@ -2,7 +2,7 @@ compilers
 make
 m4
 perl
-"python<3.9"
+python
 tar
 bc
 pkg-config

build/pkgs/configure/checksums.ini

@@ -1,4 +1,4 @@
 tarball=configure-VERSION.tar.gz
-sha1=a36c047f3e1d88350f54389386cbcbfac78afc68
-md5=63b3ba8b5d0fe6ea15e1b40395d50d47
-cksum=2980357555
+sha1=9630148997eadc90f50a99bc8902d9759cd63d4d
+md5=c73af26d82eac1af9725ca780e650ebb
+cksum=78369567