Merge #31535

.gitignore

@@ -182,6 +182,7 @@ src/*.egg-info/
 /src/bin/sage-src-env-config
 
 # Virtual environments
+/venv
 src/.env
 src/.venv
 src/env/

Makefile

@@ -109,6 +109,7 @@ distclean: build-clean
 	@echo "Deleting all remaining output from build system ..."
 	rm -rf local
 	rm -f src/bin/sage-env-config
+	rm -f prefix venv
 
 # Delete all auto-generated files which are distributed as part of the
 # source tarball

build/make/Makefile.in

@@ -535,8 +535,10 @@ $(1)-$(4)-no-deps:
 $(1)-no-deps: $(1)-$(4)-no-deps
 
 $(1)-$(4)-clean:
-	sage-spkg-uninstall $(if $(filter $(1),$(TOOLCHAIN_DEPS)),--keep-files) \
-		$(1) '$$($(4))'
+	if [ -d '$$($(4))' ]; then \
+	    sage-spkg-uninstall $(if $(filter $(1),$(TOOLCHAIN_DEPS)),--keep-files) \
+		$(1) '$$($(4))'; \
+	fi
 
 $(1)-clean: $(1)-$(4)-clean
 
@@ -657,7 +659,16 @@ $(1)-uninstall: $(1)-$(4)-uninstall
 
 $(1)-clean: $(1)-uninstall
 
+# Recursive tox invocation (note - we do not set the environment here).
+# Setting SAGE_SPKG_WHEELS is for the benefit of sagelib's tox.ini
+$(1)-tox-%: FORCE
+	$(AM_V_at)cd '$$(SAGE_ROOT)/build/pkgs/$(1)/src' && \
+		export PATH="$$(SAGE_ORIG_PATH)" && \
+		SAGE_SPKG_WHEELS=$$(SAGE_LOCAL)/var/lib/sage/wheels \
+		tox -v -v -v -e $$*
+
 .PHONY: $(1) $(1)-uninstall $(1)-build-deps $(1)-no-deps $(1)-clean
+
 endef
 
 $(foreach pkgname,$(SCRIPT_PACKAGES),\

build/pkgs/python3/spkg-build.in

@@ -84,8 +84,24 @@ sdh_configure --enable-shared $PYTHON_CONFIGURE
 # Make sure -L. is placed before -L$SAGE_LOCAL/lib so that python and extension
 # modules are linked with the right libpython; we pass this in at make time
 # only, since we don't want -L. to be saved as one of the default LDFLAGS
-# used for building third-party extension modules
-sdh_make LDFLAGS="-L. $LDFLAGS"
+# used for building third-party extension modules.
+#
+# Trac #32442: As we now install python in SAGE_VENV, not SAGE_LOCAL,
+# we need to provide paths into $SAGE_LOCAL, so that setup.py finds
+# the libraries needed for the extension modules - in particular sqlite3.
+# (The search code there does not know about CPATH and LIBRARY_PATH.)
+make_LDFLAGS="-L. -L$SAGE_LOCAL/lib $LDFLAGS"
+make_CPPFLAGS="-I$SAGE_LOCAL/include $CPPFLAGS"
+
+# Also, we need to add an rpath, like we do for SAGE_LOCAL in src/bin/sage-env.
+# SAGE_INST_LOCAL is the installation hierarchy for the current package
+# -- for python3, this is SAGE_VENV.
+make_LDFLAGS="-Wl,-rpath,$SAGE_INST_LOCAL/lib $make_LDFLAGS"
+if [ "$UNAME" = "Linux" ]; then
+    make_LDFLAGS="-Wl,-rpath-link,$SAGE_INST_LOCAL/lib $make_LDFLAGS"
+fi
+
+sdh_make LDFLAGS="$make_LDFLAGS" CPPFLAGS="$make_CPPFLAGS"
 
 if [ "$UNAME" = "Darwin" ]; then
     export DYLD_LIBRARY_PATH="."

build/pkgs/python3/spkg-configure.m4

@@ -119,6 +119,17 @@ To build Sage with a different system python, use ./configure --with-python=/pat
     ])
     AC_SUBST([PYTHON_FOR_VENV])
 
+    AS_VAR_IF([SAGE_VENV], [auto], [SAGE_VENV=$SAGE_VENV_AUTO])
+    AS_CASE([$SAGE_VENV],
+        [no],  [SAGE_VENV='${SAGE_LOCAL}'],dnl Quoted so that it is resolved at build time by shell/Makefile
+        [yes], [AS_IF([test -n "$PYTHON_FOR_VENV"], [
+                   PYTHON_VERSION=$("$PYTHON_FOR_VENV" -c "import sysconfig; print(sysconfig.get_python_version())")
+                ], [
+                   PYTHON_VERSION=$(echo $(cat build/pkgs/python3/package-version.txt))
+                ])
+                SAGE_VENV='${SAGE_LOCAL}'/var/lib/sage/venv-python$PYTHON_VERSION]
+    )
+
     dnl These temporary directories are created by the check above
     dnl and need to be cleaned up to prevent the "rm -f conftest*"
     dnl (that a bunch of other checks do) from emitting warnings about

build/pkgs/sagelib/spkg-src

@@ -15,7 +15,7 @@ fi
 # Exit on failure
 set -e
 
-cd build/pkgs/sagelib
+cd "$SAGE_ROOT"/build/pkgs/sagelib
 
 cd src
 python3 -u setup.py --no-user-cfg sdist --dist-dir "$SAGE_DISTFILES"

configure.ac

@@ -70,15 +70,24 @@ SAGE_LOCAL="$prefix"
 # This is nonstandard.
 if test "$SAGE_LOCAL" = NONE; then
     SAGE_LOCAL=local
+    if test -x "$SAGE_LOCAL"/bin/python3; then
+        # Incremental build with an existing installation of python3 spkg
+        # in SAGE_LOCAL or venv in SAGE_LOCAL.  Keep old behavior.
+        SAGE_VENV_AUTO=no
+    else
+        SAGE_VENV_AUTO=yes
+    fi
+else
+    SAGE_VENV_AUTO=no
 fi
 SAGE_SRC="$SAGE_ROOT/src"
 SAGE_SPKG_INST="$SAGE_LOCAL/var/lib/sage/installed"
 
 AC_ARG_WITH([sage-venv],
-            [AS_HELP_STRING([--with-sage-venv=SAGE_VENV],
+            [AS_HELP_STRING([--with-sage-venv={auto (default),yes,no,SAGE_VENV}],
                             [put Python packages into an installation hierarchy separate from prefix])],
             [SAGE_VENV="$withval"],
-            [SAGE_VENV='${SAGE_LOCAL}'])dnl Quoted so that it is resolved at build time by shell/Makefile
+            [SAGE_VENV="auto"])
 AC_SUBST([SAGE_VENV])
 
 #---------------------------------------------------------
@@ -493,6 +502,29 @@ AC_CONFIG_COMMANDS(mkdirs,
         SAGE_INST="$SAGE_SPKG_INST"
     ])
 
+AC_CONFIG_COMMANDS(links, [
+    dnl Create links for the convenience of users
+    SYMLINK="${ac_top_build_prefix}prefix"
+    AS_IF([test -L "$SYMLINK" -o ! -e "$SYMLINK"], [
+        AC_MSG_NOTICE([creating convenience symlink $SYMLINK -> $SAGE_LOCAL])
+        rm -f "$SYMLINK"
+        ln -sf "$SAGE_LOCAL" "$SYMLINK"
+    ], [
+        AC_MSG_NOTICE([cannot create convenience symlink $SYMLINK -> $SAGE_LOCAL because the file exists and is not a symlink; this is harmless])
+    ])
+    SYMLINK="${ac_top_build_prefix}venv"
+    AS_IF([test -L "$SYMLINK" -o ! -e "$SYMLINK"], [
+        AC_MSG_NOTICE([creating convenience symlink $SYMLINK -> $SAGE_VENV])
+        rm -f "$SYMLINK"
+        ln -sf "$SAGE_VENV" "$SYMLINK"
+    ], [
+        AC_MSG_NOTICE([cannot create convenience symlink $SYMLINK -> $SAGE_VENV because the file exists and is not a symlink; this is harmless])
+    ])
+], [
+    SAGE_LOCAL="$SAGE_LOCAL"
+    eval SAGE_VENV="$SAGE_VENV"dnl eval so as to evaluate the embedded ${SAGE_LOCAL}
+])
+
 AC_OUTPUT()
 
 SAGE_SYSTEM_PACKAGE_NOTICE()

m4/sage_spkg_collect.m4

@@ -153,8 +153,8 @@ for DIR in $SAGE_ROOT/build/pkgs/*; do
     dnl Jupyter notebook, then packages such as jupyter_core would have to be installed into
     dnl two trees.
     SPKG_TREE_VAR=SAGE_LOCAL
-    if test -f "$DIR/requirements.txt" -o -f "$DIR/install-requires.txt"; then
-        dnl A Python package
+    if test -f "$DIR/requirements.txt" -o -f "$DIR/install-requires.txt" -o "$SPKG_NAME" = python3; then
+        dnl A Python package or spkg installation of python3 itself
         SPKG_TREE_VAR=SAGE_VENV
     fi
     SAGE_PACKAGE_TREES="${SAGE_PACKAGE_TREES}$(printf '\ntrees_')${SPKG_NAME} = ${SPKG_TREE_VAR}"

pkgs/sagemath-standard/tox.ini

@@ -25,15 +25,15 @@ envlist =
     # Build dependencies according to requirements.txt (all versions fixed).
     # Use ONLY the wheels built and stored by the Sage distribution (no PyPI):
     #
-    #   ./sage -sh -c '(cd pkgs/sagelib && tox -v -v -v -e python-sagewheels-nopypi)'
+    #   ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e python-sagewheels-nopypi)'
     #
     python-sagewheels-nopypi,
     #
     # Build and test without using the concrete dependencies specified by requirements.txt,
     # using the dependencies declared in pyproject.toml and setup.cfg (install-requires) only:
     # Still use ONLY the wheels built and stored by the Sage distribution (no PyPI).
     #
-    #   ./sage -sh -c '(cd pkgs/sagelib && tox -v -v -v -e python-sagewheels-nopypi-norequirements)'
+    #   ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e python-sagewheels-nopypi-norequirements)'
     #
     python-sagewheels-nopypi-norequirements,
     #
@@ -44,7 +44,7 @@ envlist =
     # and additionally allow packages from PyPI.
     # Because all versions are fixed, we "should" end up using the prebuilt wheels.
     #
-    #   ./sage -sh -c '(cd pkgs/sagelib && tox -v -v -v -e python-sagewheels)'
+    #   ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e python-sagewheels)'
     #
     python-sagewheels,
     #
@@ -81,8 +81,8 @@ passenv =
                              PKG_CONFIG_PATH
     # Parallel build
                              SAGE_NUM_THREADS
-    # SAGE_LOCAL only for finding the wheels
-    sagewheels:              SAGE_LOCAL
+    # SAGE_VENV only for finding the wheels
+    sagewheels:              SAGE_VENV
     # Location of the wheels (needs to include a PEP 503 compliant
     # Simple Repository index, i.e., a subdirectory "simple")
     sagewheels:              SAGE_SPKG_WHEELS
@@ -92,7 +92,7 @@ setenv =
     HOME={envdir}
     # We supply pip options by environment variables so that they
     # apply both to the installation of the dependencies and of the package
-    sagewheels:              PIP_FIND_LINKS=file://{env:SAGE_SPKG_WHEELS:{env:SAGE_LOCAL:{toxinidir}/../../../../local}/var/lib/sage/wheels}
+    sagewheels:              PIP_FIND_LINKS=file://{env:SAGE_SPKG_WHEELS:{env:SAGE_VENV:{toxinidir}/../../../../venv}/var/lib/sage/wheels}
     nopypi:                  PIP_NO_INDEX=true
     # No build isolation for PEP 517 packages - use what is already in the environment
     # Note that this pip env "NO" variable uses inverted logic:

tox.ini

@@ -428,7 +428,8 @@ setenv =
     local-conda-environment:          CONDA_SAGE_ENVIRONMENT=sage-build
     local-conda-environment:            CONDA_SAGE_ENVIRONMENT_FILE=environment.yml
     local-conda-environment-optional:   CONDA_SAGE_ENVIRONMENT_FILE=environment-optional.yml
-    local-conda-environment:      SETENV_CONFIGURE=( {env:CONDA_PREFIX}/bin/conda env create -n {env:CONDA_SAGE_ENVIRONMENT} --file {env:CONDA_SAGE_ENVIRONMENT_FILE} || {env:CONDA_PREFIX}/bin/conda env update -n {env:CONDA_SAGE_ENVIRONMENT} --file {env:CONDA_SAGE_ENVIRONMENT_FILE} ) && . {env:CONDA_PREFIX}/bin/activate {env:CONDA_SAGE_ENVIRONMENT}
+    local-conda-environment-src:        CONDA_SAGE_ENVIRONMENT_DIR=src/
+    local-conda-environment:      SETENV_CONFIGURE=( {env:CONDA_PREFIX}/bin/conda env create -n {env:CONDA_SAGE_ENVIRONMENT} --file {env:CONDA_SAGE_ENVIRONMENT_DIR:}{env:CONDA_SAGE_ENVIRONMENT_FILE} || {env:CONDA_PREFIX}/bin/conda env update -n {env:CONDA_SAGE_ENVIRONMENT} --file {env:CONDA_SAGE_ENVIRONMENT_DIR:}{env:CONDA_SAGE_ENVIRONMENT_FILE} ) && . {env:CONDA_PREFIX}/bin/activate {env:CONDA_SAGE_ENVIRONMENT}
     #
     # Configuration factors
     #