From 8aa6fd96607c207a0f9563c583978cd50ddc52a7 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 7 Sep 2020 22:16:11 -0700 Subject: [PATCH 01/11] build/bin/sage-dist-helpers (sdh_pip_install): Build a wheel, store it --- build/bin/sage-dist-helpers | 27 ++++++++++++++-- build/bin/sage-pip-install | 62 +++--------------------------------- build/bin/sage-pip-uninstall | 62 ++++++++++++++++++++++++++++++++++++ build/make/install | 1 + 4 files changed, 91 insertions(+), 61 deletions(-) create mode 100755 build/bin/sage-pip-uninstall diff --git a/build/bin/sage-dist-helpers b/build/bin/sage-dist-helpers index a4a8c88a29c..603c171e070 100644 --- a/build/bin/sage-dist-helpers +++ b/build/bin/sage-dist-helpers @@ -210,10 +210,31 @@ sdh_pip_install() { local sudo="$SAGE_SUDO" local root="" fi - $sudo sage-pip-install $root "$@" || \ - sdh_die "Error installing $PKG_NAME" -} + $sudo sage-pip-uninstall "$@" || \ + sdh_die "Error uninstalling a previous version of $PKG_NAME" + + sage-python23 -m pip wheel --wheel-dir=. --no-binary :all: --verbose --no-deps --no-index --isolated --no-build-isolation "$@" || \ + sdh_die "Error building a wheel for $PKG_NAME" + wheel="" + for w in *.whl; do + if [ -n "$wheel" ]; then + sdh_die "Error: more than one wheel found after building" + fi + if [ -f "$w" ]; then + wheel="$w" + fi + done + if [ -z "$wheel" ]; then + sdh_die "Error: no wheel found after building" + fi + + $sudo sage-pip-install $root "$wheel" || \ + sdh_die "Error installing $wheel" + mkdir -p "${SAGE_DESTDIR}${SAGE_SPKG_WHEELS}" && \ + mv "$wheel" "${SAGE_DESTDIR}${SAGE_SPKG_WHEELS}/" || \ + sdh_die "Error storing $wheel" +} sdh_cmake() { echo "Configuring $PKG_NAME with cmake" diff --git a/build/bin/sage-pip-install b/build/bin/sage-pip-install index 2ff9bd43d3e..5c5279a4c40 100755 --- a/build/bin/sage-pip-install +++ b/build/bin/sage-pip-install @@ -1,8 +1,6 @@ #!/usr/bin/env bash -# This command is specifically for pip-installing from a local -# source directory, as opposed to from a package index via package -# name. That is, it is for pip-installing Sage spkgs from their -# extracted upstream sources. +# This command is specifically for pip-installing a previously +# built wheel. # # This ensures that any previous installations of the same package # are uninstalled first. @@ -17,23 +15,7 @@ # This also disables pip's version self-check. # --isolated : Don't read configuration files such as # ~/.pydistutils.cfg -# --no-build-isolation:Build the package in the usual Python environment -# (containing the dependencies) instead of an -# "isolated" environment -pip_install_flags="--ignore-installed --verbose --no-deps --no-index --isolated --no-build-isolation" - -# Consume any additional pip install arguments except the last one -while [ $# -gt 1 ]; do - pip_install_flags="$pip_install_flags $1" - shift -done - -# Last argument must be "." and will be ignored -if [ "$1" != "." ]; then - echo >&2 "$0 requires . as final argument" - exit 1 -fi - +pip_install_flags="--ignore-installed --verbose --no-deps --no-index --isolated" # Note: We need to take care to specify the full path to Sage's Python here # to emphasize that this command hould use it, and not the system Python; @@ -48,50 +30,14 @@ else PIP=pip2 fi - -# Find out the name of the package that we are installing -name="$($PYTHON setup.py --name)" - -if [ $? -ne 0 ]; then - echo >&2 "Error: could not determine package name" - exit 1 -fi - -if [ $(echo "$name" | wc -l) -gt 1 ]; then - name="$(echo "$name" | tail -1)" - echo >&2 "Warning: This package has a badly-behaved setup.py which outputs" - echo >&2 "more than the package name for 'setup.py --name'; using the last" - echo >&2 "line as the package name: $name" -fi - - # We should avoid running pip2/3 while uninstalling a package because that # is prone to race conditions. Therefore, we use a lockfile while # running pip. This is implemented in the Python script sage-flock LOCK="$SAGE_LOCAL/var/lock/$PIP.lock" -# Keep uninstalling as long as it succeeds -while true; do - out=$(sage-flock -x $LOCK $PYTHON -m pip uninstall --disable-pip-version-check -y "$name" 2>&1) - if [ $? -ne 0 ]; then - # Uninstall failed - echo >&2 "$out" - exit 1 - fi - - # Uninstall succeeded, which may mean that the package was not - # installed to begin with. - if [[ "$out" != *"not installed" ]]; then - break - fi -done - - # Finally actually do the installation (the "SHARED" tells pip2/3-lock # to apply a shared lock) -echo "Installing package $name using $PIP" - -sage-flock -s $LOCK $PYTHON -m pip install $pip_install_flags . +sage-flock -s $LOCK $PYTHON -m pip install $pip_install_flags "$@" if [ $? -ne 0 ]; then echo >&2 "Error: installing with $PIP failed" exit 3 diff --git a/build/bin/sage-pip-uninstall b/build/bin/sage-pip-uninstall new file mode 100755 index 00000000000..e46d07fe8b0 --- /dev/null +++ b/build/bin/sage-pip-uninstall @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# This command ensures that any previous installations of the same package +# are uninstalled. + +# Only argument must be "." and will be ignored. +if [ $# -gt 1 ]; then + echo >&2 "$0 requires . as only argument" + exit 1 +fi +if [ "$1" != "." ]; then + echo >&2 "$0 requires . as final argument" + exit 1 +fi + +# Note: We need to take care to specify the full path to Sage's Python here +# to emphasize that this command hould use it, and not the system Python; +# see https://trac.sagemath.org/ticket/18438 +# But now we delegate this to sage-python23. +PYTHON=sage-python23 + +# The PIP variable is only used to determine the name of the lock file. +if [ "$SAGE_PYTHON3" = yes ]; then + PIP=pip3 +else + PIP=pip2 +fi + +# Find out the name of the package that we are installing +name="$($PYTHON setup.py --name)" + +if [ $? -ne 0 ]; then + echo >&2 "Error: could not determine package name" + exit 1 +fi + +if [ $(echo "$name" | wc -l) -gt 1 ]; then + name="$(echo "$name" | tail -1)" + echo >&2 "Warning: This package has a badly-behaved setup.py which outputs" + echo >&2 "more than the package name for 'setup.py --name'; using the last" + echo >&2 "line as the package name: $name" +fi + +# We should avoid running pip2/3 while uninstalling a package because that +# is prone to race conditions. Therefore, we use a lockfile while +# running pip. This is implemented in the Python script sage-flock +LOCK="$SAGE_LOCAL/var/lock/$PIP.lock" + +# Keep uninstalling as long as it succeeds +while true; do + out=$(sage-flock -x $LOCK $PYTHON -m pip uninstall --disable-pip-version-check -y "$name" 2>&1) + if [ $? -ne 0 ]; then + # Uninstall failed + echo >&2 "$out" + exit 1 + fi + + # Uninstall succeeded, which may mean that the package was not + # installed to begin with. + if [[ "$out" != *"not installed" ]]; then + break + fi +done diff --git a/build/make/install b/build/make/install index 0d18185295f..28737726b1b 100755 --- a/build/make/install +++ b/build/make/install @@ -20,6 +20,7 @@ export SAGE_PKGCONFIG="$SAGE_LOCAL/lib/pkgconfig" export SAGE_LOGS="$SAGE_ROOT/logs/pkgs" export SAGE_SPKG_INST="$SAGE_LOCAL/var/lib/sage/installed" export SAGE_SPKG_SCRIPTS="$SAGE_LOCAL/var/lib/sage/scripts" +export SAGE_SPKG_WHEELS="$SAGE_LOCAL/var/lib/sage/wheels" if [ -z "${SAGE_ORIG_PATH_SET}" ]; then SAGE_ORIG_PATH=$PATH && export SAGE_ORIG_PATH From d369aabcda2feff14b4f1d14fe824e591d8c5c36 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 8 Sep 2020 12:38:49 -0700 Subject: [PATCH 02/11] build/bin/sage-dist-helpers (sdh_store_and_pip_install_wheel): New, factored out from sdh_pip_install --- build/bin/sage-dist-helpers | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/build/bin/sage-dist-helpers b/build/bin/sage-dist-helpers index 603c171e070..918070ec08a 100644 --- a/build/bin/sage-dist-helpers +++ b/build/bin/sage-dist-helpers @@ -1,4 +1,4 @@ -# Shell functions for making spkg-install scripts a little easier to write, +# -*- shell-script -*- functions for making spkg-install scripts a little easier to write, # eliminating duplication. All Sage helper functions begin with sdh_ (for # Sage-distribution helper). Consult the below documentation for the list of # available helper functions. @@ -205,19 +205,34 @@ sdh_pip_install() { echo "Installing $PKG_NAME" if [ -n "$SAGE_DESTDIR" ]; then local sudo="" - local root="--root=$SAGE_DESTDIR" else local sudo="$SAGE_SUDO" - local root="" fi $sudo sage-pip-uninstall "$@" || \ sdh_die "Error uninstalling a previous version of $PKG_NAME" - sage-python23 -m pip wheel --wheel-dir=. --no-binary :all: --verbose --no-deps --no-index --isolated --no-build-isolation "$@" || \ + mkdir -p dist + rm -f dist/*.whl + sage-python23 -m pip wheel --wheel-dir=dist --no-binary :all: --verbose --no-deps --no-index --isolated --no-build-isolation "$@" || \ sdh_die "Error building a wheel for $PKG_NAME" + sdh_store_and_pip_install_wheel +} + +sdh_store_and_pip_install_wheel() { + if [ -n "$SAGE_DESTDIR" ]; then + local sudo="" + # --no-warn-script-location: Suppress a warning caused by --root + local root="--root=$SAGE_DESTDIR --no-warn-script-location" + else + local sudo="$SAGE_SUDO" + local root="" + fi + if [ "$*" != "." ]; then + sdh_die "Error: sdh_store_and_pip_install_wheel requires . as only argument" + fi wheel="" - for w in *.whl; do + for w in dist/*.whl; do if [ -n "$wheel" ]; then sdh_die "Error: more than one wheel found after building" fi From 2d435ab90d74d86264d681371034700a8214e6d3 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 8 Sep 2020 12:39:16 -0700 Subject: [PATCH 03/11] build/pkgs/numpy/spkg-install.in: Install via setup.py bdist_wheel --- build/pkgs/numpy/spkg-install.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build/pkgs/numpy/spkg-install.in b/build/pkgs/numpy/spkg-install.in index 8080a3115e0..c4343132954 100644 --- a/build/pkgs/numpy/spkg-install.in +++ b/build/pkgs/numpy/spkg-install.in @@ -23,7 +23,7 @@ export NUMPY_FCONFIG="config_fc --noopt --noarch" sage-python23 setup.py \ --no-user-cfg \ - install \ - --single-version-externally-managed \ - --root "$SAGE_DESTDIR" \ - ${NUMPY_FCONFIG} || sdh_die "Error building / installing numpy" + bdist_wheel \ + ${NUMPY_FCONFIG} || sdh_die "Error building wheel for numpy" + +sdh_store_and_pip_install_wheel . From 55993b6e25560af5e582aed7fcfbb56d317b92fe Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 8 Sep 2020 12:49:07 -0700 Subject: [PATCH 04/11] build/bin/sage-dist-helpers: Fixup --- build/bin/sage-dist-helpers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/bin/sage-dist-helpers b/build/bin/sage-dist-helpers index 918070ec08a..3e1ce59ca39 100644 --- a/build/bin/sage-dist-helpers +++ b/build/bin/sage-dist-helpers @@ -216,7 +216,7 @@ sdh_pip_install() { sage-python23 -m pip wheel --wheel-dir=dist --no-binary :all: --verbose --no-deps --no-index --isolated --no-build-isolation "$@" || \ sdh_die "Error building a wheel for $PKG_NAME" - sdh_store_and_pip_install_wheel + sdh_store_and_pip_install_wheel . } sdh_store_and_pip_install_wheel() { From 0a646747dbf7c823030e1b23419348ff795a0661 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 8 Sep 2020 12:49:41 -0700 Subject: [PATCH 05/11] build/pkgs/gambit/spkg-install.in: Install via bdist_wheel --- build/pkgs/gambit/spkg-install.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/pkgs/gambit/spkg-install.in b/build/pkgs/gambit/spkg-install.in index b09d901dbd1..9e2f7964129 100644 --- a/build/pkgs/gambit/spkg-install.in +++ b/build/pkgs/gambit/spkg-install.in @@ -11,8 +11,9 @@ cd src/python rm gambit/lib/libgambit.cpp # pip doesn't work (https://github.com/gambitproject/gambit/issues/207) -sage-python23 setup.py --no-user-cfg build install +sage-python23 setup.py --no-user-cfg bdist_wheel if [ $? -ne 0 ]; then echo "Error installing Python API" exit 1 fi +sdh_store_and_pip_install_wheel . From ca586938cbdd91114a10dd3364201c8076412485 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 8 Sep 2020 12:49:59 -0700 Subject: [PATCH 06/11] build/pkgs/pillow/spkg-install.in: Install via bdist_wheel --- build/pkgs/pillow/spkg-install.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/pkgs/pillow/spkg-install.in b/build/pkgs/pillow/spkg-install.in index 18bc8c730df..97fb3066cc6 100644 --- a/build/pkgs/pillow/spkg-install.in +++ b/build/pkgs/pillow/spkg-install.in @@ -19,6 +19,6 @@ sage-python23 setup.py \ build_ext \ --debug \ --disable-jpeg \ - install \ - --single-version-externally-managed \ - --root "$SAGE_DESTDIR" || sdh_die "Error building/installing Pillow" + bdist_wheel || sdh_die "Error building/installing Pillow" + +sdh_store_and_pip_install_wheel . From 5a747c48eca1be0104c202e4922e5386aa794d08 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 9 Sep 2020 10:26:07 -0700 Subject: [PATCH 07/11] build/bin/sage-pip-{install,uninstall}: Remove pip2 support --- build/bin/sage-pip-install | 8 ++------ build/bin/sage-pip-uninstall | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/build/bin/sage-pip-install b/build/bin/sage-pip-install index 5c5279a4c40..adaa4e1f2d8 100755 --- a/build/bin/sage-pip-install +++ b/build/bin/sage-pip-install @@ -24,13 +24,9 @@ pip_install_flags="--ignore-installed --verbose --no-deps --no-index --isolated" PYTHON=sage-python23 # The PIP variable is only used to determine the name of the lock file. -if [ "$SAGE_PYTHON3" = yes ]; then - PIP=pip3 -else - PIP=pip2 -fi +PIP=pip3 -# We should avoid running pip2/3 while uninstalling a package because that +# We should avoid running pip while installing a package because that # is prone to race conditions. Therefore, we use a lockfile while # running pip. This is implemented in the Python script sage-flock LOCK="$SAGE_LOCAL/var/lock/$PIP.lock" diff --git a/build/bin/sage-pip-uninstall b/build/bin/sage-pip-uninstall index e46d07fe8b0..95c699ae87e 100755 --- a/build/bin/sage-pip-uninstall +++ b/build/bin/sage-pip-uninstall @@ -19,11 +19,7 @@ fi PYTHON=sage-python23 # The PIP variable is only used to determine the name of the lock file. -if [ "$SAGE_PYTHON3" = yes ]; then - PIP=pip3 -else - PIP=pip2 -fi +PIP=pip3 # Find out the name of the package that we are installing name="$($PYTHON setup.py --name)" @@ -40,7 +36,7 @@ if [ $(echo "$name" | wc -l) -gt 1 ]; then echo >&2 "line as the package name: $name" fi -# We should avoid running pip2/3 while uninstalling a package because that +# We should avoid running pip while uninstalling a package because that # is prone to race conditions. Therefore, we use a lockfile while # running pip. This is implemented in the Python script sage-flock LOCK="$SAGE_LOCAL/var/lock/$PIP.lock" From 9ee21109dfb001feec2b032c197defb91cc75783 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 16 Sep 2020 18:04:20 -0700 Subject: [PATCH 08/11] build/bin/sage-dist-helpers: Also use $sudo for storing the wheel file --- build/bin/sage-dist-helpers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/bin/sage-dist-helpers b/build/bin/sage-dist-helpers index 3e1ce59ca39..70815c16da1 100644 --- a/build/bin/sage-dist-helpers +++ b/build/bin/sage-dist-helpers @@ -247,7 +247,7 @@ sdh_store_and_pip_install_wheel() { $sudo sage-pip-install $root "$wheel" || \ sdh_die "Error installing $wheel" mkdir -p "${SAGE_DESTDIR}${SAGE_SPKG_WHEELS}" && \ - mv "$wheel" "${SAGE_DESTDIR}${SAGE_SPKG_WHEELS}/" || \ + $sudo mv "$wheel" "${SAGE_DESTDIR}${SAGE_SPKG_WHEELS}/" || \ sdh_die "Error storing $wheel" } From d7aac84ab4dd86ad6203a99eb82371c1d2a46363 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 16 Sep 2020 18:04:58 -0700 Subject: [PATCH 09/11] src/doc/en/developer/packaging.rst: Update sdh_... documentation --- src/doc/en/developer/packaging.rst | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/doc/en/developer/packaging.rst b/src/doc/en/developer/packaging.rst index c718c1d3644..a909cc16f22 100644 --- a/src/doc/en/developer/packaging.rst +++ b/src/doc/en/developer/packaging.rst @@ -368,12 +368,26 @@ begin with ``sdh_``, which stands for "Sage-distribution helper". arguments. If ``$SAGE_DESTDIR`` is not set then the command is run with ``$SAGE_SUDO``, if set. -- ``sdh_pip_install [...]``: Runs ``pip install`` with the given - arguments, as well as additional default arguments used for - installing packages into Sage with pip. Currently this is just a - wrapper around the ``sage-pip-install`` command. If - ``$SAGE_DESTDIR`` is not set then the command is run with - ``$SAGE_SUDO``, if set. +- ``sdh_pip_install [...]``: The equivalent of running ``pip install`` + with the given arguments, as well as additional default arguments used for + installing packages into Sage with pip. The last argument must be + ``.`` to indicate installation from the current directory. + + ``sdh_pip_install`` actually does the installation via ``pip wheel``, + creating a wheel file in ``dist/``, followed by + ``sdh_store_and_pip_install_wheel`` (see below). + +- ``sdh_store_and_pip_install_wheel .``: The current directory, + indicated by the required argument ``.``, must have a subdirectory + ``dist`` containing a unique wheel file (``*.whl``). + + This command (1) moves this wheel file to the + directory ``$SAGE_SPKG_WHEELS`` (``$SAGE_LOCAL/var/lib/sage/wheels``) + and then (2) installs the wheel in ``$SAGE_LOCAL``. + + Both of these steps, instead of writing directly into ``$SAGE_LOCAL``, + use the staging directory ``$SAGE_DESTDIR`` if set; otherwise, they + use ``$SAGE_SUDO`` (if set). - ``sdh_install [-T] SRC [SRC...] DEST``: Copies one or more files or directories given as ``SRC`` (recursively in the case of From 9b7c7a04d658838ce511e352edb9c03f42c4a7f2 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 16 Sep 2020 18:07:43 -0700 Subject: [PATCH 10/11] build/bin/sage-pip-{install,uninstall}: Fix typo in comment --- build/bin/sage-pip-install | 2 +- build/bin/sage-pip-uninstall | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/bin/sage-pip-install b/build/bin/sage-pip-install index adaa4e1f2d8..ffae57342f0 100755 --- a/build/bin/sage-pip-install +++ b/build/bin/sage-pip-install @@ -18,7 +18,7 @@ pip_install_flags="--ignore-installed --verbose --no-deps --no-index --isolated" # Note: We need to take care to specify the full path to Sage's Python here -# to emphasize that this command hould use it, and not the system Python; +# to emphasize that this command should use it, and not the system Python; # see https://trac.sagemath.org/ticket/18438 # But now we delegate this to sage-python23. PYTHON=sage-python23 diff --git a/build/bin/sage-pip-uninstall b/build/bin/sage-pip-uninstall index 95c699ae87e..3326880cf78 100755 --- a/build/bin/sage-pip-uninstall +++ b/build/bin/sage-pip-uninstall @@ -13,7 +13,7 @@ if [ "$1" != "." ]; then fi # Note: We need to take care to specify the full path to Sage's Python here -# to emphasize that this command hould use it, and not the system Python; +# to emphasize that this command should use it, and not the system Python; # see https://trac.sagemath.org/ticket/18438 # But now we delegate this to sage-python23. PYTHON=sage-python23 From 4135e8b236d2e5491b3ee96cbd00196dc5b6e036 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 16 Sep 2020 18:08:32 -0700 Subject: [PATCH 11/11] build/bin/sage-pip-install: Remove an outdated comment --- build/bin/sage-pip-install | 3 --- 1 file changed, 3 deletions(-) diff --git a/build/bin/sage-pip-install b/build/bin/sage-pip-install index ffae57342f0..077d8e0f58c 100755 --- a/build/bin/sage-pip-install +++ b/build/bin/sage-pip-install @@ -1,9 +1,6 @@ #!/usr/bin/env bash # This command is specifically for pip-installing a previously # built wheel. -# -# This ensures that any previous installations of the same package -# are uninstalled first. # Default arguments for all packages installed with `pip install` # --ignore-installed : Force pip to re-install package even if it thinks it's