From 1963257eb463dbd604e95fa69f8123089d990feb Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 27 Dec 2023 14:17:05 -0800 Subject: [PATCH 01/10] sage --package metrics: New --- build/sage_bootstrap/app.py | 62 +++++++++++++++++++++++++++++++++ build/sage_bootstrap/cmdline.py | 33 ++++++++++++++++++ build/sage_bootstrap/package.py | 21 +++++++++++ 3 files changed, 116 insertions(+) diff --git a/build/sage_bootstrap/app.py b/build/sage_bootstrap/app.py index 11600bf1b53..19ca74b6a68 100644 --- a/build/sage_bootstrap/app.py +++ b/build/sage_bootstrap/app.py @@ -24,6 +24,8 @@ import logging log = logging.getLogger() +from collections import defaultdict + from sage_bootstrap.package import Package from sage_bootstrap.tarball import Tarball, FileNotMirroredError from sage_bootstrap.updater import ChecksumUpdater, PackageUpdater @@ -366,3 +368,63 @@ def clean(self): os.remove(filepath) count += 1 print('{} files were removed from the {} directory'.format(count, SAGE_DISTFILES)) + + def metrics_cls(self, *package_classes): + """ + Show the metrics of given packages + + $ sage --package metrics :standard: + has_file_distros_arch_txt=131 + has_file_distros_conda_txt=216 + has_file_distros_debian_txt=125 + has_file_distros_fedora_txt=138 + has_file_distros_gentoo_txt=181 + has_file_distros_homebrew_txt=61 + has_file_distros_macports_txt=129 + has_file_distros_nix_txt=51 + has_file_distros_opensuse_txt=146 + has_file_distros_slackware_txt=25 + has_file_distros_void_txt=184 + has_file_patches=35 + has_file_spkg_check=59 + has_file_spkg_configure_m4=222 + has_file_spkg_install=198 + has_tarball_upstream_url=231 + line_count_file_patches=22561 + line_count_file_spkg_check=402 + line_count_file_spkg_configure_m4=2792 + line_count_file_spkg_install=2960 + packages=272 + type_standard=272 + """ + log.debug('Computing metrics') + metrics = defaultdict(int) + pc = PackageClass(*package_classes) + for package_name in pc.names: + package = Package(package_name) + metrics['packages'] += 1 + metrics['type_' + package.type] += 1 + for filenames in [['spkg-configure.m4'], + ['spkg-install', 'spkg-install.in'], + ['spkg-check', 'spkg-check.in'], + ['distros/arch.txt'], + ['distros/conda.txt'], + ['distros/debian.txt'], + ['distros/fedora.txt'], + ['distros/gentoo.txt'], + ['distros/homebrew.txt'], + ['distros/macports.txt'], + ['distros/nix.txt'], + ['distros/opensuse.txt'], + ['distros/slackware.txt'], + ['distros/void.txt'], + ['patches']]: + key = filenames[0].replace('.', '_').replace('-', '_').replace('/', '_') + metrics['has_file_' + key] += int(any(package.has_file(filename) + for filename in filenames)) + if not key.startswith('distros_'): + metrics['line_count_file_' + key] += sum(package.line_count_file(filename) + for filename in filenames) + metrics['has_tarball_upstream_url'] += int(bool(package.tarball_upstream_url)) + for key, value in sorted(metrics.items()): + print('{0}={1}'.format(key, value)) diff --git a/build/sage_bootstrap/cmdline.py b/build/sage_bootstrap/cmdline.py index e182321d7da..4f9960acfba 100644 --- a/build/sage_bootstrap/cmdline.py +++ b/build/sage_bootstrap/cmdline.py @@ -189,6 +189,27 @@ 42 files were removed from the .../upstream directory """ +epilog_metrics = \ +""" +Print metrics of given package. + +EXAMPLE: + + $ sage --package metrics :standard: + has_file_distros_arch_txt=131 + ... + has_file_distros_void_txt=184 + has_file_patches=35 + has_file_spkg_check=59 + has_file_spkg_configure_m4=222 + has_file_spkg_install=198 + has_tarball_upstream_url=231 + line_count_file_patches=22561 + ... + packages=272 + type_standard=272 +""" + def make_parser(): """ @@ -346,6 +367,16 @@ def make_parser(): formatter_class=argparse.RawDescriptionHelpFormatter, help='Remove outdated source tarballs from the upstream/ directory') + parser_metrics = subparsers.add_parser( + 'metrics', epilog=epilog_metrics, + formatter_class=argparse.RawDescriptionHelpFormatter, + help='Print metrics of given packages') + parser_metrics.add_argument( + 'package_class', metavar='[package_name|:package_type:]', + type=str, nargs='*', default=[':all:'], + help=('package name or designator for all packages of a given type ' + '(one of :all:, :standard:, :optional:, and :experimental:; default: :all:)')) + return parser @@ -403,6 +434,8 @@ def run(): app.fix_checksum_cls(*args.package_class) elif args.subcommand == 'clean': app.clean() + elif args.subcommand == 'metrics': + app.metrics_cls(*args.package_class) else: raise RuntimeError('unknown subcommand: {0}'.format(args)) diff --git a/build/sage_bootstrap/package.py b/build/sage_bootstrap/package.py index 43bfbffa47c..b4ec2d802a8 100644 --- a/build/sage_bootstrap/package.py +++ b/build/sage_bootstrap/package.py @@ -370,6 +370,27 @@ def has_file(self, filename): """ return os.path.exists(os.path.join(self.path, filename)) + def line_count_file(self, filename): + """ + Return the number of lines of the file + + Directories are traversed recursively. + + OUTPUT: + + integer; 0 if the file cannot be read, 1 if it is a symlink + """ + filename = os.path.join(self.path, filename) + if os.path.islink(filename): + return 1 + if os.path.isdir(filename): + return sum(self.line_count_file(os.path.join(filename, entry)) + for entry in os.listdir(filename)) + try: + return len(list(open(filename, "rb"))) + except OSError: + return 0 + def _init_checksum(self): """ Load the checksums from the appropriate ``checksums.ini`` file From 6e0da182e800713949e1d8e5540fbd96468b398a Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 27 Dec 2023 15:28:41 -0800 Subject: [PATCH 02/10] .github/workflows/ci-linux-incremental.yml: Compute metrics --- .github/workflows/ci-linux-incremental.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/ci-linux-incremental.yml b/.github/workflows/ci-linux-incremental.yml index d3b7801c548..2a09032ec4c 100644 --- a/.github/workflows/ci-linux-incremental.yml +++ b/.github/workflows/ci-linux-incremental.yml @@ -63,6 +63,22 @@ jobs: echo "uninstall_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([a-z0-9][_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))" >> $GITHUB_OUTPUT echo "build_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.pkgs_all_changed_files }}; do SPKG=$(echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([a-z0-9][-_.a-z0-9]*)/[^ ]* *,\2,;'); if [ -f "build/pkgs/$SPKG/checksums.ini" -o -f "build/pkgs/$SPKG/requirements.txt" -o -f "build/pkgs/$SPKG/spkg-install" ]; then echo "$SPKG-ensure"; fi; done | sort -u))" >> $GITHUB_OUTPUT cat $GITHUB_OUTPUT + - uses: actions/checkout@v4 + with: + ref: ${{ github.base_ref }} + path: worktree-base + if: github.base_ref + - name: Compute metrics + run: | + export PATH=build/bin:$PATH + if [ -d worktree-base ]; then + SAGE_ROOT=worktree-base sage-package metrics :all: > base-metrics.txt + sage-package metrics :all: > metrics.txt + diff --side-by-side base-metrics.txt metrics.txt + echo "Base: $GITHUB_BASE_REF" + else + sage-package metrics :all: + fi test: needs: [changed_files] From 22483932bcc493eaf16a4c5911e254a86a703607 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 27 Dec 2023 15:34:59 -0800 Subject: [PATCH 03/10] build/pkgs/tzdata: Update to 2023.3, change to wheel package --- build/pkgs/tzdata/checksums.ini | 10 +++++----- build/pkgs/tzdata/package-version.txt | 2 +- build/pkgs/tzdata/spkg-install.in | 2 -- 3 files changed, 6 insertions(+), 8 deletions(-) delete mode 100644 build/pkgs/tzdata/spkg-install.in diff --git a/build/pkgs/tzdata/checksums.ini b/build/pkgs/tzdata/checksums.ini index 86bf2db8de4..342e71f3761 100644 --- a/build/pkgs/tzdata/checksums.ini +++ b/build/pkgs/tzdata/checksums.ini @@ -1,5 +1,5 @@ -tarball=tzdata-VERSION.tar.gz -sha1=e244bf1bde63515d3f8a452d3bbe9f97738e1e70 -md5=2ad4652fecc6ef4f6794726d3f367363 -cksum=2894139369 -upstream_url=https://pypi.io/packages/source/t/tzdata/tzdata-VERSION.tar.gz +tarball=tzdata-VERSION-py2.py3-none-any.whl +sha1=4686c7c91a01d5af9075903937c343afa05c141b +md5=5e534124c0c916ab617421247649b193 +cksum=2929397850 +upstream_url=https://pypi.io/packages/py2.py3/t/tzdata/tzdata-VERSION-py2.py3-none-any.whl diff --git a/build/pkgs/tzdata/package-version.txt b/build/pkgs/tzdata/package-version.txt index abd454cf321..b7547c45d8d 100644 --- a/build/pkgs/tzdata/package-version.txt +++ b/build/pkgs/tzdata/package-version.txt @@ -1 +1 @@ -2022.6 +2023.3 diff --git a/build/pkgs/tzdata/spkg-install.in b/build/pkgs/tzdata/spkg-install.in deleted file mode 100644 index 37ac1a53437..00000000000 --- a/build/pkgs/tzdata/spkg-install.in +++ /dev/null @@ -1,2 +0,0 @@ -cd src -sdh_pip_install . From 134ac92e15a98d69271d8a276e3bc2f3daed67e3 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 27 Dec 2023 15:43:09 -0800 Subject: [PATCH 04/10] .github/workflows/ci-linux-incremental.yml: Compute metrics (fixup) --- .github/workflows/ci-linux-incremental.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-linux-incremental.yml b/.github/workflows/ci-linux-incremental.yml index 2a09032ec4c..158079c230c 100644 --- a/.github/workflows/ci-linux-incremental.yml +++ b/.github/workflows/ci-linux-incremental.yml @@ -72,10 +72,9 @@ jobs: run: | export PATH=build/bin:$PATH if [ -d worktree-base ]; then - SAGE_ROOT=worktree-base sage-package metrics :all: > base-metrics.txt - sage-package metrics :all: > metrics.txt - diff --side-by-side base-metrics.txt metrics.txt - echo "Base: $GITHUB_BASE_REF" + (echo "# $GITHUB_BASE_REF"; SAGE_ROOT=worktree-base sage-package metrics :all:) > base-metrics.txt + (echo "# $GITHUB_REF"; sage-package metrics :all:) > metrics.txt + diff --color --side-by-side base-metrics.txt metrics.txt || true else sage-package metrics :all: fi From d68de0285f6bf54c67e5b1833de6d14139dabc64 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 27 Dec 2023 15:52:37 -0800 Subject: [PATCH 05/10] build/sage_bootstrap/package.py: Fixups --- build/sage_bootstrap/package.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/build/sage_bootstrap/package.py b/build/sage_bootstrap/package.py index b4ec2d802a8..9363138a457 100644 --- a/build/sage_bootstrap/package.py +++ b/build/sage_bootstrap/package.py @@ -380,14 +380,15 @@ def line_count_file(self, filename): integer; 0 if the file cannot be read, 1 if it is a symlink """ - filename = os.path.join(self.path, filename) - if os.path.islink(filename): + path = os.path.join(self.path, filename) + if os.path.islink(path): return 1 - if os.path.isdir(filename): + if os.path.isdir(path): return sum(self.line_count_file(os.path.join(filename, entry)) - for entry in os.listdir(filename)) + for entry in os.listdir(path)) try: - return len(list(open(filename, "rb"))) + with open(path, "rb") as f: + return len(list(f)) except OSError: return 0 From 92c33351f7250478928a037c7d239db10d3edb4a Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 27 Dec 2023 15:58:42 -0800 Subject: [PATCH 06/10] .github/workflows/ci-linux-incremental.yml: Metrics cosmetics --- .github/workflows/ci-linux-incremental.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-linux-incremental.yml b/.github/workflows/ci-linux-incremental.yml index 158079c230c..f81ffc9b378 100644 --- a/.github/workflows/ci-linux-incremental.yml +++ b/.github/workflows/ci-linux-incremental.yml @@ -74,7 +74,7 @@ jobs: if [ -d worktree-base ]; then (echo "# $GITHUB_BASE_REF"; SAGE_ROOT=worktree-base sage-package metrics :all:) > base-metrics.txt (echo "# $GITHUB_REF"; sage-package metrics :all:) > metrics.txt - diff --color --side-by-side base-metrics.txt metrics.txt || true + diff --color=always --width=100 --side-by-side --left-column base-metrics.txt metrics.txt || true else sage-package metrics :all: fi From a817b34cd60d0149df6956ac3913f7ecb17f025a Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 6 Jan 2024 10:19:27 -0800 Subject: [PATCH 07/10] src/doc/en/developer/packaging.rst: Remove outdated example --- src/doc/en/developer/packaging.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/doc/en/developer/packaging.rst b/src/doc/en/developer/packaging.rst index 96eb302e2d3..1eaedf3e40f 100644 --- a/src/doc/en/developer/packaging.rst +++ b/src/doc/en/developer/packaging.rst @@ -1198,8 +1198,6 @@ to a new version, check whether the license changed between versions. If an upstream tarball of a package cannot be redistributed for license reasons, rename it to include the string ``do-not-distribute``. This will keep the release management scripts from uploading it to the Sage mirrors. -For an example, see the ``scipoptsuite`` package, which has an "academic" -proprietary license. Sometimes an upstream tarball contains some distributable parts using a free software license and some non-free parts. In this case, it can From c22ebf108584698828f322a417122b50a2bc572b Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 6 Jan 2024 11:52:50 -0800 Subject: [PATCH 08/10] src/doc/en/developer/packaging.rst: Fix markup --- src/doc/en/developer/packaging.rst | 70 +++++++++++++++--------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/src/doc/en/developer/packaging.rst b/src/doc/en/developer/packaging.rst index 1eaedf3e40f..76b30b0e83d 100644 --- a/src/doc/en/developer/packaging.rst +++ b/src/doc/en/developer/packaging.rst @@ -322,23 +322,23 @@ Likewise for :envvar:`CXXFLAGS`, :envvar:`FCFLAGS`, and :envvar:`F77FLAGS`. exec python3 spkg-install.py - In more detail: ``sage-bootstrap-python`` runs a version of Python - pre-installed on the machine, which is a build prerequisite of Sage. - Note that ``sage-bootstrap-python`` accepts a wide range of Python - versions, Python >= 2.6 and >= 3.4, see ``SAGE_ROOT/build/tox.ini`` - for details. You should only use ``sage-bootstrap-python`` for - installation tasks that must be able to run before Sage has made - ``python3`` available. It must not be used for running ``pip`` or - ``setup.py`` for any package. - - ``python3`` runs the version of Python managed by Sage (either its - own installation of Python 3 from an SPKG or a venv over a system - python3. You should use this if you are installing a Python package - to make sure that the libraries are installed in the right place. - - By the way, there is also a script ``sage-python``. This should be - used at runtime, for example in scripts in ``SAGE_LOCAL/bin`` which - expect Sage's Python to already be built. + In more detail: ``sage-bootstrap-python`` runs a version of Python + pre-installed on the machine, which is a build prerequisite of Sage. + Note that ``sage-bootstrap-python`` accepts a wide range of Python + versions, Python >= 2.6 and >= 3.4, see ``SAGE_ROOT/build/tox.ini`` + for details. You should only use ``sage-bootstrap-python`` for + installation tasks that must be able to run before Sage has made + ``python3`` available. It must not be used for running ``pip`` or + ``setup.py`` for any package. + + ``python3`` runs the version of Python managed by Sage (either its + own installation of Python 3 from an SPKG or a venv over a system + python3. You should use this if you are installing a Python package + to make sure that the libraries are installed in the right place. + + By the way, there is also a script ``sage-python``. This should be + used at runtime, for example in scripts in ``SAGE_LOCAL/bin`` which + expect Sage's Python to already be built. Many packages currently do not separate the build and install steps and only provide a ``spkg-install.in`` file that does both. The separation is useful in @@ -412,19 +412,19 @@ begin with ``sdh_``, which stands for "Sage-distribution helper". may be given as arguments. - ``sdh_make [...]``: Runs ``$MAKE`` with the default target. - Additional arguments to ``$MAKE`` may be given as arguments. + Additional arguments to ``$MAKE`` may be given as arguments. - ``sdh_make_install [...]``: Runs ``$MAKE install`` with DESTDIR - correctly set to a temporary install directory, for staged - installations. Additional arguments to ``$MAKE`` may be given as - arguments. If ``$SAGE_DESTDIR`` is not set then the command is run - with ``$SAGE_SUDO``, if set. + correctly set to a temporary install directory, for staged + installations. Additional arguments to ``$MAKE`` may be given as + arguments. If ``$SAGE_DESTDIR`` is not set then the command is run + with ``$SAGE_SUDO``, if set. - ``sdh_setup_bdist_wheel [...]``: Runs ``setup.py bdist_wheel`` with - the given arguments, as well as additional default arguments used for - installing packages into Sage. + the given arguments, as well as additional default arguments used for + installing packages into Sage. -- ``sdh_pip_install [...]``: The equivalent of running ``pip install`` +- ``sdh_pip_install [...]``: The equivalent of running ``pip install`` with the given arguments, as well as additional default arguments used for installing packages into Sage with pip. The last argument must be ``.`` to indicate installation from the current directory. @@ -433,17 +433,17 @@ begin with ``sdh_``, which stands for "Sage-distribution helper". creating a wheel file in ``dist/``, followed by ``sdh_store_and_pip_install_wheel`` (see below). -- ``sdh_pip_editable_install [...]``: The equivalent of running ``pip install -e`` +- ``sdh_pip_editable_install [...]``: The equivalent of running ``pip install -e`` with the given arguments, as well as additional default arguments used for installing packages into Sage with pip. The last argument must be ``.`` to indicate installation from the current directory. See `pip documentation `_ for more details concerning editable installs. -- ``sdh_pip_uninstall [...]``: Runs ``pip uninstall`` with the given arguments. +- ``sdh_pip_uninstall [...]``: Runs ``pip uninstall`` with the given arguments. If unsuccessful, it displays a warning. -- ``sdh_store_and_pip_install_wheel .``: The current directory, +- ``sdh_store_and_pip_install_wheel .``: The current directory, indicated by the required argument ``.``, must have a subdirectory ``dist`` containing a unique wheel file (``*.whl``). @@ -455,7 +455,7 @@ begin with ``sdh_``, which stands for "Sage-distribution helper". use the staging directory ``$SAGE_DESTDIR`` if set; otherwise, they use ``$SAGE_SUDO`` (if set). -- ``sdh_install [-T] SRC [SRC...] DEST``: Copies one or more files or +- ``sdh_install [-T] SRC [SRC...] DEST``: Copies one or more files or directories given as ``SRC`` (recursively in the case of directories) into the destination directory ``DEST``, while ensuring that ``DEST`` and all its parent directories exist. @@ -470,20 +470,20 @@ begin with ``sdh_``, which stands for "Sage-distribution helper". The following is automatically added to each install script, so you should not need to add it yourself. -- ``sdh_guard``: Wrapper for ``sdh_check_vars`` that checks some +- ``sdh_guard``: Wrapper for ``sdh_check_vars`` that checks some common variables without which many/most packages won't build correctly (``SAGE_ROOT``, ``SAGE_LOCAL``, ``SAGE_SHARE``). This is important to prevent installation to unintended locations. The following are also available, but rarely used. -- ``sdh_cmake [...]``: Runs ``cmake`` in the current directory with +- ``sdh_cmake [...]``: Runs ``cmake`` in the current directory with the given arguments, as well as additional arguments passed to cmake (assuming packages are using the GNUInstallDirs module) so that ``CMAKE_INSTALL_PREFIX`` and ``CMAKE_INSTALL_LIBDIR`` are set correctly. -- ``sdh_preload_lib EXECUTABLE SONAME``: (Linux only -- no-op on other +- ``sdh_preload_lib EXECUTABLE SONAME``: (Linux only -- no-op on other platforms.) Check shared libraries loaded by ``EXECUTABLE`` (may be a program or another library) for a library starting with ``SONAME``, and if found appends ``SONAME`` to the ``LD_PRELOAD`` environment variable. @@ -555,7 +555,7 @@ script template to run self-tests of the package. The format for the ``spkg-check`` is the same as ``spkg-build`` and ``spkg-install``. It is run after building and installing if the ``SAGE_CHECK`` environment variable is set, see the Sage installation guide. Ideally, upstream -has some sort of tests suite that can be run with the standard ``make +has some sort of test suite that can be run with the standard ``make check`` target. In that case, the ``spkg-check.in`` script template would simply contain: @@ -629,7 +629,8 @@ The comments may include links to GitHub Issues/PRs, as in the following example $ cat build/pkgs/packaging/install-requires.txt packaging >=18.0 - # Issue #30975: packaging 20.5 is known to work but we have to silence "DeprecationWarning: Creating a LegacyVersion" + # Issue #30975: packaging 20.5 is known to work + # but we have to silence "DeprecationWarning: Creating a LegacyVersion" The currently encoded version constraints are merely a starting point. Developers and downstream packagers are invited to refine the version @@ -650,6 +651,7 @@ The ``SPKG.rst`` file should follow this pattern: .. CODE-BLOCK:: text PACKAGE_NAME: One line description + ================================== Description ----------- From 6e0d890f69a07b8ea418e2a35ff317b209854294 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 6 Jan 2024 11:53:23 -0800 Subject: [PATCH 09/10] src/doc/en/developer/packaging.rst: Consolidate section on 'sage --package', add subsection on '... metrics' --- src/doc/en/developer/packaging.rst | 174 ++++++++++++++++++++++------- 1 file changed, 136 insertions(+), 38 deletions(-) diff --git a/src/doc/en/developer/packaging.rst b/src/doc/en/developer/packaging.rst index 76b30b0e83d..22638bedc12 100644 --- a/src/doc/en/developer/packaging.rst +++ b/src/doc/en/developer/packaging.rst @@ -1035,13 +1035,10 @@ upstream is ``$SAGE_ROOT/upstream/FoO-1.3.tar.gz``, create a new file tarball=FoO-VERSION.tar.gz Sage internally replaces the ``VERSION`` substring with the content of -``package-version.txt``. To recompute the checksums, run:: +``package-version.txt``. - [alice@localhost sage]$ sage --package fix-checksum foo - -which will modify the ``checksums.ini`` file with the correct -checksums. +.. _section-spkg-upstream-urls: Upstream URLs ------------- @@ -1068,39 +1065,33 @@ For Python packages available from PyPI, you should use an upstream_url=https://pypi.io/packages/source/m/matplotlib/matplotlib-VERSION.tar.gz -A package that has the ``upstream_url`` information can be updated by -simply typing:: - - [alice@localhost sage]$ sage --package update numpy 3.14.59 +Developers who wish to test a package update from a PR branch before +the archive is available on a Sage mirror. Sage falls back to +downloading package tarballs from the ``upstream_url`` after trying all +Sage mirrors. (This can be disabled by using ``./configure +--disable-download-from-upstream-url``.) To speed up this process, +you can trim ``upstream/mirror_list`` to fewer mirrors. -which will automatically download the archive and update the -information in ``build/pkgs/``. -For Python packages available from PyPI, there is another shortcut:: +.. _section-sage-package-command: - [alice@localhost sage]$ sage --package update-latest matplotlib - Updating matplotlib: 3.3.0 -> 3.3.1 - Downloading tarball to ...matplotlib-3.3.1.tar.bz2 - [...............................................................] +Utility script to create and maintain packages +============================================== -The ``upstream_url`` information serves yet another purpose. -Developers who wish to test a package update from a PR branch before -the archive is available on a Sage mirror can do so by configuring -their Sage tree using ``./configure ---enable-download-from-upstream-url``. Then Sage will fall back to -downloading package tarballs from the ``upstream_url`` after trying all -Sage mirrors. (To speed up this process, trim ``upstream/mirror_list`` -to fewer mirrors.) -It is then no longer necessary to manually download upstream tarballs. +The command ``sage --package`` offers a range of functionality for +creating and maintaining packages of the Sage distribution. -Utility script to create packages -================================= +Creating packages +----------------- Assuming that you have downloaded ``$SAGE_ROOT/upstream/FoO-1.3.tar.gz``, you can use:: - [alice@localhost sage]$ sage --package create foo --version 1.3 --tarball FoO-VERSION.tar.gz --type experimental + [alice@localhost sage]$ sage --package create foo \ + --version 1.3 \ + --tarball FoO-VERSION.tar.gz \ + --type experimental to create ``$SAGE_ROOT/build/pkgs/foo/package-version.txt``, ``checksums.ini``, and ``type`` in one step. @@ -1111,22 +1102,132 @@ set the ``upstream_url`` field in ``checksums.ini`` described above. For Python packages available from PyPI, you can use:: - [alice@localhost sage]$ sage --package create scikit_spatial --pypi --type optional + [alice@localhost sage]$ sage --package create scikit_spatial --pypi \ + --type optional This automatically downloads the most recent version from PyPI and also obtains most of the necessary information by querying PyPI. + The ``dependencies`` file may need editing (watch out for warnings regarding ``--no-deps`` that Sage issues during installation of the package!). Also you may want to set lower and upper bounds for acceptable package versions in the file ``install-requires.txt``. -To create a pip package rather than a normal package, you can use:: +By default, when the package is available as a platform-independent +wheel, the ``sage --package`` creates a wheel package. To create a normal package +instead (for example, when the package requires patching), you can use:: + + [alice@localhost sage]$ sage --package create scikit_spatial --pypi \ + --source normal \ + --type optional + +To create a pip package rather than a normal or wheel package, you can use:: + + [alice@localhost sage]$ sage --package create scikit_spatial --pypi \ + --source pip \ + --type optional + +When the package already exists, ``sage --package create`` overwrites it. - [alice@localhost sage]$ sage --package create scikit_spatial --pypi --source pip --type optional -To create a wheel package rather than a normal package, you can use:: +Updating packages to a new version +---------------------------------- + +A package that has the ``upstream_url`` information can be updated by +simply typing:: + + [alice@localhost sage]$ sage --package update numpy 3.14.59 + +which will automatically download the archive and update the +information in ``build/pkgs/numpy/``. + +For Python packages available from PyPI, there is another shortcut:: + + [alice@localhost sage]$ sage --package update-latest matplotlib + Updating matplotlib: 3.3.0 -> 3.3.1 + Downloading tarball to ...matplotlib-3.3.1.tar.bz2 + [...............................................................] + +If you pass the switch ``--commit``, the script will run ``git commit`` +for you. + +If you prefer to make update a package ``foo`` by making manual +changes to the files in ``build/pkgs/foo``, you will need to run:: + + [alice@localhost sage]$ sage --package fix-checksum foo + +which will modify the ``checksums.ini`` file with the correct +checksums. + - [alice@localhost sage]$ sage --package create scikit_spatial --pypi --source wheel --type optional +Obtaining package metrics +------------------------- + +The command ``sage --package metrics`` computes machine-readable +aggregated metrics for all packages in the Sage distribution or a +given list of packages:: + + [alice@localhost sage]$ sage --package metrics + has_file_distros_arch_txt=181 + has_file_distros_conda_txt=289 + has_file_distros_debian_txt=172 + has_file_distros_fedora_txt=183 + has_file_distros_gentoo_txt=211 + has_file_distros_homebrew_txt=95 + has_file_distros_macports_txt=173 + has_file_distros_nix_txt=72 + has_file_distros_opensuse_txt=206 + has_file_distros_slackware_txt=32 + has_file_distros_void_txt=221 + has_file_patches=63 + has_file_spkg_check=106 + has_file_spkg_configure_m4=262 + has_file_spkg_install=322 + has_tarball_upstream_url=291 + line_count_file_patches=31904 + line_count_file_spkg_check=585 + line_count_file_spkg_configure_m4=3337 + line_count_file_spkg_install=4342 + packages=442 + type_base=1 + type_experimental=18 + type_optional=151 + type_standard=272 + +Developers can use these metrics to monitor the complexity and quality +of the Sage distribution. Here are some examples: + +- ``has_file_patches`` indicates how many packages have non-empty + ``patches/`` directories, and ``line_count_file_patches`` gives + the total number of lines in the patch files. + + Ideally, we would not have to carry patches for a + package. For example, updating patches when a new upstream version + is released can be a maintenance burden. + + Developers can help by working with the upstream maintainers of the + package to prepare a new version that requires fewer or smaller + patches, or none at all. + +- ``line_count_spkg_install`` gives the total number of lines in + ``spkg-install`` or ``spkg-install.in`` files; see + :ref:`section-spkg-install`. + + When we carry complex ``spkg-install.in`` scripts for normal packages, + it may indicate that the upstream package's build and installation + scripts should be improved. + + Developers can help by working with the upstream maintainers of the + package to prepare an improved version. + +- ``has_file_spkg_check`` indicates how many packages have an + ``spkg-check`` or ``spkg-check.in`` file; see :ref:`section-spkg-check`. + +- ``has_file_spkg_configure_m4`` indicates how many packages + are prepared to check for an equivalent system package, and + ``has_file_distros_arch_txt``, ``has_file_distros_conda_txt`` + etc. count how many packages provide the corresponding system package + information. .. _section-manual-build: @@ -1136,9 +1237,7 @@ Building the package At this stage you have a new tarball that is not yet distributed with Sage (``FoO-1.3.tar.gz`` in the example of section -:ref:`section-directory-structure`). Now you need to manually place it -in the ``SAGE_ROOT/upstream/`` directory and run -``sage --fix-pkg-checksums`` if you have not done that yet. +:ref:`section-directory-structure`). Now you can install the package using:: @@ -1157,8 +1256,7 @@ or:: [alice@localhost sage]$ sage -f -c package_name -If all went fine, open a PR, put a link to the original tarball in -the PR and upload a branch with the code under +If all went fine, open a PR with the code under ``SAGE_ROOT/build/pkgs``. From 6731a1cf73d37d542762c68849ba22e09aa5fd1c Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 6 Jan 2024 18:32:14 -0800 Subject: [PATCH 10/10] src/doc/en/developer/packaging.rst: Unify indentation in Sphinx list --- src/doc/en/developer/packaging.rst | 114 ++++++++++++++--------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/src/doc/en/developer/packaging.rst b/src/doc/en/developer/packaging.rst index 22638bedc12..777d9203b46 100644 --- a/src/doc/en/developer/packaging.rst +++ b/src/doc/en/developer/packaging.rst @@ -424,70 +424,70 @@ begin with ``sdh_``, which stands for "Sage-distribution helper". the given arguments, as well as additional default arguments used for installing packages into Sage. -- ``sdh_pip_install [...]``: The equivalent of running ``pip install`` - with the given arguments, as well as additional default arguments used for - installing packages into Sage with pip. The last argument must be - ``.`` to indicate installation from the current directory. - - ``sdh_pip_install`` actually does the installation via ``pip wheel``, - creating a wheel file in ``dist/``, followed by - ``sdh_store_and_pip_install_wheel`` (see below). - -- ``sdh_pip_editable_install [...]``: The equivalent of running ``pip install -e`` - with the given arguments, as well as additional default arguments used for - installing packages into Sage with pip. The last argument must be - ``.`` to indicate installation from the current directory. - See `pip documentation `_ - for more details concerning editable installs. - -- ``sdh_pip_uninstall [...]``: Runs ``pip uninstall`` with the given arguments. - If unsuccessful, it displays a warning. - -- ``sdh_store_and_pip_install_wheel .``: The current directory, - indicated by the required argument ``.``, must have a subdirectory - ``dist`` containing a unique wheel file (``*.whl``). - - This command (1) moves this wheel file to the - directory ``$SAGE_SPKG_WHEELS`` (``$SAGE_LOCAL/var/lib/sage/wheels``) - and then (2) installs the wheel in ``$SAGE_LOCAL``. - - Both of these steps, instead of writing directly into ``$SAGE_LOCAL``, - use the staging directory ``$SAGE_DESTDIR`` if set; otherwise, they - use ``$SAGE_SUDO`` (if set). - -- ``sdh_install [-T] SRC [SRC...] DEST``: Copies one or more files or - directories given as ``SRC`` (recursively in the case of - directories) into the destination directory ``DEST``, while - ensuring that ``DEST`` and all its parent directories exist. - ``DEST`` should be a path under ``$SAGE_LOCAL``, generally. For - ``DESTDIR`` installs, the ``$SAGE_DESTDIR`` path is automatically - prepended to the destination. - - The ``-T`` option treats ``DEST`` as a normal file instead - (e.g. for copying a file to a different filename). All directory - components are still created in this case. +- ``sdh_pip_install [...]``: The equivalent of running ``pip install`` + with the given arguments, as well as additional default arguments used for + installing packages into Sage with pip. The last argument must be + ``.`` to indicate installation from the current directory. + + ``sdh_pip_install`` actually does the installation via ``pip wheel``, + creating a wheel file in ``dist/``, followed by + ``sdh_store_and_pip_install_wheel`` (see below). + +- ``sdh_pip_editable_install [...]``: The equivalent of running ``pip install -e`` + with the given arguments, as well as additional default arguments used for + installing packages into Sage with pip. The last argument must be + ``.`` to indicate installation from the current directory. + See `pip documentation `_ + for more details concerning editable installs. + +- ``sdh_pip_uninstall [...]``: Runs ``pip uninstall`` with the given arguments. + If unsuccessful, it displays a warning. + +- ``sdh_store_and_pip_install_wheel .``: The current directory, + indicated by the required argument ``.``, must have a subdirectory + ``dist`` containing a unique wheel file (``*.whl``). + + This command (1) moves this wheel file to the + directory ``$SAGE_SPKG_WHEELS`` (``$SAGE_LOCAL/var/lib/sage/wheels``) + and then (2) installs the wheel in ``$SAGE_LOCAL``. + + Both of these steps, instead of writing directly into ``$SAGE_LOCAL``, + use the staging directory ``$SAGE_DESTDIR`` if set; otherwise, they + use ``$SAGE_SUDO`` (if set). + +- ``sdh_install [-T] SRC [SRC...] DEST``: Copies one or more files or + directories given as ``SRC`` (recursively in the case of + directories) into the destination directory ``DEST``, while + ensuring that ``DEST`` and all its parent directories exist. + ``DEST`` should be a path under ``$SAGE_LOCAL``, generally. For + ``DESTDIR`` installs, the ``$SAGE_DESTDIR`` path is automatically + prepended to the destination. + + The ``-T`` option treats ``DEST`` as a normal file instead + (e.g. for copying a file to a different filename). All directory + components are still created in this case. The following is automatically added to each install script, so you should not need to add it yourself. -- ``sdh_guard``: Wrapper for ``sdh_check_vars`` that checks some - common variables without which many/most packages won't build - correctly (``SAGE_ROOT``, ``SAGE_LOCAL``, ``SAGE_SHARE``). This is - important to prevent installation to unintended locations. +- ``sdh_guard``: Wrapper for ``sdh_check_vars`` that checks some + common variables without which many/most packages won't build + correctly (``SAGE_ROOT``, ``SAGE_LOCAL``, ``SAGE_SHARE``). This is + important to prevent installation to unintended locations. The following are also available, but rarely used. -- ``sdh_cmake [...]``: Runs ``cmake`` in the current directory with - the given arguments, as well as additional arguments passed to - cmake (assuming packages are using the GNUInstallDirs module) so - that ``CMAKE_INSTALL_PREFIX`` and ``CMAKE_INSTALL_LIBDIR`` are set - correctly. - -- ``sdh_preload_lib EXECUTABLE SONAME``: (Linux only -- no-op on other - platforms.) Check shared libraries loaded by ``EXECUTABLE`` (may be a - program or another library) for a library starting with ``SONAME``, and - if found appends ``SONAME`` to the ``LD_PRELOAD`` environment variable. - See :trac:`24885`. +- ``sdh_cmake [...]``: Runs ``cmake`` in the current directory with + the given arguments, as well as additional arguments passed to + cmake (assuming packages are using the GNUInstallDirs module) so + that ``CMAKE_INSTALL_PREFIX`` and ``CMAKE_INSTALL_LIBDIR`` are set + correctly. + +- ``sdh_preload_lib EXECUTABLE SONAME``: (Linux only -- no-op on other + platforms.) Check shared libraries loaded by ``EXECUTABLE`` (may be a + program or another library) for a library starting with ``SONAME``, and + if found appends ``SONAME`` to the ``LD_PRELOAD`` environment variable. + See :trac:`24885`. .. _spkg-configure.m4: