diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md index 23c8526787b73..6f839bb40ff8c 100644 --- a/doc/languages-frameworks/python.section.md +++ b/doc/languages-frameworks/python.section.md @@ -1190,6 +1190,10 @@ following are specific to `buildPythonPackage`: * `pipInstallFlags ? []`: A list of strings. Arguments to be passed to `pip install`. To pass options to `python setup.py install`, use `--install-option`. E.g., `pipInstallFlags=["--install-option='--cpp_implementation'"]`. +* `pipBuildFlags ? []`: A list of strings. Arguments to be passed to `pip + wheel`. +* `pypaBuildFlags ? []`: A list of strings. Arguments to be passed to `python + -m build --wheel`. * `pythonPath ? []`: List of packages to be added into `$PYTHONPATH`. Packages in `pythonPath` are not propagated (contrary to `propagatedBuildInputs`). * `preShellHook`: Hook to execute commands before `shellHook`. @@ -1244,6 +1248,29 @@ with import {}; in python.withPackages(ps: [ ps.blaze ])).env ``` +The next example shows a non trivial overriding of the `blas` implementation to +be used through out all of the Python package set: + +```nix +python3MyBlas = pkgs.python3.override { + packageOverrides = self: super: { + # We need toPythonModule for the package set to evaluate this + blas = super.toPythonModule(super.pkgs.blas.override { + blasProvider = super.pkgs.mkl; + }); + lapack = super.toPythonModule(super.pkgs.lapack.override { + lapackProvider = super.pkgs.mkl; + }); + }; +}; +``` + +This is particularly useful for numpy and scipy users who want to gain speed +with other blas implementations. Note that using simply `scipy = +super.scipy.override { blas = super.pkgs.mkl; };` will likely result in +compilation issues, because scipy dependencies need to use the same blas +implementation as well. + #### Optional extra dependencies {#python-optional-dependencies} Some packages define optional dependencies for additional features. With @@ -1463,6 +1490,10 @@ are used in `buildPythonPackage`. - `flitBuildHook` to build a wheel using `flit`. - `pipBuildHook` to build a wheel using `pip` and PEP 517. Note a build system (e.g. `setuptools` or `flit`) should still be added as `nativeBuildInput`. +- `pypaBuildHook` to build a wheel using + [`pypa/build`](https://pypa-build.readthedocs.io/en/latest/index.html) and + PEP 517/518. Note a build system (e.g. `setuptools` or `flit`) should still + be added as `nativeBuildInput`. - `pipInstallHook` to install wheels. - `pytestCheckHook` to run tests with `pytest`. See [example usage](#using-pytestcheckhook). - `pythonCatchConflictsHook` to check whether a Python package is not already existing. diff --git a/pkgs/development/interpreters/python/hooks/default.nix b/pkgs/development/interpreters/python/hooks/default.nix index 001e477b9185b..f7e7f6ac757c5 100644 --- a/pkgs/development/interpreters/python/hooks/default.nix +++ b/pkgs/development/interpreters/python/hooks/default.nix @@ -62,6 +62,16 @@ in { }; } ./pip-build-hook.sh) {}; + pypaBuildHook = callPackage ({ makePythonHook, build, wheel }: + makePythonHook { + name = "pypa-build-hook.sh"; + propagatedBuildInputs = [ build wheel ]; + substitutions = { + inherit pythonInterpreter; + }; + } ./pypa-build-hook.sh) {}; + + pipInstallHook = callPackage ({ makePythonHook, pip }: makePythonHook { name = "pip-install-hook"; diff --git a/pkgs/development/interpreters/python/hooks/pip-build-hook.sh b/pkgs/development/interpreters/python/hooks/pip-build-hook.sh index 745f02e8c9bc3..9de4c7d1dd0dd 100644 --- a/pkgs/development/interpreters/python/hooks/pip-build-hook.sh +++ b/pkgs/development/interpreters/python/hooks/pip-build-hook.sh @@ -1,13 +1,22 @@ # Setup hook to use for pip projects echo "Sourcing pip-build-hook" +declare -a pipBuildFlags + pipBuildPhase() { echo "Executing pipBuildPhase" runHook preBuild mkdir -p dist echo "Creating a wheel..." - @pythonInterpreter@ -m pip wheel --verbose --no-index --no-deps --no-clean --no-build-isolation --wheel-dir dist . + @pythonInterpreter@ -m pip wheel \ + --verbose \ + --no-index \ + --no-deps \ + --no-clean \ + --no-build-isolation \ + --wheel-dir dist \ + $pipBuildFlags . echo "Finished creating a wheel..." runHook postBuild diff --git a/pkgs/development/interpreters/python/hooks/pypa-build-hook.sh b/pkgs/development/interpreters/python/hooks/pypa-build-hook.sh new file mode 100644 index 0000000000000..3b71300497691 --- /dev/null +++ b/pkgs/development/interpreters/python/hooks/pypa-build-hook.sh @@ -0,0 +1,19 @@ +# Setup hook to use for pypa/build projects +echo "Sourcing pypa-build-hook" + +pypaBuildPhase() { + echo "Executing pypaBuildPhase" + runHook preBuild + + echo "Creating a wheel..." + @pythonInterpreter@ -m build --no-isolation --outdir dist/ --wheel $pypaBuildFlags + echo "Finished creating a wheel..." + + runHook postBuild + echo "Finished executing pypaBuildPhase" +} + +if [ -z "${dontUsePypaBuild-}" ] && [ -z "${buildPhase-}" ]; then + echo "Using pypaBuildPhase" + buildPhase=pypaBuildPhase +fi diff --git a/pkgs/development/libraries/xsimd/default.nix b/pkgs/development/libraries/xsimd/default.nix index ec2d166ef580b..cfaf218268601 100644 --- a/pkgs/development/libraries/xsimd/default.nix +++ b/pkgs/development/libraries/xsimd/default.nix @@ -11,7 +11,9 @@ stdenv.mkDerivation rec { nativeBuildInputs = [ cmake ]; - cmakeFlags = [ "-DBUILD_TESTS=ON" ]; + cmakeFlags = [ + "-DBUILD_TESTS=${if (doCheck && stdenv.hostPlatform == stdenv.buildPlatform) then "ON" else "OFF"}" + ]; doCheck = true; nativeCheckInputs = [ gtest ]; diff --git a/pkgs/development/python-modules/meson-python/add-build-flags.sh b/pkgs/development/python-modules/meson-python/add-build-flags.sh new file mode 100644 index 0000000000000..d2535d1fd59ad --- /dev/null +++ b/pkgs/development/python-modules/meson-python/add-build-flags.sh @@ -0,0 +1,6 @@ +# Add all of mesonFlags to -Csetup-args for pypa builds +for f in $mesonFlags; do + pypaBuildFlags+=" -Csetup-args=$f" + # This requires pip>23.0.1, see: https://meson-python.readthedocs.io/en/latest/how-to-guides/config-settings.html + pipBuildFlags+=" --config-settings=setup-args=$f" +done diff --git a/pkgs/development/python-modules/meson-python/default.nix b/pkgs/development/python-modules/meson-python/default.nix index 20008b2b53410..866512a4cfdd6 100644 --- a/pkgs/development/python-modules/meson-python/default.nix +++ b/pkgs/development/python-modules/meson-python/default.nix @@ -13,13 +13,13 @@ buildPythonPackage rec { pname = "meson-python"; - version = "0.12.1"; + version = "0.13.1"; format = "pyproject"; src = fetchPypi { inherit version; pname = "meson_python"; - hash = "sha256-PVs+WB1wpYqXucEWp16Xp2zEtMfnX6Blj8g5I3Hi8sI="; + hash = "sha256-Y7MXAAFCXEL6TP7a25BRy9KJJf+O7XxA02ugCZ48dhg="; }; nativeBuildInputs = [ @@ -39,6 +39,9 @@ buildPythonPackage rec { ] ++ lib.optionals (pythonOlder "3.10") [ typing-extensions ]; + setupHooks = [ + ./add-build-flags.sh + ]; # Ugly work-around. Drop ninja dependency. # We already have ninja, but it comes without METADATA. diff --git a/pkgs/development/python-modules/pythran/default.nix b/pkgs/development/python-modules/pythran/default.nix index ef03e76947637..7a55b2003ab7d 100644 --- a/pkgs/development/python-modules/pythran/default.nix +++ b/pkgs/development/python-modules/pythran/default.nix @@ -19,13 +19,13 @@ let in buildPythonPackage rec { pname = "pythran"; - version = "0.11.0"; + version = "0.13.1"; src = fetchFromGitHub { owner = "serge-sans-paille"; repo = "pythran"; rev = version; - hash = "sha256-F9gUZOTSuiqvfGoN4yQqwUg9mnCeBntw5eHO7ZnjpzI="; + hash = "sha256-baDrReJgQXbaKA8KNhHiFjr0X34yb8WK/nUJmiM9EZs="; }; patches = [ diff --git a/pkgs/development/python-modules/scipy/default.nix b/pkgs/development/python-modules/scipy/default.nix index 1090e724a7fd3..323dc44bb1cb2 100644 --- a/pkgs/development/python-modules/scipy/default.nix +++ b/pkgs/development/python-modules/scipy/default.nix @@ -1,9 +1,15 @@ { lib , stdenv -, fetchPypi +, fetchFromGitHub +, fetchpatch +, fetchurl +, buildPackages +, writeText , python , pythonOlder , buildPythonPackage +, pypaBuildHook +, pipInstallHook , cython , gfortran , meson-python @@ -17,31 +23,71 @@ , pybind11 , pooch , libxcrypt +, xsimd +, blas +, lapack }: -buildPythonPackage rec { +let pname = "scipy"; - version = "1.10.1"; - format = "pyproject"; - - src = fetchPypi { - inherit pname version; - hash = "sha256-LPnfuAp7RYm6TEDOdYiYbW1c68VFfK0sKID2vC1C86U="; + # DON'T UPDATE THESE ATTRIBUTES MANUALLY - USE: + # + # nix-shell maintainers/scripts/update.nix --argstr package python3.pkgs.scipy + # + # Even if you do update these hashes manually, don't change their base + # (base16 or base64), because the update script uses sed regexes to replace + # them with the updated hashes. + version = "1.11.1"; + srcHash = "sha256-bgnYXe3EhzL7+Gfriz1cXCl2eYQJ8zF+rcIwHyZR8bQ="; + datasetsHashes = { + ascent = "1qjp35ncrniq9rhzb14icwwykqg2208hcssznn3hz27w39615kh3"; + ecg = "1bwbjp43b7znnwha5hv6wiz3g0bhwrpqpi75s12zidxrbwvd62pj"; + face = "11i8x29h80y7hhyqhil1fg8mxag5f827g33lhnsf44qk116hp2wx"; + }; + datasets = lib.mapAttrs ( + d: hash: fetchurl { + url = "https://raw.githubusercontent.com/scipy/dataset-${d}/main/${d}.dat"; + sha256 = hash; + } + ) datasetsHashes; + # Additional cross compilation related properties that scipy reads in scipy/meson.build + crossFileScipy = writeText "cross-file-scipy.conf" '' + [properties] + numpy-include-dir = '${numpy}/${python.sitePackages}/numpy/core/include' + pythran-include-dir = '${pythran}/${python.sitePackages}/pythran' + host-python-path = '${python.interpreter}' + host-python-version = '${python.pythonVersion}' + ''; +in buildPythonPackage { + inherit pname version; + format = "other"; + + src = fetchFromGitHub { + owner = "scipy"; + repo = pname; + rev = "v${version}"; + hash = srcHash; + fetchSubmodules = true; }; - patches = [ - # These tests require internet connection, currently impossible to disable - # them otherwise, see: - # https://github.com/scipy/scipy/pull/17965 - ./disable-datasets-tests.patch + # Helps with cross compilation, see https://github.com/scipy/scipy/pull/18167 + (fetchpatch { + url = "https://github.com/scipy/scipy/commit/dd50ac9d98dbb70625333a23e3a90e493228e3be.patch"; + hash = "sha256-Vf6/hhwu6X5s8KWhq8bUZKtSkdVu/GtEpGtj8Olxe7s="; + excludes = [ + "doc/source/dev/contributor/meson_advanced.rst" + ]; + }) ]; - nativeBuildInputs = [ cython gfortran meson-python pythran pkg-config wheel ]; + nativeBuildInputs = [ pypaBuildHook pipInstallHook cython gfortran meson-python pythran pkg-config wheel ]; buildInputs = [ - numpy.blas + blas + lapack pybind11 pooch + xsimd ] ++ lib.optionals (pythonOlder "3.9") [ libxcrypt ]; @@ -53,9 +99,29 @@ buildPythonPackage rec { doCheck = !(stdenv.isx86_64 && stdenv.isDarwin); preConfigure = '' - sed -i '0,/from numpy.distutils.core/s//import setuptools;from numpy.distutils.core/' setup.py + # Relax deps a bit + substituteInPlace pyproject.toml \ + --replace 'numpy==' 'numpy>=' + # Helps parallelization a bit export NPY_NUM_BUILD_JOBS=$NIX_BUILD_CORES - ''; + # We download manually the datasets and this variable tells the pooch + # library where these files are cached. See also: + # https://github.com/scipy/scipy/pull/18518#issuecomment-1562350648 And at: + # https://github.com/scipy/scipy/pull/17965#issuecomment-1560759962 + export XDG_CACHE_HOME=$PWD; mkdir scipy-data + '' + (lib.concatStringsSep "\n" (lib.mapAttrsToList (d: dpath: + # Actually copy the datasets + "cp ${dpath} scipy-data/${d}.dat" + ) datasets)); + + mesonFlags = [ + "-Dblas=${blas.pname}" + "-Dlapack=${lapack.pname}" + # We always run what's necessary for cross compilation, which is passing to + # meson the proper cross compilation related arguments. See also: + # https://docs.scipy.org/doc/scipy/building/cross_compilation.html + "--cross-file=${crossFileScipy}" + ]; # disable stackprotector on aarch64-darwin for now # @@ -79,17 +145,23 @@ buildPythonPackage rec { requiredSystemFeatures = [ "big-parallel" ]; # the tests need lots of CPU time passthru = { - blas = numpy.blas; + inherit blas; + updateScript = [ + ./update.sh + # Pass it this file name as argument + (builtins.unsafeGetAttrPos "pname" python.pkgs.scipy).file + ] + # Pass it the names of the datasets to update their hashes + ++ (builtins.attrNames datasetsHashes) + ; }; - setupPyBuildFlags = [ "--fcompiler='gnu95'" ]; - SCIPY_USE_G77_ABI_WRAPPER = 1; meta = with lib; { description = "SciPy (pronounced 'Sigh Pie') is open-source software for mathematics, science, and engineering"; homepage = "https://www.scipy.org/"; license = licenses.bsd3; - maintainers = [ maintainers.fridh ]; + maintainers = [ maintainers.fridh maintainers.doronbehar ]; }; } diff --git a/pkgs/development/python-modules/scipy/disable-datasets-tests.patch b/pkgs/development/python-modules/scipy/disable-datasets-tests.patch deleted file mode 100644 index a06d0d50ddf42..0000000000000 --- a/pkgs/development/python-modules/scipy/disable-datasets-tests.patch +++ /dev/null @@ -1,9 +0,0 @@ -diff --git i/scipy/datasets/meson.build w/scipy/datasets/meson.build -index 101377253..eec2feea4 100644 ---- i/scipy/datasets/meson.build -+++ w/scipy/datasets/meson.build -@@ -11,4 +11,3 @@ py3.install_sources( - subdir: 'scipy/datasets' - ) - --subdir('tests') diff --git a/pkgs/development/python-modules/scipy/update.sh b/pkgs/development/python-modules/scipy/update.sh new file mode 100755 index 0000000000000..b0d6e2da4f41d --- /dev/null +++ b/pkgs/development/python-modules/scipy/update.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p jq nix-prefetch-github + +set -euo pipefail +echoerr() { echo "$@" 1>&2; } + +fname="$1" +echoerr got fname $fname +shift +datasets="$@" +echoerr datasets are: "$@" +latest_release=$(curl --silent https://api.github.com/repos/scipy/scipy/releases/latest) +version=$(jq -r '.tag_name' <<<"$latest_release" | cut -c2-) +# Update version, if needed +if grep -q 'version = "'$version $fname; then + echo "Current version $version is the latest available, will update only datasets' hashes (don't take long)" +else + echoerr got version $version + sed -i -E 's/(version = ).*=/\1'$version'/g' $fname + # Update srcHash + srcHash='"sha256-'$(nix-prefetch-github scipy scipy --rev v${version} --fetch-submodules | jq --raw-output .sha256)'"' + sed -i 's/srcHash = .*=";/srcHash = '$srcHash';/g' $fname +fi + +for d in $datasets; do + datasetHash=$(nix-prefetch-url "https://raw.githubusercontent.com/scipy/dataset-${d}/main/${d}.dat") + sed -i 's/'$d' = "[0-9a-z]\+/'$d' = "'$datasetHash'/g' $fname + echoerr updated hash for dataset "'$d'" +done