python3.pkgs.scipy: 1.10.1 -> 1.11.1 & more

doc/languages-frameworks/python.section.md

@@ -1190,6 +1190,10 @@ following are specific to `buildPythonPackage`:
 * `pipInstallFlags ? []`: A list of strings. Arguments to be passed to `pip
   install`. To pass options to `python setup.py install`, use
   `--install-option`. E.g., `pipInstallFlags=["--install-option='--cpp_implementation'"]`.
+* `pipBuildFlags ? []`: A list of strings. Arguments to be passed to `pip
+  wheel`.
+* `pypaBuildFlags ? []`: A list of strings. Arguments to be passed to `python
+  -m build --wheel`.
 * `pythonPath ? []`: List of packages to be added into `$PYTHONPATH`. Packages
   in `pythonPath` are not propagated (contrary to `propagatedBuildInputs`).
 * `preShellHook`: Hook to execute commands before `shellHook`.
@@ -1244,6 +1248,29 @@ with import <nixpkgs> {};
 in python.withPackages(ps: [ ps.blaze ])).env
 ```
 
+The next example shows a non trivial overriding of the `blas` implementation to
+be used through out all of the Python package set:
+
+```nix
+python3MyBlas = pkgs.python3.override {
+  packageOverrides = self: super: {
+    # We need toPythonModule for the package set to evaluate this
+    blas = super.toPythonModule(super.pkgs.blas.override {
+      blasProvider = super.pkgs.mkl;
+    });
+    lapack = super.toPythonModule(super.pkgs.lapack.override {
+      lapackProvider = super.pkgs.mkl;
+    });
+  };
+};
+```
+
+This is particularly useful for numpy and scipy users who want to gain speed
+with other blas implementations. Note that using simply `scipy =
+super.scipy.override { blas = super.pkgs.mkl; };` will likely result in
+compilation issues, because scipy dependencies need to use the same blas
+implementation as well.
+
 #### Optional extra dependencies {#python-optional-dependencies}
 
 Some packages define optional dependencies for additional features. With
@@ -1463,6 +1490,10 @@ are used in `buildPythonPackage`.
 - `flitBuildHook` to build a wheel using `flit`.
 - `pipBuildHook` to build a wheel using `pip` and PEP 517. Note a build system
   (e.g. `setuptools` or `flit`) should still be added as `nativeBuildInput`.
+- `pypaBuildHook` to build a wheel using
+  [`pypa/build`](https://pypa-build.readthedocs.io/en/latest/index.html) and
+  PEP 517/518. Note a build system (e.g. `setuptools` or `flit`) should still
+  be added as `nativeBuildInput`.
 - `pipInstallHook` to install wheels.
 - `pytestCheckHook` to run tests with `pytest`. See [example usage](#using-pytestcheckhook).
 - `pythonCatchConflictsHook` to check whether a Python package is not already existing.

pkgs/development/interpreters/python/hooks/default.nix

@@ -62,6 +62,16 @@ in {
       };
     } ./pip-build-hook.sh) {};
 
+  pypaBuildHook = callPackage ({ makePythonHook, build, wheel }:
+    makePythonHook {
+      name = "pypa-build-hook.sh";
+      propagatedBuildInputs = [ build wheel ];
+      substitutions = {
+        inherit pythonInterpreter;
+      };
+    } ./pypa-build-hook.sh) {};
+
+
   pipInstallHook = callPackage ({ makePythonHook, pip }:
     makePythonHook {
       name = "pip-install-hook";

pkgs/development/interpreters/python/hooks/pip-build-hook.sh

@@ -1,13 +1,22 @@
 # Setup hook to use for pip projects
 echo "Sourcing pip-build-hook"
 
+declare -a pipBuildFlags
+
 pipBuildPhase() {
     echo "Executing pipBuildPhase"
     runHook preBuild
 
     mkdir -p dist
     echo "Creating a wheel..."
-    @pythonInterpreter@ -m pip wheel --verbose --no-index --no-deps --no-clean --no-build-isolation --wheel-dir dist .
+    @pythonInterpreter@ -m pip wheel \
+       --verbose \
+       --no-index \
+       --no-deps \
+       --no-clean \
+       --no-build-isolation \
+       --wheel-dir dist \
+       $pipBuildFlags .
     echo "Finished creating a wheel..."
 
     runHook postBuild

pkgs/development/interpreters/python/hooks/pypa-build-hook.sh

@@ -0,0 +1,19 @@
+# Setup hook to use for pypa/build projects
+echo "Sourcing pypa-build-hook"
+
+pypaBuildPhase() {
+    echo "Executing pypaBuildPhase"
+    runHook preBuild
+
+    echo "Creating a wheel..."
+    @pythonInterpreter@ -m build --no-isolation --outdir dist/ --wheel $pypaBuildFlags
+    echo "Finished creating a wheel..."
+
+    runHook postBuild
+    echo "Finished executing pypaBuildPhase"
+}
+
+if [ -z "${dontUsePypaBuild-}" ] && [ -z "${buildPhase-}" ]; then
+    echo "Using pypaBuildPhase"
+    buildPhase=pypaBuildPhase
+fi

pkgs/development/libraries/xsimd/default.nix

@@ -11,7 +11,9 @@ stdenv.mkDerivation rec {
 
   nativeBuildInputs = [ cmake ];
 
-  cmakeFlags = [ "-DBUILD_TESTS=ON" ];
+  cmakeFlags = [
+    "-DBUILD_TESTS=${if (doCheck && stdenv.hostPlatform == stdenv.buildPlatform) then "ON" else "OFF"}"
+  ];
 
   doCheck = true;
   nativeCheckInputs = [ gtest ];

pkgs/development/python-modules/meson-python/add-build-flags.sh

@@ -0,0 +1,6 @@
+# Add all of mesonFlags to -Csetup-args for pypa builds
+for f in $mesonFlags; do
+  pypaBuildFlags+=" -Csetup-args=$f"
+  # This requires pip>23.0.1, see: https://meson-python.readthedocs.io/en/latest/how-to-guides/config-settings.html
+  pipBuildFlags+=" --config-settings=setup-args=$f"
+done

pkgs/development/python-modules/meson-python/default.nix

@@ -13,13 +13,13 @@
 
 buildPythonPackage rec {
   pname = "meson-python";
-  version = "0.12.1";
+  version = "0.13.1";
   format = "pyproject";
 
   src = fetchPypi {
     inherit version;
     pname = "meson_python";
-    hash = "sha256-PVs+WB1wpYqXucEWp16Xp2zEtMfnX6Blj8g5I3Hi8sI=";
+    hash = "sha256-Y7MXAAFCXEL6TP7a25BRy9KJJf+O7XxA02ugCZ48dhg=";
   };
 
   nativeBuildInputs = [
@@ -39,6 +39,9 @@ buildPythonPackage rec {
   ] ++ lib.optionals (pythonOlder "3.10") [
     typing-extensions
   ];
+  setupHooks = [
+    ./add-build-flags.sh
+  ];
 
   # Ugly work-around. Drop ninja dependency.
   # We already have ninja, but it comes without METADATA.

pkgs/development/python-modules/pythran/default.nix

@@ -19,13 +19,13 @@ let
 
 in buildPythonPackage rec {
   pname = "pythran";
-  version = "0.11.0";
+  version = "0.13.1";
 
   src = fetchFromGitHub {
     owner = "serge-sans-paille";
     repo = "pythran";
     rev = version;
-    hash = "sha256-F9gUZOTSuiqvfGoN4yQqwUg9mnCeBntw5eHO7ZnjpzI=";
+    hash = "sha256-baDrReJgQXbaKA8KNhHiFjr0X34yb8WK/nUJmiM9EZs=";
   };
 
   patches = [

pkgs/development/python-modules/scipy/default.nix

@@ -1,9 +1,15 @@
 { lib
 , stdenv
-, fetchPypi
+, fetchFromGitHub
+, fetchpatch
+, fetchurl
+, buildPackages
+, writeText
 , python
 , pythonOlder
 , buildPythonPackage
+, pypaBuildHook
+, pipInstallHook
 , cython
 , gfortran
 , meson-python
@@ -17,31 +23,71 @@
 , pybind11
 , pooch
 , libxcrypt
+, xsimd
+, blas
+, lapack
 }:
 
-buildPythonPackage rec {
+let
   pname = "scipy";
-  version = "1.10.1";
-  format = "pyproject";
-
-  src = fetchPypi {
-    inherit pname version;
-    hash = "sha256-LPnfuAp7RYm6TEDOdYiYbW1c68VFfK0sKID2vC1C86U=";
+  # DON'T UPDATE THESE ATTRIBUTES MANUALLY - USE:
+  #
+  #     nix-shell maintainers/scripts/update.nix --argstr package python3.pkgs.scipy
+  #
+  # Even if you do update these hashes manually, don't change their base
+  # (base16 or base64), because the update script uses sed regexes to replace
+  # them with the updated hashes.
+  version = "1.11.1";
+  srcHash = "sha256-bgnYXe3EhzL7+Gfriz1cXCl2eYQJ8zF+rcIwHyZR8bQ=";
+  datasetsHashes = {
+    ascent = "1qjp35ncrniq9rhzb14icwwykqg2208hcssznn3hz27w39615kh3";
+    ecg = "1bwbjp43b7znnwha5hv6wiz3g0bhwrpqpi75s12zidxrbwvd62pj";
+    face = "11i8x29h80y7hhyqhil1fg8mxag5f827g33lhnsf44qk116hp2wx";
+  };
+  datasets = lib.mapAttrs (
+    d: hash: fetchurl {
+      url = "https://raw.githubusercontent.com/scipy/dataset-${d}/main/${d}.dat";
+      sha256 = hash;
+    }
+  ) datasetsHashes;
+  # Additional cross compilation related properties that scipy reads in scipy/meson.build
+  crossFileScipy = writeText "cross-file-scipy.conf" ''
+    [properties]
+    numpy-include-dir = '${numpy}/${python.sitePackages}/numpy/core/include'
+    pythran-include-dir = '${pythran}/${python.sitePackages}/pythran'
+    host-python-path = '${python.interpreter}'
+    host-python-version = '${python.pythonVersion}'
+  '';
+in buildPythonPackage {
+  inherit pname version;
+  format = "other";
+
+  src = fetchFromGitHub {
+    owner = "scipy";
+    repo = pname;
+    rev = "v${version}";
+    hash = srcHash;
+    fetchSubmodules = true;
   };
-
   patches = [
-    # These tests require internet connection, currently impossible to disable
-    # them otherwise, see:
-    # https://github.com/scipy/scipy/pull/17965
-    ./disable-datasets-tests.patch
+    # Helps with cross compilation, see https://github.com/scipy/scipy/pull/18167
+    (fetchpatch {
+      url = "https://github.com/scipy/scipy/commit/dd50ac9d98dbb70625333a23e3a90e493228e3be.patch";
+      hash = "sha256-Vf6/hhwu6X5s8KWhq8bUZKtSkdVu/GtEpGtj8Olxe7s=";
+      excludes = [
+        "doc/source/dev/contributor/meson_advanced.rst"
+      ];
+    })
   ];
 
-  nativeBuildInputs = [ cython gfortran meson-python pythran pkg-config wheel ];
+  nativeBuildInputs = [ pypaBuildHook pipInstallHook cython gfortran meson-python pythran pkg-config wheel ];
 
   buildInputs = [
-    numpy.blas
+    blas
+    lapack
     pybind11
     pooch
+    xsimd
   ] ++ lib.optionals (pythonOlder "3.9") [
     libxcrypt
   ];
@@ -53,9 +99,29 @@ buildPythonPackage rec {
   doCheck = !(stdenv.isx86_64 && stdenv.isDarwin);
 
   preConfigure = ''
-    sed -i '0,/from numpy.distutils.core/s//import setuptools;from numpy.distutils.core/' setup.py
+    # Relax deps a bit
+    substituteInPlace pyproject.toml \
+      --replace 'numpy==' 'numpy>='
+    # Helps parallelization a bit
     export NPY_NUM_BUILD_JOBS=$NIX_BUILD_CORES
-  '';
+    # We download manually the datasets and this variable tells the pooch
+    # library where these files are cached. See also:
+    # https://github.com/scipy/scipy/pull/18518#issuecomment-1562350648 And at:
+    # https://github.com/scipy/scipy/pull/17965#issuecomment-1560759962
+    export XDG_CACHE_HOME=$PWD; mkdir scipy-data
+  '' + (lib.concatStringsSep "\n" (lib.mapAttrsToList (d: dpath:
+    # Actually copy the datasets
+    "cp ${dpath} scipy-data/${d}.dat"
+  ) datasets));
+
+  mesonFlags = [
+    "-Dblas=${blas.pname}"
+    "-Dlapack=${lapack.pname}"
+    # We always run what's necessary for cross compilation, which is passing to
+    # meson the proper cross compilation related arguments. See also:
+    # https://docs.scipy.org/doc/scipy/building/cross_compilation.html
+    "--cross-file=${crossFileScipy}"
+  ];
 
   # disable stackprotector on aarch64-darwin for now
   #
@@ -79,17 +145,23 @@ buildPythonPackage rec {
   requiredSystemFeatures = [ "big-parallel" ]; # the tests need lots of CPU time
 
   passthru = {
-    blas = numpy.blas;
+    inherit blas;
+    updateScript = [
+      ./update.sh
+      # Pass it this file name as argument
+      (builtins.unsafeGetAttrPos "pname" python.pkgs.scipy).file
+    ]
+    # Pass it the names of the datasets to update their hashes
+    ++ (builtins.attrNames datasetsHashes)
+    ;
   };
 
-  setupPyBuildFlags = [ "--fcompiler='gnu95'" ];
-
   SCIPY_USE_G77_ABI_WRAPPER = 1;
 
   meta = with lib; {
     description = "SciPy (pronounced 'Sigh Pie') is open-source software for mathematics, science, and engineering";
     homepage = "https://www.scipy.org/";
     license = licenses.bsd3;
-    maintainers = [ maintainers.fridh ];
+    maintainers = [ maintainers.fridh maintainers.doronbehar ];
   };
 }

pkgs/development/python-modules/scipy/update.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p jq nix-prefetch-github
+
+set -euo pipefail
+echoerr() { echo "$@" 1>&2; }
+
+fname="$1"
+echoerr got fname $fname
+shift
+datasets="$@"
+echoerr datasets are: "$@"
+latest_release=$(curl --silent https://api.github.com/repos/scipy/scipy/releases/latest)
+version=$(jq -r '.tag_name' <<<"$latest_release" | cut -c2-)
+# Update version, if needed
+if grep -q 'version = "'$version $fname; then
+    echo "Current version $version is the latest available, will update only datasets' hashes (don't take long)"
+else
+    echoerr got version $version
+    sed -i -E 's/(version = ).*=/\1'$version'/g' $fname
+    # Update srcHash
+    srcHash='"sha256-'$(nix-prefetch-github scipy scipy --rev v${version} --fetch-submodules | jq --raw-output .sha256)'"'
+    sed -i 's/srcHash = .*=";/srcHash = '$srcHash';/g' $fname
+fi
+
+for d in $datasets; do
+    datasetHash=$(nix-prefetch-url "https://raw.githubusercontent.com/scipy/dataset-${d}/main/${d}.dat")
+    sed -i 's/'$d' = "[0-9a-z]\+/'$d' = "'$datasetHash'/g' $fname
+    echoerr updated hash for dataset "'$d'"
+done