Merge pull request #2862 from jfgrimm/20230117154815_new_pr_scipy

update scipy easyblock for scipy >= 1.9.0 to use meson/ninja

easybuild/easyblocks/s/scipy.py

@@ -30,34 +30,177 @@
 @author: Kenneth Hoste (Ghent University)
 @author: Pieter De Baets (Ghent University)
 @author: Jens Timmerman (Ghent University)
+@author: Jasper Grimm (University of York)
 """
+import os
+import tempfile
 from distutils.version import LooseVersion
 
-from easybuild.easyblocks.generic.fortranpythonpackage import FortranPythonPackage
-from easybuild.easyblocks.generic.pythonpackage import det_pylibdir
+import easybuild.tools.environment as env
 import easybuild.tools.toolchain as toolchain
+from easybuild.easyblocks.generic.fortranpythonpackage import FortranPythonPackage
+from easybuild.easyblocks.generic.mesonninja import MesonNinja
+from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_pylibdir
+from easybuild.framework.easyconfig import CUSTOM
+from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.filetools import change_dir, copy_dir
 
 
-class EB_scipy(FortranPythonPackage):
+class EB_scipy(FortranPythonPackage, PythonPackage, MesonNinja):
     """Support for installing the scipy Python package as part of a Python installation."""
 
+    @staticmethod
+    def extra_options():
+        """Easyconfig parameters specific to scipy."""
+        extra_vars = ({
+            'enable_slow_tests': [False, "Run scipy test suite, including tests marked as slow", CUSTOM],
+            'ignore_test_result': [None, "Run scipy test suite, but ignore test failures (True/False/None). Default "
+                                         "(None) implies True for scipy < 1.9, and False for scipy >= 1.9", CUSTOM],
+        })
+
+        return PythonPackage.extra_options(extra_vars=extra_vars)
+
     def __init__(self, *args, **kwargs):
         """Set scipy-specific test command."""
-        super(EB_scipy, self).__init__(*args, **kwargs)
-
+        # calling PythonPackage __init__ also lets MesonNinja work in an extension
+        PythonPackage.__init__(self, *args, **kwargs)
         self.testinstall = True
-        self.testcmd = "cd .. && %(python)s -c 'import numpy; import scipy; scipy.test(verbose=2)'"
+
+        if LooseVersion(self.version) >= LooseVersion('1.9'):
+            self.use_meson = True
+
+            # enforce scipy test suite results if not explicitly disabled for scipy >= 1.9
+            # strip inherited PythonPackage installopts
+            installopts = self.cfg['installopts']
+            pythonpackage_installopts = ['--no-deps', '--ignore-installed', '--no-index', '--egg',
+                                         '--zip-ok', '--no-index']
+            self.log.info("Stripping inherited PythonPackage installopts %s from installopts %s",
+                          pythonpackage_installopts, installopts)
+            for i in pythonpackage_installopts:
+                installopts = installopts.replace(i, '')
+            self.cfg['installopts'] = installopts
+
+        else:
+            self.use_meson = False
+
+        if self.cfg['ignore_test_result'] is None:
+            # automatically ignore scipy test suite results for scipy < 1.9, as we did in older easyconfigs
+            self.cfg['ignore_test_result'] = LooseVersion(self.version) < '1.9'
+            self.log.info("ignore_test_result not specified, so automatically set to %s for scipy %s",
+                          self.cfg['ignore_test_result'], self.version)
+
+        if self.cfg['ignore_test_result']:
+            # used to maintain compatibility with easyconfigs predating scipy 1.9;
+            # runs tests (serially) in a way that exits with code 0 regardless of test results,
+            # see https://github.com/easybuilders/easybuild-easyblocks/issues/2237
+            self.testcmd = "cd .. && %(python)s -c 'import numpy; import scipy; scipy.test(verbose=2)'"
+        else:
+            self.testcmd = " && ".join([
+                "cd ..",
+                "touch %(srcdir)s/.coveragerc",
+                "%(python)s %(srcdir)s/runtests.py -v --no-build --parallel %(parallel)s",
+            ])
+            if self.cfg['enable_slow_tests']:
+                self.testcmd += " -m full "
 
     def configure_step(self):
         """Custom configure step for scipy: set extra installation options when needed."""
-        super(EB_scipy, self).configure_step()
+
+        # scipy >= 1.9.0 uses Meson/Ninja
+        if self.use_meson:
+            # configure BLAS/LAPACK library to use with Meson for scipy >= 1.9.0
+            lapack_lib = self.toolchain.lapack_family()
+            if lapack_lib == toolchain.FLEXIBLAS:
+                blas_lapack = 'flexiblas'
+            elif lapack_lib == toolchain.INTELMKL:
+                blas_lapack = 'mkl'
+            elif lapack_lib == toolchain.OPENBLAS:
+                blas_lapack = 'openblas'
+            else:
+                raise EasyBuildError("Unknown BLAS/LAPACK library used: %s", lapack_lib)
+
+            for opt in ('blas', 'lapack'):
+                self.cfg.update('configopts', '-D%(opt)s=%(blas_lapack)s' % {'opt': opt, 'blas_lapack': blas_lapack})
+
+            # need to have already installed extensions in PATH, PYTHONPATH for configure/build/install steps
+            pythonpath = os.getenv('PYTHONPATH')
+            pylibdir = det_pylibdir()
+            env.setvar('PYTHONPATH', os.pathsep.join([os.path.join(self.installdir, pylibdir), pythonpath]))
+
+            path = os.getenv('PATH')
+            env.setvar('PATH', os.pathsep.join([os.path.join(self.installdir, 'bin'), path]))
+
+            MesonNinja.configure_step(self)
+
+        else:
+            # scipy < 1.9.0 uses install procedure using setup.py
+            FortranPythonPackage.configure_step(self)
 
         if LooseVersion(self.version) >= LooseVersion('0.13'):
             # in recent scipy versions, additional compilation is done in the install step,
             # which requires unsetting $LDFLAGS
             if self.toolchain.comp_family() in [toolchain.GCC, toolchain.CLANGGCC]:  # @UndefinedVariable
                 self.cfg.update('preinstallopts', "unset LDFLAGS && ")
 
+    def build_step(self):
+        """Custom build step for scipy: use ninja for scipy >= 1.9.0"""
+        if self.use_meson:
+            MesonNinja.build_step(self)
+        else:
+            FortranPythonPackage.build_step(self)
+
+    def test_step(self):
+        """Run available scipy unit tests. Adapted from numpy easyblock"""
+
+        if self.use_meson:
+            # temporarily install scipy so we can run the test suite
+            tmpdir = tempfile.mkdtemp()
+            cwd = os.getcwd()
+
+            tmp_builddir = os.path.join(tmpdir, 'build')
+            tmp_installdir = os.path.join(tmpdir, 'install')
+
+            # create a copy of the builddir
+            copy_dir(cwd, tmp_builddir)
+            change_dir(tmp_builddir)
+
+            # reconfigure (to update prefix), and install to tmpdir
+            MesonNinja.configure_step(self, cmd_prefix=tmp_installdir)
+            MesonNinja.install_step(self)
+
+            tmp_pylibdir = [os.path.join(tmp_installdir, det_pylibdir())]
+            self.prepare_python()
+
+            self.cfg['pretestopts'] = " && ".join([
+                # LDFLAGS should not be set when testing numpy/scipy, because it overwrites whatever numpy/scipy sets
+                # see http://projects.scipy.org/numpy/ticket/182
+                "unset LDFLAGS",
+                "export PYTHONPATH=%s:$PYTHONPATH" % tmp_pylibdir,
+                "",
+            ])
+            self.cfg['runtest'] = self.testcmd % {
+                'python': self.python_cmd,
+                'srcdir': self.cfg['start_dir'],
+                'parallel': self.cfg['parallel'],
+            }
+
+            MesonNinja.test_step(self)
+
+        else:
+            self.testcmd = self.testcmd % {
+                'python': '%(python)s',
+                'srcdir': self.cfg['start_dir'],
+                'parallel': self.cfg['parallel'],
+            }
+            FortranPythonPackage.test_step(self)
+
+    def install_step(self):
+        """Custom install step for scipy: use ninja for scipy >= 1.9.0"""
+        if self.use_meson:
+            MesonNinja.install_step(self)
+        else:
+            FortranPythonPackage.install_step(self)
+
     def sanity_check_step(self, *args, **kwargs):
         """Custom sanity check for scipy."""
 
@@ -68,4 +211,4 @@ def sanity_check_step(self, *args, **kwargs):
             'dirs': [det_pylibdir()],
         }
 
-        return super(EB_scipy, self).sanity_check_step(custom_paths=custom_paths)
+        return PythonPackage.sanity_check_step(self, custom_paths=custom_paths)