Merge branch '5.0.x' into cpath-mod

.github/workflows/unit_tests.yml

@@ -152,7 +152,9 @@ jobs:
           cd $HOME
           # initialize environment for modules tool
           if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
-          source $(cat $HOME/mod_init); type module
+          source $(cat $HOME/mod_init)
+          type module
+          module --version
           # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that);
           # also pick up changes to $PATH set by sourcing $MOD_INIT
           export PREFIX=/tmp/$USER/$GITHUB_SHA

RELEASE_NOTES

@@ -4,6 +4,18 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.9.4 (22 September 2024)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - set $LMOD_TERSE_DECORATIONS to 'no' to avoid additional info in output produced by 'ml --terse avail' (#4648)
+- various bug fixes, including:
+  - implement workaround for permission error when copying read-only files that have extended attributes set and using Python 3.6 (#4642)
+  - take into account alternate sysroot for /bin/bash used by run_cmd (#4646)
+
+
 v4.9.3 (14 September 2024)
 --------------------------
 

easybuild/_deprecated.py

@@ -131,7 +131,7 @@ def get_output_from_process(proc, read_size=None, asynchronous=False, print_depr
 @run_cmd_cache
 def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None,
             force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False,
-            with_hooks=True):
+            with_hooks=True, with_sysroot=True):
     """
     Run specified command (in a subshell)
     :param cmd: command to run
@@ -149,6 +149,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True
     :param stream_output: enable streaming command output to stdout
     :param asynchronous: run command asynchronously (returns subprocess.Popen instance if set to True)
     :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined)
+    :param with_sysroot: prepend sysroot to exec_cmd (if defined)
     """
 
     _log.deprecated("run_cmd is deprecated, use run_shell_cmd from easybuild.tools.run instead", '6.0')
@@ -228,6 +229,16 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True
 
     exec_cmd = "/bin/bash"
 
+    # if EasyBuild is configured to use an alternate sysroot,
+    # we should also run shell commands using the bash shell provided in there,
+    # since /bin/bash may not be compatible with the alternate sysroot
+    if with_sysroot:
+        sysroot = build_option('sysroot')
+        if sysroot:
+            sysroot_bin_bash = os.path.join(sysroot, 'bin', 'bash')
+            if os.path.exists(sysroot_bin_bash):
+                exec_cmd = sysroot_bin_bash
+
     if not shell:
         if isinstance(cmd, list):
             exec_cmd = None
@@ -237,6 +248,8 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True
         else:
             raise EasyBuildError("Don't know how to prefix with /usr/bin/env for commands of type %s", type(cmd))
 
+    _log.info("Using %s as shell for running cmd: %s", exec_cmd, cmd)
+
     if with_hooks:
         hooks = load_hooks(build_option('hooks'))
         hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': os.getcwd()})

easybuild/framework/easyblock.py

@@ -73,7 +73,7 @@
 from easybuild.tools.build_details import get_build_stats
 from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, dry_run_msg, dry_run_warning, dry_run_set_dirs
 from easybuild.tools.build_log import print_error, print_msg, print_warning
-from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES
+from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES, PYTHONPATH, EBPYTHONPREFIXES
 from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES
 from easybuild.tools.config import SEARCH_PATH_BIN_DIRS, SEARCH_PATH_LIB_DIRS
 from easybuild.tools.config import EASYBUILD_SOURCES_URL # noqa
@@ -1404,6 +1404,49 @@ def make_module_description(self):
         """
         return self.module_generator.get_description()
 
+    def make_module_pythonpath(self):
+        """
+        Add lines for module file to update $PYTHONPATH or $EBPYTHONPREFIXES,
+        if they aren't already present and the standard lib/python*/site-packages subdirectory exists
+        """
+        lines = []
+        if not os.path.isfile(os.path.join(self.installdir, 'bin', 'python')):  # only needed when not a python install
+            python_subdir_pattern = os.path.join(self.installdir, 'lib', 'python*', 'site-packages')
+            candidate_paths = (os.path.relpath(path, self.installdir) for path in glob.glob(python_subdir_pattern))
+            python_paths = [path for path in candidate_paths if re.match(r'lib/python\d+\.\d+/site-packages', path)]
+
+            # determine whether Python is a runtime dependency;
+            # if so, we assume it was installed with EasyBuild, and hence is aware of $EBPYTHONPREFIXES
+            runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)]
+
+            # don't use $EBPYTHONPREFIXES unless we can and it's preferred or necesary (due to use of multi_deps)
+            use_ebpythonprefixes = False
+            multi_deps = self.cfg['multi_deps']
+
+            if 'Python' in runtime_deps:
+                self.log.info("Found Python runtime dependency, so considering $EBPYTHONPREFIXES...")
+
+                if build_option('prefer_python_search_path') == EBPYTHONPREFIXES:
+                    self.log.info("Preferred Python search path is $EBPYTHONPREFIXES, so using that")
+                    use_ebpythonprefixes = True
+
+            elif multi_deps and 'Python' in multi_deps:
+                self.log.info("Python is listed in 'multi_deps', so using $EBPYTHONPREFIXES instead of $PYTHONPATH")
+                use_ebpythonprefixes = True
+
+            if python_paths:
+                # add paths unless they were already added
+                if use_ebpythonprefixes:
+                    path = ''  # EBPYTHONPREFIXES are relative to the install dir
+                    if path not in self.module_generator.added_paths_per_key[EBPYTHONPREFIXES]:
+                        lines.append(self.module_generator.prepend_paths(EBPYTHONPREFIXES, path))
+                else:
+                    for python_path in python_paths:
+                        if python_path not in self.module_generator.added_paths_per_key[PYTHONPATH]:
+                            lines.append(self.module_generator.prepend_paths(PYTHONPATH, python_path))
+
+        return lines
+
     def make_module_extra(self, altroot=None, altversion=None):
         """
         Set extra stuff in module file, e.g. $EBROOT*, $EBVERSION*, etc.
@@ -1452,6 +1495,9 @@ def make_module_extra(self, altroot=None, altversion=None):
                                      value, type(value))
             lines.append(self.module_generator.append_paths(key, value, allow_abs=self.cfg['allow_append_abs_path']))
 
+        # add lines to update $PYTHONPATH or $EBPYTHONPREFIXES
+        lines.extend(self.make_module_pythonpath())
+
         modloadmsg = self.cfg['modloadmsg']
         if modloadmsg:
             # add trailing newline to prevent that shell prompt is 'glued' to module load message

easybuild/framework/easyconfig/easyconfig.py

@@ -1105,15 +1105,19 @@ def filter_deps(self, deps):
 
         return retained_deps
 
-    def dependencies(self, build_only=False):
+    def dependencies(self, build_only=False, runtime_only=False):
         """
         Returns an array of parsed dependencies (after filtering, if requested)
         dependency = {'name': '', 'version': '', 'system': (False|True), 'versionsuffix': '', 'toolchain': ''}
         Iterable builddependencies are flattened when not iterating.
 
         :param build_only: only return build dependencies, discard others
+        :param runtime_only: only return runtime dependencies, discard others
         """
-        deps = self.builddependencies()
+        if runtime_only:
+            deps = []
+        else:
+            deps = self.builddependencies()
 
         if not build_only:
             # use += rather than .extend to get a new list rather than updating list of build deps in place...

easybuild/tools/config.py

@@ -96,7 +96,7 @@
 DEFAULT_ENV_FOR_SHEBANG = '/usr/bin/env'
 DEFAULT_ENVVAR_USERS_MODULES = 'HOME'
 DEFAULT_INDEX_MAX_AGE = 7 * 24 * 60 * 60  # 1 week (in seconds)
-DEFAULT_JOB_BACKEND = 'GC3Pie'
+DEFAULT_JOB_BACKEND = 'Slurm'
 DEFAULT_JOB_EB_CMD = 'eb'
 DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log")
 DEFAULT_MAX_FAIL_RATIO_PERMS = 0.5
@@ -178,6 +178,10 @@
 SEARCH_PATH_HEADER_DIRS = ["include"]
 SEARCH_PATH_LIB_DIRS = ["lib", "lib64"]
 
+PYTHONPATH = 'PYTHONPATH'
+EBPYTHONPREFIXES = 'EBPYTHONPREFIXES'
+PYTHON_SEARCH_PATH_TYPES = [PYTHONPATH, EBPYTHONPREFIXES]
+
 
 class Singleton(ABCMeta):
     """Serves as metaclass for classes that should implement the Singleton pattern.
@@ -410,6 +414,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
     OUTPUT_STYLE_AUTO: [
         'output_style',
     ],
+    PYTHONPATH: [
+        'prefer_python_search_path',
+    ]
 }
 # build option that do not have a perfectly matching command line option
 BUILD_OPTIONS_OTHER = {

easybuild/tools/filetools.py

@@ -42,11 +42,13 @@
 """
 import datetime
 import difflib
+import filecmp
 import glob
 import hashlib
 import inspect
 import itertools
 import os
+import platform
 import re
 import shutil
 import signal
@@ -61,6 +63,7 @@
 import urllib.request as std_urllib
 
 from easybuild.base import fancylogger
+from easybuild.tools import LooseVersion
 # import build_log must stay, to use of EasyBuildLog
 from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, CWD_NOTFOUND_ERROR
 from easybuild.tools.build_log import dry_run_msg, print_msg, print_warning
@@ -2427,8 +2430,42 @@ def copy_file(path, target_path, force_in_dry_run=False):
             else:
                 mkdir(os.path.dirname(target_path), parents=True)
                 if path_exists:
-                    shutil.copy2(path, target_path)
-                    _log.info("%s copied to %s", path, target_path)
+                    try:
+                        # on filesystems that support extended file attributes, copying read-only files with
+                        # shutil.copy2() will give a PermissionError, when using Python < 3.7
+                        # see https://bugs.python.org/issue24538
+                        shutil.copy2(path, target_path)
+                        _log.info("%s copied to %s", path, target_path)
+                    # catch the more general OSError instead of PermissionError,
+                    # since Python 2.7 doesn't support PermissionError
+                    except OSError as err:
+                        # if file is writable (not read-only), then we give up since it's not a simple permission error
+                        if os.path.exists(target_path) and os.stat(target_path).st_mode & stat.S_IWUSR:
+                            raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err)
+
+                        pyver = LooseVersion(platform.python_version())
+                        if pyver >= LooseVersion('3.7'):
+                            raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err)
+                        elif LooseVersion('3.7') > pyver >= LooseVersion('3'):
+                            if not isinstance(err, PermissionError):
+                                raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err)
+
+                        # double-check whether the copy actually succeeded
+                        if not os.path.exists(target_path) or not filecmp.cmp(path, target_path, shallow=False):
+                            raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err)
+
+                        try:
+                            # re-enable user write permissions in target, copy xattrs, then remove write perms again
+                            adjust_permissions(target_path, stat.S_IWUSR)
+                            shutil._copyxattr(path, target_path)
+                            adjust_permissions(target_path, stat.S_IWUSR, add=False)
+                        except OSError as err:
+                            raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err)
+
+                        msg = ("Failed to copy extended attributes from file %s to %s, due to a bug in shutil (see "
+                               "https://bugs.python.org/issue24538). Copy successful with workaround.")
+                        _log.info(msg, path, target_path)
+
                 elif os.path.islink(path):
                     if os.path.isdir(target_path):
                         target_path = os.path.join(target_path, os.path.basename(path))

easybuild/tools/job/gc3pie.py

@@ -104,6 +104,10 @@ def __init__(self, *args, **kwargs):
     def _check_version(self):
         """Check whether GC3Pie version complies with required version."""
 
+        deprecation_msg = "The GC3Pie job backend is no longer maintained and will be removed"
+        deprecation_msg += ", please use a different job backend"
+        _log.deprecated(deprecation_msg, '6.0')
+
         try:
             from pkg_resources import get_distribution, DistributionNotFound
             pkg = get_distribution('gc3pie')

easybuild/tools/module_generator.py

@@ -39,6 +39,7 @@
 import os
 import re
 import tempfile
+from collections import defaultdict
 from contextlib import contextmanager
 from easybuild.tools import LooseVersion
 from textwrap import wrap
@@ -153,7 +154,7 @@ def start_module_creation(self):
             raise EasyBuildError('Module creation already in process. '
                                  'You cannot create multiple modules at the same time!')
         # Mapping of keys/env vars to paths already added
-        self.added_paths_per_key = dict()
+        self.added_paths_per_key = defaultdict(set)
         txt = self.MODULE_SHEBANG
         if txt:
             txt += '\n'
@@ -212,7 +213,7 @@ def _filter_paths(self, key, paths):
             print_warning('Module creation has not been started. Call start_module_creation first!')
             return paths
 
-        added_paths = self.added_paths_per_key.setdefault(key, set())
+        added_paths = self.added_paths_per_key[key]
         # paths can be a string
         if isinstance(paths, str):
             if paths in added_paths:

easybuild/tools/modules.py

@@ -1589,6 +1589,9 @@ def __init__(self, *args, **kwargs):
         setvar('LMOD_REDIRECT', 'no', verbose=False)
         # disable extended defaults within Lmod (introduced and set as default in Lmod 8.0.7)
         setvar('LMOD_EXTENDED_DEFAULT', 'no', verbose=False)
+        # disabled decorations in "ml --terse avail" output
+        # (introduced in Lmod 8.8, see also https://github.com/TACC/Lmod/issues/690)
+        setvar('LMOD_TERSE_DECORATIONS', 'no', verbose=False)
 
         super(Lmod, self).__init__(*args, **kwargs)
         version = LooseVersion(self.version)

easybuild/tools/options.py

@@ -78,6 +78,7 @@
 from easybuild.tools.config import OUTPUT_STYLE_AUTO, OUTPUT_STYLES, WARN
 from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path
 from easybuild.tools.config import BuildOptions, ConfigurationVariables
+from easybuild.tools.config import PYTHON_SEARCH_PATH_TYPES, PYTHONPATH
 from easybuild.tools.configobj import ConfigObj, ConfigObjError
 from easybuild.tools.docs import FORMAT_JSON, FORMAT_MD, FORMAT_RST, FORMAT_TXT
 from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses
@@ -490,6 +491,10 @@ def override_options(self):
                                             None, 'store_true', False),
             'pre-create-installdir': ("Create installation directory before submitting build jobs",
                                       None, 'store_true', True),
+            'prefer-python-search-path': (("Prefer using specified environment variable when possible to specify where"
+                                           " Python packages were installed; see also "
+                                           "https://docs.easybuild.io/python-search-path"),
+                                          'choice', 'store_or_None', PYTHONPATH, PYTHON_SEARCH_PATH_TYPES),
             'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"),
                         None, 'store_true', False, 'p'),
             'read-only-installdir': ("Set read-only permissions on installation directory after installation",
@@ -2046,7 +2051,7 @@ def set_tmpdir(tmpdir=None, raise_error=False):
             os.chmod(tmptest_file, 0o700)
             res = run_shell_cmd(tmptest_file, fail_on_error=False, in_dry_run=True, hidden=True, stream_output=False,
                                 with_hooks=False)
-            if res.exit_code:
+            if res.exit_code != EasyBuildExit.SUCCESS:
                 msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir()
                 msg += "This can cause problems in the build process, consider using --tmpdir."
                 if raise_error:

easybuild/tools/run.py

@@ -208,6 +208,11 @@ def create_cmd_scripts(cmd_str, work_dir, env, tmpdir, out_file, err_file):
     if env is None:
         env = os.environ.copy()
 
+    # Decode any declared bash functions
+    proc = subprocess.Popen('declare -f', stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
+                            env=env, shell=True, executable='bash')
+    (bash_functions, _) = proc.communicate()
+
     env_fp = os.path.join(tmpdir, 'env.sh')
     with open(env_fp, 'w') as fid:
         # unset all environment variables in current environment first to start from a clean slate;
@@ -219,6 +224,8 @@ def create_cmd_scripts(cmd_str, work_dir, env, tmpdir, out_file, err_file):
         fid.write('\n'.join(f'export {key}={shlex.quote(value)}' for key, value in sorted(env.items())
                             if not key.endswith('%')) + '\n')
 
+        fid.write(bash_functions.decode(errors='ignore') + '\n')
+
         fid.write('\n\nPS1="eb-shell> "')
 
         # define $EB_CMD_OUT_FILE to contain path to file with command output
@@ -485,6 +492,9 @@ def to_cmd_str(cmd):
 
         if stdin:
             proc.stdin.write(stdin)
+            proc.stdin.flush()
+            if not qa_patterns:
+                proc.stdin.close()
 
         exit_code = None
         stdout, stderr = b'', b''