Merge branch 'develop' into test-suite-improvements

RELEASE_NOTES

@@ -4,6 +4,42 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.4.2 (September 7th 2021)
+---------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add per-extension timing in output produced by eb command (#3734)
+  - add definition for new toolchain nvpsmpic (NVHPC + ParaStationMPI + CUDA) (#3736)
+  - include list of missing libraries in warning about missing FFTW libraries in imkl toolchain component (#3776)
+  - check for recursive symlinks by default before copying a folder (#3784)
+  - add --filter-ecs configuration option to filter out easyconfigs from set of easyconfigs to install (#3796)
+  - check type of source_tmpl value for extensions, ensure it's a string value (not a list) (#3799)
+  - also define $BLAS_SHARED_LIBS & co in build environment (analogous to $BLAS_STATIC_LIBS) (#3800)
+  - report use of --ignore-test-failure in success message in output (#3806)
+  - add get_cuda_cc_template_value method to EasyConfig class (#3807)
+  - add support for fix_bash_shebang_for (#3808)
+  - pick up $MODULES_CMD to facilitate using Environment Modules 4.x as modules tool (#3816)
+  - use more sensible branch name for creating easyblocks PR with --new-pr (#3817)
+- various bug fixes, including:
+  - remove Python 2.6 from list of supported Python versions in setup.py (#3767)
+  - don't add directory that doesn't include any files to $PATH or $LD_LIBRARY_PATH (#3769)
+  - make logdir writable also when --stop/--fetch is used and --read-only-installdir is enabled (#3771)
+  - fix forgotten renaming of 'l' to 'char' __init__.py that is created for included Python modules (#3773)
+  - fix verify_imports by deleting all imported modules before re-importing them one by one (#3780)
+  - fix ignore_test_failure not set for Extension instances (#3782)
+  - update iompi toolchain to intel-compiler subtoolchain for oneAPI versions (>= iompi 2020.12) (#3785)
+  - don't parse patch files as easyconfigs when searching for where patch file is used (#3786)
+  - make sure git clone with a tag argument actually downloads a tag (#3795)
+  - fix CI by excluding GC3Pie 2.6.7 (which is broken with Python 2) and improve error reporting for option parsing (#3798)
+  - correctly resolve templates for patches in extensions when uploading to GitHub (#3805)
+  - add --easystack to ignored options when submitting job (#3813)
+- other changes:
+  - speed up tests by caching checked paths in set_tmpdir + less test cases for test_compiler_dependent_optarch (#3802)
+  - speed up set_parallel method in EasyBlock class (#3812)
+
+
 v4.4.1 (July 6th 2021)
 ----------------------
 

easybuild/framework/easyblock.py

@@ -212,6 +212,10 @@ def __init__(self, ec):
         self.postmsg = ''  # allow a post message to be set, which can be shown as last output
         self.current_step = None
 
+        # Create empty progress bar
+        self.progress_bar = None
+        self.pbar_task = None
+
         # list of loaded modules
         self.loaded_modules = []
 
@@ -300,6 +304,21 @@ def close_log(self):
         self.log.info("Closing log for application name %s version %s" % (self.name, self.version))
         fancylogger.logToFile(self.logfile, enable=False)
 
+    def set_progress_bar(self, progress_bar, task_id):
+        """
+        Set progress bar, the progress bar is needed when writing messages so
+        that the progress counter is always at the bottom
+        """
+        self.progress_bar = progress_bar
+        self.pbar_task = task_id
+
+    def advance_progress(self, tick=1.0):
+        """
+        Advance the progress bar forward with `tick`
+        """
+        if self.progress_bar and self.pbar_task is not None:
+            self.progress_bar.advance(self.pbar_task, tick)
+
     #
     # DRY RUN UTILITIES
     #
@@ -593,7 +612,13 @@ def fetch_extension_sources(self, skip_checksums=False):
                         default_source_tmpl = resolve_template('%(name)s-%(version)s.tar.gz', template_values)
 
                         # if no sources are specified via 'sources', fall back to 'source_tmpl'
-                        src_fn = ext_options.get('source_tmpl', default_source_tmpl)
+                        src_fn = ext_options.get('source_tmpl')
+                        if src_fn is None:
+                            src_fn = default_source_tmpl
+                        elif not isinstance(src_fn, string_type):
+                            error_msg = "source_tmpl value must be a string! (found value of type '%s'): %s"
+                            raise EasyBuildError(error_msg, type(src_fn).__name__, src_fn)
+
                         src_path = self.obtain_file(src_fn, extension=True, urls=source_urls,
                                                     force_download=force_download)
                         if src_path:
@@ -1785,18 +1810,19 @@ def set_parallel(self):
         """Set 'parallel' easyconfig parameter to determine how many cores can/should be used for parallel builds."""
         # set level of parallelism for build
         par = build_option('parallel')
-        if self.cfg['parallel'] is not None:
-            if par is None:
-                par = self.cfg['parallel']
-                self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par)
-            else:
-                par = min(int(par), int(self.cfg['parallel']))
-                self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par)
-        else:
+        cfg_par = self.cfg['parallel']
+        if cfg_par is None:
             self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par)
+        elif par is None:
+            par = cfg_par
+            self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par)
+        else:
+            par = min(int(par), int(cfg_par))
+            self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par)
 
-        self.cfg['parallel'] = det_parallelism(par=par, maxpar=self.cfg['maxparallel'])
-        self.log.info("Setting parallelism: %s" % self.cfg['parallel'])
+        par = det_parallelism(par, maxpar=self.cfg['maxparallel'])
+        self.log.info("Setting parallelism: %s" % par)
+        self.cfg['parallel'] = par
 
     def remove_module_file(self):
         """Remove module file (if it exists), and check for ghost installation directory (and deal with it)."""
@@ -2412,6 +2438,7 @@ def extensions_step(self, fetch=False, install=True):
 
             tup = (ext.name, ext.version or '', idx + 1, exts_cnt)
             print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent)
+            start_time = datetime.now()
 
             if self.dry_run:
                 tup = (ext.name, ext.version, ext.__class__.__name__)
@@ -2432,11 +2459,19 @@ def extensions_step(self, fetch=False, install=True):
 
             # real work
             if install:
-                ext.prerun()
-                txt = ext.run()
-                if txt:
-                    self.module_extra_extensions += txt
-                ext.postrun()
+                try:
+                    ext.prerun()
+                    txt = ext.run()
+                    if txt:
+                        self.module_extra_extensions += txt
+                    ext.postrun()
+                finally:
+                    if not self.dry_run:
+                        ext_duration = datetime.now() - start_time
+                        if ext_duration.total_seconds() >= 1:
+                            print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent)
+                        elif self.logdebug or build_option('trace'):
+                            print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent)
 
         # cleanup (unload fake module, remove fake module dir)
         if fake_mod_data:
@@ -2469,7 +2504,7 @@ def package_step(self):
 
     def fix_shebang(self):
         """Fix shebang lines for specified files."""
-        for lang in ['perl', 'python']:
+        for lang in ['bash', 'perl', 'python']:
             shebang_regex = re.compile(r'^#![ ]*.*[/ ]%s.*' % lang)
             fix_shebang_for = self.cfg['fix_%s_shebang_for' % lang]
             if fix_shebang_for:
@@ -3552,6 +3587,8 @@ def run_all_steps(self, run_test_cases):
             return True
 
         steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt())
+        # Calculate progress bar tick
+        tick = 1.0 / float(len(steps))
 
         print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent)
         trace_msg("installation prefix: %s" % self.installdir)
@@ -3590,6 +3627,7 @@ def run_all_steps(self, run_test_cases):
                                 print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent)
                             elif self.logdebug or build_option('trace'):
                                 print_msg("... (took < 1 sec)", log=self.log, silent=self.silent)
+                self.advance_progress(tick)
 
         except StopException:
             pass
@@ -3615,7 +3653,7 @@ def print_dry_run_note(loc, silent=True):
     dry_run_msg(msg, silent=silent)
 
 
-def build_and_install_one(ecdict, init_env):
+def build_and_install_one(ecdict, init_env, progress_bar=None, task_id=None):
     """
     Build the software
     :param ecdict: dictionary contaning parsed easyconfig + metadata
@@ -3663,6 +3701,11 @@ def build_and_install_one(ecdict, init_env):
         print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg),
                     silent=silent)
 
+    # Setup progress bar
+    if progress_bar and task_id is not None:
+        app.set_progress_bar(progress_bar, task_id)
+        _log.info("Updated progress bar instance for easyblock %s", easyblock)
+
     # application settings
     stop = build_option('stop')
     if stop is not None:
@@ -3688,15 +3731,21 @@ def build_and_install_one(ecdict, init_env):
 
             if os.path.exists(app.installdir) and build_option('read_only_installdir') and (
                     build_option('rebuild') or build_option('force')):
+                enabled_write_permissions = True
                 # re-enable write permissions so we can install additional modules
                 adjust_permissions(app.installdir, stat.S_IWUSR, add=True, recursive=True)
+            else:
+                enabled_write_permissions = False
 
         result = app.run_all_steps(run_test_cases=run_test_cases)
 
         if not dry_run:
             # also add any extension easyblocks used during the build for reproducibility
             if app.ext_instances:
                 copy_easyblocks_for_reprod(app.ext_instances, reprod_dir)
+            # If not already done remove the granted write permissions if we did so
+            if enabled_write_permissions and os.lstat(app.installdir)[stat.ST_MODE] & stat.S_IWUSR:
+                adjust_permissions(app.installdir, stat.S_IWUSR, add=False, recursive=True)
 
     except EasyBuildError as err:
         first_n = 300
@@ -3713,28 +3762,37 @@ def build_and_install_one(ecdict, init_env):
 
     # successful (non-dry-run) build
     if result and not dry_run:
+        def ensure_writable_log_dir(log_dir):
+            """Make sure we can write into the log dir"""
+            if build_option('read_only_installdir'):
+                # temporarily re-enable write permissions for copying log/easyconfig to install dir
+                if os.path.exists(log_dir):
+                    adjust_permissions(log_dir, stat.S_IWUSR, add=True, recursive=True)
+                else:
+                    parent_dir = os.path.dirname(log_dir)
+                    if os.path.exists(parent_dir):
+                        adjust_permissions(parent_dir, stat.S_IWUSR, add=True, recursive=False)
+                        mkdir(log_dir, parents=True)
+                        adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False)
+                    else:
+                        mkdir(log_dir, parents=True)
+                        adjust_permissions(log_dir, stat.S_IWUSR, add=True, recursive=True)
 
         if app.cfg['stop']:
             ended = 'STOPPED'
             if app.builddir is not None:
                 new_log_dir = os.path.join(app.builddir, config.log_path(ec=app.cfg))
             else:
                 new_log_dir = os.path.dirname(app.logfile)
+            ensure_writable_log_dir(new_log_dir)
 
         # if we're only running the sanity check, we should not copy anything new to the installation directory
         elif build_option('sanity_check_only'):
             _log.info("Only running sanity check, so skipping build stats, easyconfigs archive, reprod files...")
 
         else:
             new_log_dir = os.path.join(app.installdir, config.log_path(ec=app.cfg))
-            if build_option('read_only_installdir'):
-                # temporarily re-enable write permissions for copying log/easyconfig to install dir
-                if os.path.exists(new_log_dir):
-                    adjust_permissions(new_log_dir, stat.S_IWUSR, add=True, recursive=True)
-                else:
-                    adjust_permissions(app.installdir, stat.S_IWUSR, add=True, recursive=False)
-                    mkdir(new_log_dir, parents=True)
-                    adjust_permissions(app.installdir, stat.S_IWUSR, add=False, recursive=False)
+            ensure_writable_log_dir(new_log_dir)
 
             # collect build stats
             _log.info("Collecting build stats...")

easybuild/framework/easyconfig/default.py

@@ -95,6 +95,8 @@
     'easybuild_version': [None, "EasyBuild-version this spec-file was written for", BUILD],
     'enhance_sanity_check': [False, "Indicate that additional sanity check commands & paths should enhance "
                              "the existin sanity check, not replace it", BUILD],
+    'fix_bash_shebang_for': [None, "List of files for which Bash shebang should be fixed "
+                                   "to '#!/usr/bin/env bash' (glob patterns supported)", BUILD],
     'fix_perl_shebang_for': [None, "List of files for which Perl shebang should be fixed "
                                    "to '#!/usr/bin/env perl' (glob patterns supported)", BUILD],
     'fix_python_shebang_for': [None, "List of files for which Python shebang should be fixed "

easybuild/framework/easyconfig/easyconfig.py

@@ -59,7 +59,7 @@
 from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT
 from easybuild.framework.easyconfig.parser import DEPRECATED_PARAMETERS, REPLACED_PARAMETERS
 from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig
-from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, template_constant_dict
+from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC, template_constant_dict
 from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg
 from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG
 from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN
@@ -559,6 +559,13 @@ def disable_templating(self):
         finally:
             self.enable_templating = old_enable_templating
 
+    def __str__(self):
+        """Return a string representation of this EasyConfig instance"""
+        if self.path:
+            return '%s EasyConfig @ %s' % (self.name, self.path)
+        else:
+            return 'Raw %s EasyConfig' % self.name
+
     def filename(self):
         """Determine correct filename for this easyconfig file."""
 
@@ -1803,6 +1810,25 @@ def asdict(self):
             res[key] = value
         return res
 
+    def get_cuda_cc_template_value(self, key):
+        """
+        Get template value based on --cuda-compute-capabilities EasyBuild configuration option
+        and cuda_compute_capabilities easyconfig parameter.
+        Returns user-friendly error message in case neither are defined,
+        or if an unknown key is used.
+        """
+        if key.startswith('cuda_') and any(x[0] == key for x in TEMPLATE_NAMES_DYNAMIC):
+            try:
+                return self.template_values[key]
+            except KeyError:
+                error_msg = "Template value '%s' is not defined!\n"
+                error_msg += "Make sure that either the --cuda-compute-capabilities EasyBuild configuration "
+                error_msg += "option is set, or that the cuda_compute_capabilities easyconfig parameter is defined."
+                raise EasyBuildError(error_msg, key)
+        else:
+            error_msg = "%s is not a template value based on --cuda-compute-capabilities/cuda_compute_capabilities"
+            raise EasyBuildError(error_msg, key)
+
 
 def det_installversion(version, toolchain_name, toolchain_version, prefix, suffix):
     """Deprecated 'det_installversion' function, to determine exact install version, based on supplied parameters."""

easybuild/framework/easyconfig/tools.py

@@ -37,6 +37,7 @@
 :author: Ward Poelmans (Ghent University)
 """
 import copy
+import fnmatch
 import glob
 import os
 import re
@@ -360,6 +361,10 @@ def det_easyconfig_paths(orig_paths):
             # if no easyconfigs are specified, use all the ones touched in the PR
             ec_files = [path for path in pr_files if path.endswith('.eb')]
 
+    filter_ecs = build_option('filter_ecs')
+    if filter_ecs:
+        ec_files = [ec for ec in ec_files
+                    if not any(fnmatch.fnmatch(ec, filter_spec) for filter_spec in filter_ecs)]
     if ec_files and robot_path:
         ignore_subdirs = build_option('ignore_dirs')
         if not build_option('consider_archived_easyconfigs'):