From 3abbd19be949f363e2d1008ecbce49a321beb27c Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 10 Jun 2021 08:53:39 +0200 Subject: [PATCH 01/91] Add per-extension timing --- easybuild/framework/easyblock.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index d561df3ed7..90eeda1ac8 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2366,6 +2366,7 @@ def extensions_step(self, fetch=False, install=True): tup = (ext.name, ext.version or '', idx + 1, exts_cnt) print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent) + start_time = datetime.now() if self.dry_run: tup = (ext.name, ext.version, ext.__class__.__name__) @@ -2386,11 +2387,19 @@ def extensions_step(self, fetch=False, install=True): # real work if install: - ext.prerun() - txt = ext.run() - if txt: - self.module_extra_extensions += txt - ext.postrun() + try: + ext.prerun() + txt = ext.run() + if txt: + self.module_extra_extensions += txt + ext.postrun() + finally: + if not self.dry_run: + ext_duration = datetime.now() - start_time + if ext_duration.total_seconds() >= 1: + print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent) + elif self.logdebug or build_option('trace'): + print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent) # cleanup (unload fake module, remove fake module dir) if fake_mod_data: From 665bf2f294bcf2ba9e6a99352ff23028c2144bfb Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Thu, 10 Jun 2021 16:51:47 +0200 Subject: [PATCH 02/91] add toolchain npsmpic --- easybuild/toolchains/npsmpic.py | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 easybuild/toolchains/npsmpic.py diff --git a/easybuild/toolchains/npsmpic.py b/easybuild/toolchains/npsmpic.py new file mode 100644 index 0000000000..33ee2bb06a --- /dev/null +++ b/easybuild/toolchains/npsmpic.py @@ -0,0 +1,41 @@ +## +# Copyright 2016-2016 Ghent University +# Copyright 2016-2016 Forschungszentrum Juelich +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for npsmpi compiler toolchain (includes NVHPC and ParaStationMPI, and CUDA as dependency). + +@author: Damian Alvarez (Forschungszentrum Juelich) +""" + +from easybuild.toolchains.nvhpc import NvhpcToolchain +# We pull in MPI and CUDA at once so this maps nicely to HMNS +from easybuild.toolchains.mpi.psmpi import Psmpi +from easybuild.toolchains.compiler.cuda import Cuda + +# Order matters! +class Npsmpic(NvhpcToolchain, Cuda, Psmpi): + """Compiler toolchain with NVHPC and ParaStationMPI, with CUDA as dependency.""" + NAME = 'npsmpic' + SUBTOOLCHAIN = NvhpcToolchain.NAME From f3c05415fd0cf413c353273980ec4001d5c570d1 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Thu, 10 Jun 2021 17:19:17 +0200 Subject: [PATCH 03/91] update npsmpic toolchain --- easybuild/toolchains/npsmpic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/toolchains/npsmpic.py b/easybuild/toolchains/npsmpic.py index 33ee2bb06a..ed79ce2306 100644 --- a/easybuild/toolchains/npsmpic.py +++ b/easybuild/toolchains/npsmpic.py @@ -34,6 +34,7 @@ from easybuild.toolchains.mpi.psmpi import Psmpi from easybuild.toolchains.compiler.cuda import Cuda + # Order matters! class Npsmpic(NvhpcToolchain, Cuda, Psmpi): """Compiler toolchain with NVHPC and ParaStationMPI, with CUDA as dependency.""" From 419879e39e1c784e2de82036b2be6993d2bdbce7 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Fri, 11 Jun 2021 09:52:41 +0200 Subject: [PATCH 04/91] update nvpsmpic toolchain --- easybuild/toolchains/{npsmpic.py => nvpsmpic.py} | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) rename easybuild/toolchains/{npsmpic.py => nvpsmpic.py} (87%) diff --git a/easybuild/toolchains/npsmpic.py b/easybuild/toolchains/nvpsmpic.py similarity index 87% rename from easybuild/toolchains/npsmpic.py rename to easybuild/toolchains/nvpsmpic.py index ed79ce2306..de530f055c 100644 --- a/easybuild/toolchains/npsmpic.py +++ b/easybuild/toolchains/nvpsmpic.py @@ -1,6 +1,6 @@ ## -# Copyright 2016-2016 Ghent University -# Copyright 2016-2016 Forschungszentrum Juelich +# Copyright 2016-2021 Ghent University +# Copyright 2016-2021 Forschungszentrum Juelich # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -27,6 +27,7 @@ EasyBuild support for npsmpi compiler toolchain (includes NVHPC and ParaStationMPI, and CUDA as dependency). @author: Damian Alvarez (Forschungszentrum Juelich) +@author: Sebastian Achilles (Forschungszentrum Juelich) """ from easybuild.toolchains.nvhpc import NvhpcToolchain @@ -36,7 +37,7 @@ # Order matters! -class Npsmpic(NvhpcToolchain, Cuda, Psmpi): +class NVpsmpic(NvhpcToolchain, Cuda, Psmpi): """Compiler toolchain with NVHPC and ParaStationMPI, with CUDA as dependency.""" - NAME = 'npsmpic' + NAME = 'nvpsmpic' SUBTOOLCHAIN = NvhpcToolchain.NAME From 5415187fa37db2620e345540062c1794331250ef Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Mon, 28 Jun 2021 10:42:26 +0200 Subject: [PATCH 05/91] update nvpsmpic toolchain --- easybuild/toolchains/nvpsmpic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/toolchains/nvpsmpic.py b/easybuild/toolchains/nvpsmpic.py index de530f055c..538651076d 100644 --- a/easybuild/toolchains/nvpsmpic.py +++ b/easybuild/toolchains/nvpsmpic.py @@ -30,14 +30,14 @@ @author: Sebastian Achilles (Forschungszentrum Juelich) """ -from easybuild.toolchains.nvhpc import NvhpcToolchain +from easybuild.toolchains.nvhpc import NVHPCToolchain # We pull in MPI and CUDA at once so this maps nicely to HMNS from easybuild.toolchains.mpi.psmpi import Psmpi from easybuild.toolchains.compiler.cuda import Cuda # Order matters! -class NVpsmpic(NvhpcToolchain, Cuda, Psmpi): +class NVpsmpic(NVHPCToolchain, Cuda, Psmpi): """Compiler toolchain with NVHPC and ParaStationMPI, with CUDA as dependency.""" NAME = 'nvpsmpic' - SUBTOOLCHAIN = NvhpcToolchain.NAME + SUBTOOLCHAIN = NVHPCToolchain.NAME From 2bd3b537c62d40d5404fb7a92440f139f15c9cf5 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Mon, 28 Jun 2021 11:41:04 +0200 Subject: [PATCH 06/91] update nvpsmpic toolchain --- easybuild/toolchains/nvpsmpic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/toolchains/nvpsmpic.py b/easybuild/toolchains/nvpsmpic.py index 538651076d..e4120344c6 100644 --- a/easybuild/toolchains/nvpsmpic.py +++ b/easybuild/toolchains/nvpsmpic.py @@ -26,8 +26,8 @@ """ EasyBuild support for npsmpi compiler toolchain (includes NVHPC and ParaStationMPI, and CUDA as dependency). -@author: Damian Alvarez (Forschungszentrum Juelich) -@author: Sebastian Achilles (Forschungszentrum Juelich) +:author: Damian Alvarez (Forschungszentrum Juelich) +:author: Sebastian Achilles (Forschungszentrum Juelich) """ from easybuild.toolchains.nvhpc import NVHPCToolchain From 068bad130bd1194e6bc37e7144a09420505780e9 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Jul 2021 12:46:09 +0200 Subject: [PATCH 07/91] Make logdir writable also when --stop/--fetch is given --- easybuild/framework/easyblock.py | 31 +++++++++++++++++++++++-------- test/framework/toy_build.py | 13 +++++++++++-- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 015aa18691..415d66da40 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3690,8 +3690,11 @@ def build_and_install_one(ecdict, init_env): if os.path.exists(app.installdir) and build_option('read_only_installdir') and ( build_option('rebuild') or build_option('force')): + enabled_write_permissions = True # re-enable write permissions so we can install additional modules adjust_permissions(app.installdir, stat.S_IWUSR, add=True, recursive=True) + else: + enabled_write_permissions = False result = app.run_all_steps(run_test_cases=run_test_cases) @@ -3699,6 +3702,9 @@ def build_and_install_one(ecdict, init_env): # also add any extension easyblocks used during the build for reproducibility if app.ext_instances: copy_easyblocks_for_reprod(app.ext_instances, reprod_dir) + # If not already done remove the granted write permissions if we did so + if enabled_write_permissions and os.lstat(app.installdir)[stat.ST_MODE] & stat.S_IWUSR: + adjust_permissions(app.installdir, stat.S_IWUSR, add=False, recursive=True) except EasyBuildError as err: first_n = 300 @@ -3715,6 +3721,21 @@ def build_and_install_one(ecdict, init_env): # successful (non-dry-run) build if result and not dry_run: + def ensure_writable_log_dir(log_dir): + """Make sure we can write into the log dir""" + if build_option('read_only_installdir'): + # temporarily re-enable write permissions for copying log/easyconfig to install dir + if os.path.exists(log_dir): + adjust_permissions(log_dir, stat.S_IWUSR, add=True, recursive=True) + else: + parent_dir = os.path.dirname(log_dir) + if os.path.exists(parent_dir): + adjust_permissions(parent_dir, stat.S_IWUSR, add=True, recursive=False) + mkdir(log_dir, parents=True) + adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False) + else: + mkdir(log_dir, parents=True) + adjust_permissions(log_dir, stat.S_IWUSR, add=True, recursive=True) if app.cfg['stop']: ended = 'STOPPED' @@ -3722,6 +3743,7 @@ def build_and_install_one(ecdict, init_env): new_log_dir = os.path.join(app.builddir, config.log_path(ec=app.cfg)) else: new_log_dir = os.path.dirname(app.logfile) + ensure_writable_log_dir(new_log_dir) # if we're only running the sanity check, we should not copy anything new to the installation directory elif build_option('sanity_check_only'): @@ -3729,14 +3751,7 @@ def build_and_install_one(ecdict, init_env): else: new_log_dir = os.path.join(app.installdir, config.log_path(ec=app.cfg)) - if build_option('read_only_installdir'): - # temporarily re-enable write permissions for copying log/easyconfig to install dir - if os.path.exists(new_log_dir): - adjust_permissions(new_log_dir, stat.S_IWUSR, add=True, recursive=True) - else: - adjust_permissions(app.installdir, stat.S_IWUSR, add=True, recursive=False) - mkdir(new_log_dir, parents=True) - adjust_permissions(app.installdir, stat.S_IWUSR, add=False, recursive=False) + ensure_writable_log_dir(new_log_dir) # collect build stats _log.info("Collecting build stats...") diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 51d3f1a977..10c855aeaf 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -112,7 +112,7 @@ def check_toy(self, installpath, outtxt, version='0.0', versionprefix='', versio full_version = ''.join([versionprefix, version, versionsuffix]) # check for success - success = re.compile(r"COMPLETED: Installation ended successfully \(took .* secs?\)") + success = re.compile(r"COMPLETED: Installation (ended|STOPPED) successfully \(took .* secs?\)") self.assertTrue(success.search(outtxt), "COMPLETED message found in '%s" % outtxt) # if the module exists, it should be fine @@ -615,7 +615,16 @@ def test_toy_permissions_installdir(self): # 2. Existing build with --rebuild -> Reinstall and set read-only # 3. Existing build with --force -> Reinstall and set read-only # 4-5: Same as 2-3 but with --skip - for extra_args in ([], ['--rebuild'], ['--force'], ['--skip', '--rebuild'], ['--skip', '--force']): + # 6. Existing build with --fetch -> Test that logs can be written + test_cases = ( + [], + ['--rebuild'], + ['--force'], + ['--skip', '--rebuild'], + ['--skip', '--force'], + ['--rebuild', '--fetch'], + ) + for extra_args in test_cases: self.mock_stdout(True) self.test_toy_build(ec_file=test_ec, extra_args=['--read-only-installdir'] + extra_args, force=False) self.mock_stdout(False) From 65a7d91291bb97a64c05b0bbf30468d10c6becd4 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Jul 2021 14:46:42 +0200 Subject: [PATCH 08/91] Add __str__ method to EasyConfig We often print an easyconfig which is currently shown as "" This is not helpful at all. Instead show something like "GCC EasyConfig @ /tmp/GCC.eb" --- easybuild/framework/easyconfig/easyconfig.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 382a1c18bf..47be31903d 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -559,6 +559,13 @@ def disable_templating(self): finally: self.enable_templating = old_enable_templating + def __str__(self): + """Return a string representation of this EasyConfig instance""" + if self.path: + return '%s EasyConfig @ %s' % (self.name, self.path) + else: + return 'Raw %s EasyConfig' % self.name + def filename(self): """Determine correct filename for this easyconfig file.""" From 003b7f58624b724979f00ba6d752e990c0282ef4 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Jul 2021 14:52:56 +0200 Subject: [PATCH 09/91] Explicitely print for unset MODULEPATH --- easybuild/tools/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 4ee0cd8674..d8df2b0db2 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -801,7 +801,7 @@ def run_module(self, *args, **kwargs): else: args = list(args) - self.log.debug('Current MODULEPATH: %s' % os.environ.get('MODULEPATH', '')) + self.log.debug('Current MODULEPATH: %s' % os.environ.get('MODULEPATH', '')) # restore selected original environment variables before running module command environ = os.environ.copy() From 7a5b63fa1b9f2269400f2ceb634a7fbe3eb665d1 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Jul 2021 16:12:34 +0200 Subject: [PATCH 10/91] Print caught error for failed toy build --- test/framework/toy_build.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 10c855aeaf..cad57faaa7 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -106,21 +106,26 @@ def tearDown(self): if os.path.exists(self.dummylogfn): os.remove(self.dummylogfn) - def check_toy(self, installpath, outtxt, version='0.0', versionprefix='', versionsuffix=''): + def check_toy(self, installpath, outtxt, version='0.0', versionprefix='', versionsuffix='', error=None): """Check whether toy build succeeded.""" full_version = ''.join([versionprefix, version, versionsuffix]) + if error is not None: + error_msg = '\nNote: Caught error: %s' % error + else: + error_msg = '' + # check for success success = re.compile(r"COMPLETED: Installation (ended|STOPPED) successfully \(took .* secs?\)") - self.assertTrue(success.search(outtxt), "COMPLETED message found in '%s" % outtxt) + self.assertTrue(success.search(outtxt), "COMPLETED message found in '%s'%s" % (outtxt, error_msg)) # if the module exists, it should be fine toy_module = os.path.join(installpath, 'modules', 'all', 'toy', full_version) msg = "module for toy build toy/%s found (path %s)" % (full_version, toy_module) if get_module_syntax() == 'Lua': toy_module += '.lua' - self.assertTrue(os.path.exists(toy_module), msg) + self.assertTrue(os.path.exists(toy_module), msg + error_msg) # module file is symlinked according to moduleclass toy_module_symlink = os.path.join(installpath, 'modules', 'tools', 'toy', full_version) @@ -183,7 +188,7 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True raise myerr if verify: - self.check_toy(self.test_installpath, outtxt, versionsuffix=versionsuffix) + self.check_toy(self.test_installpath, outtxt, versionsuffix=versionsuffix, error=myerr) if test_readme: # make sure postinstallcmds were used From c866662b91739f11936928690eabadf8d68422b7 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Jul 2021 16:42:53 +0200 Subject: [PATCH 11/91] Pass modtool=None for --fetch Avoids failure with modules-env due to having no $MODULEPATH but an (otherwise inexistant) modules tool --- test/framework/utilities.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/framework/utilities.py b/test/framework/utilities.py index d36c9da53a..4a82aaf6a8 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -295,7 +295,13 @@ def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbos env_before = copy.deepcopy(os.environ) try: - main(args=args, logfile=logfile, do_build=do_build, testing=testing, modtool=self.modtool) + if '--fetch' in args: + # The config sets modules_tool to None if --fetch is specified, + # so do the same here to keep the behavior consistent + modtool = None + else: + modtool = self.modtool + main(args=args, logfile=logfile, do_build=do_build, testing=testing, modtool=modtool) except SystemExit as err: if raise_systemexit: raise err From 4e83468df97cafbd78cd13deffb3f7fa7897932b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 22 Jul 2021 16:11:04 +0200 Subject: [PATCH 12/91] Add has_recursive_symlinks function --- easybuild/tools/filetools.py | 20 ++++++++++++++++++ test/framework/filetools.py | 40 ++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 1cb65f826f..ef021455a2 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -44,6 +44,7 @@ import hashlib import imp import inspect +import itertools import os import re import shutil @@ -2340,6 +2341,25 @@ def copy_files(paths, target_path, force_in_dry_run=False, target_single_file=Fa raise EasyBuildError("One or more files to copy should be specified!") +def has_recursive_symlinks(path): + """ + Check the given directory for recursive symlinks. + + That means symlinks to folders inside the path which would cause infinite loops when traversed regularily. + + :param path: Path to directory to check + """ + for dirpath, dirnames, filenames in os.walk(path, followlinks=True): + for name in itertools.chain(dirnames, filenames): + fullpath = os.path.join(dirpath, name) + if os.path.islink(fullpath): + linkpath = os.path.realpath(fullpath) + fullpath += os.sep # To catch the case where both are equal + if fullpath.startswith(linkpath + os.sep): + return True + return False + + def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **kwargs): """ Copy a directory from specified location to specified location diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 444a6986e3..5d959eb091 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1790,6 +1790,46 @@ def test_copy_files(self): regex = re.compile("^copied 2 files to .*/target") self.assertTrue(regex.match(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_has_recursive_symlinks(self): + """Test has_recursive_symlinks function""" + test_folder = tempfile.mkdtemp() + self.assertFalse(ft.has_recursive_symlinks(test_folder)) + # Clasic Loop: Symlink to . + os.symlink('.', os.path.join(test_folder, 'self_link_dot')) + self.assertTrue(ft.has_recursive_symlinks(test_folder)) + # Symlink to self + test_folder = tempfile.mkdtemp() + os.symlink('self_link', os.path.join(test_folder, 'self_link')) + self.assertTrue(ft.has_recursive_symlinks(test_folder)) + # Symlink from 2 folders up + test_folder = tempfile.mkdtemp() + sub_folder = os.path.join(test_folder, 'sub1', 'sub2') + os.makedirs(sub_folder) + os.symlink(os.path.join('..', '..'), os.path.join(sub_folder, 'uplink')) + self.assertTrue(ft.has_recursive_symlinks(test_folder)) + # Non-issue: Symlink to sibling folders + test_folder = tempfile.mkdtemp() + sub_folder = os.path.join(test_folder, 'sub1', 'sub2') + os.makedirs(sub_folder) + sibling_folder = os.path.join(test_folder, 'sub1', 'sibling') + os.mkdir(sibling_folder) + os.symlink('sibling', os.path.join(test_folder, 'sub1', 'sibling_link')) + os.symlink(os.path.join('..', 'sibling'), os.path.join(test_folder, sub_folder, 'sibling_link')) + self.assertFalse(ft.has_recursive_symlinks(test_folder)) + # Tricky case: Sibling symlink to folder starting with the same name + os.mkdir(os.path.join(test_folder, 'sub11')) + os.symlink(os.path.join('..', 'sub11'), os.path.join(test_folder, 'sub1', 'trick_link')) + self.assertFalse(ft.has_recursive_symlinks(test_folder)) + # Symlink cycle: sub1/cycle_2 -> sub2, sub2/cycle_1 -> sub1, ... + test_folder = tempfile.mkdtemp() + sub_folder1 = os.path.join(test_folder, 'sub1') + sub_folder2 = sub_folder = os.path.join(test_folder, 'sub2') + os.mkdir(sub_folder1) + os.mkdir(sub_folder2) + os.symlink(os.path.join('..', 'sub2'), os.path.join(sub_folder1, 'cycle_1')) + os.symlink(os.path.join('..', 'sub1'), os.path.join(sub_folder2, 'cycle_2')) + self.assertTrue(ft.has_recursive_symlinks(test_folder)) + def test_copy_dir(self): """Test copy_dir function.""" testdir = os.path.dirname(os.path.abspath(__file__)) From 9fd71570f18da696b2b7185ecbfcaf7a588dedf5 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 23 Jul 2021 14:05:47 +0200 Subject: [PATCH 13/91] Check for recursive symlinks by default before copying a folder Avoids endless loops and creating infinite length paths when `symlinks=True` is NOT also passed to copy_dir --- easybuild/tools/filetools.py | 16 ++++++++++++++-- test/framework/filetools.py | 9 +++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index ef021455a2..28d825709b 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2356,11 +2356,13 @@ def has_recursive_symlinks(path): linkpath = os.path.realpath(fullpath) fullpath += os.sep # To catch the case where both are equal if fullpath.startswith(linkpath + os.sep): + _log.info("Recursive symlink detected at %s", fullpath) return True return False -def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **kwargs): +def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, check_for_recursive_symlinks=True, + **kwargs): """ Copy a directory from specified location to specified location @@ -2368,6 +2370,7 @@ def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **k :param target_path: path to copy the directory to :param force_in_dry_run: force running the command during dry run :param dirs_exist_ok: boolean indicating whether it's OK if the target directory already exists + :param check_for_recursive_symlinks: If symlink arg is not given or False check for recursive symlinks first shutil.copytree is used if the target path does not exist yet; if the target path already exists, the 'copy' function will be used to copy the contents of @@ -2379,6 +2382,13 @@ def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **k dry_run_msg("copied directory %s to %s" % (path, target_path)) else: try: + if check_for_recursive_symlinks and not kwargs.get('symlinks'): + if has_recursive_symlinks(path): + raise EasyBuildError("Recursive symlinks detected in %s. " + "Will not try copying this unless `symlinks=True` is passed", + path) + else: + _log.debug("No recursive symlinks in %s", path) if not dirs_exist_ok and os.path.exists(target_path): raise EasyBuildError("Target location %s to copy %s to already exists", target_path, path) @@ -2406,7 +2416,9 @@ def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **k paths_to_copy = [os.path.join(path, x) for x in entries] copy(paths_to_copy, target_path, - force_in_dry_run=force_in_dry_run, dirs_exist_ok=dirs_exist_ok, **kwargs) + force_in_dry_run=force_in_dry_run, dirs_exist_ok=dirs_exist_ok, + check_for_recursive_symlinks=False, # Don't check again + **kwargs) else: # if dirs_exist_ok is not enabled or target directory doesn't exist, just use shutil.copytree diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 5d959eb091..6e5d0748a6 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1901,6 +1901,15 @@ def ignore_func(_, names): ft.mkdir(subdir) ft.copy_dir(srcdir, target_dir, symlinks=True, dirs_exist_ok=True) + # Detect recursive symlinks by default instead of infinite loop during copy + ft.remove_dir(target_dir) + os.symlink('.', os.path.join(subdir, 'recursive_link')) + self.assertErrorRegex(EasyBuildError, 'Recursive symlinks detected', ft.copy_dir, srcdir, target_dir) + self.assertFalse(os.path.exists(target_dir)) + # Ok for symlinks=True + ft.copy_dir(srcdir, target_dir, symlinks=True) + self.assertTrue(os.path.exists(target_dir)) + # also test behaviour of copy_file under --dry-run build_options = { 'extended_dry_run': True, From 1ae90cccc24077d32878ed9aecfec4905f29c25d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 28 Jul 2021 09:38:18 +0200 Subject: [PATCH 14/91] Don't parse patch files as EasyConfigs when searching for patch usage Exclude *.patch files during search Remove C&P bug that needlessly reads the file a third time --- easybuild/tools/github.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 8259de1587..750f77163b 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1073,7 +1073,8 @@ def find_software_name_for_patch(patch_name, ec_dirs): for ec_dir in ec_dirs: for (dirpath, _, filenames) in os.walk(ec_dir): for fn in filenames: - if fn != 'TEMPLATE.eb' and not fn.endswith('.py'): + # TODO: In EasyBuild 5.x only check for '*.eb' files + if fn != 'TEMPLATE.eb' and os.path.splitext(fn) not in ('.py', '.patch'): path = os.path.join(dirpath, fn) rawtxt = read_file(path) if 'patches' in rawtxt: @@ -1083,7 +1084,6 @@ def find_software_name_for_patch(patch_name, ec_dirs): for idx, path in enumerate(all_ecs): if soft_name: break - rawtxt = read_file(path) try: ecs = process_easyconfig(path, validate=False) for ec in ecs: From ffcd3715a6a980d30d22fef1bc8cc5400d3d499e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 28 Jul 2021 10:04:23 +0200 Subject: [PATCH 15/91] Speed up EC-for-patch search by smart sorting ECs --- easybuild/tools/github.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 750f77163b..dab4b5c522 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1080,6 +1080,24 @@ def find_software_name_for_patch(patch_name, ec_dirs): if 'patches' in rawtxt: all_ecs.append(path) + # Usual patch names are -_fix_foo.patch + # So search those ECs first + patch_stem = os.path.splitext(patch_name)[0] + # Extract possible sw name and version according to above scheme + # Those might be the same as the whole patch stem, which is OK + possible_sw_name = patch_stem.split('-')[0].lower() + possible_sw_name_version = patch_stem.split('_')[0].lower() + + def ec_key(path): + filename = os.path.basename(path).lower() + # Put files with one of those as the prefix first, then sort by name + return ( + not filename.startswith(possible_sw_name_version), + not filename.startswith(possible_sw_name), + filename + ) + all_ecs.sort(key=ec_key) + nr_of_ecs = len(all_ecs) for idx, path in enumerate(all_ecs): if soft_name: From f6c16190b175f07ccf6fb1350970e370df042894 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 19 Jul 2021 16:23:12 +0200 Subject: [PATCH 16/91] Add reproducer test for #3779 --- easybuild/tools/include.py | 4 ++-- test/framework/options.py | 48 +++++++++++++++++++++++++++++++------- 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py index 34390d632a..7de349e5b3 100644 --- a/easybuild/tools/include.py +++ b/easybuild/tools/include.py @@ -180,8 +180,8 @@ def include_easyblocks(tmpdir, paths): if not os.path.exists(target_path): symlink(easyblock_module, target_path) - included_ebs = [x for x in os.listdir(easyblocks_dir) if x not in ['__init__.py', 'generic']] - included_generic_ebs = [x for x in os.listdir(os.path.join(easyblocks_dir, 'generic')) if x != '__init__.py'] + included_ebs = sorted(x for x in os.listdir(easyblocks_dir) if x not in ['__init__.py', 'generic']) + included_generic_ebs = sorted(x for x in os.listdir(os.path.join(easyblocks_dir, 'generic')) if x != '__init__.py') _log.debug("Included generic easyblocks: %s", included_generic_ebs) _log.debug("Included software-specific easyblocks: %s", included_ebs) diff --git a/test/framework/options.py b/test/framework/options.py index b001b0546e..e2a017823e 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -34,6 +34,7 @@ import stat import sys import tempfile +import textwrap from distutils.version import LooseVersion from unittest import TextTestRunner @@ -3232,13 +3233,29 @@ def test_xxx_include_easyblocks(self): sys.modules[pkg].__path__.remove(path) # include extra test easyblocks - foo_txt = '\n'.join([ - 'from easybuild.framework.easyblock import EasyBlock', - 'class EB_foo(EasyBlock):', - ' pass', - '' - ]) + # Make them inherit from each other to trigger a known issue with changed imports, see #3779 + # Choose naming so that order of naming is different than inheritance order + afoo_txt = textwrap.dedent(""" + from easybuild.framework.easyblock import EasyBlock + class EB_afoo(EasyBlock): + def __init__(self, *args, **kwargs): + super(EB_afoo, self).__init__(*args, **kwargs) + """) + write_file(os.path.join(self.test_prefix, 'afoo.py'), afoo_txt) + foo_txt = textwrap.dedent(""" + from easybuild.easyblocks.zfoo import EB_zfoo + class EB_foo(EB_zfoo): + def __init__(self, *args, **kwargs): + super(EB_foo, self).__init__(*args, **kwargs) + """) write_file(os.path.join(self.test_prefix, 'foo.py'), foo_txt) + zfoo_txt = textwrap.dedent(""" + from easybuild.easyblocks.afoo import EB_afoo + class EB_zfoo(EB_afoo): + def __init__(self, *args, **kwargs): + super(EB_zfoo, self).__init__(*args, **kwargs) + """) + write_file(os.path.join(self.test_prefix, 'zfoo.py'), zfoo_txt) # clear log write_file(self.logfile, '') @@ -3256,11 +3273,26 @@ def test_xxx_include_easyblocks(self): foo_regex = re.compile(r"^\|-- EB_foo \(easybuild.easyblocks.foo @ %s\)" % path_pattern, re.M) self.assertTrue(foo_regex.search(logtxt), "Pattern '%s' found in: %s" % (foo_regex.pattern, logtxt)) + ec_txt = '\n'.join([ + 'easyblock = "EB_foo"', + 'name = "pi"', + 'version = "3.14"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = SYSTEM', + ]) + ec = EasyConfig(path=None, rawtxt=ec_txt) + # easyblock is found via get_easyblock_class - klass = get_easyblock_class('EB_foo') - self.assertTrue(issubclass(klass, EasyBlock), "%s is an EasyBlock derivative class" % klass) + for name in ('EB_afoo', 'EB_foo', 'EB_zfoo'): + klass = get_easyblock_class(name) + self.assertTrue(issubclass(klass, EasyBlock), "%s (%s) is an EasyBlock derivative class" % (klass, name)) + + eb_inst = klass(ec) + self.assertTrue(eb_inst is not None, "Instantiating the injected class %s works" % name) # 'undo' import of foo easyblock + del sys.modules['easybuild.easyblocks.afoo'] del sys.modules['easybuild.easyblocks.foo'] # must be run after test for --list-easyblocks, hence the '_xxx_' From 993d43e02cd42d0d1d4c8de19074d60ed47c363b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 19 Jul 2021 16:26:59 +0200 Subject: [PATCH 17/91] Delete all modules to be imported first before importing them one by one Avoids errors on on Python2 when we delete a subclass after importing a module with a class depending on it. Fixes #3779 --- easybuild/tools/include.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py index 7de349e5b3..31aecb9997 100644 --- a/easybuild/tools/include.py +++ b/easybuild/tools/include.py @@ -128,15 +128,17 @@ def set_up_eb_package(parent_path, eb_pkg_name, subpkgs=None, pkg_init_body=None def verify_imports(pymods, pypkg, from_path): """Verify that import of specified modules from specified package and expected location works.""" - for pymod in pymods: - pymod_spec = '%s.%s' % (pypkg, pymod) - + pymod_specs = ['%s.%s' % (pypkg, pymod) for pymod in pymods] + for pymod_spec in pymod_specs: # force re-import if the specified modules was already imported; # this is required to ensure that an easyblock that is included via --include-easyblocks-from-pr # gets preference over one that is included via --include-easyblocks if pymod_spec in sys.modules: del sys.modules[pymod_spec] + # After all modules to be reloaded have been removed, import them again + # Note that removing them here may delete transitively loaded modules and not import them again + for pymod_spec in pymod_specs: try: pymod = __import__(pymod_spec, fromlist=[pypkg]) # different types of exceptions may be thrown, not only ImportErrors From c61efb438d6ee3a43bd30919e73e80d656ebdada Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 22 Jul 2021 09:00:40 +0200 Subject: [PATCH 18/91] Delete all imported easyblocks from test --- test/framework/options.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index e2a017823e..f40829b34b 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3291,9 +3291,9 @@ def __init__(self, *args, **kwargs): eb_inst = klass(ec) self.assertTrue(eb_inst is not None, "Instantiating the injected class %s works" % name) - # 'undo' import of foo easyblock - del sys.modules['easybuild.easyblocks.afoo'] - del sys.modules['easybuild.easyblocks.foo'] + # 'undo' import of the easyblocks + for name in ('afoo', 'foo', 'zfoo'): + del sys.modules['easybuild.easyblocks.' + name] # must be run after test for --list-easyblocks, hence the '_xxx_' # cleaning up the imported easyblocks is quite difficult... From 07b671aa4227de2f19674d6e460d479ab96b07a6 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Aug 2021 12:37:47 +0200 Subject: [PATCH 19/91] Clone with tags as the explicit target Avoids accidentally cloning a branch with the same name as that would take preference for the --branch option of git clone --- easybuild/tools/filetools.py | 2 +- test/framework/filetools.py | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 1cb65f826f..7c1e201674 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2472,7 +2472,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): clone_cmd = ['git', 'clone'] if tag: - clone_cmd.extend(['--branch', tag]) + clone_cmd.extend(['--branch', 'refs/tags/' + tag]) if recursive: clone_cmd.append('--recursive') diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 444a6986e3..58cfbb3c6a 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2491,16 +2491,22 @@ def test_get_source_tarball_from_git(self): git_config = { 'repo_name': 'testrepository', 'url': 'https://github.com/easybuilders', - 'tag': 'main', + 'tag': 'tag_for_tests', } target_dir = os.path.join(self.test_prefix, 'target') try: ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) # (only) tarball is created in specified target dir - self.assertTrue(os.path.isfile(os.path.join(target_dir, 'test.tar.gz'))) + test_file = os.path.join(target_dir, 'test.tar.gz') + self.assertTrue(os.path.isfile(test_file)) self.assertEqual(os.listdir(target_dir), ['test.tar.gz']) + # Check that we indeed downloaded the tag and not a branch + extracted_dir = tempfile.mkdtemp(prefix='extracted_dir') + target_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False) + self.assertTrue(os.path.isfile(os.path.join(target_dir, 'this-is-a-tag.txt'))) + del git_config['tag'] git_config['commit'] = '8456f86' ft.get_source_tarball_from_git('test2.tar.gz', target_dir, git_config) @@ -2516,7 +2522,7 @@ def test_get_source_tarball_from_git(self): git_config = { 'repo_name': 'testrepository', 'url': 'git@github.com:easybuilders', - 'tag': 'master', + 'tag': 'tag_for_tests', } args = ['test.tar.gz', self.test_prefix, git_config] @@ -2570,10 +2576,10 @@ def run_check(): git_config = { 'repo_name': 'testrepository', 'url': 'git@github.com:easybuilders', - 'tag': 'master', + 'tag': 'tag_for_tests', } expected = '\n'.join([ - r' running command "git clone --branch master git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --branch refs/tags/tag_for_tests git@github.com:easybuilders/testrepository.git"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", @@ -2582,7 +2588,7 @@ def run_check(): git_config['recursive'] = True expected = '\n'.join([ - r' running command "git clone --branch master --recursive git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --branch refs/tags/tag_for_tests --recursive git@github.com:easybuilders/testrepository.git"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", @@ -2591,7 +2597,7 @@ def run_check(): git_config['keep_git_dir'] = True expected = '\n'.join([ - r' running command "git clone --branch master --recursive git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --branch refs/tags/tag_for_tests --recursive git@github.com:easybuilders/testrepository.git"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz testrepository"', r" \(in .*/tmp.*\)", From 956fa9c49a7572fdafc810a2f48262bc7eeacc19 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Aug 2021 12:46:00 +0200 Subject: [PATCH 20/91] Speed up git checkouts Use shallow checkouts if .git folder is not required Don't download submodules if we do that again for a potentially other version --- easybuild/tools/filetools.py | 12 +++++++++--- test/framework/filetools.py | 8 ++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 7c1e201674..dfd2012e56 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2471,11 +2471,17 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # compose 'git clone' command, and run it clone_cmd = ['git', 'clone'] + if not keep_git_dir: + # Speed up cloning by only fetching the most recent commit, not the whole history + # When we don't want to keep the .git folder there won't be a difference in the result + clone_cmd.extend(['--depth', '1']) + if tag: clone_cmd.extend(['--branch', 'refs/tags/' + tag]) - - if recursive: - clone_cmd.append('--recursive') + if recursive: + clone_cmd.append('--recursive') + else: + clone_cmd.append('--no-checkout') # We do that manually below clone_cmd.append('%s/%s.git' % (url, repo_name)) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 58cfbb3c6a..6e4a9c06e5 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2579,7 +2579,7 @@ def run_check(): 'tag': 'tag_for_tests', } expected = '\n'.join([ - r' running command "git clone --branch refs/tags/tag_for_tests git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --depth 1 --branch refs/tags/tag_for_tests git@github.com:easybuilders/testrepository.git"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", @@ -2588,7 +2588,7 @@ def run_check(): git_config['recursive'] = True expected = '\n'.join([ - r' running command "git clone --branch refs/tags/tag_for_tests --recursive git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --depth 1 --branch refs/tags/tag_for_tests --recursive git@github.com:easybuilders/testrepository.git"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", @@ -2608,7 +2608,7 @@ def run_check(): del git_config['tag'] git_config['commit'] = '8456f86' expected = '\n'.join([ - r' running command "git clone --recursive git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --depth 1 --no-checkout git@github.com:easybuilders/testrepository.git"', r" \(in .*/tmp.*\)", r' running command "git checkout 8456f86 && git submodule update --init --recursive"', r" \(in testrepository\)", @@ -2619,7 +2619,7 @@ def run_check(): del git_config['recursive'] expected = '\n'.join([ - r' running command "git clone git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --depth 1 --no-checkout git@github.com:easybuilders/testrepository.git"', r" \(in .*/tmp.*\)", r' running command "git checkout 8456f86"', r" \(in testrepository\)", From d1fa2a4669112c4dcc86450ba1c2394668b3525e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Aug 2021 12:50:12 +0200 Subject: [PATCH 21/91] Slightly enhance test Check for return value of get_source_tarball_from_git and use context manager --- test/framework/filetools.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 6e4a9c06e5..41c88b0afb 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2496,9 +2496,10 @@ def test_get_source_tarball_from_git(self): target_dir = os.path.join(self.test_prefix, 'target') try: - ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) + res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) # (only) tarball is created in specified target dir test_file = os.path.join(target_dir, 'test.tar.gz') + self.assertEqual(res, test_file) self.assertTrue(os.path.isfile(test_file)) self.assertEqual(os.listdir(target_dir), ['test.tar.gz']) @@ -2509,8 +2510,10 @@ def test_get_source_tarball_from_git(self): del git_config['tag'] git_config['commit'] = '8456f86' - ft.get_source_tarball_from_git('test2.tar.gz', target_dir, git_config) - self.assertTrue(os.path.isfile(os.path.join(target_dir, 'test2.tar.gz'))) + res = ft.get_source_tarball_from_git('test2.tar.gz', target_dir, git_config) + test_file = os.path.join(target_dir, 'test2.tar.gz') + self.assertEqual(res, test_file) + self.assertTrue(os.path.isfile(test_file)) self.assertEqual(sorted(os.listdir(target_dir)), ['test.tar.gz', 'test2.tar.gz']) except EasyBuildError as err: @@ -2559,13 +2562,10 @@ def test_get_source_tarball_from_git(self): def run_check(): """Helper function to run get_source_tarball_from_git & check dry run output""" - self.mock_stdout(True) - self.mock_stderr(True) - res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) - stdout = self.get_stdout() - stderr = self.get_stderr() - self.mock_stdout(False) - self.mock_stderr(False) + with self.mocked_stdout_stderr(): + res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) + stdout = self.get_stdout() + stderr = self.get_stderr() self.assertEqual(stderr, '') regex = re.compile(expected) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) From 15799c0e33bb4a4629dca4baeeceec41a5285f5d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Aug 2021 15:16:57 +0200 Subject: [PATCH 22/91] Move dry-run tests before real tests --- test/framework/filetools.py | 130 +++++++++++++++++++----------------- 1 file changed, 67 insertions(+), 63 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 41c88b0afb..245eab6981 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2488,71 +2488,8 @@ def test_diff_files(self): def test_get_source_tarball_from_git(self): """Test get_source_tarball_from_git function.""" - git_config = { - 'repo_name': 'testrepository', - 'url': 'https://github.com/easybuilders', - 'tag': 'tag_for_tests', - } target_dir = os.path.join(self.test_prefix, 'target') - try: - res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) - # (only) tarball is created in specified target dir - test_file = os.path.join(target_dir, 'test.tar.gz') - self.assertEqual(res, test_file) - self.assertTrue(os.path.isfile(test_file)) - self.assertEqual(os.listdir(target_dir), ['test.tar.gz']) - - # Check that we indeed downloaded the tag and not a branch - extracted_dir = tempfile.mkdtemp(prefix='extracted_dir') - target_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False) - self.assertTrue(os.path.isfile(os.path.join(target_dir, 'this-is-a-tag.txt'))) - - del git_config['tag'] - git_config['commit'] = '8456f86' - res = ft.get_source_tarball_from_git('test2.tar.gz', target_dir, git_config) - test_file = os.path.join(target_dir, 'test2.tar.gz') - self.assertEqual(res, test_file) - self.assertTrue(os.path.isfile(test_file)) - self.assertEqual(sorted(os.listdir(target_dir)), ['test.tar.gz', 'test2.tar.gz']) - - except EasyBuildError as err: - if "Network is down" in str(err): - print("Ignoring download error in test_get_source_tarball_from_git, working offline?") - else: - raise err - - git_config = { - 'repo_name': 'testrepository', - 'url': 'git@github.com:easybuilders', - 'tag': 'tag_for_tests', - } - args = ['test.tar.gz', self.test_prefix, git_config] - - for key in ['repo_name', 'url', 'tag']: - orig_value = git_config.pop(key) - if key == 'tag': - error_pattern = "Neither tag nor commit found in git_config parameter" - else: - error_pattern = "%s not specified in git_config parameter" % key - self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) - git_config[key] = orig_value - - git_config['commit'] = '8456f86' - error_pattern = "Tag and commit are mutually exclusive in git_config parameter" - self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) - del git_config['commit'] - - git_config['unknown'] = 'foobar' - error_pattern = "Found one or more unexpected keys in 'git_config' specification" - self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) - del git_config['unknown'] - - args[0] = 'test.txt' - error_pattern = "git_config currently only supports filename ending in .tar.gz" - self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) - args[0] = 'test.tar.gz' - # only test in dry run mode, i.e. check which commands would be executed without actually running them build_options = { 'extended_dry_run': True, @@ -2628,6 +2565,73 @@ def run_check(): ]) run_check() + # Test with real data + init_config() + git_config = { + 'repo_name': 'testrepository', + 'url': 'https://github.com/easybuilders', + 'tag': 'tag_for_tests', + } + + try: + res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) + # (only) tarball is created in specified target dir + test_file = os.path.join(target_dir, 'test.tar.gz') + self.assertEqual(res, test_file) + self.assertTrue(os.path.isfile(test_file)) + self.assertEqual(os.listdir(target_dir), ['test.tar.gz']) + + # Check that we indeed downloaded the tag and not a branch + extracted_dir = tempfile.mkdtemp(prefix='extracted_dir') + target_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False) + self.assertTrue(os.path.isfile(os.path.join(target_dir, 'this-is-a-tag.txt'))) + + del git_config['tag'] + git_config['commit'] = '8456f86' + res = ft.get_source_tarball_from_git('test2.tar.gz', target_dir, git_config) + test_file = os.path.join(target_dir, 'test2.tar.gz') + self.assertEqual(res, test_file) + self.assertTrue(os.path.isfile(test_file)) + self.assertEqual(sorted(os.listdir(target_dir)), ['test.tar.gz', 'test2.tar.gz']) + + except EasyBuildError as err: + if "Network is down" in str(err): + print("Ignoring download error in test_get_source_tarball_from_git, working offline?") + else: + raise err + + git_config = { + 'repo_name': 'testrepository', + 'url': 'git@github.com:easybuilders', + 'tag': 'tag_for_tests', + } + args = ['test.tar.gz', self.test_prefix, git_config] + + for key in ['repo_name', 'url', 'tag']: + orig_value = git_config.pop(key) + if key == 'tag': + error_pattern = "Neither tag nor commit found in git_config parameter" + else: + error_pattern = "%s not specified in git_config parameter" % key + self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) + git_config[key] = orig_value + + git_config['commit'] = '8456f86' + error_pattern = "Tag and commit are mutually exclusive in git_config parameter" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) + del git_config['commit'] + + git_config['unknown'] = 'foobar' + error_pattern = "Found one or more unexpected keys in 'git_config' specification" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) + del git_config['unknown'] + + args[0] = 'test.txt' + error_pattern = "git_config currently only supports filename ending in .tar.gz" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) + args[0] = 'test.tar.gz' + + def test_is_sha256_checksum(self): """Test for is_sha256_checksum function.""" a_sha256_checksum = '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc' From 95e05c88d3089b48225d73bc036dd24d366b45cb Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Aug 2021 15:22:37 +0200 Subject: [PATCH 23/91] Fix line length --- test/framework/filetools.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 245eab6981..542a7c91e2 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2515,54 +2515,55 @@ def run_check(): 'url': 'git@github.com:easybuilders', 'tag': 'tag_for_tests', } + git_repo = {'git_repo': 'git@github.com:easybuilders/testrepository.git'} # Just to make the below shorter expected = '\n'.join([ - r' running command "git clone --depth 1 --branch refs/tags/tag_for_tests git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --depth 1 --branch refs/tags/tag_for_tests %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", - ]) + ]) % git_repo run_check() git_config['recursive'] = True expected = '\n'.join([ - r' running command "git clone --depth 1 --branch refs/tags/tag_for_tests --recursive git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --depth 1 --branch refs/tags/tag_for_tests --recursive %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", - ]) + ]) % git_repo run_check() git_config['keep_git_dir'] = True expected = '\n'.join([ - r' running command "git clone --branch refs/tags/tag_for_tests --recursive git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --branch refs/tags/tag_for_tests --recursive %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz testrepository"', r" \(in .*/tmp.*\)", - ]) + ]) % git_repo run_check() del git_config['keep_git_dir'] del git_config['tag'] git_config['commit'] = '8456f86' expected = '\n'.join([ - r' running command "git clone --depth 1 --no-checkout git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --depth 1 --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "git checkout 8456f86 && git submodule update --init --recursive"', r" \(in testrepository\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", - ]) + ]) % git_repo run_check() del git_config['recursive'] expected = '\n'.join([ - r' running command "git clone --depth 1 --no-checkout git@github.com:easybuilders/testrepository.git"', + r' running command "git clone --depth 1 --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "git checkout 8456f86"', r" \(in testrepository\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", - ]) + ]) % git_repo run_check() # Test with real data @@ -2631,7 +2632,6 @@ def run_check(): self.assertErrorRegex(EasyBuildError, error_pattern, ft.get_source_tarball_from_git, *args) args[0] = 'test.tar.gz' - def test_is_sha256_checksum(self): """Test for is_sha256_checksum function.""" a_sha256_checksum = '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc' From fafaf309a082fe9baf01d4006fd8034acef1f842 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Aug 2021 17:16:25 +0200 Subject: [PATCH 24/91] Fix the cloning of tags Can't use refs/tags/xxx for git clone so clone it assuming it is a tag and check afterwards. Fall back to fetching the full history and checking out the tag and submodules manually --- easybuild/tools/filetools.py | 27 +++++++++++++++++++++++---- test/framework/filetools.py | 26 ++++++++++++++++++-------- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index dfd2012e56..df23ec331d 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2477,7 +2477,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): clone_cmd.extend(['--depth', '1']) if tag: - clone_cmd.extend(['--branch', 'refs/tags/' + tag]) + clone_cmd.extend(['--branch', tag]) if recursive: clone_cmd.append('--recursive') else: @@ -2487,7 +2487,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): tmpdir = tempfile.mkdtemp() cwd = change_dir(tmpdir) - run.run_cmd(' '.join(clone_cmd), log_all=True, log_ok=False, simple=False, regexp=False) + run.run_cmd(' '.join(clone_cmd), log_all=True, simple=True, regexp=False) # if a specific commit is asked for, check it out if commit: @@ -2495,14 +2495,33 @@ def get_source_tarball_from_git(filename, targetdir, git_config): if recursive: checkout_cmd.extend(['&&', 'git', 'submodule', 'update', '--init', '--recursive']) - run.run_cmd(' '.join(checkout_cmd), log_all=True, log_ok=False, simple=False, regexp=False, path=repo_name) + run.run_cmd(' '.join(checkout_cmd), log_all=True, simple=True, regexp=False, path=repo_name) + elif not build_option('extended_dry_run'): + # If we wanted to get a tag make sure we actually got a tag and not a branch with the same name + # This doesn't make sense in dry-run mode as we don't have anything to check + cmd = 'git describe --exact-match --tags HEAD' + # Note: Disable logging to also disable the error handling in run_cmd + (out, ec) = run.run_cmd(cmd, log_ok=False, log_all=False, regexp=False, path=repo_name) + if ec != 0 or tag not in out.splitlines(): + cmds = [] + if not keep_git_dir: + # Make the repo unshallow, same as git fetch --unshallow in git 1.8.3+ + # The first fetch seemingly does nothing, no idea why. + cmds.append('git fetch --depth=2147483647 && git fetch --depth=2147483647') + cmds.append('git checkout refs/tags/' + tag) + # Clean all untracked files, e.g. from left-over submodules + cmds.append('git clean --force -d -x') + if recursive: + cmds.append('git submodule update --init --recursive') + for cmd in cmds: + run.run_cmd(cmd, log_all=True, simple=True, regexp=False, path=repo_name) # create an archive and delete the git repo directory if keep_git_dir: tar_cmd = ['tar', 'cfvz', targetpath, repo_name] else: tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name] - run.run_cmd(' '.join(tar_cmd), log_all=True, log_ok=False, simple=False, regexp=False) + run.run_cmd(' '.join(tar_cmd), log_all=True, simple=True, regexp=False) # cleanup (repo_name dir does not exist in dry run mode) change_dir(cwd) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 542a7c91e2..722fbf54da 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2517,7 +2517,7 @@ def run_check(): } git_repo = {'git_repo': 'git@github.com:easybuilders/testrepository.git'} # Just to make the below shorter expected = '\n'.join([ - r' running command "git clone --depth 1 --branch refs/tags/tag_for_tests %(git_repo)s"', + r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", @@ -2526,7 +2526,7 @@ def run_check(): git_config['recursive'] = True expected = '\n'.join([ - r' running command "git clone --depth 1 --branch refs/tags/tag_for_tests --recursive %(git_repo)s"', + r' running command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", @@ -2535,7 +2535,7 @@ def run_check(): git_config['keep_git_dir'] = True expected = '\n'.join([ - r' running command "git clone --branch refs/tags/tag_for_tests --recursive %(git_repo)s"', + r' running command "git clone --branch tag_for_tests --recursive %(git_repo)s"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz testrepository"', r" \(in .*/tmp.*\)", @@ -2566,12 +2566,12 @@ def run_check(): ]) % git_repo run_check() - # Test with real data + # Test with real data. init_config() git_config = { 'repo_name': 'testrepository', 'url': 'https://github.com/easybuilders', - 'tag': 'tag_for_tests', + 'tag': 'branch_tag_for_test', } try: @@ -2581,11 +2581,21 @@ def run_check(): self.assertEqual(res, test_file) self.assertTrue(os.path.isfile(test_file)) self.assertEqual(os.listdir(target_dir), ['test.tar.gz']) + # Check that we indeed downloaded the right tag + extracted_dir = tempfile.mkdtemp(prefix='extracted_dir') + extracted_repo_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False) + self.assertTrue(os.path.isfile(os.path.join(extracted_repo_dir, 'this-is-a-branch.txt'))) + os.remove(test_file) - # Check that we indeed downloaded the tag and not a branch + # use a tag that clashes with a branch name and make sure this is handled correctly + git_config['tag'] = 'tag_for_tests' + res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) + self.assertEqual(res, test_file) + self.assertTrue(os.path.isfile(test_file)) + # Check that we indeed downloaded the tag and not the branch extracted_dir = tempfile.mkdtemp(prefix='extracted_dir') - target_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False) - self.assertTrue(os.path.isfile(os.path.join(target_dir, 'this-is-a-tag.txt'))) + extracted_repo_dir = ft.extract_file(test_file, extracted_dir, change_into_dir=False) + self.assertTrue(os.path.isfile(os.path.join(extracted_repo_dir, 'this-is-a-tag.txt'))) del git_config['tag'] git_config['commit'] = '8456f86' From c352a9a588c964fd73e557c37f8584f77937886f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Aug 2021 17:36:40 +0200 Subject: [PATCH 25/91] Print a warning if the slow path is entered --- easybuild/tools/filetools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index df23ec331d..983abdcb76 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2503,6 +2503,9 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # Note: Disable logging to also disable the error handling in run_cmd (out, ec) = run.run_cmd(cmd, log_ok=False, log_all=False, regexp=False, path=repo_name) if ec != 0 or tag not in out.splitlines(): + print_warning('Tag %s was not downloaded in the first try due to %s/%s containing a branch' + ' with the same name. You might want to alert the maintainers of %s about that issue.', + tag, url, repo_name, repo_name) cmds = [] if not keep_git_dir: # Make the repo unshallow, same as git fetch --unshallow in git 1.8.3+ From 6276d912e188c2ab35238eba3040e9c3493e39d3 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Aug 2021 17:40:29 +0200 Subject: [PATCH 26/91] Fix getting extension from a file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mikael Öhman --- easybuild/tools/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index dab4b5c522..8c261c65df 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1074,7 +1074,7 @@ def find_software_name_for_patch(patch_name, ec_dirs): for (dirpath, _, filenames) in os.walk(ec_dir): for fn in filenames: # TODO: In EasyBuild 5.x only check for '*.eb' files - if fn != 'TEMPLATE.eb' and os.path.splitext(fn) not in ('.py', '.patch'): + if fn != 'TEMPLATE.eb' and os.path.splitext(fn)[1] not in ('.py', '.patch'): path = os.path.join(dirpath, fn) rawtxt = read_file(path) if 'patches' in rawtxt: From fa60ab455c89bb60cb03b19147b6795f5cc77fc9 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Aug 2021 18:19:28 +0200 Subject: [PATCH 27/91] Don't search hidden folders for ECs --- easybuild/tools/github.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 8c261c65df..c9e5e3d623 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -839,7 +839,7 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ # copy easyconfig files to right place target_dir = os.path.join(git_working_dir, pr_target_repo) print_msg("copying files to %s..." % target_dir) - file_info = COPY_FUNCTIONS[pr_target_repo](ec_paths, os.path.join(git_working_dir, pr_target_repo)) + file_info = COPY_FUNCTIONS[pr_target_repo](ec_paths, target_dir) # figure out commit message to use if commit_msg: @@ -1071,7 +1071,9 @@ def find_software_name_for_patch(patch_name, ec_dirs): all_ecs = [] for ec_dir in ec_dirs: - for (dirpath, _, filenames) in os.walk(ec_dir): + for (dirpath, dirnames, filenames) in os.walk(ec_dir): + # Don't visit any hidden folders, such as .git + dirnames[:] = [i for i in dirnames if not i.startswith('.')] for fn in filenames: # TODO: In EasyBuild 5.x only check for '*.eb' files if fn != 'TEMPLATE.eb' and os.path.splitext(fn)[1] not in ('.py', '.patch'): From 6850f104efc9775322ec25d081a7c50e9b67d830 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 5 Aug 2021 16:03:18 +0200 Subject: [PATCH 28/91] Use the ignore-dirs option instead --- easybuild/tools/github.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index c9e5e3d623..d49cbc2f5e 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1069,11 +1069,13 @@ def find_software_name_for_patch(patch_name, ec_dirs): soft_name = None + ignore_dirs = build_option('ignore_dirs') all_ecs = [] for ec_dir in ec_dirs: for (dirpath, dirnames, filenames) in os.walk(ec_dir): - # Don't visit any hidden folders, such as .git - dirnames[:] = [i for i in dirnames if not i.startswith('.')] + # Exclude ignored dirs + if ignore_dirs: + dirnames[:] = [i for i in dirnames if i not in ignore_dirs] for fn in filenames: # TODO: In EasyBuild 5.x only check for '*.eb' files if fn != 'TEMPLATE.eb' and os.path.splitext(fn)[1] not in ('.py', '.patch'): From afd47593574c3a84b18a9cb61c90a1a40a0d22f1 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 5 Aug 2021 16:26:34 +0200 Subject: [PATCH 29/91] Add --filter-ecs parameter to remove ECs from the build set Removes ECs to be build from the ECs passed on the cmdline or implicitely (and more useful) selected via the --from-pr option Accepts a list of glob style patterns --- easybuild/framework/easyconfig/tools.py | 5 +++++ easybuild/tools/config.py | 1 + easybuild/tools/options.py | 3 +++ test/framework/robot.py | 17 +++++++++++++++++ 4 files changed, 26 insertions(+) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index d78ff96e7c..5f2acb9aa5 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -37,6 +37,7 @@ :author: Ward Poelmans (Ghent University) """ import copy +import fnmatch import glob import os import re @@ -360,6 +361,10 @@ def det_easyconfig_paths(orig_paths): # if no easyconfigs are specified, use all the ones touched in the PR ec_files = [path for path in pr_files if path.endswith('.eb')] + filter_ecs = build_option('filter_ecs') + if filter_ecs: + ec_files = [ec for ec in ec_files + if not any(fnmatch.fnmatch(ec, filter_spec) for filter_spec in filter_ecs)] if ec_files and robot_path: ignore_subdirs = build_option('ignore_dirs') if not build_option('consider_archived_easyconfigs'): diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 23f3c97f54..8f660331f5 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -180,6 +180,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'envvars_user_modules', 'extra_modules', 'filter_deps', + 'filter_ecs', 'filter_env_vars', 'hide_deps', 'hide_toolchains', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 7dadcd415d..647c10c383 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -396,6 +396,9 @@ def override_options(self): 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, " "because equivalent OS packages are installed. (e.g. --filter-deps=zlib,ncurses)", 'strlist', 'extend', None), + 'filter-ecs': ("List of easyconfigs (given as glob patterns) to *ignore* when given on command line " + "or auto-selected when building with --from-pr. (e.g. --filter-ecs=*intel*)", + 'strlist', 'extend', None), 'filter-env-vars': ("List of names of environment variables that should *not* be defined/updated by " "module files generated by EasyBuild", 'strlist', 'extend', None), 'fixed-installdir-naming-scheme': ("Use fixed naming scheme for installation directories", None, diff --git a/test/framework/robot.py b/test/framework/robot.py index f55a6e822b..4add2ada9f 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -670,6 +670,23 @@ def test_det_easyconfig_paths(self): regex = re.compile(r"^ \* \[.\] .*/__archive__/.*/intel-2012a.eb \(module: intel/2012a\)", re.M) self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + args = [ + os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), + os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), + os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0-gompi-2018a.eb'), + '--dry-run', + '--robot', + '--tmpdir=%s' % self.test_prefix, + '--filter-ecs=*oy-0.0.eb,*-test.eb', + ] + outtxt = self.eb_main(args, raise_error=True) + + regex = re.compile(r"^ \* \[.\] .*/toy-0.0-gompi-2018a.eb \(module: toy/0.0-gompi-2018a\)", re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + for ec in ('toy-0.0.eb', 'toy-0.0-gompi-2018a-test.eb'): + regex = re.compile(r"^ \* \[.\] .*/%s \(module:" % ec, re.M) + self.assertFalse(regex.search(outtxt), "%s should be fitered in %s" % (ec, outtxt)) + def test_search_paths(self): """Test search_paths command line argument.""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') From 999deb36e9c398060952e490c09132335a717161 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 5 Aug 2021 16:32:19 +0200 Subject: [PATCH 30/91] Mock and catch warning message --- test/framework/filetools.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 722fbf54da..4ea4738881 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2589,7 +2589,10 @@ def run_check(): # use a tag that clashes with a branch name and make sure this is handled correctly git_config['tag'] = 'tag_for_tests' - res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) + with self.mocked_stdout_stderr(): + res = ft.get_source_tarball_from_git('test.tar.gz', target_dir, git_config) + stderr = self.get_stderr() + self.assertIn('Tag tag_for_tests was not downloaded in the first try', stderr) self.assertEqual(res, test_file) self.assertTrue(os.path.isfile(test_file)) # Check that we indeed downloaded the tag and not the branch From 2c84fea1d3608c50de03538c939f9f7944da940a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 10 Aug 2021 16:41:15 +0200 Subject: [PATCH 31/91] Only catch our own errors when failing to parse config options --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 7dadcd415d..678363ab35 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1392,7 +1392,7 @@ def parse_options(args=None, with_include=True): eb_go = EasyBuildOptions(usage=usage, description=description, prog='eb', envvar_prefix=CONFIG_ENV_VAR_PREFIX, go_args=eb_args, error_env_options=True, error_env_option_method=raise_easybuilderror, with_include=with_include) - except Exception as err: + except EasyBuildError as err: raise EasyBuildError("Failed to parse configuration options: %s" % err) return eb_go From 055b28a2a11070692bf9251e68748a2a95ef40bf Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 10 Aug 2021 16:42:32 +0200 Subject: [PATCH 32/91] Fix CI on Python 2 --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e5d99f3e1e..8730d3f102 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,7 +49,8 @@ PyYAML; python_version >= '2.7' pycodestyle; python_version < '2.7' flake8; python_version >= '2.7' -GC3Pie +# 2.6.7 uses invalid Python 2 syntax +GC3Pie!=2.6.7 python-graph-dot python-hglib requests From be485dc47664054e72afa3215f9fc873a0f4d7ff Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 11 Aug 2021 11:38:40 +0200 Subject: [PATCH 33/91] Only exclude latest GC3Pie for python2 --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8730d3f102..e63085eb51 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,7 +50,8 @@ pycodestyle; python_version < '2.7' flake8; python_version >= '2.7' # 2.6.7 uses invalid Python 2 syntax -GC3Pie!=2.6.7 +GC3Pie!=2.6.7; python_version < '3.0' +GC3Pie; python_version >= '3.0' python-graph-dot python-hglib requests From 88acb30df50c3acff684528d0bf907cc53138dab Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Aug 2021 16:50:44 +0200 Subject: [PATCH 34/91] check type of source_tmpl value for extensions, ensure it's a string value (not a list) --- easybuild/framework/easyblock.py | 8 +++++++- test/framework/easyblock.py | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 3b1ffa3096..a0e496cd9a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -593,7 +593,13 @@ def fetch_extension_sources(self, skip_checksums=False): default_source_tmpl = resolve_template('%(name)s-%(version)s.tar.gz', template_values) # if no sources are specified via 'sources', fall back to 'source_tmpl' - src_fn = ext_options.get('source_tmpl', default_source_tmpl) + src_fn = ext_options.get('source_tmpl') + if src_fn is None: + src_fn = default_source_tmpl + elif not isinstance(src_fn, string_type): + error_msg = "source_tmpl value must be a string! (found value of type '%s'): %s" + raise EasyBuildError(error_msg, type(src_fn).__name__, src_fn) + src_path = self.obtain_file(src_fn, extension=True, urls=source_urls, force_download=force_download) if src_path: diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 21873a1662..18bfcf07b0 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1052,6 +1052,33 @@ def test_init_extensions(self): error_pattern = "ConfigureMake easyblock can not be used to install extensions" self.assertErrorRegex(EasyBuildError, error_pattern, eb.init_ext_instances) + def test_extension_source_tmpl(self): + """Test type checking for 'source_tmpl' value of an extension.""" + self.contents = '\n'.join([ + "easyblock = 'ConfigureMake'", + "name = 'toy'", + "version = '0.0'", + "homepage = 'https://example.com'", + "description = 'test'", + "toolchain = SYSTEM", + "exts_list = [", + " ('bar', '0.0', {", + " 'source_tmpl': [SOURCE_TAR_GZ],", + " }),", + "]", + ]) + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + + error_pattern = r"source_tmpl value must be a string! " + error_pattern += r"\(found value of type 'list'\): \['bar-0\.0\.tar\.gz'\]" + self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) + + self.contents = self.contents.replace("'source_tmpl': [SOURCE_TAR_GZ]", "'source_tmpl': SOURCE_TAR_GZ") + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + eb.fetch_step() + def test_skip_extensions_step(self): """Test the skip_extensions_step""" From 9ed8415afb710e4cab5fb107fdf91bc0d92c1247 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Aug 2021 18:33:50 +0200 Subject: [PATCH 35/91] enhance test to check for fixed bug reported in #3781 --- test/framework/github.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/framework/github.py b/test/framework/github.py index 34f2f7f885..b280fc5448 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -595,6 +595,15 @@ def test_github_find_patches(self): reg = re.compile(r'[1-9]+ of [1-9]+ easyconfigs checked') self.assertTrue(re.search(reg, txt)) + self.assertEqual(gh.find_software_name_for_patch('test.patch', []), None) + + # check behaviour of find_software_name_for_patch when non-UTF8 patch files are present + non_utf8_patch = os.path.join(self.test_prefix, 'problem.patch') + with open(non_utf8_patch, 'wb') as fp: + fp.write(bytes("+ ximage->byte_order=T1_byte_order; /* Set t1lib\xb4s byteorder */\n", 'iso_8859_1')) + + self.assertEqual(gh.find_software_name_for_patch('test.patch', [self.test_prefix]), None) + def test_github_det_commit_status(self): """Test det_commit_status function.""" From 822750dc0b2d01729e561cf154d4837adcfdbef1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Aug 2021 19:30:03 +0200 Subject: [PATCH 36/91] also define $BLAS_SHARED_LIBS & co in build environment (analogous to $BLAS_STATIC_LIBS) --- easybuild/toolchains/fft/fftw.py | 8 +- easybuild/tools/toolchain/constants.py | 26 +++++- easybuild/tools/toolchain/fft.py | 2 + easybuild/tools/toolchain/linalg.py | 10 +++ easybuild/tools/toolchain/variables.py | 10 ++- test/framework/toolchain.py | 106 ++++++++++++++++++++++++- 6 files changed, 152 insertions(+), 10 deletions(-) diff --git a/easybuild/toolchains/fft/fftw.py b/easybuild/toolchains/fft/fftw.py index 4b6c32dcb9..ece375253a 100644 --- a/easybuild/toolchains/fft/fftw.py +++ b/easybuild/toolchains/fft/fftw.py @@ -71,7 +71,7 @@ def _set_fft_variables(self): # TODO can these be replaced with the FFT ones? self.variables.join('FFTW_INC_DIR', 'FFT_INC_DIR') self.variables.join('FFTW_LIB_DIR', 'FFT_LIB_DIR') - if 'FFT_STATIC_LIBS' in self.variables: - self.variables.join('FFTW_STATIC_LIBS', 'FFT_STATIC_LIBS') - if 'FFT_STATIC_LIBS_MT' in self.variables: - self.variables.join('FFTW_STATIC_LIBS_MT', 'FFT_STATIC_LIBS_MT') + + for key in ('SHARED_LIBS', 'SHARED_LIBS_MT', 'STATIC_LIBS', 'STATIC_LIBS_MT'): + if 'FFT_' + key in self.variables: + self.variables.join('FFTW_' + key, 'FFT_' + key) diff --git a/easybuild/tools/toolchain/constants.py b/easybuild/tools/toolchain/constants.py index 307827dd1e..b334d17ae9 100644 --- a/easybuild/tools/toolchain/constants.py +++ b/easybuild/tools/toolchain/constants.py @@ -30,8 +30,8 @@ """ from easybuild.tools.variables import AbsPathList -from easybuild.tools.toolchain.variables import LinkLibraryPaths, IncludePaths, CommandFlagList, CommaStaticLibs -from easybuild.tools.toolchain.variables import FlagList, LibraryList +from easybuild.tools.toolchain.variables import CommandFlagList, CommaSharedLibs, CommaStaticLibs +from easybuild.tools.toolchain.variables import FlagList, IncludePaths, LibraryList, LinkLibraryPaths COMPILER_VARIABLES = [ @@ -114,6 +114,10 @@ ('LIBBLAS', 'BLAS libraries'), ('LIBBLAS_MT', 'multithreaded BLAS libraries'), ], + CommaSharedLibs: [ + ('BLAS_SHARED_LIBS', 'Comma-separated list of shared BLAS libraries'), + ('BLAS_MT_SHARED_LIBS', 'Comma-separated list of shared multithreaded BLAS libraries'), + ], CommaStaticLibs: [ ('BLAS_STATIC_LIBS', 'Comma-separated list of static BLAS libraries'), ('BLAS_MT_STATIC_LIBS', 'Comma-separated list of static multithreaded BLAS libraries'), @@ -132,6 +136,12 @@ ('LIBLAPACK', 'LAPACK libraries'), ('LIBLAPACK_MT', 'multithreaded LAPACK libraries'), ], + CommaSharedLibs: [ + ('LAPACK_SHARED_LIBS', 'Comma-separated list of shared LAPACK libraries'), + ('LAPACK_MT_SHARED_LIBS', 'Comma-separated list of shared LAPACK libraries'), + ('BLAS_LAPACK_SHARED_LIBS', 'Comma-separated list of shared BLAS and LAPACK libraries'), + ('BLAS_LAPACK_MT_SHARED_LIBS', 'Comma-separated list of shared BLAS and LAPACK libraries'), + ], CommaStaticLibs: [ ('LAPACK_STATIC_LIBS', 'Comma-separated list of static LAPACK libraries'), ('LAPACK_MT_STATIC_LIBS', 'Comma-separated list of static LAPACK libraries'), @@ -166,6 +176,10 @@ ('LIBSCALAPACK', 'SCALAPACK libraries'), ('LIBSCALAPACK_MT', 'multithreaded SCALAPACK libraries'), ], + CommaSharedLibs: [ + ('SCALAPACK_SHARED_LIBS', 'Comma-separated list of shared SCALAPACK libraries'), + ('SCALAPACK_MT_SHARED_LIBS', 'Comma-separated list of shared SCALAPACK libraries'), + ], CommaStaticLibs: [ ('SCALAPACK_STATIC_LIBS', 'Comma-separated list of static SCALAPACK libraries'), ('SCALAPACK_MT_STATIC_LIBS', 'Comma-separated list of static SCALAPACK libraries'), @@ -181,6 +195,10 @@ ('LIBFFT', 'FFT libraries'), ('LIBFFT_MT', 'Multithreaded FFT libraries'), ], + CommaSharedLibs: [ + ('FFT_SHARED_LIBS', 'Comma-separated list of shared FFT libraries'), + ('FFT_SHARED_LIBS_MT', 'Comma-separated list of shared multithreaded FFT libraries'), + ], CommaStaticLibs: [ ('FFT_STATIC_LIBS', 'Comma-separated list of static FFT libraries'), ('FFT_STATIC_LIBS_MT', 'Comma-separated list of static multithreaded FFT libraries'), @@ -192,6 +210,10 @@ ('FFTW_LIB_DIR', 'FFTW library directory'), ('FFTW_INC_DIR', 'FFTW include directory'), ], + CommaSharedLibs: [ + ('FFTW_SHARED_LIBS', 'Comma-separated list of shared FFTW libraries'), + ('FFTW_SHARED_LIBS_MT', 'Comma-separated list of shared multithreaded FFTW libraries'), + ], CommaStaticLibs: [ ('FFTW_STATIC_LIBS', 'Comma-separated list of static FFTW libraries'), ('FFTW_STATIC_LIBS_MT', 'Comma-separated list of static multithreaded FFTW libraries'), diff --git a/easybuild/tools/toolchain/fft.py b/easybuild/tools/toolchain/fft.py index 4facafc7c2..1e39953ba7 100644 --- a/easybuild/tools/toolchain/fft.py +++ b/easybuild/tools/toolchain/fft.py @@ -68,7 +68,9 @@ def _set_fft_variables(self): if getattr(self, 'LIB_MULTITHREAD', None) is not None: self.variables.nappend('LIBFFT_MT', self.LIB_MULTITHREAD) + self.variables.join('FFT_SHARED_LIBS', 'LIBFFT') self.variables.join('FFT_STATIC_LIBS', 'LIBFFT') + self.variables.join('FFT_SHARED_LIBS_MT', 'LIBFFT_MT') self.variables.join('FFT_STATIC_LIBS_MT', 'LIBFFT_MT') for root in self.get_software_root(self.FFT_MODULE_NAME): diff --git a/easybuild/tools/toolchain/linalg.py b/easybuild/tools/toolchain/linalg.py index 7d27350996..2e1a46db41 100644 --- a/easybuild/tools/toolchain/linalg.py +++ b/easybuild/tools/toolchain/linalg.py @@ -127,7 +127,9 @@ def _set_blas_variables(self): self.variables.nappend('LIBBLAS', self.LIB_EXTRA, position=20) self.variables.nappend('LIBBLAS_MT', self.LIB_EXTRA, position=20) + self.variables.join('BLAS_SHARED_LIBS', 'LIBBLAS') self.variables.join('BLAS_STATIC_LIBS', 'LIBBLAS') + self.variables.join('BLAS_MT_SHARED_LIBS', 'LIBBLAS_MT') self.variables.join('BLAS_MT_STATIC_LIBS', 'LIBBLAS_MT') for root in self.get_software_root(self.BLAS_MODULE_NAME): self.variables.append_exists('BLAS_LIB_DIR', root, self.BLAS_LIB_DIR) @@ -147,7 +149,9 @@ def _set_lapack_variables(self): self.variables.join('LIBLAPACK_MT_ONLY', 'LIBBLAS_MT') self.variables.join('LIBLAPACK', 'LIBBLAS') self.variables.join('LIBLAPACK_MT', 'LIBBLAS_MT') + self.variables.join('LAPACK_SHARED_LIBS', 'BLAS_SHARED_LIBS') self.variables.join('LAPACK_STATIC_LIBS', 'BLAS_STATIC_LIBS') + self.variables.join('LAPACK_MT_SHARED_LIBS', 'BLAS_MT_SHARED_LIBS') self.variables.join('LAPACK_MT_STATIC_LIBS', 'BLAS_MT_STATIC_LIBS') self.variables.join('LAPACK_LIB_DIR', 'BLAS_LIB_DIR') self.variables.join('LAPACK_INC_DIR', 'BLAS_INC_DIR') @@ -183,7 +187,9 @@ def _set_lapack_variables(self): self.variables.nappend('LIBLAPACK', self.LIB_EXTRA, position=20) self.variables.nappend('LIBLAPACK_MT', self.LIB_EXTRA, position=20) + self.variables.join('LAPACK_SHARED_LIBS', 'LIBLAPACK') self.variables.join('LAPACK_STATIC_LIBS', 'LIBLAPACK') + self.variables.join('LAPACK_MT_SHARED_LIBS', 'LIBLAPACK_MT') self.variables.join('LAPACK_MT_STATIC_LIBS', 'LIBLAPACK_MT') for root in self.get_software_root(self.LAPACK_MODULE_NAME): @@ -192,7 +198,9 @@ def _set_lapack_variables(self): self.variables.join('BLAS_LAPACK_LIB_DIR', 'LAPACK_LIB_DIR', 'BLAS_LIB_DIR') self.variables.join('BLAS_LAPACK_INC_DIR', 'LAPACK_INC_DIR', 'BLAS_INC_DIR') + self.variables.join('BLAS_LAPACK_SHARED_LIBS', 'LAPACK_SHARED_LIBS', 'BLAS_SHARED_LIBS') self.variables.join('BLAS_LAPACK_STATIC_LIBS', 'LAPACK_STATIC_LIBS', 'BLAS_STATIC_LIBS') + self.variables.join('BLAS_LAPACK_MT_SHARED_LIBS', 'LAPACK_MT_SHARED_LIBS', 'BLAS_MT_SHARED_LIBS') self.variables.join('BLAS_LAPACK_MT_STATIC_LIBS', 'LAPACK_MT_STATIC_LIBS', 'BLAS_MT_STATIC_LIBS') # add general dependency variables @@ -293,7 +301,9 @@ def _set_scalapack_variables(self): self.variables.nappend('LIBSCALAPACK', self.LIB_EXTRA, position=20) self.variables.nappend('LIBSCALAPACK_MT', self.LIB_EXTRA, position=20) + self.variables.join('SCALAPACK_SHARED_LIBS', 'LIBSCALAPACK') self.variables.join('SCALAPACK_STATIC_LIBS', 'LIBSCALAPACK') + self.variables.join('SCALAPACK_MT_SHARED_LIBS', 'LIBSCALAPACK_MT') self.variables.join('SCALAPACK_MT_STATIC_LIBS', 'LIBSCALAPACK_MT') for root in self.get_software_root(self.SCALAPACK_MODULE_NAME): self.variables.append_exists('SCALAPACK_LIB_DIR', root, self.SCALAPACK_LIB_DIR) diff --git a/easybuild/tools/toolchain/variables.py b/easybuild/tools/toolchain/variables.py index 30a365ad5b..4becb8986b 100644 --- a/easybuild/tools/toolchain/variables.py +++ b/easybuild/tools/toolchain/variables.py @@ -111,8 +111,16 @@ def change(self, separator=None, separator_begin_end=None, prefix=None, prefix_b self.END.PREFIX = prefix_begin_end +class CommaSharedLibs(LibraryList): + """Comma-separated list of shared libraries""" + SEPARATOR = ',' + + PREFIX = 'lib' + SUFFIX = '.so' + + class CommaStaticLibs(LibraryList): - """Comma-separated list""" + """Comma-separated list of static libraries""" SEPARATOR = ',' PREFIX = 'lib' diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 0fa2ff2d56..c02c2b8f42 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1453,6 +1453,8 @@ def test_old_new_iccifort(self): libblas_mt_intel3 += " -Wl,--end-group -Wl,-Bdynamic -liomp5 -lguide -lpthread" # no -lguide + blas_static_libs_intel4 = 'libmkl_intel_lp64.a,libmkl_sequential.a,libmkl_core.a' + blas_shared_libs_intel4 = blas_static_libs_intel4.replace('.a', '.so') libblas_intel4 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_sequential -lmkl_core" libblas_intel4 += " -Wl,--end-group -Wl,-Bdynamic" libblas_mt_intel4 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" @@ -1469,18 +1471,83 @@ def test_old_new_iccifort(self): libscalack_intel4 = "-lmkl_scalapack_lp64 -lmkl_blacs_intelmpi_lp64 -lmkl_intel_lp64 -lmkl_sequential " libscalack_intel4 += "-lmkl_core" - libblas_mt_fosscuda = "-lopenblas -lgfortran -lpthread" + blas_static_libs_fosscuda = "libopenblas.a,libgfortran.a" + blas_shared_libs_fosscuda = blas_static_libs_fosscuda.replace('.a', '.so') + blas_mt_static_libs_fosscuda = blas_static_libs_fosscuda + ",libpthread.a" + blas_mt_shared_libs_fosscuda = blas_mt_static_libs_fosscuda.replace('.a', '.so') + libblas_fosscuda = "-lopenblas -lgfortran" + libblas_mt_fosscuda = libblas_fosscuda + " -lpthread" + + fft_static_libs_fosscuda = "libfftw3.a" + fft_shared_libs_fosscuda = fft_static_libs_fosscuda.replace('.a', '.so') + fft_mt_static_libs_fosscuda = "libfftw3.a,libpthread.a" + fft_mt_shared_libs_fosscuda = fft_mt_static_libs_fosscuda.replace('.a', '.so') + fft_mt_static_libs_fosscuda_omp = "libfftw3_omp.a,libfftw3.a,libpthread.a" + fft_mt_shared_libs_fosscuda_omp = fft_mt_static_libs_fosscuda_omp.replace('.a', '.so') + libfft_fosscuda = "-lfftw3" + libfft_mt_fosscuda = libfft_fosscuda + " -lpthread" + libfft_mt_fosscuda_omp = "-lfftw3_omp " + libfft_fosscuda + " -lpthread" + + lapack_static_libs_fosscuda = "libopenblas.a,libgfortran.a" + lapack_shared_libs_fosscuda = lapack_static_libs_fosscuda.replace('.a', '.so') + lapack_mt_static_libs_fosscuda = lapack_static_libs_fosscuda + ",libpthread.a" + lapack_mt_shared_libs_fosscuda = lapack_mt_static_libs_fosscuda.replace('.a', '.so') + liblapack_fosscuda = "-lopenblas -lgfortran" + liblapack_mt_fosscuda = liblapack_fosscuda + " -lpthread" + libscalack_fosscuda = "-lscalapack -lopenblas -lgfortran" - libfft_mt_fosscuda = "-lfftw3_omp -lfftw3 -lpthread" + libscalack_mt_fosscuda = libscalack_fosscuda + " -lpthread" + scalapack_static_libs_fosscuda = "libscalapack.a,libopenblas.a,libgfortran.a" + scalapack_shared_libs_fosscuda = scalapack_static_libs_fosscuda.replace('.a', '.so') + scalapack_mt_static_libs_fosscuda = "libscalapack.a,libopenblas.a,libgfortran.a,libpthread.a" + scalapack_mt_shared_libs_fosscuda = scalapack_mt_static_libs_fosscuda.replace('.a', '.so') tc = self.get_toolchain('fosscuda', version='2018a') tc.prepare() + self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBBLAS'], libblas_fosscuda) self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_fosscuda) + + self.assertEqual(os.environ['LAPACK_SHARED_LIBS'], lapack_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_STATIC_LIBS'], lapack_static_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_SHARED_LIBS'], lapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_STATIC_LIBS'], lapack_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBLAPACK'], liblapack_fosscuda) + self.assertEqual(os.environ['LIBLAPACK_MT'], liblapack_mt_fosscuda) + + self.assertEqual(os.environ['BLAS_LAPACK_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + + self.assertEqual(os.environ['FFT_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFT_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBFFT'], libfft_fosscuda) + self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda) + self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_fosscuda) + self.assertEqual(os.environ['LIBSCALAPACK_MT'], libscalack_mt_fosscuda) + self.assertEqual(os.environ['SCALAPACK_SHARED_LIBS'], scalapack_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_STATIC_LIBS'], scalapack_static_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_SHARED_LIBS'], scalapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_STATIC_LIBS'], scalapack_mt_static_libs_fosscuda) self.modtool.purge() tc = self.get_toolchain('intel', version='2018a') tc.prepare() + self.assertEqual(os.environ.get('BLAS_SHARED_LIBS', "(not set)"), blas_shared_libs_intel4) + self.assertEqual(os.environ.get('BLAS_STATIC_LIBS', "(not set)"), blas_static_libs_intel4) + self.assertEqual(os.environ.get('LAPACK_SHARED_LIBS', "(not set)"), blas_shared_libs_intel4) + self.assertEqual(os.environ.get('LAPACK_STATIC_LIBS', "(not set)"), blas_static_libs_intel4) self.assertEqual(os.environ.get('LIBBLAS', "(not set)"), libblas_intel4) self.assertEqual(os.environ.get('LIBBLAS_MT', "(not set)"), libblas_mt_intel4) self.assertEqual(os.environ.get('LIBFFT', "(not set)"), libfft_intel4) @@ -1517,9 +1584,42 @@ def test_old_new_iccifort(self): tc = self.get_toolchain('fosscuda', version='2018a') tc.set_options({'openmp': True}) tc.prepare() + self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBBLAS'], libblas_fosscuda) self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_fosscuda) - self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda) + + self.assertEqual(os.environ['LAPACK_SHARED_LIBS'], lapack_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_STATIC_LIBS'], lapack_static_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_SHARED_LIBS'], lapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_STATIC_LIBS'], lapack_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBLAPACK'], liblapack_fosscuda) + self.assertEqual(os.environ['LIBLAPACK_MT'], liblapack_mt_fosscuda) + + self.assertEqual(os.environ['BLAS_LAPACK_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + + self.assertEqual(os.environ['FFT_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFT_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda_omp) + self.assertEqual(os.environ['FFT_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda_omp) + self.assertEqual(os.environ['FFTW_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda_omp) + self.assertEqual(os.environ['FFTW_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda_omp) + self.assertEqual(os.environ['LIBFFT'], libfft_fosscuda) + self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda_omp) + self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_fosscuda) + self.assertEqual(os.environ['LIBSCALAPACK_MT'], libscalack_mt_fosscuda) + self.assertEqual(os.environ['SCALAPACK_SHARED_LIBS'], scalapack_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_STATIC_LIBS'], scalapack_static_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_SHARED_LIBS'], scalapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_STATIC_LIBS'], scalapack_mt_static_libs_fosscuda) def test_standalone_iccifort(self): """Test whether standalone installation of iccifort matches the iccifort toolchain definition.""" From 703813e49e4982901642f05aa4a62ff45483df8c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Aug 2021 19:32:51 +0200 Subject: [PATCH 37/91] only run additional check for find_software_name_for_patch when testing with Python 3 --- test/framework/github.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/framework/github.py b/test/framework/github.py index b280fc5448..423f860a00 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -597,12 +597,13 @@ def test_github_find_patches(self): self.assertEqual(gh.find_software_name_for_patch('test.patch', []), None) - # check behaviour of find_software_name_for_patch when non-UTF8 patch files are present - non_utf8_patch = os.path.join(self.test_prefix, 'problem.patch') - with open(non_utf8_patch, 'wb') as fp: - fp.write(bytes("+ ximage->byte_order=T1_byte_order; /* Set t1lib\xb4s byteorder */\n", 'iso_8859_1')) + # check behaviour of find_software_name_for_patch when non-UTF8 patch files are present (only with Python 3) + if sys.version_info[0] >= 3: + non_utf8_patch = os.path.join(self.test_prefix, 'problem.patch') + with open(non_utf8_patch, 'wb') as fp: + fp.write(bytes("+ ximage->byte_order=T1_byte_order; /* Set t1lib\xb4s byteorder */\n", 'iso_8859_1')) - self.assertEqual(gh.find_software_name_for_patch('test.patch', [self.test_prefix]), None) + self.assertEqual(gh.find_software_name_for_patch('test.patch', [self.test_prefix]), None) def test_github_det_commit_status(self): """Test det_commit_status function.""" From b2662fb1a1d205c23e638e65fd2b1d575db38c38 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 12 Aug 2021 09:42:59 +0200 Subject: [PATCH 38/91] don't hardcode .so, use get_shared_lib_ext function --- easybuild/tools/toolchain/variables.py | 3 ++- test/framework/toolchain.py | 23 +++++++++++++---------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/toolchain/variables.py b/easybuild/tools/toolchain/variables.py index 4becb8986b..6758af24de 100644 --- a/easybuild/tools/toolchain/variables.py +++ b/easybuild/tools/toolchain/variables.py @@ -30,6 +30,7 @@ """ from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.variables import StrList, AbsPathList @@ -116,7 +117,7 @@ class CommaSharedLibs(LibraryList): SEPARATOR = ',' PREFIX = 'lib' - SUFFIX = '.so' + SUFFIX = '.' + get_shared_lib_ext() class CommaStaticLibs(LibraryList): diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index c02c2b8f42..0bf963857c 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -51,6 +51,7 @@ from easybuild.tools.filetools import read_file, symlink, write_file, which from easybuild.tools.py2vs3 import string_type from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.toolchain.mpi import get_mpi_cmd_template from easybuild.tools.toolchain.toolchain import env_vars_external_module from easybuild.tools.toolchain.utilities import get_toolchain, search_toolchain @@ -1448,13 +1449,15 @@ def test_old_new_iccifort(self): self.setup_sandbox_for_intel_fftw(self.test_prefix, imklver='10.2.6.038') self.modtool.prepend_module_path(self.test_prefix) + shlib_ext = get_shared_lib_ext() + # incl. -lguide libblas_mt_intel3 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" libblas_mt_intel3 += " -Wl,--end-group -Wl,-Bdynamic -liomp5 -lguide -lpthread" # no -lguide blas_static_libs_intel4 = 'libmkl_intel_lp64.a,libmkl_sequential.a,libmkl_core.a' - blas_shared_libs_intel4 = blas_static_libs_intel4.replace('.a', '.so') + blas_shared_libs_intel4 = blas_static_libs_intel4.replace('.a', '.' + shlib_ext) libblas_intel4 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_sequential -lmkl_core" libblas_intel4 += " -Wl,--end-group -Wl,-Bdynamic" libblas_mt_intel4 = "-Wl,-Bstatic -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core" @@ -1472,35 +1475,35 @@ def test_old_new_iccifort(self): libscalack_intel4 += "-lmkl_core" blas_static_libs_fosscuda = "libopenblas.a,libgfortran.a" - blas_shared_libs_fosscuda = blas_static_libs_fosscuda.replace('.a', '.so') + blas_shared_libs_fosscuda = blas_static_libs_fosscuda.replace('.a', '.' + shlib_ext) blas_mt_static_libs_fosscuda = blas_static_libs_fosscuda + ",libpthread.a" - blas_mt_shared_libs_fosscuda = blas_mt_static_libs_fosscuda.replace('.a', '.so') + blas_mt_shared_libs_fosscuda = blas_mt_static_libs_fosscuda.replace('.a', '.' + shlib_ext) libblas_fosscuda = "-lopenblas -lgfortran" libblas_mt_fosscuda = libblas_fosscuda + " -lpthread" fft_static_libs_fosscuda = "libfftw3.a" - fft_shared_libs_fosscuda = fft_static_libs_fosscuda.replace('.a', '.so') + fft_shared_libs_fosscuda = fft_static_libs_fosscuda.replace('.a', '.' + shlib_ext) fft_mt_static_libs_fosscuda = "libfftw3.a,libpthread.a" - fft_mt_shared_libs_fosscuda = fft_mt_static_libs_fosscuda.replace('.a', '.so') + fft_mt_shared_libs_fosscuda = fft_mt_static_libs_fosscuda.replace('.a', '.' + shlib_ext) fft_mt_static_libs_fosscuda_omp = "libfftw3_omp.a,libfftw3.a,libpthread.a" - fft_mt_shared_libs_fosscuda_omp = fft_mt_static_libs_fosscuda_omp.replace('.a', '.so') + fft_mt_shared_libs_fosscuda_omp = fft_mt_static_libs_fosscuda_omp.replace('.a', '.' + shlib_ext) libfft_fosscuda = "-lfftw3" libfft_mt_fosscuda = libfft_fosscuda + " -lpthread" libfft_mt_fosscuda_omp = "-lfftw3_omp " + libfft_fosscuda + " -lpthread" lapack_static_libs_fosscuda = "libopenblas.a,libgfortran.a" - lapack_shared_libs_fosscuda = lapack_static_libs_fosscuda.replace('.a', '.so') + lapack_shared_libs_fosscuda = lapack_static_libs_fosscuda.replace('.a', '.' + shlib_ext) lapack_mt_static_libs_fosscuda = lapack_static_libs_fosscuda + ",libpthread.a" - lapack_mt_shared_libs_fosscuda = lapack_mt_static_libs_fosscuda.replace('.a', '.so') + lapack_mt_shared_libs_fosscuda = lapack_mt_static_libs_fosscuda.replace('.a', '.' + shlib_ext) liblapack_fosscuda = "-lopenblas -lgfortran" liblapack_mt_fosscuda = liblapack_fosscuda + " -lpthread" libscalack_fosscuda = "-lscalapack -lopenblas -lgfortran" libscalack_mt_fosscuda = libscalack_fosscuda + " -lpthread" scalapack_static_libs_fosscuda = "libscalapack.a,libopenblas.a,libgfortran.a" - scalapack_shared_libs_fosscuda = scalapack_static_libs_fosscuda.replace('.a', '.so') + scalapack_shared_libs_fosscuda = scalapack_static_libs_fosscuda.replace('.a', '.' + shlib_ext) scalapack_mt_static_libs_fosscuda = "libscalapack.a,libopenblas.a,libgfortran.a,libpthread.a" - scalapack_mt_shared_libs_fosscuda = scalapack_mt_static_libs_fosscuda.replace('.a', '.so') + scalapack_mt_shared_libs_fosscuda = scalapack_mt_static_libs_fosscuda.replace('.a', '.' + shlib_ext) tc = self.get_toolchain('fosscuda', version='2018a') tc.prepare() From 9ebdeb96022305e7f878eb20281c28c2223ffe9f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 12 Aug 2021 18:55:10 +0200 Subject: [PATCH 39/91] Avoid checking executable rights of subdirs of temp dirs already checked `run_cmd` is one of the slowest things we can do as it forks the whole process. In the check when setting a temp dir we run a dummy file to see if the folder is actually executable. This means running this check at least twice per test case and another time for each init_config call in the test amounting to e.g. 263 calls in a single test (test_compiler_dependent_optarch) So cache the results and don't repeat the run_cmd if it already succeeded for a parent folder. Likely no change at all in regular mode but really helps for the tests --- easybuild/tools/options.py | 41 ++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 678363ab35..7d6d0f117d 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1756,24 +1756,31 @@ def set_tmpdir(tmpdir=None, raise_error=False): # reset to make sure tempfile picks up new temporary directory to use tempfile.tempdir = None - # test if temporary directory allows to execute files, warn if it doesn't - try: - fd, tmptest_file = tempfile.mkstemp() - os.close(fd) - os.chmod(tmptest_file, 0o700) - if not run_cmd(tmptest_file, simple=True, log_ok=False, regexp=False, force_in_dry_run=True, trace=False, - stream_output=False): - msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir() - msg += "This can cause problems in the build process, consider using --tmpdir." - if raise_error: - raise EasyBuildError(msg) + # Skip the executable check if it already succeeded for any parent folder + # Especially important for the unit test suite, less so for actual execution + executable_tmp_paths = getattr(set_tmpdir, 'executable_tmp_paths', []) + if not any(current_tmpdir.startswith(path) for path in executable_tmp_paths): + # test if temporary directory allows to execute files, warn if it doesn't + try: + fd, tmptest_file = tempfile.mkstemp() + os.close(fd) + os.chmod(tmptest_file, 0o700) + if not run_cmd(tmptest_file, simple=True, log_ok=False, regexp=False, force_in_dry_run=True, trace=False, + stream_output=False): + msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir() + msg += "This can cause problems in the build process, consider using --tmpdir." + if raise_error: + raise EasyBuildError(msg) + else: + _log.warning(msg) else: - _log.warning(msg) - else: - _log.debug("Temporary directory %s allows to execute files, good!" % tempfile.gettempdir()) - os.remove(tmptest_file) + _log.debug("Temporary directory %s allows to execute files, good!" % tempfile.gettempdir()) + # Put this folder into the cache + executable_tmp_paths.append(current_tmpdir) + set_tmpdir.executable_tmp_paths = executable_tmp_paths + os.remove(tmptest_file) - except OSError as err: - raise EasyBuildError("Failed to test whether temporary directory allows to execute files: %s", err) + except OSError as err: + raise EasyBuildError("Failed to test whether temporary directory allows to execute files: %s", err) return current_tmpdir From 8e60b7511ea0e93f922027f9122bc8da49f7d5d0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 12 Aug 2021 19:09:26 +0200 Subject: [PATCH 40/91] Reduce number of combinations tested in test_compiler_dependent_optarch The full product of options is not required to cover all cases. Instead run only with varying options per active toolchain and a single case (actually 2) for PGI This cuts down the combinations from 216 to 20 and runtime from ~1m to ~7s --- test/framework/toolchain.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 0bf963857c..11a6c083d0 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -752,17 +752,28 @@ def test_compiler_dependent_optarch(self): intel_options = [('intelflag', 'intelflag'), ('GENERIC', 'xSSE2'), ('', '')] gcc_options = [('gccflag', 'gccflag'), ('march=nocona', 'march=nocona'), ('', '')] gcccore_options = [('gcccoreflag', 'gcccoreflag'), ('GENERIC', 'march=x86-64 -mtune=generic'), ('', '')] - toolchains = [ - ('iccifort', '2018.1.163'), - ('GCC', '6.4.0-2.28'), - ('GCCcore', '6.2.0'), - ('PGI', '16.7-GCC-5.4.0-2.26'), - ] - enabled = [True, False] - test_cases = product(intel_options, gcc_options, gcccore_options, toolchains, enabled) + tc_intel = ('iccifort', '2018.1.163') + tc_gcc = ('GCC', '6.4.0-2.28') + tc_gcccore = ('GCCcore', '6.2.0') + tc_pgi = ('PGI', '16.7-GCC-5.4.0-2.26') + enabled = [True, False] - for intel_flags, gcc_flags, gcccore_flags, (toolchain_name, toolchain_ver), enable in test_cases: + test_cases = [] + for i, (tc, options) in enumerate(zip((tc_intel, tc_gcc, tc_gcccore), + (intel_options, gcc_options, gcccore_options))): + # Vary only the compiler specific option + for opt in options: + new_value = [intel_options[0], gcc_options[0], gcccore_options[0], tc] + new_value[i] = opt + test_cases.append(new_value) + # Add one case for PGI + test_cases.append((intel_options[0], gcc_options[0], gcccore_options[0], tc_pgi)) + + # Run each for enabled and disabled + test_cases = list(product(test_cases, enabled)) + + for (intel_flags, gcc_flags, gcccore_flags, (toolchain_name, toolchain_ver)), enable in test_cases: intel_flags, intel_flags_exp = intel_flags gcc_flags, gcc_flags_exp = gcc_flags From 0653e2d69afed02f4664119502b4be2753bdf80f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 17 Aug 2021 12:00:34 +0200 Subject: [PATCH 41/91] Correctly resolve templates for patches in extensions when uploading to github --- easybuild/tools/github.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index d49cbc2f5e..e703b94cc9 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1013,10 +1013,11 @@ def is_patch_for(patch_name, ec): patches = copy.copy(ec['patches']) - for ext in ec['exts_list']: + for ext in ec.get_ref('exts_list'): if isinstance(ext, (list, tuple)) and len(ext) == 3 and isinstance(ext[2], dict): + templates = {'name': ext[0], 'version': ext[1]} ext_options = ext[2] - patches.extend(ext_options.get('patches', [])) + patches.extend(p % templates for p in ext_options.get('patches', [])) for patch in patches: if isinstance(patch, (tuple, list)): From 5d7623c1a0f6e936d56f1a1126cc782bea412463 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 17 Aug 2021 12:13:22 +0200 Subject: [PATCH 42/91] Extend patch search logic to components --- easybuild/tools/github.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index e703b94cc9..8f96f4f0e6 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -34,6 +34,7 @@ import getpass import glob import functools +import itertools import os import random import re @@ -1013,11 +1014,13 @@ def is_patch_for(patch_name, ec): patches = copy.copy(ec['patches']) - for ext in ec.get_ref('exts_list'): - if isinstance(ext, (list, tuple)) and len(ext) == 3 and isinstance(ext[2], dict): - templates = {'name': ext[0], 'version': ext[1]} - ext_options = ext[2] - patches.extend(p % templates for p in ext_options.get('patches', [])) + with ec.disable_templating(): + for ext in itertools.chain(ec['exts_list'], ec.get('components', [])): + if isinstance(ext, (list, tuple)) and len(ext) == 3 and isinstance(ext[2], dict): + templates = {'name': ext[0], 'version': ext[1]} + ext_options = ext[2] + patches.extend(p[0] % templates if isinstance(p, (tuple, list)) else p % templates + for p in ext_options.get('patches', [])) for patch in patches: if isinstance(patch, (tuple, list)): From d8c463f8d1e9091e1d8b3ea8814b06d28d1f3882 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 17 Aug 2021 16:05:02 +0200 Subject: [PATCH 43/91] Update test to use real EC instances --- test/framework/github.py | 106 +++++++++++++----- .../easyblocks/generic/pythonbundle.py | 4 + 2 files changed, 83 insertions(+), 27 deletions(-) diff --git a/test/framework/github.py b/test/framework/github.py index 423f860a00..e6c4abf4a5 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -33,12 +33,14 @@ import random import re import sys +import textwrap from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from time import gmtime from unittest import TextTestRunner import easybuild.tools.testing from easybuild.base.rest import RestClient +from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.framework.easyconfig.tools import categorize_files_by_type from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, module_classes, update_build_option @@ -803,53 +805,103 @@ def test_github_det_patch_specs(self): """Test for det_patch_specs function.""" patch_paths = [os.path.join(self.test_prefix, p) for p in ['1.patch', '2.patch', '3.patch']] - file_info = {'ecs': [ - {'name': 'A', 'patches': ['1.patch'], 'exts_list': []}, - {'name': 'B', 'patches': [], 'exts_list': []}, - ] - } + file_info = {'ecs': []} + + rawtxt = textwrap.dedent(""" + easyblock = 'ConfigureMake' + name = 'A' + version = '42' + homepage = 'http://foo.com/' + description = '' + toolchain = {"name":"GCC", "version": "4.6.3"} + + patches = ['1.patch'] + """) + file_info['ecs'].append(EasyConfig(None, rawtxt=rawtxt)) + rawtxt = textwrap.dedent(""" + easyblock = 'ConfigureMake' + name = 'B' + version = '42' + homepage = 'http://foo.com/' + description = '' + toolchain = {"name":"GCC", "version": "4.6.3"} + """) + file_info['ecs'].append(EasyConfig(None, rawtxt=rawtxt)) + error_pattern = "Failed to determine software name to which patch file .*/2.patch relates" self.mock_stdout(True) self.assertErrorRegex(EasyBuildError, error_pattern, gh.det_patch_specs, patch_paths, file_info, []) self.mock_stdout(False) - file_info['ecs'].append({'name': 'C', 'patches': [('3.patch', 'subdir'), '2.patch'], 'exts_list': []}) + rawtxt = textwrap.dedent(""" + easyblock = 'ConfigureMake' + name = 'C' + version = '42' + homepage = 'http://foo.com/' + description = '' + toolchain = {"name":"GCC", "version": "4.6.3"} + + patches = [('3.patch', 'subdir'), '2.patch'] + """) + file_info['ecs'].append(EasyConfig(None, rawtxt=rawtxt)) self.mock_stdout(True) res = gh.det_patch_specs(patch_paths, file_info, []) self.mock_stdout(False) - self.assertEqual(len(res), 3) - self.assertEqual(os.path.basename(res[0][0]), '1.patch') - self.assertEqual(res[0][1], 'A') - self.assertEqual(os.path.basename(res[1][0]), '2.patch') - self.assertEqual(res[1][1], 'C') - self.assertEqual(os.path.basename(res[2][0]), '3.patch') - self.assertEqual(res[2][1], 'C') + self.assertEqual([i[0] for i in res], patch_paths) + self.assertEqual([i[1] for i in res], ['A', 'C', 'C']) # check if patches for extensions are found - file_info['ecs'][-1] = { - 'name': 'patched_ext', - 'patches': [], - 'exts_list': [ + rawtxt = textwrap.dedent(""" + easyblock = 'ConfigureMake' + name = 'patched_ext' + version = '42' + homepage = 'http://foo.com/' + description = '' + toolchain = {"name":"GCC", "version": "4.6.3"} + + exts_list = [ 'foo', ('bar', '1.2.3'), ('patched', '4.5.6', { - 'patches': [('2.patch', 1), '3.patch'], + 'patches': [('%(name)s-2.patch', 1), '%(name)s-3.patch'], }), - ], - } + ] + """) + patch_paths[1:3] = [os.path.join(self.test_prefix, p) for p in ['patched-2.patch', 'patched-3.patch']] + file_info['ecs'][-1] = EasyConfig(None, rawtxt=rawtxt) + + self.mock_stdout(True) + res = gh.det_patch_specs(patch_paths, file_info, []) + self.mock_stdout(False) + + self.assertEqual([i[0] for i in res], patch_paths) + self.assertEqual([i[1] for i in res], ['A', 'patched_ext', 'patched_ext']) + + # check if patches for components are found + rawtxt = textwrap.dedent(""" + easyblock = 'PythonBundle' + name = 'patched_bundle' + version = '42' + homepage = 'http://foo.com/' + description = '' + toolchain = {"name":"GCC", "version": "4.6.3"} + + components = [ + ('bar', '1.2.3'), + ('patched', '4.5.6', { + 'patches': [('%(name)s-2.patch', 1), '%(name)s-3.patch'], + }), + ] + """) + file_info['ecs'][-1] = EasyConfig(None, rawtxt=rawtxt) self.mock_stdout(True) res = gh.det_patch_specs(patch_paths, file_info, []) self.mock_stdout(False) - self.assertEqual(len(res), 3) - self.assertEqual(os.path.basename(res[0][0]), '1.patch') - self.assertEqual(res[0][1], 'A') - self.assertEqual(os.path.basename(res[1][0]), '2.patch') - self.assertEqual(res[1][1], 'patched_ext') - self.assertEqual(os.path.basename(res[2][0]), '3.patch') - self.assertEqual(res[2][1], 'patched_ext') + self.assertEqual([i[0] for i in res], patch_paths) + self.assertEqual([i[1] for i in res], ['A', 'patched_bundle', 'patched_bundle']) def test_github_restclient(self): """Test use of RestClient.""" diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py b/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py index 7146b8ecd3..0321602f3f 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py @@ -28,6 +28,7 @@ @author: Miguel Dias Costa (National University of Singapore) """ from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easyconfig import CUSTOM class PythonBundle(EasyBlock): @@ -37,4 +38,7 @@ class PythonBundle(EasyBlock): def extra_options(extra_vars=None): if extra_vars is None: extra_vars = {} + extra_vars.update({ + 'components': [(), "List of components to install: tuples w/ name, version and easyblock to use", CUSTOM], + }) return EasyBlock.extra_options(extra_vars) From 0ffd0c9b2f503bb2a06344d7bae8668716ea6845 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 18 Aug 2021 11:19:05 +0200 Subject: [PATCH 44/91] report use of --ignore-test-failure in success message in output --- easybuild/main.py | 5 ++++- test/framework/toy_build.py | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 8cbff81b41..748c068376 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -527,7 +527,10 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)]) overall_success = correct_builds_cnt == len(ordered_ecs) - success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs)) + success_msg = "Build succeeded " + if build_option('ignore_test_failure'): + success_msg += "(with --ignore-test-failure) " + success_msg += "for %s out of %s" % (correct_builds_cnt, len(ordered_ecs)) repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 51d3f1a977..b0ad961027 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3491,6 +3491,14 @@ def test_toy_build_sanity_check_linked_libs(self): self.test_toy_build(ec_file=test_ec, extra_args=args, force=False, raise_error=True, verbose=False, verify=False) + def test_toy_ignore_test_failure(self): + """Check whether use of --ignore-test-failure is mentioned in build output.""" + args = ['--ignore-test-failure'] + stdout, stderr = self.run_test_toy_build_with_output(extra_args=args, verify=False, testing=False) + + self.assertTrue("Build succeeded (with --ignore-test-failure) for 1 out of 1" in stdout) + self.assertFalse(stderr) + def suite(): """ return all the tests in this file """ From 2b5524a9da98f44b38378b22b35807684f87aad1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 19 Aug 2021 17:47:54 +0200 Subject: [PATCH 45/91] add comments to clarify use of function attribute to cache checked paths in set_tmpdir --- easybuild/tools/options.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 7d6d0f117d..9709fb71ea 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1756,10 +1756,13 @@ def set_tmpdir(tmpdir=None, raise_error=False): # reset to make sure tempfile picks up new temporary directory to use tempfile.tempdir = None + # cache for checked paths, via function attribute + executable_tmp_paths = getattr(set_tmpdir, 'executable_tmp_paths', []) + # Skip the executable check if it already succeeded for any parent folder # Especially important for the unit test suite, less so for actual execution - executable_tmp_paths = getattr(set_tmpdir, 'executable_tmp_paths', []) if not any(current_tmpdir.startswith(path) for path in executable_tmp_paths): + # test if temporary directory allows to execute files, warn if it doesn't try: fd, tmptest_file = tempfile.mkstemp() @@ -1775,9 +1778,13 @@ def set_tmpdir(tmpdir=None, raise_error=False): _log.warning(msg) else: _log.debug("Temporary directory %s allows to execute files, good!" % tempfile.gettempdir()) + # Put this folder into the cache executable_tmp_paths.append(current_tmpdir) + + # set function attribute so we can retrieve cache later set_tmpdir.executable_tmp_paths = executable_tmp_paths + os.remove(tmptest_file) except OSError as err: From 9e11d2ecf5fcce25b7008e93db46e5949901f937 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 20 Aug 2021 14:32:18 +0200 Subject: [PATCH 46/91] add get_cuda_cc_template_value method to EasyConfig class --- easybuild/framework/easyconfig/easyconfig.py | 21 +++++++- test/framework/easyconfig.py | 50 ++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 55948e11a2..6597c00aac 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -59,7 +59,7 @@ from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT from easybuild.framework.easyconfig.parser import DEPRECATED_PARAMETERS, REPLACED_PARAMETERS from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig -from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, template_constant_dict +from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC, template_constant_dict from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN @@ -1803,6 +1803,25 @@ def asdict(self): res[key] = value return res + def get_cuda_cc_template_value(self, key): + """ + Get template value based on --cuda-compute-capabilities EasyBuild configuration option + and cuda_compute_capabilities easyconfig parameter. + Returns user-friendly error message in case neither are defined, + or if an unknown key is used. + """ + if key.startswith('cuda_') and key in [x for (x, _) in TEMPLATE_NAMES_DYNAMIC]: + if key in self.template_values: + return self.template_values[key] + else: + error_msg = "(get_cuda_cc_template_value) Template value '%s' is not defined!\n" + error_msg += "Make sure that either the --cuda-compute-capabilities EasyBuild configuration " + error_msg += "option is set, or that the cuda_compute_capabilities easyconfig parameter is defined." + raise EasyBuildError(error_msg, key) + else: + error_msg = "%s is not a template value baed on --cuda-compute-capabilities/cuda_compute_capabilities" + raise EasyBuildError(error_msg, key) + def det_installversion(version, toolchain_name, toolchain_version, prefix, suffix): """Deprecated 'det_installversion' function, to determine exact install version, based on supplied parameters.""" diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index bd0fd80a3c..d904814d4e 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4483,6 +4483,56 @@ def test_easyconfig_import(self): error_pattern = r"Failed to copy '.*' easyconfig parameter" self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, test_ec) + def test_get_cuda_cc_template_value(self): + """ + Test getting template value based on --cuda-compute-capabilities / cuda_compute_capabilities. + """ + self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.14"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = SYSTEM', + ]) + self.prep() + ec = EasyConfig(self.eb_file) + + error_pattern = "foobar is not a template value baed on --cuda-compute-capabilities/cuda_compute_capabilities" + self.assertErrorRegex(EasyBuildError, error_pattern, ec.get_cuda_cc_template_value, 'foobar') + + error_pattern = r"\(get_cuda_cc_template_value\) Template value '%s' is not defined!\n" + error_pattern += r"Make sure that either the --cuda-compute-capabilities EasyBuild configuration " + error_pattern += "option is set, or that the cuda_compute_capabilities easyconfig parameter is defined." + cuda_template_values = { + 'cuda_compute_capabilities': '6.5,7.0', + 'cuda_cc_space_sep': '6.5 7.0', + 'cuda_cc_semicolon_sep': '6.5;7.0', + 'cuda_sm_comma_sep': 'sm_65,sm_70', + 'cuda_sm_space_sep': 'sm_65 sm_70', + } + for key in cuda_template_values: + self.assertErrorRegex(EasyBuildError, error_pattern % key, ec.get_cuda_cc_template_value, key) + + update_build_option('cuda_compute_capabilities', ['6.5', '7.0']) + ec = EasyConfig(self.eb_file) + + for key in cuda_template_values: + self.assertEqual(ec.get_cuda_cc_template_value(key), cuda_template_values[key]) + + update_build_option('cuda_compute_capabilities', None) + ec = EasyConfig(self.eb_file) + + for key in cuda_template_values: + self.assertErrorRegex(EasyBuildError, error_pattern % key, ec.get_cuda_cc_template_value, key) + + self.contents += "\ncuda_compute_capabilities = ['6.5', '7.0']" + self.prep() + ec = EasyConfig(self.eb_file) + + for key in cuda_template_values: + self.assertEqual(ec.get_cuda_cc_template_value(key), cuda_template_values[key]) + def suite(): """ returns all the testcases in this module """ From 2d993457d5a95c06d6023fec02b649bf1719a4ff Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 20 Aug 2021 15:54:39 +0200 Subject: [PATCH 47/91] fix typo in error message --- easybuild/framework/easyconfig/easyconfig.py | 2 +- test/framework/easyconfig.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 6597c00aac..322c9546e2 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1819,7 +1819,7 @@ def get_cuda_cc_template_value(self, key): error_msg += "option is set, or that the cuda_compute_capabilities easyconfig parameter is defined." raise EasyBuildError(error_msg, key) else: - error_msg = "%s is not a template value baed on --cuda-compute-capabilities/cuda_compute_capabilities" + error_msg = "%s is not a template value based on --cuda-compute-capabilities/cuda_compute_capabilities" raise EasyBuildError(error_msg, key) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index d904814d4e..64cb3ea1ae 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4498,7 +4498,7 @@ def test_get_cuda_cc_template_value(self): self.prep() ec = EasyConfig(self.eb_file) - error_pattern = "foobar is not a template value baed on --cuda-compute-capabilities/cuda_compute_capabilities" + error_pattern = "foobar is not a template value based on --cuda-compute-capabilities/cuda_compute_capabilities" self.assertErrorRegex(EasyBuildError, error_pattern, ec.get_cuda_cc_template_value, 'foobar') error_pattern = r"\(get_cuda_cc_template_value\) Template value '%s' is not defined!\n" From 204bcf168f4e2f0958176182ad343fcafe1bd7bc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 21 Aug 2021 12:08:32 +0200 Subject: [PATCH 48/91] apply @Flamefire's suggested code changes Co-authored-by: Alexander Grund --- easybuild/framework/easyconfig/easyconfig.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 322c9546e2..155dc487f1 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1810,10 +1810,10 @@ def get_cuda_cc_template_value(self, key): Returns user-friendly error message in case neither are defined, or if an unknown key is used. """ - if key.startswith('cuda_') and key in [x for (x, _) in TEMPLATE_NAMES_DYNAMIC]: - if key in self.template_values: + if key.startswith('cuda_') and any(x[0] == key for x in TEMPLATE_NAMES_DYNAMIC): + try: return self.template_values[key] - else: + except KeyError: error_msg = "(get_cuda_cc_template_value) Template value '%s' is not defined!\n" error_msg += "Make sure that either the --cuda-compute-capabilities EasyBuild configuration " error_msg += "option is set, or that the cuda_compute_capabilities easyconfig parameter is defined." From f9db1772f9efa235119b8e505a98fe5009dd090b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 21 Aug 2021 12:10:37 +0200 Subject: [PATCH 49/91] drop '(get_cuda_cc_template_value)' from error message --- easybuild/framework/easyconfig/easyconfig.py | 2 +- test/framework/easyconfig.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 155dc487f1..d0470018e8 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1814,7 +1814,7 @@ def get_cuda_cc_template_value(self, key): try: return self.template_values[key] except KeyError: - error_msg = "(get_cuda_cc_template_value) Template value '%s' is not defined!\n" + error_msg = "Template value '%s' is not defined!\n" error_msg += "Make sure that either the --cuda-compute-capabilities EasyBuild configuration " error_msg += "option is set, or that the cuda_compute_capabilities easyconfig parameter is defined." raise EasyBuildError(error_msg, key) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 64cb3ea1ae..96ce325e6e 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4501,7 +4501,7 @@ def test_get_cuda_cc_template_value(self): error_pattern = "foobar is not a template value based on --cuda-compute-capabilities/cuda_compute_capabilities" self.assertErrorRegex(EasyBuildError, error_pattern, ec.get_cuda_cc_template_value, 'foobar') - error_pattern = r"\(get_cuda_cc_template_value\) Template value '%s' is not defined!\n" + error_pattern = r"Template value '%s' is not defined!\n" error_pattern += r"Make sure that either the --cuda-compute-capabilities EasyBuild configuration " error_pattern += "option is set, or that the cuda_compute_capabilities easyconfig parameter is defined." cuda_template_values = { From 178f546c4731c5635e94de1e3a409630a5a4ed10 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 23 Aug 2021 18:10:55 +0000 Subject: [PATCH 50/91] added bash as an option in the fix_*_shebang_for --- easybuild/framework/easyblock.py | 2 +- easybuild/framework/easyconfig/default.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 93810300b4..d6d8b622f1 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2466,7 +2466,7 @@ def package_step(self): def fix_shebang(self): """Fix shebang lines for specified files.""" - for lang in ['perl', 'python']: + for lang in ['bash', 'perl', 'python']: shebang_regex = re.compile(r'^#![ ]*.*[/ ]%s.*' % lang) fix_shebang_for = self.cfg['fix_%s_shebang_for' % lang] if fix_shebang_for: diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 5279332541..70571c12d5 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -95,6 +95,8 @@ 'easybuild_version': [None, "EasyBuild-version this spec-file was written for", BUILD], 'enhance_sanity_check': [False, "Indicate that additional sanity check commands & paths should enhance " "the existin sanity check, not replace it", BUILD], + 'fix_bash_shebang_for': [None, "List of files for which Bash shebang should be fixed " + "to '#!/usr/bin/env bash' (glob patterns supported)", BUILD], 'fix_perl_shebang_for': [None, "List of files for which Perl shebang should be fixed " "to '#!/usr/bin/env perl' (glob patterns supported)", BUILD], 'fix_python_shebang_for': [None, "List of files for which Python shebang should be fixed " From fe60a29569abe7113270e6ca37e243ae27621ee8 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 23 Aug 2021 18:52:33 +0000 Subject: [PATCH 51/91] refactored test_fix_shebang to reduce code duplication, added tests for bash --- test/framework/toy_build.py | 101 ++++++++++++++++++++---------------- 1 file changed, 55 insertions(+), 46 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 51d3f1a977..698494ec55 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2937,6 +2937,7 @@ def test_fix_shebang(self): # copy of bin/toy to use in fix_python_shebang_for and fix_perl_shebang_for " 'cp -a %(installdir)s/bin/toy %(installdir)s/bin/toy.python',", " 'cp -a %(installdir)s/bin/toy %(installdir)s/bin/toy.perl',", + " 'cp -a %(installdir)s/bin/toy %(installdir)s/bin/toy.sh',", # hardcoded path to bin/python " 'echo \"#!/usr/bin/python\\n# test\" > %(installdir)s/bin/t1.py',", @@ -2973,9 +2974,26 @@ def test_fix_shebang(self): # shebang bash " 'echo \"#!/usr/bin/env bash\\n# test\" > %(installdir)s/bin/b2.sh',", + # tests for bash shebang + # hardcoded path to bin/bash + " 'echo \"#!/bin/bash\\n# test\" > %(installdir)s/bin/t1.sh',", + # hardcoded path to usr/bin/bash + " 'echo \"#!/usr/bin/bash\\n# test\" > %(installdir)s/bin/t2.sh',", + # already OK, should remain the same + " 'echo \"#!/usr/bin/env bash\\n# test\" > %(installdir)s/bin/t3.sh',", + # shebang with space, should strip the space + " 'echo \"#! /usr/bin/env bash\\n# test\" > %(installdir)s/bin/t4.sh',", + # no shebang sh + " 'echo \"# test\" > %(installdir)s/bin/t5.sh',", + # shebang python + " 'echo \"#!/usr/bin/env python\\n# test\" > %(installdir)s/bin/b1.py',", + # shebang perl + " 'echo \"#!/usr/bin/env perl\\n# test\" > %(installdir)s/bin/b1.pl',", + "]", - "fix_python_shebang_for = ['bin/t1.py', 'bin/*.py', 'nosuchdir/*.py', 'bin/toy.python', 'bin/b1.sh']", - "fix_perl_shebang_for = ['bin/*.pl', 'bin/b2.sh', 'bin/toy.perl']", + "fix_python_shebang_for = ['bin/t1.py', 'bin/t*.py', 'nosuchdir/*.py', 'bin/toy.python', 'bin/b1.sh']", + "fix_perl_shebang_for = ['bin/t*.pl', 'bin/b2.sh', 'bin/toy.perl']", + "fix_bash_shebang_for = ['bin/t*.sh', 'bin/b1.py', 'bin/b1.pl', 'bin/toy.sh']", ]) write_file(test_ec, test_ec_txt) self.test_toy_build(ec_file=test_ec, raise_error=True) @@ -2984,36 +3002,32 @@ def test_fix_shebang(self): # bin/toy and bin/toy2 should *not* be patched, since they're binary files toy_txt = read_file(os.path.join(toy_bindir, 'toy'), mode='rb') - for fn in ['toy.perl', 'toy.python']: + for fn in ['toy.sh', 'toy.perl', 'toy.python']: fn_txt = read_file(os.path.join(toy_bindir, fn), mode='rb') # no shebang added self.assertFalse(fn_txt.startswith(b"#!/")) # exact same file as original binary (untouched) self.assertEqual(toy_txt, fn_txt) + regexes = { } # no re.M, this should match at start of file! - py_shebang_regex = re.compile(r'^#!/usr/bin/env python\n# test$') - for pybin in ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py', 't7.py']: - pybin_path = os.path.join(toy_bindir, pybin) - pybin_txt = read_file(pybin_path) - self.assertTrue(py_shebang_regex.match(pybin_txt), - "Pattern '%s' found in %s: %s" % (py_shebang_regex.pattern, pybin_path, pybin_txt)) + regexes['py'] = re.compile(r'^#!/usr/bin/env python\n# test$') + regexes['pl'] = re.compile(r'^#!/usr/bin/env perl\n# test$') + regexes['sh'] = re.compile(r'^#!/usr/bin/env bash\n# test$') - # no re.M, this should match at start of file! - perl_shebang_regex = re.compile(r'^#!/usr/bin/env perl\n# test$') - for perlbin in ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl', 't6.pl', 't7.pl']: - perlbin_path = os.path.join(toy_bindir, perlbin) - perlbin_txt = read_file(perlbin_path) - self.assertTrue(perl_shebang_regex.match(perlbin_txt), - "Pattern '%s' found in %s: %s" % (perl_shebang_regex.pattern, perlbin_path, perlbin_txt)) - - # There are 2 bash files which shouldn't be influenced by fix_shebang - bash_shebang_regex = re.compile(r'^#!/usr/bin/env bash\n# test$') - for bashbin in ['b1.sh', 'b2.sh']: - bashbin_path = os.path.join(toy_bindir, bashbin) - bashbin_txt = read_file(bashbin_path) - self.assertTrue(bash_shebang_regex.match(bashbin_txt), - "Pattern '%s' found in %s: %s" % (bash_shebang_regex.pattern, bashbin_path, bashbin_txt)) + + # all scripts should have a shebang that matches their extension + scripts = { } + scripts['py'] = ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py', 't7.py', 'b1.py'] + scripts['pl'] = ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl', 't6.pl', 't7.pl', 'b1.pl'] + scripts['sh'] = ['t1.sh', 't2.sh', 't3.sh', 't4.sh', 't5.sh', 'b1.sh', 'b2.sh'] + + for ext in ['sh', 'pl', 'py']: + for script in scripts[ext]: + bin_path = os.path.join(toy_bindir, script) + bin_txt = read_file(bin_path) + self.assertTrue(regexes[ext].match(bin_txt), + "Pattern '%s' found in %s: %s" % (regexes[ext].pattern, bin_path, bin_txt)) # now test with a custom env command extra_args = ['--env-for-shebang=/usr/bin/env -S'] @@ -3023,36 +3037,31 @@ def test_fix_shebang(self): # bin/toy and bin/toy2 should *not* be patched, since they're binary files toy_txt = read_file(os.path.join(toy_bindir, 'toy'), mode='rb') - for fn in ['toy.perl', 'toy.python']: + for fn in ['toy.sh', 'toy.perl', 'toy.python']: fn_txt = read_file(os.path.join(toy_bindir, fn), mode='rb') # no shebang added self.assertFalse(fn_txt.startswith(b"#!/")) # exact same file as original binary (untouched) self.assertEqual(toy_txt, fn_txt) + regexes_S = { } # no re.M, this should match at start of file! - py_shebang_regex = re.compile(r'^#!/usr/bin/env -S python\n# test$') - for pybin in ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py', 't7.py']: - pybin_path = os.path.join(toy_bindir, pybin) - pybin_txt = read_file(pybin_path) - self.assertTrue(py_shebang_regex.match(pybin_txt), - "Pattern '%s' found in %s: %s" % (py_shebang_regex.pattern, pybin_path, pybin_txt)) + regexes_S['py'] = re.compile(r'^#!/usr/bin/env -S python\n# test$') + regexes_S['pl'] = re.compile(r'^#!/usr/bin/env -S perl\n# test$') + regexes_S['sh'] = re.compile(r'^#!/usr/bin/env -S bash\n# test$') + + for ext in ['sh', 'pl', 'py']: + for script in scripts[ext]: + bin_path = os.path.join(toy_bindir, script) + bin_txt = read_file(bin_path) + # the scripts b1.py, b1.pl, b1.sh, b2.sh should keep their original shebang + if script.startswith('b'): + self.assertTrue(regexes[ext].match(bin_txt), + "Pattern '%s' found in %s: %s" % (regexes[ext].pattern, bin_path, bin_txt)) + else: + self.assertTrue(regexes_S[ext].match(bin_txt), + "Pattern '%s' found in %s: %s" % (regexes_S[ext].pattern, bin_path, bin_txt)) - # no re.M, this should match at start of file! - perl_shebang_regex = re.compile(r'^#!/usr/bin/env -S perl\n# test$') - for perlbin in ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl', 't6.pl', 't7.pl']: - perlbin_path = os.path.join(toy_bindir, perlbin) - perlbin_txt = read_file(perlbin_path) - self.assertTrue(perl_shebang_regex.match(perlbin_txt), - "Pattern '%s' found in %s: %s" % (perl_shebang_regex.pattern, perlbin_path, perlbin_txt)) - - # There are 2 bash files which shouldn't be influenced by fix_shebang - bash_shebang_regex = re.compile(r'^#!/usr/bin/env bash\n# test$') - for bashbin in ['b1.sh', 'b2.sh']: - bashbin_path = os.path.join(toy_bindir, bashbin) - bashbin_txt = read_file(bashbin_path) - self.assertTrue(bash_shebang_regex.match(bashbin_txt), - "Pattern '%s' found in %s: %s" % (bash_shebang_regex.pattern, bashbin_path, bashbin_txt)) def test_toy_system_toolchain_alias(self): """Test use of 'system' toolchain alias.""" From 080acd34159c0241175484e1f0c92c4813c25aba Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 23 Aug 2021 18:57:02 +0000 Subject: [PATCH 52/91] appeasing hound --- test/framework/toy_build.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 698494ec55..d253b0f66b 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3009,15 +3009,14 @@ def test_fix_shebang(self): # exact same file as original binary (untouched) self.assertEqual(toy_txt, fn_txt) - regexes = { } + regexes = {} # no re.M, this should match at start of file! regexes['py'] = re.compile(r'^#!/usr/bin/env python\n# test$') regexes['pl'] = re.compile(r'^#!/usr/bin/env perl\n# test$') regexes['sh'] = re.compile(r'^#!/usr/bin/env bash\n# test$') - # all scripts should have a shebang that matches their extension - scripts = { } + scripts = {} scripts['py'] = ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py', 't7.py', 'b1.py'] scripts['pl'] = ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl', 't6.pl', 't7.pl', 'b1.pl'] scripts['sh'] = ['t1.sh', 't2.sh', 't3.sh', 't4.sh', 't5.sh', 'b1.sh', 'b2.sh'] @@ -3044,7 +3043,7 @@ def test_fix_shebang(self): # exact same file as original binary (untouched) self.assertEqual(toy_txt, fn_txt) - regexes_S = { } + regexes_S = {} # no re.M, this should match at start of file! regexes_S['py'] = re.compile(r'^#!/usr/bin/env -S python\n# test$') regexes_S['pl'] = re.compile(r'^#!/usr/bin/env -S perl\n# test$') From a2cb1bc652bd8af3da7ceedee63c9cb5500ce4cf Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 23 Aug 2021 20:35:37 +0000 Subject: [PATCH 53/91] removed extra blank line --- test/framework/toy_build.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index d253b0f66b..4e55a3e9f0 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3061,7 +3061,6 @@ def test_fix_shebang(self): self.assertTrue(regexes_S[ext].match(bin_txt), "Pattern '%s' found in %s: %s" % (regexes_S[ext].pattern, bin_path, bin_txt)) - def test_toy_system_toolchain_alias(self): """Test use of 'system' toolchain alias.""" toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') From dda3a48aec2dd6ce813565d3724c1a9cfdb86e66 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 30 Aug 2021 16:46:04 +0200 Subject: [PATCH 54/91] Speedup EasyBlock.set_parallel - Small refactoring to reduce dictionary access to cfg[parallel] option - Cache default parallel value in det_parallelism to avoid systemcalls --- easybuild/framework/easyblock.py | 16 +++++++------ easybuild/tools/systemtools.py | 41 ++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index d8829d8587..22a13e630a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1791,18 +1791,20 @@ def set_parallel(self): """Set 'parallel' easyconfig parameter to determine how many cores can/should be used for parallel builds.""" # set level of parallelism for build par = build_option('parallel') - if self.cfg['parallel'] is not None: + cfg_par = self.cfg['parallel'] + if cfg_par is None: + self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par) + else: if par is None: - par = self.cfg['parallel'] + par = cfg_par self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par) else: - par = min(int(par), int(self.cfg['parallel'])) + par = min(int(par), int(cfg_par)) self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par) - else: - self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par) - self.cfg['parallel'] = det_parallelism(par=par, maxpar=self.cfg['maxparallel']) - self.log.info("Setting parallelism: %s" % self.cfg['parallel']) + par = det_parallelism(par, maxpar=self.cfg['maxparallel']) + self.log.info("Setting parallelism: %s" % par) + self.cfg['parallel'] = par def remove_module_file(self): """Remove module file (if it exists), and check for ghost installation directory (and deal with it).""" diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 108cdb4571..16d471f37c 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -976,31 +976,42 @@ def det_parallelism(par=None, maxpar=None): Determine level of parallelism that should be used. Default: educated guess based on # cores and 'ulimit -u' setting: min(# cores, ((ulimit -u) - 15) // 6) """ - if par is not None: - if not isinstance(par, int): + def get_default_parallelism(): + try: + # Get cache value if any + par = det_parallelism._default_parallelism + except AttributeError: + # No cache -> Calculate value from current system values + par = get_avail_core_count() + # check ulimit -u + out, ec = run_cmd('ulimit -u', force_in_dry_run=True, trace=False, stream_output=False) try: - par = int(par) + if out.startswith("unlimited"): + maxuserproc = 2 ** 32 - 1 + else: + maxuserproc = int(out) except ValueError as err: - raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err) - else: - par = get_avail_core_count() - # check ulimit -u - out, ec = run_cmd('ulimit -u', force_in_dry_run=True, trace=False, stream_output=False) - try: - if out.startswith("unlimited"): - out = 2 ** 32 - 1 - maxuserproc = int(out) + raise EasyBuildError("Failed to determine max user processes (%s, %s): %s", ec, out, err) # assume 6 processes per build thread + 15 overhead - par_guess = int((maxuserproc - 15) // 6) + par_guess = (maxuserproc - 15) // 6 if par_guess < par: par = par_guess _log.info("Limit parallel builds to %s because max user processes is %s" % (par, out)) + # Cache value + det_parallelism._default_parallelism = par + return par + + if par is None: + par = get_default_parallelism() + else: + try: + par = int(par) except ValueError as err: - raise EasyBuildError("Failed to determine max user processes (%s, %s): %s", ec, out, err) + raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err) if maxpar is not None and maxpar < par: _log.info("Limiting parallellism from %s to %s" % (par, maxpar)) - par = min(par, maxpar) + par = maxpar return par From 0f19f1c664160d7411bb49d80687d4c80c5af8c3 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 30 Aug 2021 18:14:59 +0200 Subject: [PATCH 55/91] handle test failure --- test/framework/easyconfig.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 96ce325e6e..48aaeb86a4 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3052,6 +3052,10 @@ def test_template_constant_dict(self): self.assertEqual(res, expected) # mock get_avail_core_count which is used by set_parallel -> det_parallelism + try: + del st.det_parallelism._default_parallelism # Remove cache value + except AttributeError: + pass # Ignore if not present orig_get_avail_core_count = st.get_avail_core_count st.get_avail_core_count = lambda: 42 From 04204a14d44ef2ccc469e8362ec1034b997a3a94 Mon Sep 17 00:00:00 2001 From: Christoph Siegert Date: Tue, 31 Aug 2021 14:53:30 +0200 Subject: [PATCH 56/91] add --easystack to ignore_opts for submit_job() --- easybuild/tools/parallelbuild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py index df151aa3a1..0621a29516 100644 --- a/easybuild/tools/parallelbuild.py +++ b/easybuild/tools/parallelbuild.py @@ -127,7 +127,7 @@ def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True): curdir = os.getcwd() # regex pattern for options to ignore (help options can't reach here) - ignore_opts = re.compile('^--robot$|^--job|^--try-.*$') + ignore_opts = re.compile('^--robot$|^--job|^--try-.*$|^--easystack$') # generate_cmd_line returns the options in form --longopt=value opts = [o for o in cmd_line_opts if not ignore_opts.match(o.split('=')[0])] From a99001b2faa24846c4727604ca24a5f6a7e69734 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 31 Aug 2021 17:29:14 +0200 Subject: [PATCH 57/91] trivial style fixes --- easybuild/framework/easyblock.py | 11 +++++------ easybuild/tools/systemtools.py | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 22a13e630a..436ac373e3 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1794,13 +1794,12 @@ def set_parallel(self): cfg_par = self.cfg['parallel'] if cfg_par is None: self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par) + elif par is None: + par = cfg_par + self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par) else: - if par is None: - par = cfg_par - self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par) - else: - par = min(int(par), int(cfg_par)) - self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par) + par = min(int(par), int(cfg_par)) + self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par) par = det_parallelism(par, maxpar=self.cfg['maxparallel']) self.log.info("Setting parallelism: %s" % par) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 16d471f37c..a43338e147 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -996,7 +996,7 @@ def get_default_parallelism(): par_guess = (maxuserproc - 15) // 6 if par_guess < par: par = par_guess - _log.info("Limit parallel builds to %s because max user processes is %s" % (par, out)) + _log.info("Limit parallel builds to %s because max user processes is %s", par, out) # Cache value det_parallelism._default_parallelism = par return par @@ -1010,7 +1010,7 @@ def get_default_parallelism(): raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err) if maxpar is not None and maxpar < par: - _log.info("Limiting parallellism from %s to %s" % (par, maxpar)) + _log.info("Limiting parallellism from %s to %s", par, maxpar) par = maxpar return par From 6a831521af6be42d103acdf157421d6d680fe502 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 1 Sep 2021 15:48:31 +0200 Subject: [PATCH 58/91] trivial code style tweaking in get_source_tarball_from_git --- easybuild/tools/filetools.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 983abdcb76..635e897ee9 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2481,7 +2481,8 @@ def get_source_tarball_from_git(filename, targetdir, git_config): if recursive: clone_cmd.append('--recursive') else: - clone_cmd.append('--no-checkout') # We do that manually below + # checkout is done separately below for specific commits + clone_cmd.append('--no-checkout') clone_cmd.append('%s/%s.git' % (url, repo_name)) @@ -2496,6 +2497,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): checkout_cmd.extend(['&&', 'git', 'submodule', 'update', '--init', '--recursive']) run.run_cmd(' '.join(checkout_cmd), log_all=True, simple=True, regexp=False, path=repo_name) + elif not build_option('extended_dry_run'): # If we wanted to get a tag make sure we actually got a tag and not a branch with the same name # This doesn't make sense in dry-run mode as we don't have anything to check @@ -2507,10 +2509,13 @@ def get_source_tarball_from_git(filename, targetdir, git_config): ' with the same name. You might want to alert the maintainers of %s about that issue.', tag, url, repo_name, repo_name) cmds = [] + if not keep_git_dir: - # Make the repo unshallow, same as git fetch --unshallow in git 1.8.3+ - # The first fetch seemingly does nothing, no idea why. + # make the repo unshallow first; + # this is equivalent with 'git fetch -unshallow' in Git 1.8.3+ + # (first fetch seems to do nothing, unclear why) cmds.append('git fetch --depth=2147483647 && git fetch --depth=2147483647') + cmds.append('git checkout refs/tags/' + tag) # Clean all untracked files, e.g. from left-over submodules cmds.append('git clean --force -d -x') From e1bed5f198bb4aae86ddd845b94e1b6a91f3b5d4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 2 Sep 2021 10:40:11 +0200 Subject: [PATCH 59/91] clarify use of both exts_list and components in is_patch_for --- easybuild/tools/github.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 8f96f4f0e6..a495e8de9a 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1015,12 +1015,13 @@ def is_patch_for(patch_name, ec): patches = copy.copy(ec['patches']) with ec.disable_templating(): - for ext in itertools.chain(ec['exts_list'], ec.get('components', [])): - if isinstance(ext, (list, tuple)) and len(ext) == 3 and isinstance(ext[2], dict): - templates = {'name': ext[0], 'version': ext[1]} - ext_options = ext[2] + # take into account both list of extensions (via exts_list) and components (cfr. Bundle easyblock) + for entry in itertools.chain(ec['exts_list'], ec.get('components', [])): + if isinstance(entry, (list, tuple)) and len(entry) == 3 and isinstance(entry[2], dict): + templates = {'name': entry[0], 'version': entry[1]} + options = entry[2] patches.extend(p[0] % templates if isinstance(p, (tuple, list)) else p % templates - for p in ext_options.get('patches', [])) + for p in options.get('patches', [])) for patch in patches: if isinstance(patch, (tuple, list)): From e51c5f9104206eea87e5575d5bbd97b167deff44 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 2 Sep 2021 10:58:02 +0200 Subject: [PATCH 60/91] pick up $MODULES_CMD to facilitate using Environment Modules 4.x as modules tool (fixes #3815) --- easybuild/tools/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index d8df2b0db2..5e48591be5 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -187,7 +187,6 @@ def __init__(self, mod_paths=None, testing=False): self.cmd = env_cmd_path self.log.debug("Set %s command via environment variable %s: %s", self.NAME, self.COMMAND_ENVIRONMENT, self.cmd) - # check whether paths obtained via $PATH and $LMOD_CMD are different elif cmd_path != env_cmd_path: self.log.debug("Different paths found for %s command '%s' via which/$PATH and $%s: %s vs %s", self.NAME, self.COMMAND, self.COMMAND_ENVIRONMENT, cmd_path, env_cmd_path) @@ -1315,6 +1314,7 @@ class EnvironmentModules(EnvironmentModulesTcl): """Interface to environment modules 4.0+""" NAME = "Environment Modules v4" COMMAND = os.path.join(os.getenv('MODULESHOME', 'MODULESHOME_NOT_DEFINED'), 'libexec', 'modulecmd.tcl') + COMMAND_ENVIRONMENT = 'MODULES_CMD' REQ_VERSION = '4.0.0' MAX_VERSION = None VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' From f232500a7f2224b9713078ba3a6c88a3bcdc03d7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 2 Sep 2021 16:58:01 +0200 Subject: [PATCH 61/91] use more sensible branch name for creating easyblocks PR with --new-pr --- easybuild/tools/github.py | 2 ++ test/framework/options.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index d49cbc2f5e..a381034816 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -901,6 +901,8 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ if pr_branch is None: if ec_paths and pr_target_repo == GITHUB_EASYCONFIGS_REPO: label = file_info['ecs'][0].name + re.sub('[.-]', '', file_info['ecs'][0].version) + elif pr_target_repo == GITHUB_EASYBLOCKS_REPO and paths.get('py_files'): + label = os.path.splitext(os.path.basename(paths['py_files'][0]))[0] else: label = ''.join(random.choice(ascii_letters) for _ in range(10)) pr_branch = '%s_new_pr_%s' % (time.strftime("%Y%m%d%H%M%S"), label) diff --git a/test/framework/options.py b/test/framework/options.py index f40829b34b..c9a64bfd0e 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4020,7 +4020,7 @@ def test_new_branch_github(self): regexs = [ r"^== fetching branch 'develop' from https://github.com/easybuilders/easybuild-easyconfigs.git\.\.\.", r"^== copying files to .*/easybuild-easyconfigs\.\.\.", - r"^== pushing branch '.*' to remote '.*' \(%s\) \[DRY RUN\]" % remote, + r"^== pushing branch '[0-9]{14}_new_pr_toy00' to remote '.*' \(%s\) \[DRY RUN\]" % remote, ] self._assert_regexs(regexs, txt) @@ -4041,7 +4041,7 @@ def test_new_branch_github(self): regexs = [ r"^== fetching branch 'develop' from https://github.com/easybuilders/easybuild-easyblocks.git\.\.\.", r"^== copying files to .*/easybuild-easyblocks\.\.\.", - r"^== pushing branch '.*' to remote '.*' \(%s\) \[DRY RUN\]" % remote, + r"^== pushing branch '[0-9]{14}_new_pr_toy' to remote '.*' \(%s\) \[DRY RUN\]" % remote, ] self._assert_regexs(regexs, txt) @@ -4068,7 +4068,7 @@ def test_new_branch_github(self): regexs = [ r"^== fetching branch 'develop' from https://github.com/easybuilders/easybuild-framework.git\.\.\.", r"^== copying files to .*/easybuild-framework\.\.\.", - r"^== pushing branch '.*' to remote '.*' \(%s\) \[DRY RUN\]" % remote, + r"^== pushing branch '[0-9]{14}_new_pr_[A-Za-z]{10}' to remote '.*' \(%s\) \[DRY RUN\]" % remote, ] self._assert_regexs(regexs, txt) From a4cbc73f0158b8f670971a660ae39ab370e23934 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 6 Sep 2021 10:27:14 +0800 Subject: [PATCH 62/91] prepare release notes for EasyBuild v4.4.2 + bump version to 4.4.2 --- RELEASE_NOTES | 36 ++++++++++++++++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 402f9a8e9a..ee11381f72 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,42 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.4.2 (September 7th 2021) +--------------------------- + +update/bugfix release + +- add definition for new toolchain nvpsmpic (NVHPC + ParaStationMPI) (#3736) +- various enhancements, including: + - add per-extension timing in output produced by eb command (#3734) + - include list of missing libraries in warning about missing FFTW libraries in imkl toolchain component (#3776) + - check for recursive symlinks by default before copying a folder (#3784) + - add --filter-ecs options to filter out easyconfigs from set of easyconfigs to install (#3796) + - check type of source_tmpl value for extensions, ensure it's a string value (not a list) (#3799) + - also define $BLAS_SHARED_LIBS & co in build environment (analogous to $BLAS_STATIC_LIBS) (#3800) + - report use of --ignore-test-failure in success message in output (#3806) + - add get_cuda_cc_template_value method to EasyConfig class (#3807) + - add support for fix_bash_shebang_for (#3808) + - pick up $MODULES_CMD to facilitate using Environment Modules 4.x as modules tool (#3816) + - use more sensible branch name for creating easyblocks PR with --new-pr (#3817) +- various bug fixes, including: + - remove Python 2.6 from list of supported Python versions in setup.py (#3767) + - don't add directory that doesn't include any files to $PATH or $LD_LIBRARY_PATH (#3769) + - make logdir writable also when --stop/--fetch is used and --read-only-installdir is enabled (#3771) + - fix forgotten renaming of 'l' to 'char' __init__.py that is created for included Python modules (#3773) + - fix verify_imports by deleting all imported modules before re-importing them one by one (#3780) + - fix ignore_test_failure not set for Extensions (#3782) + - update iompi toolchain to intel-compiler subtoolchain for oneAPI versions (>= iompi 2020.12) (#3785) + - don't parse patch files as easyconfigs when searching for where patch file is used (#3786) + - make sure git clone with a tag argument actually downloads a tag (#3795) + - fix CI by excluding GC3Pie 2.6.7 (which is broken with Python 2) and improve error reporting for option parsing (#3798) + - speed up tests by caching checked paths in set_tmpdir + less test cases for test_compiler_dependent_optarch (#3802) + - correctly resolve templates for patches in extensions when uploading to GitHub (#3805) + - add --easystack to ignore_opts for submit_job() (#3813) +- other changes: + - speed up set_parallel method in EasyBlock class (#3812) + + v4.4.1 (July 6th 2021) ---------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 6e9468b67c..70aa973100 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.4.2.dev0') +VERSION = LooseVersion('4.4.2') UNKNOWN = 'UNKNOWN' From def7bd4b5ed7aa92b0220fd49f91fb558697c0a3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 6 Sep 2021 09:04:00 +0200 Subject: [PATCH 63/91] minor tweaks to v4.4.2 release notes --- RELEASE_NOTES | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index ee11381f72..e622dd198b 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -9,12 +9,12 @@ v4.4.2 (September 7th 2021) update/bugfix release -- add definition for new toolchain nvpsmpic (NVHPC + ParaStationMPI) (#3736) - various enhancements, including: - add per-extension timing in output produced by eb command (#3734) + - add definition for new toolchain nvpsmpic (NVHPC + ParaStationMPI + CUDA) (#3736) - include list of missing libraries in warning about missing FFTW libraries in imkl toolchain component (#3776) - check for recursive symlinks by default before copying a folder (#3784) - - add --filter-ecs options to filter out easyconfigs from set of easyconfigs to install (#3796) + - add --filter-ecs configuration option to filter out easyconfigs from set of easyconfigs to install (#3796) - check type of source_tmpl value for extensions, ensure it's a string value (not a list) (#3799) - also define $BLAS_SHARED_LIBS & co in build environment (analogous to $BLAS_STATIC_LIBS) (#3800) - report use of --ignore-test-failure in success message in output (#3806) @@ -28,15 +28,15 @@ update/bugfix release - make logdir writable also when --stop/--fetch is used and --read-only-installdir is enabled (#3771) - fix forgotten renaming of 'l' to 'char' __init__.py that is created for included Python modules (#3773) - fix verify_imports by deleting all imported modules before re-importing them one by one (#3780) - - fix ignore_test_failure not set for Extensions (#3782) + - fix ignore_test_failure not set for Extension instances (#3782) - update iompi toolchain to intel-compiler subtoolchain for oneAPI versions (>= iompi 2020.12) (#3785) - don't parse patch files as easyconfigs when searching for where patch file is used (#3786) - make sure git clone with a tag argument actually downloads a tag (#3795) - fix CI by excluding GC3Pie 2.6.7 (which is broken with Python 2) and improve error reporting for option parsing (#3798) - - speed up tests by caching checked paths in set_tmpdir + less test cases for test_compiler_dependent_optarch (#3802) - correctly resolve templates for patches in extensions when uploading to GitHub (#3805) - - add --easystack to ignore_opts for submit_job() (#3813) + - add --easystack to ignored options when submitting job (#3813) - other changes: + - speed up tests by caching checked paths in set_tmpdir + less test cases for test_compiler_dependent_optarch (#3802) - speed up set_parallel method in EasyBlock class (#3812) From 3d673307d9863b6aba16ed0f6e8e2c8d9975c16e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 Sep 2021 09:02:35 +0200 Subject: [PATCH 64/91] bump version to 4.4.3dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 70aa973100..2216e1f42d 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.4.2') +VERSION = LooseVersion('4.4.3.dev0') UNKNOWN = 'UNKNOWN' From df39e6e3fb61b728c00bedde9401da9ee87ee747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= Date: Wed, 8 Sep 2021 08:58:06 +0200 Subject: [PATCH 65/91] Add progressbar to installation procedure Utilize [`tqdm`](https://github.com/tqdm/tqdm) to track progress when installing EasyBuilds. This require a few changes to allow `print_msg` and the progressbar to interact. --- easybuild/framework/easyblock.py | 57 ++++++++++++++++++++++---------- easybuild/main.py | 18 +++++++--- easybuild/tools/build_log.py | 6 +++- requirements.txt | 2 ++ 4 files changed, 60 insertions(+), 23 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index ee6d2eae49..43a4286b97 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -212,6 +212,9 @@ def __init__(self, ec): self.postmsg = '' # allow a post message to be set, which can be shown as last output self.current_step = None + # Create empty progress bar + self.progressbar = None + # list of loaded modules self.loaded_modules = [] @@ -300,6 +303,13 @@ def close_log(self): self.log.info("Closing log for application name %s version %s" % (self.name, self.version)) fancylogger.logToFile(self.logfile, enable=False) + def set_progressbar(self, progressbar): + """ + Set progress bar, the progress bar is needed when writing messages so + that the progress counter is always at the bottom + """ + self.progressbar = progressbar + # # DRY RUN UTILITIES # @@ -318,7 +328,7 @@ def dry_run_msg(self, msg, *args): """Print dry run message.""" if args: msg = msg % args - dry_run_msg(msg, silent=self.silent) + dry_run_msg(msg, silent=self.silent, progressbar=self.progressbar) # # FETCH UTILITY FUNCTIONS @@ -1637,7 +1647,8 @@ def skip_extensions(self): self.log.debug("exit code: %s, stdout/err: %s", ec, cmdstdouterr) res.append(ext_inst) else: - print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log) + print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log, + progressbar=self.progressbar) self.ext_instances = res @@ -1741,7 +1752,8 @@ def handle_iterate_opts(self): self.log.debug("Found list for %s: %s", opt, self.iter_opts[opt]) if self.iter_opts: - print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent) + print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent, + progressbar=self.progressbar) self.log.info("Current iteration index: %s", self.iter_idx) # pop first element from all iterative easyconfig parameters as next value to use @@ -1882,7 +1894,8 @@ def check_readiness_step(self): hidden = LooseVersion(self.modules_tool.version) < LooseVersion('7.0.0') self.mod_file_backup = back_up_file(self.mod_filepath, hidden=hidden, strip_fn=strip_fn) - print_msg("backup of existing module file stored at %s" % self.mod_file_backup, log=self.log) + print_msg("backup of existing module file stored at %s" % self.mod_file_backup, log=self.log, + progressbar=self.progressbar) # check if main install needs to be skipped # - if a current module can be found, skip is ok @@ -2418,7 +2431,7 @@ def extensions_step(self, fetch=False, install=True): change_dir(self.orig_workdir) tup = (ext.name, ext.version or '', idx + 1, exts_cnt) - print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent) + print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent, progressbar=self.progressbar) start_time = datetime.now() if self.dry_run: @@ -2450,9 +2463,11 @@ def extensions_step(self, fetch=False, install=True): if not self.dry_run: ext_duration = datetime.now() - start_time if ext_duration.total_seconds() >= 1: - print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent) + print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent, + progressbar=self.progressbar) elif self.logdebug or build_option('trace'): - print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent) + print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent, + progressbar=self.progressbar) # cleanup (unload fake module, remove fake module dir) if fake_mod_data: @@ -3250,7 +3265,7 @@ def make_module_step(self, fake=False): else: diff_msg += 'no differences found' self.log.info(diff_msg) - print_msg(diff_msg, log=self.log) + print_msg(diff_msg, log=self.log, progressbar=self.progressbar) self.invalidate_module_caches(modpath) @@ -3569,7 +3584,8 @@ def run_all_steps(self, run_test_cases): steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt()) - print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent) + print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent, + progressbar=self.progressbar) trace_msg("installation prefix: %s" % self.installdir) ignore_locks = build_option('ignore_locks') @@ -3589,12 +3605,12 @@ def run_all_steps(self, run_test_cases): try: for (step_name, descr, step_methods, skippable) in steps: if self.skip_step(step_name, skippable): - print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent) + print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent, progressbar=self.progressbar) else: if self.dry_run: self.dry_run_msg("%s... [DRY RUN]\n", descr) else: - print_msg("%s..." % descr, log=self.log, silent=self.silent) + print_msg("%s..." % descr, log=self.log, silent=self.silent, progressbar=self.progressbar) self.current_step = step_name start_time = datetime.now() try: @@ -3603,9 +3619,11 @@ def run_all_steps(self, run_test_cases): if not self.dry_run: step_duration = datetime.now() - start_time if step_duration.total_seconds() >= 1: - print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent) + print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent, + progressbar=self.progressbar) elif self.logdebug or build_option('trace'): - print_msg("... (took < 1 sec)", log=self.log, silent=self.silent) + print_msg("... (took < 1 sec)", log=self.log, silent=self.silent, + progressbar=self.progressbar) except StopException: pass @@ -3631,7 +3649,7 @@ def print_dry_run_note(loc, silent=True): dry_run_msg(msg, silent=silent) -def build_and_install_one(ecdict, init_env): +def build_and_install_one(ecdict, init_env, progressbar=None): """ Build the software :param ecdict: dictionary contaning parsed easyconfig + metadata @@ -3649,7 +3667,7 @@ def build_and_install_one(ecdict, init_env): if dry_run: dry_run_msg('', silent=silent) - print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent) + print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent, progressbar=progressbar) if dry_run: # print note on interpreting dry run output (argument is reference to location of dry run messages) @@ -3674,6 +3692,7 @@ def build_and_install_one(ecdict, init_env): try: app_class = get_easyblock_class(easyblock, name=name) app = app_class(ecdict['ec']) + app.set_progressbar(progressbar) _log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock)) except EasyBuildError as err: print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg), @@ -3856,7 +3875,8 @@ def ensure_writable_log_dir(log_dir): application_log = app.logfile req_time = time2str(end_timestamp - start_timestamp) - print_msg("%s: Installation %s %s (took %s)" % (summary, ended, succ, req_time), log=_log, silent=silent) + print_msg("%s: Installation %s %s (took %s)" % (summary, ended, succ, req_time), log=_log, silent=silent, + progressbar=progressbar) # check for errors if run.errors_found_in_log > 0: @@ -3864,7 +3884,7 @@ def ensure_writable_log_dir(log_dir): "build logs, please verify the build.", run.errors_found_in_log) if app.postmsg: - print_msg("\nWARNING: %s\n" % app.postmsg, log=_log, silent=silent) + print_msg("\nWARNING: %s\n" % app.postmsg, log=_log, silent=silent, progressbar=progressbar) if dry_run: # print note on interpreting dry run output (argument is reference to location of dry run messages) @@ -3878,7 +3898,8 @@ def ensure_writable_log_dir(log_dir): if application_log: # there may be multiple log files, or the file name may be different due to zipping logs = glob.glob('%s*' % application_log) - print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent) + print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent, + progressbar=progressbar) del app diff --git a/easybuild/main.py b/easybuild/main.py index 748c068376..1da4362597 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -39,6 +39,7 @@ import os import stat import sys +import tqdm import traceback # IMPORTANT this has to be the first easybuild import as it customises the logging @@ -98,7 +99,7 @@ def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing= return [(ec_file, generated)] -def build_and_install_software(ecs, init_session_state, exit_on_failure=True): +def build_and_install_software(ecs, init_session_state, exit_on_failure=True, progress=None): """ Build and install software for all provided parsed easyconfig files. @@ -113,9 +114,11 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): res = [] for ec in ecs: + if progress: + progress.set_description("Installing %s" % ec['short_mod_name']) ec_res = {} try: - (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env) + (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, progressbar=progress) ec_res['log_file'] = app_log if not ec_res['success']: ec_res['err'] = EasyBuildError(err) @@ -153,6 +156,8 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): raise EasyBuildError(test_msg) res.append((ec, ec_res)) + if progress: + progress.update() return res @@ -520,8 +525,13 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # build software, will exit when errors occurs (except when testing) if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) - - ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) + # Create progressbar around software to install + progress_bar = tqdm.tqdm(total=len(ordered_ecs), desc="EasyBuild", + leave=False, unit='EB') + ecs_with_res = build_and_install_software( + ordered_ecs, init_session_state, exit_on_failure=exit_on_failure, + progress=progress_bar) + progress_bar.close() else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index 2cf97c5f2d..2242a78dea 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -258,6 +258,7 @@ def print_msg(msg, *args, **kwargs): prefix = kwargs.pop('prefix', True) newline = kwargs.pop('newline', True) stderr = kwargs.pop('stderr', False) + pbar = kwargs.pop('progressbar', None) if kwargs: raise EasyBuildError("Unknown named arguments passed to print_msg: %s", kwargs) @@ -272,6 +273,8 @@ def print_msg(msg, *args, **kwargs): if stderr: sys.stderr.write(msg) + elif pbar: + pbar.write(msg, end='') else: sys.stdout.write(msg) @@ -304,6 +307,7 @@ def dry_run_msg(msg, *args, **kwargs): msg = msg % args silent = kwargs.pop('silent', False) + pbar = kwargs.pop('progressbar', None) if kwargs: raise EasyBuildError("Unknown named arguments passed to dry_run_msg: %s", kwargs) @@ -311,7 +315,7 @@ def dry_run_msg(msg, *args, **kwargs): if dry_run_var is not None: msg = dry_run_var[0].sub(dry_run_var[1], msg) - print_msg(msg, silent=silent, prefix=False) + print_msg(msg, silent=silent, prefix=False, progressbar=pbar) def dry_run_warning(msg, *args, **kwargs): diff --git a/requirements.txt b/requirements.txt index e63085eb51..77d393c398 100644 --- a/requirements.txt +++ b/requirements.txt @@ -62,3 +62,5 @@ archspec; python_version >= '2.7' # cryptography 3.4.0 no longer supports Python 2.7 cryptography==3.3.2; python_version == '2.7' cryptography; python_version >= '3.5' + +tqdm; python_version >= '2.7' From 671e7faceda523c0a7511268bb363c15cb572d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= Date: Wed, 8 Sep 2021 09:06:58 +0200 Subject: [PATCH 66/91] Moved initialization of easyblock progressbar Moved initialization outside try block and added logging for easier future debugging --- easybuild/framework/easyblock.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 43a4286b97..9bdd64ed68 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3692,12 +3692,15 @@ def build_and_install_one(ecdict, init_env, progressbar=None): try: app_class = get_easyblock_class(easyblock, name=name) app = app_class(ecdict['ec']) - app.set_progressbar(progressbar) _log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock)) except EasyBuildError as err: print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg), silent=silent) + # Setup progressbar + if progressbar: + app.set_progressbar(progressbar) + _log.info("Updated progressbar instance for easyblock %s" % easyblock) # application settings stop = build_option('stop') if stop is not None: From 9f5fff0b41c4027a3dea6d03ed1b079ecc8d3155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Nordmoen?= Date: Thu, 9 Sep 2021 08:52:42 +0200 Subject: [PATCH 67/91] Moved to `rich` library and more fine grained ticks --- easybuild/framework/easyblock.py | 64 ++++++++++++++++---------------- easybuild/main.py | 29 +++++++++------ easybuild/tools/build_log.py | 6 +-- requirements.txt | 2 +- 4 files changed, 53 insertions(+), 48 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 9bdd64ed68..8f4c4a713b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -214,6 +214,7 @@ def __init__(self, ec): # Create empty progress bar self.progressbar = None + self.pbar_task = None # list of loaded modules self.loaded_modules = [] @@ -303,12 +304,20 @@ def close_log(self): self.log.info("Closing log for application name %s version %s" % (self.name, self.version)) fancylogger.logToFile(self.logfile, enable=False) - def set_progressbar(self, progressbar): + def set_progressbar(self, progressbar, task_id): """ Set progress bar, the progress bar is needed when writing messages so that the progress counter is always at the bottom """ self.progressbar = progressbar + self.pbar_task = task_id + + def advance_progress(self, tick=1.0): + """ + Advance the progress bar forward with `tick` + """ + if self.progressbar and self.pbar_task is not None: + self.progressbar.advance(self.pbar_task, tick) # # DRY RUN UTILITIES @@ -328,7 +337,7 @@ def dry_run_msg(self, msg, *args): """Print dry run message.""" if args: msg = msg % args - dry_run_msg(msg, silent=self.silent, progressbar=self.progressbar) + dry_run_msg(msg, silent=self.silent) # # FETCH UTILITY FUNCTIONS @@ -1647,8 +1656,7 @@ def skip_extensions(self): self.log.debug("exit code: %s, stdout/err: %s", ec, cmdstdouterr) res.append(ext_inst) else: - print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log, - progressbar=self.progressbar) + print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log) self.ext_instances = res @@ -1752,8 +1760,7 @@ def handle_iterate_opts(self): self.log.debug("Found list for %s: %s", opt, self.iter_opts[opt]) if self.iter_opts: - print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent, - progressbar=self.progressbar) + print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent) self.log.info("Current iteration index: %s", self.iter_idx) # pop first element from all iterative easyconfig parameters as next value to use @@ -1894,8 +1901,7 @@ def check_readiness_step(self): hidden = LooseVersion(self.modules_tool.version) < LooseVersion('7.0.0') self.mod_file_backup = back_up_file(self.mod_filepath, hidden=hidden, strip_fn=strip_fn) - print_msg("backup of existing module file stored at %s" % self.mod_file_backup, log=self.log, - progressbar=self.progressbar) + print_msg("backup of existing module file stored at %s" % self.mod_file_backup, log=self.log) # check if main install needs to be skipped # - if a current module can be found, skip is ok @@ -2431,7 +2437,7 @@ def extensions_step(self, fetch=False, install=True): change_dir(self.orig_workdir) tup = (ext.name, ext.version or '', idx + 1, exts_cnt) - print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent, progressbar=self.progressbar) + print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent) start_time = datetime.now() if self.dry_run: @@ -2463,11 +2469,9 @@ def extensions_step(self, fetch=False, install=True): if not self.dry_run: ext_duration = datetime.now() - start_time if ext_duration.total_seconds() >= 1: - print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent, - progressbar=self.progressbar) + print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent) elif self.logdebug or build_option('trace'): - print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent, - progressbar=self.progressbar) + print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent) # cleanup (unload fake module, remove fake module dir) if fake_mod_data: @@ -3265,7 +3269,7 @@ def make_module_step(self, fake=False): else: diff_msg += 'no differences found' self.log.info(diff_msg) - print_msg(diff_msg, log=self.log, progressbar=self.progressbar) + print_msg(diff_msg, log=self.log) self.invalidate_module_caches(modpath) @@ -3583,9 +3587,10 @@ def run_all_steps(self, run_test_cases): return True steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt()) + # Calculate progress bar tick + tick = 1.0 / float(len(steps)) - print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent, - progressbar=self.progressbar) + print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent) trace_msg("installation prefix: %s" % self.installdir) ignore_locks = build_option('ignore_locks') @@ -3605,12 +3610,12 @@ def run_all_steps(self, run_test_cases): try: for (step_name, descr, step_methods, skippable) in steps: if self.skip_step(step_name, skippable): - print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent, progressbar=self.progressbar) + print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent) else: if self.dry_run: self.dry_run_msg("%s... [DRY RUN]\n", descr) else: - print_msg("%s..." % descr, log=self.log, silent=self.silent, progressbar=self.progressbar) + print_msg("%s..." % descr, log=self.log, silent=self.silent) self.current_step = step_name start_time = datetime.now() try: @@ -3619,11 +3624,10 @@ def run_all_steps(self, run_test_cases): if not self.dry_run: step_duration = datetime.now() - start_time if step_duration.total_seconds() >= 1: - print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent, - progressbar=self.progressbar) + print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent) elif self.logdebug or build_option('trace'): - print_msg("... (took < 1 sec)", log=self.log, silent=self.silent, - progressbar=self.progressbar) + print_msg("... (took < 1 sec)", log=self.log, silent=self.silent) + self.advance_progress(tick) except StopException: pass @@ -3649,7 +3653,7 @@ def print_dry_run_note(loc, silent=True): dry_run_msg(msg, silent=silent) -def build_and_install_one(ecdict, init_env, progressbar=None): +def build_and_install_one(ecdict, init_env, progressbar=None, task_id=None): """ Build the software :param ecdict: dictionary contaning parsed easyconfig + metadata @@ -3667,7 +3671,7 @@ def build_and_install_one(ecdict, init_env, progressbar=None): if dry_run: dry_run_msg('', silent=silent) - print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent, progressbar=progressbar) + print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent) if dry_run: # print note on interpreting dry run output (argument is reference to location of dry run messages) @@ -3698,8 +3702,8 @@ def build_and_install_one(ecdict, init_env, progressbar=None): silent=silent) # Setup progressbar - if progressbar: - app.set_progressbar(progressbar) + if progressbar and task_id is not None: + app.set_progressbar(progressbar, task_id) _log.info("Updated progressbar instance for easyblock %s" % easyblock) # application settings stop = build_option('stop') @@ -3878,8 +3882,7 @@ def ensure_writable_log_dir(log_dir): application_log = app.logfile req_time = time2str(end_timestamp - start_timestamp) - print_msg("%s: Installation %s %s (took %s)" % (summary, ended, succ, req_time), log=_log, silent=silent, - progressbar=progressbar) + print_msg("%s: Installation %s %s (took %s)" % (summary, ended, succ, req_time), log=_log, silent=silent) # check for errors if run.errors_found_in_log > 0: @@ -3887,7 +3890,7 @@ def ensure_writable_log_dir(log_dir): "build logs, please verify the build.", run.errors_found_in_log) if app.postmsg: - print_msg("\nWARNING: %s\n" % app.postmsg, log=_log, silent=silent, progressbar=progressbar) + print_msg("\nWARNING: %s\n" % app.postmsg, log=_log, silent=silent) if dry_run: # print note on interpreting dry run output (argument is reference to location of dry run messages) @@ -3901,8 +3904,7 @@ def ensure_writable_log_dir(log_dir): if application_log: # there may be multiple log files, or the file name may be different due to zipping logs = glob.glob('%s*' % application_log) - print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent, - progressbar=progressbar) + print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent) del app diff --git a/easybuild/main.py b/easybuild/main.py index 1da4362597..987819fb0d 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -39,7 +39,6 @@ import os import stat import sys -import tqdm import traceback # IMPORTANT this has to be the first easybuild import as it customises the logging @@ -74,6 +73,7 @@ from easybuild.tools.parallelbuild import submit_jobs from easybuild.tools.repository.repository import init_repository from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state +from rich.progress import Progress, TextColumn, BarColumn, TimeElapsedColumn _log = None @@ -112,13 +112,17 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, pr # e.g. via easyconfig.handle_allowed_system_deps init_env = copy.deepcopy(os.environ) + # Initialize progress bar with overall installation task + if progress: + task_id = progress.add_task("", total=len(ecs)) res = [] for ec in ecs: if progress: - progress.set_description("Installing %s" % ec['short_mod_name']) + progress.update(task_id, description=ec['short_mod_name']) ec_res = {} try: - (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, progressbar=progress) + (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, progressbar=progress, + task_id=task_id) ec_res['log_file'] = app_log if not ec_res['success']: ec_res['err'] = EasyBuildError(err) @@ -156,8 +160,6 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, pr raise EasyBuildError(test_msg) res.append((ec, ec_res)) - if progress: - progress.update() return res @@ -526,12 +528,17 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) # Create progressbar around software to install - progress_bar = tqdm.tqdm(total=len(ordered_ecs), desc="EasyBuild", - leave=False, unit='EB') - ecs_with_res = build_and_install_software( - ordered_ecs, init_session_state, exit_on_failure=exit_on_failure, - progress=progress_bar) - progress_bar.close() + progress_bar = Progress( + TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"), + BarColumn(), + "[progress.percentage]{task.percentage:>3.1f}%", + "•", + TimeElapsedColumn() + ) + with progress_bar: + ecs_with_res = build_and_install_software( + ordered_ecs, init_session_state, exit_on_failure=exit_on_failure, + progress=progress_bar) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index 2242a78dea..2cf97c5f2d 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -258,7 +258,6 @@ def print_msg(msg, *args, **kwargs): prefix = kwargs.pop('prefix', True) newline = kwargs.pop('newline', True) stderr = kwargs.pop('stderr', False) - pbar = kwargs.pop('progressbar', None) if kwargs: raise EasyBuildError("Unknown named arguments passed to print_msg: %s", kwargs) @@ -273,8 +272,6 @@ def print_msg(msg, *args, **kwargs): if stderr: sys.stderr.write(msg) - elif pbar: - pbar.write(msg, end='') else: sys.stdout.write(msg) @@ -307,7 +304,6 @@ def dry_run_msg(msg, *args, **kwargs): msg = msg % args silent = kwargs.pop('silent', False) - pbar = kwargs.pop('progressbar', None) if kwargs: raise EasyBuildError("Unknown named arguments passed to dry_run_msg: %s", kwargs) @@ -315,7 +311,7 @@ def dry_run_msg(msg, *args, **kwargs): if dry_run_var is not None: msg = dry_run_var[0].sub(dry_run_var[1], msg) - print_msg(msg, silent=silent, prefix=False, progressbar=pbar) + print_msg(msg, silent=silent, prefix=False) def dry_run_warning(msg, *args, **kwargs): diff --git a/requirements.txt b/requirements.txt index 77d393c398..3f82c41b26 100644 --- a/requirements.txt +++ b/requirements.txt @@ -63,4 +63,4 @@ archspec; python_version >= '2.7' cryptography==3.3.2; python_version == '2.7' cryptography; python_version >= '3.5' -tqdm; python_version >= '2.7' +rich; python_version >= '2.7' From 9b0a12d99aad17a6a093fb671e6c3affcc7dbd10 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Sep 2021 14:05:43 +0200 Subject: [PATCH 68/91] use Rich as optional dependency for showing progress bar --- easybuild/framework/easyblock.py | 21 +++++++------ easybuild/main.py | 54 +++++++++++++++++++++----------- 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 8f4c4a713b..58bed6afde 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -213,7 +213,7 @@ def __init__(self, ec): self.current_step = None # Create empty progress bar - self.progressbar = None + self.progress_bar = None self.pbar_task = None # list of loaded modules @@ -304,20 +304,20 @@ def close_log(self): self.log.info("Closing log for application name %s version %s" % (self.name, self.version)) fancylogger.logToFile(self.logfile, enable=False) - def set_progressbar(self, progressbar, task_id): + def set_progress_bar(self, progress_bar, task_id): """ Set progress bar, the progress bar is needed when writing messages so that the progress counter is always at the bottom """ - self.progressbar = progressbar + self.progress_bar = progress_bar self.pbar_task = task_id def advance_progress(self, tick=1.0): """ Advance the progress bar forward with `tick` """ - if self.progressbar and self.pbar_task is not None: - self.progressbar.advance(self.pbar_task, tick) + if self.progress_bar and self.pbar_task is not None: + self.progress_bar.advance(self.pbar_task, tick) # # DRY RUN UTILITIES @@ -3653,7 +3653,7 @@ def print_dry_run_note(loc, silent=True): dry_run_msg(msg, silent=silent) -def build_and_install_one(ecdict, init_env, progressbar=None, task_id=None): +def build_and_install_one(ecdict, init_env, progress_bar=None, task_id=None): """ Build the software :param ecdict: dictionary contaning parsed easyconfig + metadata @@ -3701,10 +3701,11 @@ def build_and_install_one(ecdict, init_env, progressbar=None, task_id=None): print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg), silent=silent) - # Setup progressbar - if progressbar and task_id is not None: - app.set_progressbar(progressbar, task_id) - _log.info("Updated progressbar instance for easyblock %s" % easyblock) + # Setup progress bar + if progress_bar and task_id is not None: + app.set_progress_bar(progress_bar, task_id) + _log.info("Updated progress bar instance for easyblock %s", easyblock) + # application settings stop = build_option('stop') if stop is not None: diff --git a/easybuild/main.py b/easybuild/main.py index 987819fb0d..1a5e427917 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -73,7 +73,13 @@ from easybuild.tools.parallelbuild import submit_jobs from easybuild.tools.repository.repository import init_repository from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state -from rich.progress import Progress, TextColumn, BarColumn, TimeElapsedColumn + +try: + from rich.progress import Progress, TextColumn, BarColumn, TimeElapsedColumn + HAVE_RICH = True +except ImportError: + HAVE_RICH = False + _log = None @@ -99,13 +105,14 @@ def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing= return [(ec_file, generated)] -def build_and_install_software(ecs, init_session_state, exit_on_failure=True, progress=None): +def build_and_install_software(ecs, init_session_state, exit_on_failure=True, progress_bar=None): """ Build and install software for all provided parsed easyconfig files. :param ecs: easyconfig files to install software with :param init_session_state: initial session state, to use in test reports :param exit_on_failure: whether or not to exit on installation failure + :param progress_bar: ProgressBar instance to use to report progress """ # obtain a copy of the starting environment so each build can start afresh # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since @@ -113,15 +120,20 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, pr init_env = copy.deepcopy(os.environ) # Initialize progress bar with overall installation task - if progress: - task_id = progress.add_task("", total=len(ecs)) + if progress_bar: + task_id = progress_bar.add_task("", total=len(ecs)) + else: + task_id = None + res = [] for ec in ecs: - if progress: - progress.update(task_id, description=ec['short_mod_name']) + + if progress_bar: + progress_bar.update(task_id, description=ec['short_mod_name']) + ec_res = {} try: - (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, progressbar=progress, + (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, progress_bar=progress_bar, task_id=task_id) ec_res['log_file'] = app_log if not ec_res['success']: @@ -527,18 +539,22 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # build software, will exit when errors occurs (except when testing) if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) - # Create progressbar around software to install - progress_bar = Progress( - TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"), - BarColumn(), - "[progress.percentage]{task.percentage:>3.1f}%", - "•", - TimeElapsedColumn() - ) - with progress_bar: - ecs_with_res = build_and_install_software( - ordered_ecs, init_session_state, exit_on_failure=exit_on_failure, - progress=progress_bar) + + if HAVE_RICH: + # Create progressbar around software to install + progress_bar = Progress( + TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"), + BarColumn(), + "[progress.percentage]{task.percentage:>3.1f}%", + "•", + TimeElapsedColumn() + ) + with progress_bar: + ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, + exit_on_failure=exit_on_failure, + progress_bar=progress_bar) + else: + ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] From 33f8e546ae43e933ca27434b6aa0c03aff157324 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Sep 2021 14:27:12 +0200 Subject: [PATCH 69/91] add create_progress_bar function in new easybuild.tools.output module --- easybuild/main.py | 29 ++++----------- easybuild/tools/output.py | 75 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 22 deletions(-) create mode 100644 easybuild/tools/output.py diff --git a/easybuild/main.py b/easybuild/main.py index 1a5e427917..9c2243a50c 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -68,18 +68,13 @@ from easybuild.tools.hooks import START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import set_up_configuration, use_color +from easybuild.tools.output import create_progress_bar from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs from easybuild.tools.package.utilities import check_pkg_support from easybuild.tools.parallelbuild import submit_jobs from easybuild.tools.repository.repository import init_repository from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state -try: - from rich.progress import Progress, TextColumn, BarColumn, TimeElapsedColumn - HAVE_RICH = True -except ImportError: - HAVE_RICH = False - _log = None @@ -112,7 +107,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, pr :param ecs: easyconfig files to install software with :param init_session_state: initial session state, to use in test reports :param exit_on_failure: whether or not to exit on installation failure - :param progress_bar: ProgressBar instance to use to report progress + :param progress_bar: progress bar to use to report progress """ # obtain a copy of the starting environment so each build can start afresh # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since @@ -540,21 +535,11 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) - if HAVE_RICH: - # Create progressbar around software to install - progress_bar = Progress( - TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"), - BarColumn(), - "[progress.percentage]{task.percentage:>3.1f}%", - "•", - TimeElapsedColumn() - ) - with progress_bar: - ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, - exit_on_failure=exit_on_failure, - progress_bar=progress_bar) - else: - ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) + progress_bar = create_progress_bar() + with progress_bar: + ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, + exit_on_failure=exit_on_failure, + progress_bar=progress_bar) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py new file mode 100644 index 0000000000..f20f027d31 --- /dev/null +++ b/easybuild/tools/output.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# # +# Copyright 2021-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Tools for controlling output to terminal produced by EasyBuild. + +:author: Kenneth Hoste (Ghent University) +:author: Jørgen Nordmoen (University of Oslo) +""" +try: + from rich.progress import Progress, TextColumn, BarColumn, TimeElapsedColumn + HAVE_RICH = True +except ImportError: + HAVE_RICH = False + + +class DummyProgress(object): + """Shim for Rich's Progress class.""" + + # __enter__ and __exit__ must be implemented to allow use as context manager + def __enter__(self, *args, **kwargs): + pass + + def __exit__(self, *args, **kwargs): + pass + + # dummy implementations for methods supported by rich.progress.Progress class + def add_task(self, *args, **kwargs): + pass + + def update(self, *args, **kwargs): + pass + + +def create_progress_bar(): + """ + Create progress bar to display overall progress. + + Returns rich.progress.Progress instance if the Rich Python package is available, + or a shim DummyProgress instance otherwise. + """ + if HAVE_RICH: + progress_bar = Progress( + TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"), + BarColumn(), + "[progress.percentage]{task.percentage:>3.1f}%", + "•", + TimeElapsedColumn() + ) + else: + progress_bar = DummyProgress() + + return progress_bar From d779a618d3fe70afc0b7f68c277bd4f55be36862 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Sep 2021 14:33:20 +0200 Subject: [PATCH 70/91] fix requirements.txt: Rich only supports Python 3.6+ --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3f82c41b26..591fc502f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -63,4 +63,5 @@ archspec; python_version >= '2.7' cryptography==3.3.2; python_version == '2.7' cryptography; python_version >= '3.5' -rich; python_version >= '2.7' +# rich is only supported for Python 3.6+ +rich; python_version >= '3.6' From 771f95d9108401582bea4a52399f8ac84b9a7515 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Sep 2021 16:44:10 +0200 Subject: [PATCH 71/91] use transient progress bar, so it disappears when installation is complete --- easybuild/tools/output.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index f20f027d31..359e7746d2 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -67,7 +67,8 @@ def create_progress_bar(): BarColumn(), "[progress.percentage]{task.percentage:>3.1f}%", "•", - TimeElapsedColumn() + TimeElapsedColumn(), + transient=True, ) else: progress_bar = DummyProgress() From 33df74b6de0e2fe400c588dfba37dfecd928e980 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Sep 2021 17:05:24 +0200 Subject: [PATCH 72/91] use random spinner in progress bar --- easybuild/tools/output.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 359e7746d2..831e0674bb 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -29,8 +29,10 @@ :author: Kenneth Hoste (Ghent University) :author: Jørgen Nordmoen (University of Oslo) """ +import random + try: - from rich.progress import Progress, TextColumn, BarColumn, TimeElapsedColumn + from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn HAVE_RICH = True except ImportError: HAVE_RICH = False @@ -62,11 +64,15 @@ def create_progress_bar(): or a shim DummyProgress instance otherwise. """ if HAVE_RICH: + + # pick random spinner, from a selected subset of available spinner (see 'python3 -m rich.spinner') + spinner = random.choice(('aesthetic', 'arc', 'bounce', 'dots', 'line', 'monkey', 'point', 'simpleDots')) + progress_bar = Progress( TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"), BarColumn(), "[progress.percentage]{task.percentage:>3.1f}%", - "•", + SpinnerColumn(spinner), TimeElapsedColumn(), transient=True, ) From f714ea6132362500e3ac6f09f422e8cfbb829b10 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Sep 2021 17:45:54 +0200 Subject: [PATCH 73/91] add configuration option to allow disabling progress bar --- easybuild/tools/config.py | 1 + easybuild/tools/options.py | 7 ++++--- easybuild/tools/output.py | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 8f660331f5..94b02d4424 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -292,6 +292,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'map_toolchains', 'modules_tool_version_check', 'pre_create_installdir', + 'show_progress_bar', ], WARN: [ 'check_ebroot_env_vars', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index b1aa1dd3cb..64948af45c 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -406,6 +406,8 @@ def override_options(self): 'force-download': ("Force re-downloading of sources and/or patches, " "even if they are available already in source path", 'choice', 'store_or_None', DEFAULT_FORCE_DOWNLOAD, FORCE_DOWNLOAD_CHOICES), + 'generate-devel-module': ("Generate a develop module file, implies --force if disabled", + None, 'store_true', True), 'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None), 'group-writable-installdir': ("Enable group write permissions on installation directory after installation", None, 'store_true', False), @@ -468,13 +470,12 @@ def override_options(self): None, 'store_true', False), 'set-default-module': ("Set the generated module as default", None, 'store_true', False), 'set-gid-bit': ("Set group ID bit on newly created directories", None, 'store_true', False), + 'show-progress-bar': ("Show progress bar in terminal output", None, 'store_true', True), 'silence-deprecation-warnings': ("Silence specified deprecation warnings", 'strlist', 'extend', None), - 'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False), 'skip-extensions': ("Skip installation of extensions", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), 'skip-test-step': ("Skip running the test step (e.g. unit tests)", None, 'store_true', False), - 'generate-devel-module': ("Generate a develop module file, implies --force if disabled", - None, 'store_true', True), + 'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False), 'sysroot': ("Location root directory of system, prefix for standard paths like /usr/lib and /usr/include", None, 'store', None), 'trace': ("Provide more information in output to stdout on progress", None, 'store_true', False, 'T'), diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 831e0674bb..35c6d962a3 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -31,6 +31,8 @@ """ import random +from easybuild.tools.config import build_option + try: from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn HAVE_RICH = True @@ -63,7 +65,7 @@ def create_progress_bar(): Returns rich.progress.Progress instance if the Rich Python package is available, or a shim DummyProgress instance otherwise. """ - if HAVE_RICH: + if HAVE_RICH and build_option('show_progress_bar'): # pick random spinner, from a selected subset of available spinner (see 'python3 -m rich.spinner') spinner = random.choice(('aesthetic', 'arc', 'bounce', 'dots', 'line', 'monkey', 'point', 'simpleDots')) From 37e6877942f46277285c51e3706b1fd7369e8e3f Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sun, 12 Sep 2021 08:32:27 +0100 Subject: [PATCH 74/91] expand progress bar to full screen width --- easybuild/tools/output.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 35c6d962a3..7c936a7261 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -72,11 +72,12 @@ def create_progress_bar(): progress_bar = Progress( TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"), - BarColumn(), + BarColumn(bar_width=None), "[progress.percentage]{task.percentage:>3.1f}%", SpinnerColumn(spinner), TimeElapsedColumn(), transient=True, + expand=True, ) else: progress_bar = DummyProgress() From 681e53f227fe2a839c906b811bec77e83b940fb0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 12 Sep 2021 10:07:31 +0200 Subject: [PATCH 75/91] reorder progress bar components --- easybuild/tools/output.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 7c936a7261..52ee836b50 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -71,10 +71,10 @@ def create_progress_bar(): spinner = random.choice(('aesthetic', 'arc', 'bounce', 'dots', 'line', 'monkey', 'point', 'simpleDots')) progress_bar = Progress( - TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total})"), - BarColumn(bar_width=None), - "[progress.percentage]{task.percentage:>3.1f}%", SpinnerColumn(spinner), + "[progress.percentage]{task.percentage:>3.1f}%", + TextColumn("[bold blue]Installing {task.description} ({task.completed:.0f}/{task.total} done)"), + BarColumn(bar_width=None), TimeElapsedColumn(), transient=True, expand=True, From 2db1befd6b45981db1fb33a9720294db7b1352d3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 12 Sep 2021 12:20:23 +0200 Subject: [PATCH 76/91] add support for checking required/optional EasyBuild dependencies via 'eb --check-eb-deps' --- easybuild/main.py | 5 ++ easybuild/tools/modules.py | 13 +++- easybuild/tools/options.py | 2 + easybuild/tools/systemtools.py | 124 ++++++++++++++++++++++++++++++++- test/framework/options.py | 22 ++++++ 5 files changed, 162 insertions(+), 4 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 9c2243a50c..84b02dc186 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -73,6 +73,7 @@ from easybuild.tools.package.utilities import check_pkg_support from easybuild.tools.parallelbuild import submit_jobs from easybuild.tools.repository.repository import init_repository +from easybuild.tools.systemtools import check_easybuild_deps from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state @@ -259,6 +260,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename, terse=options.terse) + if options.check_eb_deps: + print(check_easybuild_deps(modtool)) + # GitHub options that warrant a silent cleanup & exit if options.check_github: check_github() @@ -297,6 +301,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # non-verbose cleanup after handling GitHub integration stuff or printing terse info early_stop_options = [ options.add_pr_labels, + options.check_eb_deps, options.check_github, options.create_index, options.install_github_token, diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 5e48591be5..0860b810d8 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -207,6 +207,15 @@ def __init__(self, mod_paths=None, testing=False): self.set_and_check_version() self.supports_depends_on = False + def __str__(self): + """String representation of this ModulesTool instance.""" + res = self.NAME + if self.version: + res += ' ' + self.version + else: + res += ' (unknown version)' + return res + def buildstats(self): """Return tuple with data to be included in buildstats""" return (self.NAME, self.cmd, self.version) @@ -1177,7 +1186,7 @@ def update(self): class EnvironmentModulesC(ModulesTool): """Interface to (C) environment modules (modulecmd).""" - NAME = "Environment Modules v3" + NAME = "Environment Modules" COMMAND = "modulecmd" REQ_VERSION = '3.2.10' MAX_VERSION = '3.99' @@ -1312,7 +1321,7 @@ def remove_module_path(self, path, set_mod_paths=True): class EnvironmentModules(EnvironmentModulesTcl): """Interface to environment modules 4.0+""" - NAME = "Environment Modules v4" + NAME = "Environment Modules" COMMAND = os.path.join(os.getenv('MODULESHOME', 'MODULESHOME_NOT_DEFINED'), 'libexec', 'modulecmd.tcl') COMMAND_ENVIRONMENT = 'MODULES_CMD' REQ_VERSION = '4.0.0' diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 64948af45c..759e8d8e42 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -617,6 +617,8 @@ def informative_options(self): 'avail-hooks': ("Show list of known hooks", None, 'store_true', False), 'avail-toolchain-opts': ("Show options for toolchain", 'str', 'store', None), 'check-conflicts': ("Check for version conflicts in dependency graphs", None, 'store_true', False), + 'check-eb-deps': ("Check presence and version of (required and optional) EasyBuild dependencies", + None, 'store_true', False), 'dep-graph': ("Create dependency graph", None, 'store', None, {'metavar': 'depgraph.'}), 'dump-env-script': ("Dump source script to set up build environment based on toolchain/dependencies", None, 'store_true', False), diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index a43338e147..bf96fb7745 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -34,6 +34,7 @@ import grp # @UnresolvedImport import os import platform +import pkg_resources import pwd import re import struct @@ -152,6 +153,36 @@ RPM = 'rpm' DPKG = 'dpkg' +SYSTEM_TOOLS = ('7z', 'bunzip2', DPKG, 'gunzip', 'make', 'patch', RPM, 'sed', 'tar', 'unxz', 'unzip') + +OPT_DEPS = { + 'archspec': "determining name of CPU microarchitecture", + 'autopep8': "auto-formatting for dumped easyconfigs", + 'GC3Pie': "backend for --job", + 'GitPython': "GitHub integration + using Git repository as easyconfigs archive", + 'graphviz-python': "rendering dependency graph with Graphviz: --dep-graph", + 'keyring': "storing GitHub token", + 'pep8': "fallback for code style checking: --check-style, --check-contrib", + 'pycodestyle': "code style checking: --check-style, --check-contrib", + 'pysvn': "using SVN repository as easyconfigs archive", + 'python-graph-core': "creating dependency graph: --dep-graph", + 'python-graph-dot': "saving dependency graph as dot file: --dep-graph", + 'python-hglib': "using Mercurial repository as easyconfigs archive", + 'requests': "fallback library for downloading files", + 'Rich': "eb command rich terminal output", + 'PyYAML': "easystack files and .yeb easyconfig format", +} + +OPT_DEP_PKG_NAMES = { + 'GC3Pie': 'gc3libs', + 'GitPython': 'git', + 'graphviz-python': 'gv', + 'python-graph-core': 'pygraph.classes.digraph', + 'python-graph-dot': 'pygraph.readwrite.dot', + 'python-hglib': 'hglib', + 'PyYAML': 'yaml', +} + class SystemToolsException(Exception): """raised when systemtools fails""" @@ -722,14 +753,14 @@ def check_os_dependency(dep): return found -def get_tool_version(tool, version_option='--version'): +def get_tool_version(tool, version_option='--version', ignore_ec=False): """ Get output of running version option for specific command line tool. Output is returned as a single-line string (newlines are replaced by '; '). """ out, ec = run_cmd(' '.join([tool, version_option]), simple=False, log_ok=False, force_in_dry_run=True, trace=False, stream_output=False) - if ec: + if not ignore_ec and ec: _log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, out)) return UNKNOWN else: @@ -1103,3 +1134,92 @@ def pick_dep_version(dep_version): raise EasyBuildError("Unknown value type for version: %s (%s), should be string value", typ, dep_version) return result + + +def check_easybuild_deps(modtool): + """ + Check presence and version of required and optional EasyBuild dependencies, and report back to terminal. + """ + version_regex = re.compile(r'\s(?P[0-9][0-9.]+[a-z]*)') + + def extract_version(tool): + """Helper function to extract (only) version for specific command line tool.""" + out = get_tool_version(tool, ignore_ec=True) + res = version_regex.search(out) + if res: + version = res.group('version') + else: + version = "UNKNOWN version" + + return version + + python_version = extract_version(sys.executable) + + opt_dep_versions = {} + for key in OPT_DEPS: + + pkg = OPT_DEP_PKG_NAMES.get(key, key.lower()) + + try: + mod = __import__(pkg) + except ImportError: + mod = None + + if mod: + try: + dep_version = pkg_resources.get_distribution(pkg).version + except pkg_resources.DistributionNotFound: + try: + dep_version = pkg_resources.get_distribution(key).version + except pkg_resources.DistributionNotFound: + if hasattr(mod, '__version__'): + dep_version = mod.__version__ + else: + dep_version = '(unknown version)' + else: + dep_version = '(NOT AVAILABLE)' + + opt_dep_versions[key] = dep_version + + lines = [ + '', + "Required dependencies:", + "----------------------", + '', + "* Python %s" % python_version, + "* %s (modules tool)" % modtool, + '', + "Optional dependencies:", + "----------------------", + '', + ] + for pkg in sorted(opt_dep_versions, key=lambda x: x.lower()): + line = "* %s %s" % (pkg, opt_dep_versions[pkg]) + line = line.ljust(40) + " [%s]" % OPT_DEPS[pkg] + lines.append(line) + + lines.extend([ + '', + "System tools:", + "-------------", + '', + ]) + + tools = list(SYSTEM_TOOLS) + ['Slurm'] + cmds = {'Slurm': 'sbatch'} + + for tool in sorted(tools, key=lambda x: x.lower()): + line = "* %s " % tool + cmd = cmds.get(tool, tool) + if which(cmd): + version = extract_version(cmd) + if version.startswith('UNKNOWN'): + line += "(available, %s)" % version + else: + line += version + else: + line += "(NOT AVAILABLE)" + + lines.append(line) + + return '\n'.join(lines) diff --git a/test/framework/options.py b/test/framework/options.py index c9a64bfd0e..de338e4045 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5845,6 +5845,28 @@ def test_show_system_info(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + def test_check_eb_deps(self): + """Test for --check-eb-deps.""" + txt, _ = self._run_mock_eb(['--check-eb-deps'], raise_error=True) + patterns = [ + r"^Required dependencies:", + r"^\* Python [23][0-9.]+$", + r"^\* [A-Za-z ]+ [0-9.]+ \(modules tool\)$", + r"^Optional dependencies:", + r"^\* archspec ([0-9.]+|\(NOT AVAILABLE\))+\s+\[determining name of CPU microarchitecture\]$", + r"^\* GitPython ([0-9.]+|\(NOT AVAILABLE\))+\s+\[GitHub integration .*\]$", + r"^\* Rich ([0-9.]+|\(NOT AVAILABLE\))+\s+\[eb command rich terminal output\]$", + r"^System tools:", + r"^\* make ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", + r"^\* patch ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", + r"^\* sed ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", + r"^\* Slurm ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", + ] + + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + def test_tmp_logdir(self): """Test use of --tmp-logdir.""" From c97661a0efb2107cbd60b9c186dfd5cf35c5c8ce Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 12 Sep 2021 14:19:45 +0200 Subject: [PATCH 77/91] avoid making setuptools a required dependency by only using pkg_resources if it's available --- easybuild/tools/systemtools.py | 49 ++++++++++++++++++++++++++-------- test/framework/options.py | 8 +++--- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index bf96fb7745..f8a115ba0f 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -34,7 +34,6 @@ import grp # @UnresolvedImport import os import platform -import pkg_resources import pwd import re import struct @@ -43,6 +42,14 @@ from ctypes.util import find_library from socket import gethostname +# pkg_resources is provided by the setuptools Python package, +# which we really want to keep as an *optional* dependency +try: + import pkg_resources + HAVE_PKG_RESOURCES = True +except ImportError: + HAVE_PKG_RESOURCES = False + try: # only needed on macOS, may not be available on Linux import ctypes.macholib.dyld @@ -162,6 +169,7 @@ 'GitPython': "GitHub integration + using Git repository as easyconfigs archive", 'graphviz-python': "rendering dependency graph with Graphviz: --dep-graph", 'keyring': "storing GitHub token", + 'pbs-python': "using Torque as --job backend", 'pep8': "fallback for code style checking: --check-style, --check-contrib", 'pycodestyle': "code style checking: --check-style, --check-contrib", 'pysvn': "using SVN repository as easyconfigs archive", @@ -171,12 +179,14 @@ 'requests': "fallback library for downloading files", 'Rich': "eb command rich terminal output", 'PyYAML': "easystack files and .yeb easyconfig format", + 'setuptools': "obtaining information on Python packages via pkg_resources module", } OPT_DEP_PKG_NAMES = { 'GC3Pie': 'gc3libs', 'GitPython': 'git', 'graphviz-python': 'gv', + 'pbs-python': 'pbs', 'python-graph-core': 'pygraph.classes.digraph', 'python-graph-dot': 'pygraph.readwrite.dot', 'python-hglib': 'hglib', @@ -1136,6 +1146,30 @@ def pick_dep_version(dep_version): return result +def det_pypkg_version(pkg_name, imported_pkg, import_name=None): + """Determine version of a Python package.""" + + version = None + + if HAVE_PKG_RESOURCES: + if import_name: + try: + version = pkg_resources.get_distribution(import_name).version + except pkg_resources.DistributionNotFound as err: + _log.debug("%s Python package not found: %s", import_name, err) + + if version is None: + try: + version = pkg_resources.get_distribution(pkg_name).version + except pkg_resources.DistributionNotFound as err: + _log.debug("%s Python package not found: %s", pkg_name, err) + + if version is None and hasattr(imported_pkg, '__version__'): + version = imported_pkg.__version__ + + return version + + def check_easybuild_deps(modtool): """ Check presence and version of required and optional EasyBuild dependencies, and report back to terminal. @@ -1166,16 +1200,9 @@ def extract_version(tool): mod = None if mod: - try: - dep_version = pkg_resources.get_distribution(pkg).version - except pkg_resources.DistributionNotFound: - try: - dep_version = pkg_resources.get_distribution(key).version - except pkg_resources.DistributionNotFound: - if hasattr(mod, '__version__'): - dep_version = mod.__version__ - else: - dep_version = '(unknown version)' + dep_version = det_pypkg_version(key, mod, import_name=pkg) + if dep_version is None: + dep_version = '(unknown version)' else: dep_version = '(NOT AVAILABLE)' diff --git a/test/framework/options.py b/test/framework/options.py index de338e4045..b3e60904f0 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5848,14 +5848,16 @@ def test_show_system_info(self): def test_check_eb_deps(self): """Test for --check-eb-deps.""" txt, _ = self._run_mock_eb(['--check-eb-deps'], raise_error=True) + opt_dep_version_pattern = r'([0-9.]+|\(NOT AVAILABLE\)|\(unknown version\))' patterns = [ r"^Required dependencies:", r"^\* Python [23][0-9.]+$", r"^\* [A-Za-z ]+ [0-9.]+ \(modules tool\)$", r"^Optional dependencies:", - r"^\* archspec ([0-9.]+|\(NOT AVAILABLE\))+\s+\[determining name of CPU microarchitecture\]$", - r"^\* GitPython ([0-9.]+|\(NOT AVAILABLE\))+\s+\[GitHub integration .*\]$", - r"^\* Rich ([0-9.]+|\(NOT AVAILABLE\))+\s+\[eb command rich terminal output\]$", + r"^\* archspec %s\s+\[determining name of CPU microarchitecture\]$" % opt_dep_version_pattern, + r"^\* GitPython %s\s+\[GitHub integration .*\]$" % opt_dep_version_pattern, + r"^\* Rich %s\s+\[eb command rich terminal output\]$" % opt_dep_version_pattern, + r"^\* setuptools %s\s+\[obtaining information on Python packages .*\]$" % opt_dep_version_pattern, r"^System tools:", r"^\* make ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", r"^\* patch ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", From 331820fa9edbe6500b2b589f0666fca5c089b6b4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 12 Sep 2021 15:45:03 +0200 Subject: [PATCH 78/91] add print_checks function in easybuild.tools.output and leverage it to produce rich output for --check-eb-deps --- easybuild/main.py | 4 +- easybuild/tools/output.py | 71 ++++++++++++++++++++++++++++ easybuild/tools/systemtools.py | 86 ++++++++++++++++++---------------- test/framework/options.py | 31 ++++++------ 4 files changed, 136 insertions(+), 56 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 84b02dc186..1e5792fd0e 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -68,7 +68,7 @@ from easybuild.tools.hooks import START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import set_up_configuration, use_color -from easybuild.tools.output import create_progress_bar +from easybuild.tools.output import create_progress_bar, print_checks from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs from easybuild.tools.package.utilities import check_pkg_support from easybuild.tools.parallelbuild import submit_jobs @@ -261,7 +261,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): terse=options.terse) if options.check_eb_deps: - print(check_easybuild_deps(modtool)) + print_checks(check_easybuild_deps(modtool)) # GitHub options that warrant a silent cleanup & exit if options.check_github: diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 35c6d962a3..e5d72119d0 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -32,8 +32,11 @@ import random from easybuild.tools.config import build_option +from easybuild.tools.py2vs3 import OrderedDict try: + from rich.console import Console + from rich.table import Table from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn HAVE_RICH = True except ImportError: @@ -82,3 +85,71 @@ def create_progress_bar(): progress_bar = DummyProgress() return progress_bar + + +def print_checks(checks_data): + """Print overview of checks that were made.""" + + col_titles = checks_data.pop('col_titles', ('name', 'info', 'description')) + + col2_label = col_titles[1] + + if HAVE_RICH: + console = Console() + # don't use console.print, which causes SyntaxError in Python 2 + console_print = getattr(console, 'print') + console_print('') + + for section in checks_data: + section_checks = checks_data[section] + + if HAVE_RICH: + table = Table(title=section) + table.add_column(col_titles[0]) + table.add_column(col_titles[1]) + # only add 3rd column if there's any information to include in it + if any(x[1] for x in section_checks.values()): + table.add_column(col_titles[2]) + else: + lines = [ + '', + section + ':', + '-' * (len(section) + 1), + '', + ] + + if isinstance(section_checks, OrderedDict): + check_names = section_checks.keys() + else: + check_names = sorted(section_checks, key=lambda x: x.lower()) + + if HAVE_RICH: + for check_name in check_names: + (info, descr) = checks_data[section][check_name] + if info is None: + info = ':yellow_circle: [yellow]%s?!' % col2_label + elif info is False: + info = ':cross_mark: [red]not found' + else: + info = ':white_heavy_check_mark: [green]%s' % info + if descr: + table.add_row(check_name.rstrip(':'), info, descr) + else: + table.add_row(check_name.rstrip(':'), info) + else: + for check_name in check_names: + (info, descr) = checks_data[section][check_name] + if info is None: + info = '(found, UNKNOWN %s)' % col2_label + elif info is False: + info = '(NOT FOUND)' + line = "* %s %s" % (check_name, info) + if descr: + line = line.ljust(40) + '[%s]' % descr + lines.append(line) + lines.append('') + + if HAVE_RICH: + console_print(table) + else: + print('\n'.join(lines)) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index f8a115ba0f..2c7052f4ef 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -59,7 +59,7 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import is_readable, read_file, which -from easybuild.tools.py2vs3 import string_type +from easybuild.tools.py2vs3 import OrderedDict, string_type from easybuild.tools.run import run_cmd @@ -160,7 +160,24 @@ RPM = 'rpm' DPKG = 'dpkg' -SYSTEM_TOOLS = ('7z', 'bunzip2', DPKG, 'gunzip', 'make', 'patch', RPM, 'sed', 'tar', 'unxz', 'unzip') +SYSTEM_TOOLS = { + '7z': "extracting sources (.iso)", + 'bunzip2': "decompressing sources (.bz2, .tbz, .tbz2, ...)", + DPKG: "checking OS dependencies (Debian, Ubuntu, ...)", + 'gunzip': "decompressing source files (.gz, .tgz, ...)", + 'make': "build tool", + 'patch': "applying patch files", + RPM: "checking OS dependencies (CentOS, RHEL, OpenSuSE, SLES, ...)", + 'sed': "runtime patching", + 'Slurm': "backend for --job (sbatch command)", + 'tar': "unpacking source files (.tar)", + 'unxz': "decompressing source files (.xz, .txz)", + 'unzip': "decompressing files (.zip)", +} + +SYSTEM_TOOL_CMDS = { + 'Slurm': 'sbatch', +} OPT_DEPS = { 'archspec': "determining name of CPU microarchitecture", @@ -1176,6 +1193,8 @@ def check_easybuild_deps(modtool): """ version_regex = re.compile(r'\s(?P[0-9][0-9.]+[a-z]*)') + checks_data = OrderedDict() + def extract_version(tool): """Helper function to extract (only) version for specific command line tool.""" out = get_tool_version(tool, ignore_ec=True) @@ -1201,52 +1220,39 @@ def extract_version(tool): if mod: dep_version = det_pypkg_version(key, mod, import_name=pkg) - if dep_version is None: - dep_version = '(unknown version)' else: - dep_version = '(NOT AVAILABLE)' + dep_version = False opt_dep_versions[key] = dep_version - lines = [ - '', - "Required dependencies:", - "----------------------", - '', - "* Python %s" % python_version, - "* %s (modules tool)" % modtool, - '', - "Optional dependencies:", - "----------------------", - '', - ] - for pkg in sorted(opt_dep_versions, key=lambda x: x.lower()): - line = "* %s %s" % (pkg, opt_dep_versions[pkg]) - line = line.ljust(40) + " [%s]" % OPT_DEPS[pkg] - lines.append(line) - - lines.extend([ - '', - "System tools:", - "-------------", - '', - ]) - - tools = list(SYSTEM_TOOLS) + ['Slurm'] - cmds = {'Slurm': 'sbatch'} - - for tool in sorted(tools, key=lambda x: x.lower()): - line = "* %s " % tool - cmd = cmds.get(tool, tool) + checks_data['col_titles'] = ('name', 'version', 'used for') + + req_deps_key = "Required dependencies" + checks_data[req_deps_key] = OrderedDict() + checks_data[req_deps_key]['Python'] = (python_version, None) + checks_data[req_deps_key]['modules tool:'] = (str(modtool), None) + + opt_deps_key = "Optional dependencies" + checks_data[opt_deps_key] = {} + + for pkg in opt_dep_versions: + checks_data[opt_deps_key][pkg] = (opt_dep_versions[pkg], OPT_DEPS[pkg]) + + sys_tools_key = "System tools" + checks_data[sys_tools_key] = {} + + for tool in SYSTEM_TOOLS: + tool_info = None + cmd = SYSTEM_TOOL_CMDS.get(tool, tool) if which(cmd): version = extract_version(cmd) if version.startswith('UNKNOWN'): - line += "(available, %s)" % version + tool_info = None else: - line += version + tool_info = version else: - line += "(NOT AVAILABLE)" + tool_info = False - lines.append(line) + checks_data[sys_tools_key][tool] = (tool_info, None) - return '\n'.join(lines) + return checks_data diff --git a/test/framework/options.py b/test/framework/options.py index b3e60904f0..2dda9e9a1c 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5848,21 +5848,24 @@ def test_show_system_info(self): def test_check_eb_deps(self): """Test for --check-eb-deps.""" txt, _ = self._run_mock_eb(['--check-eb-deps'], raise_error=True) - opt_dep_version_pattern = r'([0-9.]+|\(NOT AVAILABLE\)|\(unknown version\))' + + # keep in mind that these patterns should match with both normal output and Rich output! + opt_dep_info_pattern = r'([0-9.]+|\(NOT FOUND\)|not found|\(unknown version\))' + tool_info_pattern = r'([0-9.]+|\(NOT FOUND\)|not found|\(found, UNKNOWN version\)|version\?\!)' patterns = [ - r"^Required dependencies:", - r"^\* Python [23][0-9.]+$", - r"^\* [A-Za-z ]+ [0-9.]+ \(modules tool\)$", - r"^Optional dependencies:", - r"^\* archspec %s\s+\[determining name of CPU microarchitecture\]$" % opt_dep_version_pattern, - r"^\* GitPython %s\s+\[GitHub integration .*\]$" % opt_dep_version_pattern, - r"^\* Rich %s\s+\[eb command rich terminal output\]$" % opt_dep_version_pattern, - r"^\* setuptools %s\s+\[obtaining information on Python packages .*\]$" % opt_dep_version_pattern, - r"^System tools:", - r"^\* make ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", - r"^\* patch ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", - r"^\* sed ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", - r"^\* Slurm ([0-9.]+|\(NOT AVAILABLE\)|\(available, UNKNOWN version\))$", + r"Required dependencies", + r"Python.* [23][0-9.]+", + r"modules tool.* [A-Za-z0-9.\s-]+", + r"Optional dependencies", + r"archspec.* %s.*determining name" % opt_dep_info_pattern, + r"GitPython.* %s.*GitHub integration" % opt_dep_info_pattern, + r"Rich.* %s.*eb command rich terminal output" % opt_dep_info_pattern, + r"setuptools.* %s.*information on Python packages" % opt_dep_info_pattern, + r"System tools", + r"make.* %s" % tool_info_pattern, + r"patch.* %s" % tool_info_pattern, + r"sed.* %s" % tool_info_pattern, + r"Slurm.* %s" % tool_info_pattern, ] for pattern in patterns: From 93c3277f3e0d9c9b2715a9472c950aa8a8aff2ae Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Sep 2021 09:04:47 +0200 Subject: [PATCH 79/91] silence the Hound on accessing console.print method via getattr --- easybuild/tools/output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index e5d72119d0..bd629f3bd3 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -97,7 +97,7 @@ def print_checks(checks_data): if HAVE_RICH: console = Console() # don't use console.print, which causes SyntaxError in Python 2 - console_print = getattr(console, 'print') + console_print = getattr(console, 'print') # noqa: B009 console_print('') for section in checks_data: From 79cc83b96d758eb4f6ef623b492d8fcd28a5655f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Sep 2021 09:50:41 +0200 Subject: [PATCH 80/91] ensure that path configuration options have absolute path values --- easybuild/tools/options.py | 32 ++++++++++++++++++++++++++++++++ test/framework/options.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 64948af45c..47804a28ab 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1047,8 +1047,40 @@ def _postprocess_checks(self): self.log.info("Checks on configuration options passed") + def _ensure_abs_path(self, opt_name): + """Ensure that path value for specified configuration option is an absolute path.""" + + def _ensure_abs_path(opt_name, path): + """Helper function to make path value for a configuration option an absolute path.""" + if os.path.isabs(path): + abs_path = path + else: + abs_path = os.path.abspath(path) + self.log.info("Relative path value for '%s' configuration option resolved to absolute path: %s", + path, abs_path) + return abs_path + + opt_val = getattr(self.options, opt_name) + if opt_val: + if isinstance(opt_val, string_type): + setattr(self.options, opt_name, _ensure_abs_path(opt_name, opt_val)) + elif isinstance(opt_val, list): + abs_paths = [_ensure_abs_path(opt_name, p) for p in opt_val] + setattr(self.options, opt_name, abs_paths) + else: + error_msg = "Don't know how to ensure absolute path(s) for '%s' configuration option (value type: %s)" + raise EasyBuildError(error_msg, opt_name, type(opt_val)) + def _postprocess_config(self): """Postprocessing of configuration options""" + + # resolve relative paths for configuration options that specify a location + path_opt_names = ('buildpath', 'containerpath', 'git_working_dirs_path', 'installpath', + 'installpath_modules', 'installpath_software', 'prefix', 'packagepath', + 'repositorypath', 'robot_paths', 'sourcepath') + for opt_name in path_opt_names: + self._ensure_abs_path(opt_name) + if self.options.prefix is not None: # prefix applies to all paths, and repository has to be reinitialised to take new repositorypath in account # in the legacy-style configuration, repository is initialised in configuration file itself diff --git a/test/framework/options.py b/test/framework/options.py index c9a64bfd0e..045e868790 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6233,6 +6233,42 @@ def test_accept_eula_for(self): self.eb_main(args, do_build=True, raise_error=True) self.assertTrue(os.path.exists(toy_modfile)) + def test_config_abs_path(self): + """Test ensuring of absolute path values for path configuration options.""" + + test_topdir = os.path.join(self.test_prefix, 'test_topdir') + test_subdir = os.path.join(test_topdir, 'test_middle_dir', 'test_subdir') + mkdir(test_subdir, parents=True) + change_dir(test_subdir) + + # a relative path specified in a configuration file is positively weird, but fine :) + cfgfile = os.path.join(self.test_prefix, 'test.cfg') + cfgtxt = '\n'.join([ + "[config]", + "containerpath = ..", + ]) + write_file(cfgfile, cfgtxt) + + os.environ['EASYBUILD_INSTALLPATH'] = '../..' + + args = [ + '--configfiles=%s' % cfgfile, + '--prefix=..', + '--sourcepath=.', + '--show-config', + ] + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) + + patterns = [ + r"^containerpath\s+\(F\) = .*/test_topdir/test_middle_dir$", + r"^installpath\s+\(E\) = .*/test_topdir$", + r"^prefix\s+\(C\) = .*/test_topdir/test_middle_dir$", + r"^sourcepath\s+\(C\) = .*/test_topdir/test_middle_dir/test_subdir$", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (pattern, txt)) + # end-to-end testing of unknown filename def test_easystack_wrong_read(self): """Test for --easystack when wrong name is provided""" From 3252979f29b1b92a7223f5163cef02c7c03fa564 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Sep 2021 12:16:03 +0200 Subject: [PATCH 81/91] handle repositorypath as special case when ensuring absolute path values --- easybuild/tools/options.py | 38 ++++++++++++++++++++++++-------------- test/framework/config.py | 2 +- test/framework/options.py | 2 ++ 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 47804a28ab..8d7b3af441 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1047,25 +1047,25 @@ def _postprocess_checks(self): self.log.info("Checks on configuration options passed") + def get_cfg_opt_abs_path(self, opt_name, path): + """Get path value of configuration option as absolute path.""" + if os.path.isabs(path): + abs_path = path + else: + abs_path = os.path.abspath(path) + self.log.info("Relative path value for '%s' configuration option resolved to absolute path: %s", + path, abs_path) + return abs_path + def _ensure_abs_path(self, opt_name): """Ensure that path value for specified configuration option is an absolute path.""" - def _ensure_abs_path(opt_name, path): - """Helper function to make path value for a configuration option an absolute path.""" - if os.path.isabs(path): - abs_path = path - else: - abs_path = os.path.abspath(path) - self.log.info("Relative path value for '%s' configuration option resolved to absolute path: %s", - path, abs_path) - return abs_path - opt_val = getattr(self.options, opt_name) if opt_val: if isinstance(opt_val, string_type): - setattr(self.options, opt_name, _ensure_abs_path(opt_name, opt_val)) + setattr(self.options, opt_name, self.get_cfg_opt_abs_path(opt_name, opt_val)) elif isinstance(opt_val, list): - abs_paths = [_ensure_abs_path(opt_name, p) for p in opt_val] + abs_paths = [self.get_cfg_opt_abs_path(opt_name, p) for p in opt_val] setattr(self.options, opt_name, abs_paths) else: error_msg = "Don't know how to ensure absolute path(s) for '%s' configuration option (value type: %s)" @@ -1075,9 +1075,19 @@ def _postprocess_config(self): """Postprocessing of configuration options""" # resolve relative paths for configuration options that specify a location - path_opt_names = ('buildpath', 'containerpath', 'git_working_dirs_path', 'installpath', + path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath', 'installpath_modules', 'installpath_software', 'prefix', 'packagepath', - 'repositorypath', 'robot_paths', 'sourcepath') + 'robot_paths', 'sourcepath'] + + # repositorypath is a special case: only first part is a path; + # 2nd (optional) part is a relative subdir and should not be resolved to an absolute path! + repositorypath = self.options.repositorypath + if isinstance(repositorypath, (list, tuple)) and len(repositorypath) == 2: + abs_path = self.get_cfg_opt_abs_path('repositorypath', repositorypath[0]) + self.options.repositorypath = (abs_path, repositorypath[1]) + else: + path_opt_names.append('repositorypath') + for opt_name in path_opt_names: self._ensure_abs_path(opt_name) diff --git a/test/framework/config.py b/test/framework/config.py index cb13d348a5..0c4489a412 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -302,7 +302,7 @@ def test_generaloption_config_file(self): self.assertEqual(install_path('mod'), installpath_modules), # via config file self.assertEqual(source_paths(), [testpath2]) # via command line self.assertEqual(build_path(), testpath1) # via config file - self.assertEqual(get_repositorypath(), [os.path.join(topdir, 'ebfiles_repo'), 'somesubdir']) # via config file + self.assertEqual(get_repositorypath(), (os.path.join(topdir, 'ebfiles_repo'), 'somesubdir')) # via config file # hardcoded first entry self.assertEqual(options.robot_paths[0], '/tmp/foo') diff --git a/test/framework/options.py b/test/framework/options.py index 045e868790..a8bc6fe985 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6246,6 +6246,7 @@ def test_config_abs_path(self): cfgtxt = '\n'.join([ "[config]", "containerpath = ..", + "repositorypath = /apps/easyconfigs_archive, somesubdir", ]) write_file(cfgfile, cfgtxt) @@ -6263,6 +6264,7 @@ def test_config_abs_path(self): r"^containerpath\s+\(F\) = .*/test_topdir/test_middle_dir$", r"^installpath\s+\(E\) = .*/test_topdir$", r"^prefix\s+\(C\) = .*/test_topdir/test_middle_dir$", + r"^repositorypath\s+\(F\) = \('/apps/easyconfigs_archive', ' somesubdir'\)$", r"^sourcepath\s+\(C\) = .*/test_topdir/test_middle_dir/test_subdir$", ] for pattern in patterns: From 25470adfb03bd407732055be8b74d2e31b899964 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Sep 2021 16:40:37 +0200 Subject: [PATCH 82/91] collapse OPT_DEPS and OPT_DEP_PKG_NAMES to EASYBUILD_OPTIONAL_DEPENDENCIES --- easybuild/tools/systemtools.py | 57 ++++++++++++++-------------------- 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 2c7052f4ef..52c6ee63c2 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -179,35 +179,24 @@ 'Slurm': 'sbatch', } -OPT_DEPS = { - 'archspec': "determining name of CPU microarchitecture", - 'autopep8': "auto-formatting for dumped easyconfigs", - 'GC3Pie': "backend for --job", - 'GitPython': "GitHub integration + using Git repository as easyconfigs archive", - 'graphviz-python': "rendering dependency graph with Graphviz: --dep-graph", - 'keyring': "storing GitHub token", - 'pbs-python': "using Torque as --job backend", - 'pep8': "fallback for code style checking: --check-style, --check-contrib", - 'pycodestyle': "code style checking: --check-style, --check-contrib", - 'pysvn': "using SVN repository as easyconfigs archive", - 'python-graph-core': "creating dependency graph: --dep-graph", - 'python-graph-dot': "saving dependency graph as dot file: --dep-graph", - 'python-hglib': "using Mercurial repository as easyconfigs archive", - 'requests': "fallback library for downloading files", - 'Rich': "eb command rich terminal output", - 'PyYAML': "easystack files and .yeb easyconfig format", - 'setuptools': "obtaining information on Python packages via pkg_resources module", -} - -OPT_DEP_PKG_NAMES = { - 'GC3Pie': 'gc3libs', - 'GitPython': 'git', - 'graphviz-python': 'gv', - 'pbs-python': 'pbs', - 'python-graph-core': 'pygraph.classes.digraph', - 'python-graph-dot': 'pygraph.readwrite.dot', - 'python-hglib': 'hglib', - 'PyYAML': 'yaml', +EASYBUILD_OPTIONAL_DEPENDENCIES = { + 'archspec': (None, "determining name of CPU microarchitecture"), + 'autopep8': (None, "auto-formatting for dumped easyconfigs"), + 'GC3Pie': ('gc3libs', "backend for --job"), + 'GitPython': ('git', "GitHub integration + using Git repository as easyconfigs archive"), + 'graphviz-python': ('gv', "rendering dependency graph with Graphviz: --dep-graph"), + 'keyring': (None, "storing GitHub token"), + 'pbs-python': ('pbs', "using Torque as --job backend"), + 'pep8': (None, "fallback for code style checking: --check-style, --check-contrib"), + 'pycodestyle': (None, "code style checking: --check-style, --check-contrib"), + 'pysvn': (None, "using SVN repository as easyconfigs archive"), + 'python-graph-core': ('pygraph.classes.digraph', "creating dependency graph: --dep-graph"), + 'python-graph-dot': ('pygraph.readwrite.dot', "saving dependency graph as dot file: --dep-graph"), + 'python-hglib': ('hglib', "using Mercurial repository as easyconfigs archive"), + 'requests': (None, "fallback library for downloading files"), + 'Rich': (None, "eb command rich terminal output"), + 'PyYAML': ('yaml', "easystack files and .yeb easyconfig format"), + 'setuptools': ('pkg_resources', "obtaining information on Python packages via pkg_resources module"), } @@ -1209,9 +1198,11 @@ def extract_version(tool): python_version = extract_version(sys.executable) opt_dep_versions = {} - for key in OPT_DEPS: + for key in EASYBUILD_OPTIONAL_DEPENDENCIES: - pkg = OPT_DEP_PKG_NAMES.get(key, key.lower()) + pkg = EASYBUILD_OPTIONAL_DEPENDENCIES[key][0] + if pkg is None: + pkg = key.lower() try: mod = __import__(pkg) @@ -1235,8 +1226,8 @@ def extract_version(tool): opt_deps_key = "Optional dependencies" checks_data[opt_deps_key] = {} - for pkg in opt_dep_versions: - checks_data[opt_deps_key][pkg] = (opt_dep_versions[pkg], OPT_DEPS[pkg]) + for key in opt_dep_versions: + checks_data[opt_deps_key][key] = (opt_dep_versions[key], EASYBUILD_OPTIONAL_DEPENDENCIES[key][1]) sys_tools_key = "System tools" checks_data[sys_tools_key] = {} From dc40a2b7f671ffbcaa2824728142cfc544e59cbe Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Sep 2021 17:13:18 +0200 Subject: [PATCH 83/91] fix log message in get_cfg_opt_abs_path --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 8d7b3af441..3dc979b149 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1054,7 +1054,7 @@ def get_cfg_opt_abs_path(self, opt_name, path): else: abs_path = os.path.abspath(path) self.log.info("Relative path value for '%s' configuration option resolved to absolute path: %s", - path, abs_path) + opt_name, abs_path) return abs_path def _ensure_abs_path(self, opt_name): From 5b87003ea590fdcd2d570d6e3a911345fb3048de Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Sep 2021 17:40:19 +0200 Subject: [PATCH 84/91] also ensure absolute paths for 'robot' configuration option + enhance test_config_abs_path to also check on robot and robot-paths configuration options --- easybuild/tools/options.py | 4 ++-- test/framework/options.py | 32 ++++++++++++++++++++++++++++---- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 3dc979b149..a0592bc2fb 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1077,7 +1077,7 @@ def _postprocess_config(self): # resolve relative paths for configuration options that specify a location path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath', 'installpath_modules', 'installpath_software', 'prefix', 'packagepath', - 'robot_paths', 'sourcepath'] + 'robot', 'robot_paths', 'sourcepath'] # repositorypath is a special case: only first part is a path; # 2nd (optional) part is a relative subdir and should not be resolved to an absolute path! @@ -1133,7 +1133,7 @@ def _postprocess_config(self): # paths specified to --robot have preference over --robot-paths # keep both values in sync if robot is enabled, which implies enabling dependency resolver - self.options.robot_paths = [os.path.abspath(path) for path in self.options.robot + self.options.robot_paths] + self.options.robot_paths = self.options.robot + self.options.robot_paths self.options.robot = self.options.robot_paths # Update the search_paths (if any) to absolute paths diff --git a/test/framework/options.py b/test/framework/options.py index a8bc6fe985..0425402049 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6250,7 +6250,10 @@ def test_config_abs_path(self): ]) write_file(cfgfile, cfgtxt) + # relative paths in environment variables is also weird, + # but OK for the sake of testing... os.environ['EASYBUILD_INSTALLPATH'] = '../..' + os.environ['EASYBUILD_ROBOT_PATHS'] = '../..' args = [ '--configfiles=%s' % cfgfile, @@ -6258,19 +6261,40 @@ def test_config_abs_path(self): '--sourcepath=.', '--show-config', ] + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) patterns = [ - r"^containerpath\s+\(F\) = .*/test_topdir/test_middle_dir$", - r"^installpath\s+\(E\) = .*/test_topdir$", - r"^prefix\s+\(C\) = .*/test_topdir/test_middle_dir$", + r"^containerpath\s+\(F\) = /.*/test_topdir/test_middle_dir$", + r"^installpath\s+\(E\) = /.*/test_topdir$", + r"^prefix\s+\(C\) = /.*/test_topdir/test_middle_dir$", r"^repositorypath\s+\(F\) = \('/apps/easyconfigs_archive', ' somesubdir'\)$", - r"^sourcepath\s+\(C\) = .*/test_topdir/test_middle_dir/test_subdir$", + r"^sourcepath\s+\(C\) = /.*/test_topdir/test_middle_dir/test_subdir$", + r"^robot-paths\s+\(E\) = /.*/test_topdir$", ] for pattern in patterns: regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (pattern, txt)) + # if --robot is also used, that wins and $EASYBUILD_ROBOT_PATHS doesn't matter anymore + change_dir(test_subdir) + args.append('--robot=..:.') + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) + + patterns.pop(-1) + robot_value_pattern = ', '.join([ + r'/.*/test_topdir/test_middle_dir', # via --robot (first path) + r'/.*/test_topdir/test_middle_dir/test_subdir', # via --robot (second path) + r'/.*/test_topdir', # via $EASYBUILD_ROBOT_PATHS + ]) + patterns.extend([ + r"^robot-paths\s+\(C\) = %s$" % robot_value_pattern, + r"^robot\s+\(C\) = %s$" % robot_value_pattern, + ]) + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (pattern, txt)) + # end-to-end testing of unknown filename def test_easystack_wrong_read(self): """Test for --easystack when wrong name is provided""" From 74175a78cd6754c9420c8d18329c119bed2c772b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Sep 2021 18:37:08 +0200 Subject: [PATCH 85/91] fix comment in test_config_abs_path --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 0425402049..c547f2328d 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6276,7 +6276,7 @@ def test_config_abs_path(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (pattern, txt)) - # if --robot is also used, that wins and $EASYBUILD_ROBOT_PATHS doesn't matter anymore + # paths specified via --robot have precedence over those specified via $EASYBUILD_ROBOT_PATHS change_dir(test_subdir) args.append('--robot=..:.') txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True) From 593105e058c29d68719ad7f65ace4428beddf2eb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Sep 2021 19:37:39 +0200 Subject: [PATCH 86/91] add --output-style configuration option, which can be used to disable use of Rich or type of any colored output --- easybuild/tools/config.py | 32 ++++++++++++++++++++++++++++++++ easybuild/tools/options.py | 6 +++++- easybuild/tools/output.py | 20 ++++++++++++-------- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 94b02d4424..fc583e7c4f 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -48,6 +48,12 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.py2vs3 import ascii_letters, create_base_metaclass, string_type +try: + import rich + HAVE_RICH = True +except ImportError: + HAVE_RICH = False + _log = fancylogger.getLogger('config', fname=False) @@ -137,6 +143,13 @@ LOCAL_VAR_NAMING_CHECKS = [LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG, LOCAL_VAR_NAMING_CHECK_WARN] +OUTPUT_STYLE_AUTO = 'auto' +OUTPUT_STYLE_BASIC = 'basic' +OUTPUT_STYLE_NO_COLOR = 'no_color' +OUTPUT_STYLE_RICH = 'rich' +OUTPUT_STYLES = (OUTPUT_STYLE_AUTO, OUTPUT_STYLE_BASIC, OUTPUT_STYLE_NO_COLOR, OUTPUT_STYLE_RICH) + + class Singleton(ABCMeta): """Serves as metaclass for classes that should implement the Singleton pattern. @@ -342,6 +355,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_WAIT_ON_LOCK_INTERVAL: [ 'wait_on_lock_interval', ], + OUTPUT_STYLE_AUTO: [ + 'output_style', + ], } # build option that do not have a perfectly matching command line option BUILD_OPTIONS_OTHER = { @@ -688,6 +704,22 @@ def get_module_syntax(): return ConfigurationVariables()['module_syntax'] +def get_output_style(): + """Return output style to use.""" + output_style = build_option('output_style') + + if output_style == OUTPUT_STYLE_AUTO: + if HAVE_RICH: + output_style = OUTPUT_STYLE_RICH + else: + output_style = OUTPUT_STYLE_BASIC + + if output_style == OUTPUT_STYLE_RICH and not HAVE_RICH: + raise EasyBuildError("Can't use '%s' output style, Rich Python package is not available!", OUTPUT_STYLE_RICH) + + return output_style + + def log_file_format(return_directory=False, ec=None, date=None, timestamp=None): """ Return the format for the logfile or the directory diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 759e8d8e42..0d4cec6787 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -69,7 +69,8 @@ from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS -from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS, WARN +from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS +from easybuild.tools.config import OUTPUT_STYLE_AUTO, OUTPUT_STYLES, WARN from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path from easybuild.tools.configobj import ConfigObj, ConfigObjError from easybuild.tools.docs import FORMAT_TXT, FORMAT_RST @@ -448,6 +449,9 @@ def override_options(self): 'optarch': ("Set architecture optimization, overriding native architecture optimizations", None, 'store', None), 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_TXT, FORMAT_RST]), + 'output-style': ("Control output style; auto implies using Rich if available to produce rich output, " + "with fallback to basic colored output", + 'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES), 'parallel': ("Specify (maximum) level of parallellism used during build procedure", 'int', 'store', None), 'pre-create-installdir': ("Create installation directory before submitting build jobs", diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 5b9c9ba3dd..fb9ad176c4 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -31,16 +31,15 @@ """ import random -from easybuild.tools.config import build_option +from easybuild.tools.config import OUTPUT_STYLE_RICH, build_option, get_output_style from easybuild.tools.py2vs3 import OrderedDict try: from rich.console import Console from rich.table import Table from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn - HAVE_RICH = True except ImportError: - HAVE_RICH = False + pass class DummyProgress(object): @@ -61,6 +60,11 @@ def update(self, *args, **kwargs): pass +def use_rich(): + """Return whether or not to use Rich to produce rich output.""" + return get_output_style() == OUTPUT_STYLE_RICH + + def create_progress_bar(): """ Create progress bar to display overall progress. @@ -68,7 +72,7 @@ def create_progress_bar(): Returns rich.progress.Progress instance if the Rich Python package is available, or a shim DummyProgress instance otherwise. """ - if HAVE_RICH and build_option('show_progress_bar'): + if use_rich() and build_option('show_progress_bar'): # pick random spinner, from a selected subset of available spinner (see 'python3 -m rich.spinner') spinner = random.choice(('aesthetic', 'arc', 'bounce', 'dots', 'line', 'monkey', 'point', 'simpleDots')) @@ -95,7 +99,7 @@ def print_checks(checks_data): col2_label = col_titles[1] - if HAVE_RICH: + if use_rich(): console = Console() # don't use console.print, which causes SyntaxError in Python 2 console_print = getattr(console, 'print') # noqa: B009 @@ -104,7 +108,7 @@ def print_checks(checks_data): for section in checks_data: section_checks = checks_data[section] - if HAVE_RICH: + if use_rich(): table = Table(title=section) table.add_column(col_titles[0]) table.add_column(col_titles[1]) @@ -124,7 +128,7 @@ def print_checks(checks_data): else: check_names = sorted(section_checks, key=lambda x: x.lower()) - if HAVE_RICH: + if use_rich(): for check_name in check_names: (info, descr) = checks_data[section][check_name] if info is None: @@ -150,7 +154,7 @@ def print_checks(checks_data): lines.append(line) lines.append('') - if HAVE_RICH: + if use_rich(): console_print(table) else: print('\n'.join(lines)) From f538359a8ce653f14e534562639de1c6ebfacf2d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Sep 2021 21:25:09 +0200 Subject: [PATCH 87/91] silence the Hound on unused import of rich in tools.config --- easybuild/tools/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index fc583e7c4f..18902ae799 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -49,7 +49,7 @@ from easybuild.tools.py2vs3 import ascii_letters, create_base_metaclass, string_type try: - import rich + import rich # noqa HAVE_RICH = True except ImportError: HAVE_RICH = False From 476a826c6f18ffbe610fd4c45f7fb782580a450e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 14 Sep 2021 09:07:05 +0200 Subject: [PATCH 88/91] add tests for function provided by easybuild.tools.output --- test/framework/output.py | 110 +++++++++++++++++++++++++++++++++++++++ test/framework/suite.py | 3 +- 2 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 test/framework/output.py diff --git a/test/framework/output.py b/test/framework/output.py new file mode 100644 index 0000000000..b69a90cd35 --- /dev/null +++ b/test/framework/output.py @@ -0,0 +1,110 @@ +# # +# Copyright 2021-2021 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Unit tests for functionality in easybuild.tools.output + +@author: Kenneth Hoste (Ghent University) +""" +import sys +from unittest import TextTestRunner +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered + +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option, get_output_style, update_build_option +from easybuild.tools.output import DummyProgress, create_progress_bar, use_rich + +try: + import rich.progress + HAVE_RICH = True +except ImportError: + HAVE_RICH = False + + +class OutputTest(EnhancedTestCase): + """Tests for functions controlling terminal output.""" + + def test_create_progress_bar(self): + """Test create_progress_bar function.""" + + progress_bar = create_progress_bar() + if HAVE_RICH: + progress_bar_class = rich.progress.ProgressBar + else: + progress_bar_class = DummyProgress + self.assertTrue(isinstance(progress_bar, progress_bar_class)) + + update_build_option('output_style', 'basic') + self.assertTrue(isinstance(progress_bar, DummyProgress)) + + update_build_option('output_style', 'rich') + self.assertTrue(isinstance(progress_bar, progress_bar_class)) + + update_build_option('show_progress_bar', False) + self.assertTrue(isinstance(progress_bar, DummyProgress)) + + def test_get_output_style(self): + """Test get_output_style function.""" + + self.assertEqual(build_option('output_style'), 'auto') + + for style in (None, 'auto'): + if style: + update_build_option('output_style', style) + + if HAVE_RICH: + self.assertEqual(get_output_style(), 'rich') + else: + self.assertEqual(get_output_style(), 'basic') + + test_styles = ['basic', 'no_color'] + if HAVE_RICH: + test_styles.append('rich') + + for style in test_styles: + update_build_option('output_style', style) + self.assertEqual(get_output_style(), style) + + if not HAVE_RICH: + update_build_option('output_style', 'rich') + error_pattern = "Can't use 'rich' output style, Rich Python package is not available!" + self.assertErrorRegex(EasyBuildError, error_pattern, get_output_style) + + def test_use_rich(self): + """Test use_rich function.""" + try: + import rich # noqa + self.assertTrue(use_rich()) + except ImportError: + self.assertFalse(use_rich()) + + +def suite(): + """ returns all the testcases in this module """ + return TestLoaderFiltered().loadTestsFromTestCase(OutputTest, sys.argv[1:]) + + +if __name__ == '__main__': + res = TextTestRunner(verbosity=1).run(suite()) + sys.exit(len(res.failures)) diff --git a/test/framework/suite.py b/test/framework/suite.py index 1633bba103..80bce4983f 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -66,6 +66,7 @@ import test.framework.modules as m import test.framework.modulestool as mt import test.framework.options as o +import test.framework.output as ou import test.framework.parallelbuild as p import test.framework.package as pkg import test.framework.repository as r @@ -120,7 +121,7 @@ # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, lic, f_c, - tw, p, i, pkg, d, env, et, y, st, h, ct, lib, u, es] + tw, p, i, pkg, d, env, et, y, st, h, ct, lib, u, es, ou] SUITE = unittest.TestSuite([x.suite() for x in tests]) res = unittest.TextTestRunner().run(SUITE) From 4c9f72dc0eea2e31cc9ae62384292bc9d6a2e822 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 14 Sep 2021 09:49:23 +0200 Subject: [PATCH 89/91] fix test_create_progress_bar --- test/framework/output.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/test/framework/output.py b/test/framework/output.py index b69a90cd35..174f8164d2 100644 --- a/test/framework/output.py +++ b/test/framework/output.py @@ -48,20 +48,27 @@ class OutputTest(EnhancedTestCase): def test_create_progress_bar(self): """Test create_progress_bar function.""" - progress_bar = create_progress_bar() if HAVE_RICH: - progress_bar_class = rich.progress.ProgressBar + expected_progress_bar_class = rich.progress.Progress else: - progress_bar_class = DummyProgress - self.assertTrue(isinstance(progress_bar, progress_bar_class)) + expected_progress_bar_class = DummyProgress + + progress_bar = create_progress_bar() + error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class) + self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg) update_build_option('output_style', 'basic') + progress_bar = create_progress_bar() self.assertTrue(isinstance(progress_bar, DummyProgress)) - update_build_option('output_style', 'rich') - self.assertTrue(isinstance(progress_bar, progress_bar_class)) + if HAVE_RICH: + update_build_option('output_style', 'rich') + progress_bar = create_progress_bar() + error_msg = "%s should be instance of class %s" % (progress_bar, expected_progress_bar_class) + self.assertTrue(isinstance(progress_bar, expected_progress_bar_class), error_msg) update_build_option('show_progress_bar', False) + progress_bar = create_progress_bar() self.assertTrue(isinstance(progress_bar, DummyProgress)) def test_get_output_style(self): From 6934dd84c5d1de61851bf9cd73d2791a206ea90a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 14 Sep 2021 10:37:23 +0200 Subject: [PATCH 90/91] handle ensuring of absolute paths in 'robot' configuration value separately, due to special treatment needed for --robot argument --- easybuild/tools/options.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index a0592bc2fb..06ce7f6013 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1074,10 +1074,12 @@ def _ensure_abs_path(self, opt_name): def _postprocess_config(self): """Postprocessing of configuration options""" - # resolve relative paths for configuration options that specify a location + # resolve relative paths for configuration options that specify a location; + # ensuring absolute paths for 'robot' is handled separately below, + # because we need to be careful with the argument pass to --robot path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath', 'installpath_modules', 'installpath_software', 'prefix', 'packagepath', - 'robot', 'robot_paths', 'sourcepath'] + 'robot_paths', 'sourcepath'] # repositorypath is a special case: only first part is a path; # 2nd (optional) part is a relative subdir and should not be resolved to an absolute path! @@ -1133,7 +1135,7 @@ def _postprocess_config(self): # paths specified to --robot have preference over --robot-paths # keep both values in sync if robot is enabled, which implies enabling dependency resolver - self.options.robot_paths = self.options.robot + self.options.robot_paths + self.options.robot_paths = [os.path.abspath(p) for p in self.options.robot + self.options.robot_paths] self.options.robot = self.options.robot_paths # Update the search_paths (if any) to absolute paths From 94e9876038530c89dcd3ad6b81287ee15cb0264d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 15 Sep 2021 09:59:41 +0200 Subject: [PATCH 91/91] only call os.path.abspath on paths passed to robot paths, robot_paths is already handled --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 06ce7f6013..614ba78f97 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1135,7 +1135,7 @@ def _postprocess_config(self): # paths specified to --robot have preference over --robot-paths # keep both values in sync if robot is enabled, which implies enabling dependency resolver - self.options.robot_paths = [os.path.abspath(p) for p in self.options.robot + self.options.robot_paths] + self.options.robot_paths = [os.path.abspath(p) for p in self.options.robot] + self.options.robot_paths self.options.robot = self.options.robot_paths # Update the search_paths (if any) to absolute paths