From 6b779256c85c89a3aa32b357c16efa1ff881f442 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 21 Apr 2021 08:17:29 +0200 Subject: [PATCH 01/19] add support for installing extensions in parallel --- easybuild/framework/easyblock.py | 196 +++++++++++++++++++++++++------ easybuild/framework/extension.py | 51 +++++++- 2 files changed, 210 insertions(+), 37 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 77e0df72a5..4b3777db34 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1620,6 +1620,166 @@ def skip_extensions(self): self.ext_instances = res + def install_extensions(self, install=True, parallel=False): + """ + Install extensions. + + :param install: actually install extensions, don't just prepare environment for installing + :param parallel: install extensions in parallel + + """ + self.log.debug("List of loaded modules: %s", self.modules_tool.list()) + + if parallel: + self.install_extensions_parallel(install=install) + else: + self.install_extensions_sequential(install=install) + + def install_extensions_sequential(self, install=True): + """ + Install extensions sequentially. + + :param install: actually install extensions, don't just prepare environment for installing + """ + self.log.info("Installing extensions sequentially...") + + exts_cnt = len(self.ext_instances) + for idx, ext in enumerate(self.ext_instances): + + self.log.debug("Starting extension %s" % ext.name) + + # always go back to original work dir to avoid running stuff from a dir that no longer exists + change_dir(self.orig_workdir) + + tup = (ext.name, ext.version or '', idx + 1, exts_cnt) + print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent, log=self.log) + + if self.dry_run: + tup = (ext.name, ext.version, ext.__class__.__name__) + msg = "\n* installing extension %s %s using '%s' easyblock\n" % tup + self.dry_run_msg(msg) + + self.log.debug("List of loaded modules: %s", self.modules_tool.list()) + + # prepare toolchain build environment, but only when not doing a dry run + # since in that case the build environment is the same as for the parent + if self.dry_run: + self.dry_run_msg("defining build environment based on toolchain (options) and dependencies...") + else: + # don't reload modules for toolchain, there is no need since they will be loaded already; + # the (fake) module for the parent software gets loaded before installing extensions + ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, + rpath_filter_dirs=self.rpath_filter_dirs) + + # real work + if install: + ext.prerun() + txt = ext.run() + if txt: + self.module_extra_extensions += txt + ext.postrun() + + def install_extensions_parallel(self, install=True): + """ + Install extensions in parallel. + + :param install: actually install extensions, don't just prepare environment for installing + """ + self.log.info("Installing extensions in parallel...") + + running_exts = [] + installed_ext_names = [] + + all_ext_names = [x['name'] for x in self.exts_all] + self.log.debug("List of names of all extensions: %s", all_ext_names) + + # take into account that some extensions may be installed already + to_install_ext_names = [x.name for x in self.ext_instances] + installed_ext_names = [n for n in all_ext_names if n not in to_install_ext_names] + + exts_cnt = len(all_ext_names) + exts_queue = self.ext_instances[:] + + iter_id = 0 + while exts_queue or running_exts: + + iter_id += 1 + + # always go back to original work dir to avoid running stuff from a dir that no longer exists + change_dir(self.orig_workdir) + + # check for extension installations that have completed + if running_exts: + self.log.info("Checking for completed extension installations (%d running)...", len(running_exts)) + for ext in running_exts[:]: + if self.dry_run or ext.async_cmd_check(): + self.log.info("Installation of %s completed!", ext.name) + ext.postrun() + running_exts.remove(ext) + installed_ext_names.append(ext.name) + else: + self.log.debug("Installation of %s is still running...", ext.name) + + # print progress info every now and then + if iter_id % 1 == 0: + msg = "%d out of %d extensions installed (%d queued, %d running: %s)" + installed_cnt, queued_cnt, running_cnt = len(installed_ext_names), len(exts_queue), len(running_exts) + if running_cnt <= 3: + running_ext_names = ', '.join(x.name for x in running_exts) + else: + running_ext_names = ', '.join(x.name for x in running_exts[:3]) + ", ..." + print_msg(msg % (installed_cnt, exts_cnt, queued_cnt, running_cnt, running_ext_names), log=self.log) + + # try to start as many extension installations as we can, taking into account number of available cores, + # but only consider first 100 extensions still in the queue + max_iter = min(100, len(exts_queue)) + + for _ in range(max_iter): + + if not (exts_queue and len(running_exts) < self.cfg['parallel']): + break + + # check whether extension at top of the queue is ready to install + ext = exts_queue.pop(0) + + pending_deps = [x for x in ext.required_deps if x not in installed_ext_names] + + if self.dry_run: + tup = (ext.name, ext.version, ext.__class__.__name__) + msg = "\n* installing extension %s %s using '%s' easyblock\n" % tup + self.dry_run_msg(msg) + running_exts.append(ext) + + # if some of the required dependencies are not installed yet, requeue this extension + elif pending_deps: + + # make sure all required dependencies are actually going to be installed, + # to avoid getting stuck in an infinite loop! + missing_deps = [x for x in ext.required_deps if x not in all_ext_names] + if missing_deps: + raise EasyBuildError("Missing required dependencies for %s are not going to be installed: %s", + ext.name, ', '.join(missing_deps)) + else: + self.log.info("Required dependencies missing for extension %s (%s), adding it back to queue...", + ext.name, ', '.join(pending_deps)) + # purposely adding extension back in the queue at Nth place rather than at the end, + # since we assume that the required dependencies will be installed soon... + exts_queue.insert(max_iter, ext) + + else: + tup = (ext.name, ext.version or '') + print_msg("starting installation of extension %s %s..." % tup, silent=self.silent, log=self.log) + + # don't reload modules for toolchain, there is no need since they will be loaded already; + # the (fake) module for the parent software gets loaded before installing extensions + ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, + rpath_filter_dirs=self.rpath_filter_dirs) + if install: + ext.prerun() + ext.run(asynchronous=True) + running_exts.append(ext) + self.log.debug("Started installation of extension %s in the background...", ext.name) + # # MISCELLANEOUS UTILITY FUNCTIONS # @@ -2318,41 +2478,7 @@ def extensions_step(self, fetch=False, install=True): if self.skip: self.skip_extensions() - exts_cnt = len(self.ext_instances) - for idx, ext in enumerate(self.ext_instances): - - self.log.debug("Starting extension %s" % ext.name) - - # always go back to original work dir to avoid running stuff from a dir that no longer exists - change_dir(self.orig_workdir) - - tup = (ext.name, ext.version or '', idx + 1, exts_cnt) - print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent) - - if self.dry_run: - tup = (ext.name, ext.version, cls.__name__) - msg = "\n* installing extension %s %s using '%s' easyblock\n" % tup - self.dry_run_msg(msg) - - self.log.debug("List of loaded modules: %s", self.modules_tool.list()) - - # prepare toolchain build environment, but only when not doing a dry run - # since in that case the build environment is the same as for the parent - if self.dry_run: - self.dry_run_msg("defining build environment based on toolchain (options) and dependencies...") - else: - # don't reload modules for toolchain, there is no need since they will be loaded already; - # the (fake) module for the parent software gets loaded before installing extensions - ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, - rpath_filter_dirs=self.rpath_filter_dirs) - - # real work - if install: - ext.prerun() - txt = ext.run() - if txt: - self.module_extra_extensions += txt - ext.postrun() + self.install_extensions(install=install) # cleanup (unload fake module, remove fake module dir) if fake_mod_data: diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 569a3bb414..6b70afd966 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -40,7 +40,7 @@ from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.tools.build_log import EasyBuildError, raise_nosupport from easybuild.tools.filetools import change_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import complete_cmd, get_output_from_process, run_cmd from easybuild.tools.py2vs3 import string_type @@ -138,6 +138,12 @@ def __init__(self, mself, ext, extra_params=None): key, name, version, value) self.sanity_check_fail_msgs = [] + self.async_cmd_info = None + self.async_cmd_output = None + self.async_cmd_check_cnt = None + # initial read size should be relatively small, + # to avoid hanging for a long time until desired output is available in async_cmd_check + self.async_cmd_read_size = 1024 @property def name(self): @@ -159,7 +165,7 @@ def prerun(self): """ pass - def run(self): + def run(self, *args, **kwargs): """ Actual installation of a extension. """ @@ -171,6 +177,47 @@ def postrun(self): """ pass + def async_cmd_start(self, cmd, inp=None): + """ + Start installation asynchronously using specified command. + """ + self.async_cmd_output = '' + self.async_cmd_check_cnt = 0 + self.async_cmd_info = run_cmd(cmd, log_all=True, simple=False, inp=inp, regexp=False, asynchronous=True) + + def async_cmd_check(self): + """ + Check progress of installation command that was started asynchronously. + + :return: True if command completed, False otherwise + """ + if self.async_cmd_info is None: + raise EasyBuildError("No installation command running asynchronously for %s", self.name) + else: + self.log.debug("Checking on installation of extension %s...", self.name) + proc = self.async_cmd_info[0] + # use small read size, to avoid waiting for a long time until sufficient output is produced + self.async_cmd_output += get_output_from_process(proc, read_size=self.async_cmd_read_size) + ec = proc.poll() + if ec is None: + res = False + self.async_cmd_check_cnt += 1 + # increase read size after sufficient checks, + # to avoid that installation hangs due to output buffer filling up... + if self.async_cmd_check_cnt % 10 == 0 and self.async_cmd_read_size < (1024 ** 2): + self.async_cmd_read_size *= 2 + else: + self.log.debug("Completing installation of extension %s...", self.name) + self.async_cmd_output, _ = complete_cmd(*self.async_cmd_info, output=self.async_cmd_output) + res = True + + return res + + @property + def required_deps(self): + """Return list of required dependencies for this extension.""" + raise NotImplementedError("Don't know how to determine required dependencies for %s" % self.name) + @property def toolchain(self): """ From 6053c0cb1d768d0e5970e1cf1ebf21dafa2d9e13 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Oct 2021 14:36:44 +0200 Subject: [PATCH 02/19] also update extensions progress bar when installing extensions in parallel --- easybuild/framework/easyblock.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e3b34f5c2b..a2383b6a0d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1733,6 +1733,21 @@ def install_extensions_parallel(self, install=True): exts_cnt = len(all_ext_names) exts_queue = self.ext_instances[:] + start_progress_bar(PROGRESS_BAR_EXTENSIONS, exts_cnt) + + def update_exts_progress_bar(running_exts, progress_size): + """Helper function to update extensions progress bar.""" + running_exts_cnt = len(running_exts) + if running_exts_cnt > 1: + progress_label = "Installing %d extensions: " % running_exts_cnt + elif running_exts_cnt == 1: + progress_label = "Installing extension " + else: + progress_label = "Not installing extensions (yet)" + + progress_label += ' '.join(e.name for e in running_exts) + update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=progress_label, progress_size=progress_size) + iter_id = 0 while exts_queue or running_exts: @@ -1750,6 +1765,7 @@ def install_extensions_parallel(self, install=True): ext.postrun() running_exts.remove(ext) installed_ext_names.append(ext.name) + update_exts_progress_bar(running_exts, 1) else: self.log.debug("Installation of %s is still running...", ext.name) @@ -1811,7 +1827,10 @@ def install_extensions_parallel(self, install=True): ext.prerun() ext.run(asynchronous=True) running_exts.append(ext) - self.log.debug("Started installation of extension %s in the background...", ext.name) + self.log.info("Started installation of extension %s in the background...", ext.name) + update_exts_progress_bar(running_exts, 0) + + stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False) # # MISCELLANEOUS UTILITY FUNCTIONS From 3fe1f688b3903433c36ed538a18fdf7a693dc2e9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Oct 2021 14:59:01 +0200 Subject: [PATCH 03/19] mark support for installing extensions in parallel as experimental + add --parallel-extensions-install configuration option to opt-in to it --- easybuild/framework/easyblock.py | 3 ++- easybuild/tools/config.py | 1 + easybuild/tools/options.py | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a2383b6a0d..dbc31f1ca6 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1646,7 +1646,8 @@ def install_extensions(self, install=True, parallel=False): """ self.log.debug("List of loaded modules: %s", self.modules_tool.list()) - if parallel: + if build_option('parallel_extensions_install') and parallel: + self.log.experimental("installing extensions in parallel") self.install_extensions_parallel(install=install) else: self.install_extensions_sequential(install=install) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 18902ae799..32642ac224 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -269,6 +269,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'module_extensions', 'module_only', 'package', + 'parallel_extensions_install', 'read_only_installdir', 'remove_ghost_install_dirs', 'rebuild', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index e4fc7661a4..7ef8e7232e 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -454,6 +454,8 @@ def override_options(self): 'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES), 'parallel': ("Specify (maximum) level of parallellism used during build procedure", 'int', 'store', None), + 'parallel-extensions-install': ("Install list of extensions in parallel (if supported)", + None, 'store_true', False), 'pre-create-installdir': ("Create installation directory before submitting build jobs", None, 'store_true', True), 'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"), From 2e5c6ab8db7440b56a063cf3672280205c96a691 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Oct 2021 15:14:24 +0200 Subject: [PATCH 04/19] start extensions progress bar a bit earlier, also mention preparatory steps (like creating of Extension instances) --- easybuild/framework/easyblock.py | 17 +++++++++-------- easybuild/tools/output.py | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index dbc31f1ca6..636c0d30a6 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1661,7 +1661,6 @@ def install_extensions_sequential(self, install=True): self.log.info("Installing extensions sequentially...") exts_cnt = len(self.ext_instances) - start_progress_bar(PROGRESS_BAR_EXTENSIONS, exts_cnt) for idx, ext in enumerate(self.ext_instances): @@ -1670,7 +1669,7 @@ def install_extensions_sequential(self, install=True): # always go back to original work dir to avoid running stuff from a dir that no longer exists change_dir(self.orig_workdir) - progress_label = "Installing '%s' extension" % ext.name + progress_label = "Installing '%s' extension (%s/%s)" % (ext.name, idx + 1, exts_cnt) update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=progress_label) tup = (ext.name, ext.version or '', idx + 1, exts_cnt) @@ -1711,8 +1710,6 @@ def install_extensions_sequential(self, install=True): elif self.logdebug or build_option('trace'): print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent) - stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False) - def install_extensions_parallel(self, install=True): """ Install extensions in parallel. @@ -1734,8 +1731,6 @@ def install_extensions_parallel(self, install=True): exts_cnt = len(all_ext_names) exts_queue = self.ext_instances[:] - start_progress_bar(PROGRESS_BAR_EXTENSIONS, exts_cnt) - def update_exts_progress_bar(running_exts, progress_size): """Helper function to update extensions progress bar.""" running_exts_cnt = len(running_exts) @@ -1747,6 +1742,7 @@ def update_exts_progress_bar(running_exts, progress_size): progress_label = "Not installing extensions (yet)" progress_label += ' '.join(e.name for e in running_exts) + progress_label += "(%d/%d done)" % (len(installed_ext_names), exts_cnt) update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=progress_label, progress_size=progress_size) iter_id = 0 @@ -1831,8 +1827,6 @@ def update_exts_progress_bar(running_exts, progress_size): self.log.info("Started installation of extension %s in the background...", ext.name) update_exts_progress_bar(running_exts, 0) - stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False) - # # MISCELLANEOUS UTILITY FUNCTIONS # @@ -2587,9 +2581,12 @@ def extensions_step(self, fetch=False, install=True): fake_mod_data = self.load_fake_module(purge=True, extra_modules=build_dep_mods) + start_progress_bar(PROGRESS_BAR_EXTENSIONS, len(self.cfg['exts_list'])) + self.prepare_for_extensions() if fetch: + update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="fetching extension sources/patches", progress_size=0) self.exts = self.collect_exts_file_info(fetch_files=True) self.exts_all = self.exts[:] # retain a copy of all extensions, regardless of filtering/skipping @@ -2603,9 +2600,11 @@ def extensions_step(self, fetch=False, install=True): self.clean_up_fake_module(fake_mod_data) raise EasyBuildError("ERROR: No default extension class set for %s", self.name) + update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="creating Extension instances", progress_size=0) self.init_ext_instances() if self.skip: + update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="skipping installed extensions", progress_size=0) self.skip_extensions() self.install_extensions(install=install) @@ -2614,6 +2613,8 @@ def extensions_step(self, fetch=False, install=True): if fake_mod_data: self.clean_up_fake_module(fake_mod_data) + stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False) + def package_step(self): """Package installed software (e.g., into an RPM), if requested, using selected package tool.""" diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 23270fecc4..ff8250b7b2 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -242,7 +242,7 @@ def extensions_progress_bar(): Get progress bar to show progress for installing extensions. """ progress_bar = Progress( - TextColumn("[bold blue]{task.description} ({task.completed}/{task.total})"), + TextColumn("[bold blue]{task.description}"), BarColumn(), TimeElapsedColumn(), ) From 3748d9c444a290c3b7649821376a34c504b10854 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Oct 2021 15:26:00 +0200 Subject: [PATCH 05/19] add and use update_exts_progress_bar method to EasyBlock --- easybuild/framework/easyblock.py | 36 +++++++++++++++++++------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 636c0d30a6..39b06987dc 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1669,8 +1669,8 @@ def install_extensions_sequential(self, install=True): # always go back to original work dir to avoid running stuff from a dir that no longer exists change_dir(self.orig_workdir) - progress_label = "Installing '%s' extension (%s/%s)" % (ext.name, idx + 1, exts_cnt) - update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=progress_label) + progress_info = "Installing '%s' extension (%s/%s)" % (ext.name, idx + 1, exts_cnt) + self.update_exts_progress_bar(progress_info) tup = (ext.name, ext.version or '', idx + 1, exts_cnt) print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent, log=self.log) @@ -1710,6 +1710,8 @@ def install_extensions_sequential(self, install=True): elif self.logdebug or build_option('trace'): print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent) + self.update_exts_progress_bar(progress_info, progress_size=1) + def install_extensions_parallel(self, install=True): """ Install extensions in parallel. @@ -1731,19 +1733,19 @@ def install_extensions_parallel(self, install=True): exts_cnt = len(all_ext_names) exts_queue = self.ext_instances[:] - def update_exts_progress_bar(running_exts, progress_size): + def update_exts_progress_bar_helper(running_exts, progress_size): """Helper function to update extensions progress bar.""" running_exts_cnt = len(running_exts) if running_exts_cnt > 1: - progress_label = "Installing %d extensions: " % running_exts_cnt + progress_info = "Installing %d extensions: " % running_exts_cnt elif running_exts_cnt == 1: - progress_label = "Installing extension " + progress_info = "Installing extension " else: - progress_label = "Not installing extensions (yet)" + progress_info = "Not installing extensions (yet)" - progress_label += ' '.join(e.name for e in running_exts) - progress_label += "(%d/%d done)" % (len(installed_ext_names), exts_cnt) - update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=progress_label, progress_size=progress_size) + progress_info += ' '.join(e.name for e in running_exts) + progress_info += "(%d/%d done)" % (len(installed_ext_names), exts_cnt) + self.update_exts_progress_bar(progress_info, progress_size=progress_size) iter_id = 0 while exts_queue or running_exts: @@ -1762,7 +1764,7 @@ def update_exts_progress_bar(running_exts, progress_size): ext.postrun() running_exts.remove(ext) installed_ext_names.append(ext.name) - update_exts_progress_bar(running_exts, 1) + update_exts_progress_bar_helper(running_exts, 1) else: self.log.debug("Installation of %s is still running...", ext.name) @@ -1825,7 +1827,7 @@ def update_exts_progress_bar(running_exts, progress_size): ext.run(asynchronous=True) running_exts.append(ext) self.log.info("Started installation of extension %s in the background...", ext.name) - update_exts_progress_bar(running_exts, 0) + update_exts_progress_bar_helper(running_exts, 0) # # MISCELLANEOUS UTILITY FUNCTIONS @@ -2560,6 +2562,12 @@ def init_ext_instances(self): self.ext_instances.append(inst) + def update_exts_progress_bar(self, info, progress_size=0): + """ + Update extensions progress bar with specified info and amount of progress made + """ + update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=info, progress_size=progress_size) + def extensions_step(self, fetch=False, install=True): """ After make install, run this. @@ -2586,7 +2594,7 @@ def extensions_step(self, fetch=False, install=True): self.prepare_for_extensions() if fetch: - update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="fetching extension sources/patches", progress_size=0) + self.update_exts_progress_bar("fetching extension sources/patches") self.exts = self.collect_exts_file_info(fetch_files=True) self.exts_all = self.exts[:] # retain a copy of all extensions, regardless of filtering/skipping @@ -2600,11 +2608,11 @@ def extensions_step(self, fetch=False, install=True): self.clean_up_fake_module(fake_mod_data) raise EasyBuildError("ERROR: No default extension class set for %s", self.name) - update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="creating Extension instances", progress_size=0) + self.update_exts_progress_bar("creating internal datastructures") self.init_ext_instances() if self.skip: - update_progress_bar(PROGRESS_BAR_EXTENSIONS, label="skipping installed extensions", progress_size=0) + self.update_exts_progress_bar("skipping install extensions") self.skip_extensions() self.install_extensions(install=install) From 45a2627a382df986bc28d2330fb6a8734e04f0ac Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Oct 2021 16:57:38 +0200 Subject: [PATCH 06/19] fix formatting for extension progress bar when installing extensions in parallel --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 39b06987dc..e0fca04973 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1743,8 +1743,8 @@ def update_exts_progress_bar_helper(running_exts, progress_size): else: progress_info = "Not installing extensions (yet)" - progress_info += ' '.join(e.name for e in running_exts) - progress_info += "(%d/%d done)" % (len(installed_ext_names), exts_cnt) + progress_info += ', '.join(e.name for e in running_exts) + progress_info += " (%d/%d done)" % (len(installed_ext_names), exts_cnt) self.update_exts_progress_bar(progress_info, progress_size=progress_size) iter_id = 0 From 78faeabafa37aef7fc9b8a3c5a66ad1a388e6ae9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Oct 2021 19:55:25 +0200 Subject: [PATCH 07/19] update extensions progress bar with more detailed info when creating Extension instances + checking for extensions to skip --- easybuild/framework/easyblock.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e0fca04973..b22322544c 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1616,14 +1616,18 @@ def skip_extensions(self): - use this to detect existing extensions and to remove them from self.ext_instances - based on initial R version """ + self.update_exts_progress_bar("skipping installed extensions") + # obtaining untemplated reference value is required here to support legacy string templates like name/version exts_filter = self.cfg.get_ref('exts_filter') if not exts_filter or len(exts_filter) == 0: raise EasyBuildError("Skipping of extensions, but no exts_filter set in easyconfig") + exts_cnt = len(self.ext_instances) + res = [] - for ext_inst in self.ext_instances: + for idx, ext_inst in enumerate(self.ext_instances): cmd, stdin = resolve_exts_filter_template(exts_filter, ext_inst) (cmdstdouterr, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, regexp=False) self.log.info("exts_filter result %s %s", cmdstdouterr, ec) @@ -1634,6 +1638,8 @@ def skip_extensions(self): else: print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log) + self.update_exts_progress_bar("skipping installed extensions (%d/%d checked)" % (idx + 1, exts_cnt)) + self.ext_instances = res def install_extensions(self, install=True, parallel=False): @@ -2477,6 +2483,8 @@ def init_ext_instances(self): """ exts_list = self.cfg.get_ref('exts_list') + self.update_exts_progress_bar("creating internal datastructures for extensions") + # early exit if there are no extensions if not exts_list: return @@ -2500,7 +2508,9 @@ def init_ext_instances(self): error_msg = "Improper default extension class specification, should be string: %s (%s)" raise EasyBuildError(error_msg, exts_defaultclass, type(exts_defaultclass)) - for ext in self.exts: + exts_cnt = len(self.exts) + + for idx, ext in enumerate(self.exts): ext_name = ext['name'] self.log.debug("Creating class instance for extension %s...", ext_name) @@ -2561,6 +2571,9 @@ def init_ext_instances(self): self.log.debug("Installing extension %s with class %s (from %s)", ext_name, class_name, mod_path) self.ext_instances.append(inst) + pbar_label = "creating internal datastructures for extensions " + pbar_label += "(%d/%d done)" % (idx + 1, exts_cnt) + self.update_exts_progress_bar(pbar_label) def update_exts_progress_bar(self, info, progress_size=0): """ @@ -2608,11 +2621,9 @@ def extensions_step(self, fetch=False, install=True): self.clean_up_fake_module(fake_mod_data) raise EasyBuildError("ERROR: No default extension class set for %s", self.name) - self.update_exts_progress_bar("creating internal datastructures") self.init_ext_instances() if self.skip: - self.update_exts_progress_bar("skipping install extensions") self.skip_extensions() self.install_extensions(install=install) From fdc8a1aa15dc92dcf6e9623eac7912515d0c653a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 22 Oct 2021 10:04:26 +0200 Subject: [PATCH 08/19] only update extensions progress bar in init_ext_instances if there actually are extensions --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index b22322544c..e4b1254d0e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2483,8 +2483,6 @@ def init_ext_instances(self): """ exts_list = self.cfg.get_ref('exts_list') - self.update_exts_progress_bar("creating internal datastructures for extensions") - # early exit if there are no extensions if not exts_list: return @@ -2510,6 +2508,8 @@ def init_ext_instances(self): exts_cnt = len(self.exts) + self.update_exts_progress_bar("creating internal datastructures for extensions") + for idx, ext in enumerate(self.exts): ext_name = ext['name'] self.log.debug("Creating class instance for extension %s...", ext_name) From 85273d039ccb8926d95102045321efd7005e111e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 22 Oct 2021 14:10:18 +0200 Subject: [PATCH 09/19] use check_async_cmd in Extension.async_cmd_check --- easybuild/framework/extension.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index eda2393fa0..024261c332 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -40,7 +40,7 @@ from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.tools.build_log import EasyBuildError, raise_nosupport from easybuild.tools.filetools import change_dir -from easybuild.tools.run import complete_cmd, get_output_from_process, run_cmd +from easybuild.tools.run import check_async_cmd, complete_cmd, run_cmd from easybuild.tools.py2vs3 import string_type @@ -196,23 +196,21 @@ def async_cmd_check(self): raise EasyBuildError("No installation command running asynchronously for %s", self.name) else: self.log.debug("Checking on installation of extension %s...", self.name) - proc = self.async_cmd_info[0] # use small read size, to avoid waiting for a long time until sufficient output is produced - self.async_cmd_output += get_output_from_process(proc, read_size=self.async_cmd_read_size) - ec = proc.poll() - if ec is None: - res = False + res = check_async_cmd(*self.async_cmd_info, output_read_size=self.async_cmd_read_size) + self.async_cmd_output += res['output'] + if res['done']: + self.log.info("Installation of extension %s completed!", self.name) + else: self.async_cmd_check_cnt += 1 + self.log.debug("Installation of extension %s still running (checked %d times)", + self.name, self.async_cmd_check_cnt) # increase read size after sufficient checks, # to avoid that installation hangs due to output buffer filling up... if self.async_cmd_check_cnt % 10 == 0 and self.async_cmd_read_size < (1024 ** 2): self.async_cmd_read_size *= 2 - else: - self.log.debug("Completing installation of extension %s...", self.name) - self.async_cmd_output, _ = complete_cmd(*self.async_cmd_info, output=self.async_cmd_output) - res = True - return res + return res['done'] @property def required_deps(self): From b2938dde42c2848df277fc8b2db401ff80f68dfe Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 22 Oct 2021 14:27:29 +0200 Subject: [PATCH 10/19] remove import for unused complete_cmd from framework/extension.py --- easybuild/framework/extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 024261c332..f78d1c63e6 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -40,7 +40,7 @@ from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.tools.build_log import EasyBuildError, raise_nosupport from easybuild.tools.filetools import change_dir -from easybuild.tools.run import check_async_cmd, complete_cmd, run_cmd +from easybuild.tools.run import check_async_cmd, run_cmd from easybuild.tools.py2vs3 import string_type From 447c1da420e7ae24e73adef41cfc6e74f05c3ce1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 25 Oct 2021 09:53:07 +0200 Subject: [PATCH 11/19] fix occasional failure in test_run_cmd_async --- test/framework/run.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/framework/run.py b/test/framework/run.py index a6a88b638c..24128aad6b 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -616,6 +616,9 @@ def test_run_cmd_async(self): cmd_info = run_cmd(error_test_cmd, asynchronous=True) res = check_async_cmd(*cmd_info, fail_on_error=False) + # keep checking until command is fully done + while not res['done']: + res = check_async_cmd(*cmd_info, fail_on_error=False) self.assertEqual(res, {'done': True, 'exit_code': 123, 'output': "FAIL!\n"}) # also test with a command that produces a lot of output, From d82ade9348cce77368ea1baca5a37de9a33a6b29 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 25 Oct 2021 09:53:26 +0200 Subject: [PATCH 12/19] check early for opt-in to using experimental feature when --parallel-extensions-install is used --- easybuild/tools/options.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 99ed51721b..2142a50f4a 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -892,6 +892,10 @@ def postprocess(self): # set tmpdir self.tmpdir = set_tmpdir(self.options.tmpdir) + # early check for opt-in to installing extensions in parallel (experimental feature) + if self.options.parallel_extensions_install: + self.log.experimental("installing extensions in parallel") + # take --include options into account (unless instructed otherwise) if self.with_include: self._postprocess_include() From 42c0bb3ad4fac2acb2d1ae6fc8601c4fe3ffa802 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 25 Oct 2021 09:57:07 +0200 Subject: [PATCH 13/19] tweak extensions progress bar label to also show 'X/Y done' when installing extensions in parallel --- easybuild/framework/easyblock.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e4b1254d0e..2f31ea464b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1743,14 +1743,16 @@ def update_exts_progress_bar_helper(running_exts, progress_size): """Helper function to update extensions progress bar.""" running_exts_cnt = len(running_exts) if running_exts_cnt > 1: - progress_info = "Installing %d extensions: " % running_exts_cnt + progress_info = "Installing %d extensions" % running_exts_cnt elif running_exts_cnt == 1: progress_info = "Installing extension " else: progress_info = "Not installing extensions (yet)" - progress_info += ', '.join(e.name for e in running_exts) - progress_info += " (%d/%d done)" % (len(installed_ext_names), exts_cnt) + if running_exts_cnt: + progress_info += " (%d/%d done): " % (len(installed_ext_names), exts_cnt) + progress_info += ', '.join(e.name for e in running_exts) + self.update_exts_progress_bar(progress_info, progress_size=progress_size) iter_id = 0 From c5d598f19cc9824ea6b945605132eb12f992fe22 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Oct 2021 14:20:28 +0200 Subject: [PATCH 14/19] inject short sleep before checking status of failing asynchronous command --- test/framework/run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/run.py b/test/framework/run.py index 24128aad6b..da4448d741 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -611,6 +611,7 @@ def test_run_cmd_async(self): # check asynchronous running of failing command error_test_cmd = "echo 'FAIL!' >&2; exit 123" cmd_info = run_cmd(error_test_cmd, asynchronous=True) + time.sleep(1) error_pattern = 'cmd ".*" exited with exit code 123' self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info) From 73af4253f6d642ba8162ef1bf8e5a32d30e9ecd4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Oct 2021 14:58:40 +0200 Subject: [PATCH 15/19] drop 'parallel' argument for install_extensions, to avoid having to opt-in to support for installing extensions in parallel in various easyblocks --- easybuild/framework/easyblock.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 2f31ea464b..468eddf764 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1642,17 +1642,16 @@ def skip_extensions(self): self.ext_instances = res - def install_extensions(self, install=True, parallel=False): + def install_extensions(self, install=True): """ Install extensions. :param install: actually install extensions, don't just prepare environment for installing - :param parallel: install extensions in parallel """ self.log.debug("List of loaded modules: %s", self.modules_tool.list()) - if build_option('parallel_extensions_install') and parallel: + if build_option('parallel_extensions_install'): self.log.experimental("installing extensions in parallel") self.install_extensions_parallel(install=install) else: From 16e02eae6fce1e3d8878ac68efc135f83046c9f3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Oct 2021 15:25:16 +0200 Subject: [PATCH 16/19] add run_async method to install extension asynchronously --- easybuild/framework/easyblock.py | 2 +- easybuild/framework/extension.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 468eddf764..ff8a561226 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1831,7 +1831,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size): rpath_filter_dirs=self.rpath_filter_dirs) if install: ext.prerun() - ext.run(asynchronous=True) + ext.run_async() running_exts.append(ext) self.log.info("Started installation of extension %s in the background...", ext.name) update_exts_progress_bar_helper(running_exts, 0) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index f78d1c63e6..251eed6afe 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -168,10 +168,16 @@ def prerun(self): def run(self, *args, **kwargs): """ - Actual installation of a extension. + Actual installation of an extension. """ pass + def run_async(self, *args, **kwargs): + """ + Asynchronous installation of an extension. + """ + raise NotImplementedError + def postrun(self): """ Stuff to do after installing a extension. From 60c5d1537b3bfbb0f1b3a85676331df952e154f0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Oct 2021 20:07:35 +0200 Subject: [PATCH 17/19] move printing of progress info on installing extensions in parallel after every iteration, and only when not showing progress bars --- easybuild/framework/easyblock.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index ff8a561226..58f4303aa3 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -91,7 +91,7 @@ from easybuild.tools.modules import Lmod, curr_module_paths, invalidate_module_caches_for, get_software_root from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ALL, PROGRESS_BAR_EASYCONFIG, PROGRESS_BAR_EXTENSIONS -from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar +from easybuild.tools.output import show_progress_bars, start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.package.utilities import package from easybuild.tools.py2vs3 import extract_method_name, string_type from easybuild.tools.repository.repository import init_repository @@ -1775,16 +1775,6 @@ def update_exts_progress_bar_helper(running_exts, progress_size): else: self.log.debug("Installation of %s is still running...", ext.name) - # print progress info every now and then - if iter_id % 1 == 0: - msg = "%d out of %d extensions installed (%d queued, %d running: %s)" - installed_cnt, queued_cnt, running_cnt = len(installed_ext_names), len(exts_queue), len(running_exts) - if running_cnt <= 3: - running_ext_names = ', '.join(x.name for x in running_exts) - else: - running_ext_names = ', '.join(x.name for x in running_exts[:3]) + ", ..." - print_msg(msg % (installed_cnt, exts_cnt, queued_cnt, running_cnt, running_ext_names), log=self.log) - # try to start as many extension installations as we can, taking into account number of available cores, # but only consider first 100 extensions still in the queue max_iter = min(100, len(exts_queue)) @@ -1836,6 +1826,16 @@ def update_exts_progress_bar_helper(running_exts, progress_size): self.log.info("Started installation of extension %s in the background...", ext.name) update_exts_progress_bar_helper(running_exts, 0) + # print progress info after every iteration (unless that info is already shown via progress bar) + if not show_progress_bars(): + msg = "%d out of %d extensions installed (%d queued, %d running: %s)" + installed_cnt, queued_cnt, running_cnt = len(installed_ext_names), len(exts_queue), len(running_exts) + if running_cnt <= 3: + running_ext_names = ', '.join(x.name for x in running_exts) + else: + running_ext_names = ', '.join(x.name for x in running_exts[:3]) + ", ..." + print_msg(msg % (installed_cnt, exts_cnt, queued_cnt, running_cnt, running_ext_names), log=self.log) + # # MISCELLANEOUS UTILITY FUNCTIONS # From 3e186fb061c7e862e9966be31e959125b6a477a1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Oct 2021 20:09:48 +0200 Subject: [PATCH 18/19] return True in Extension.async_cmd_check if async_cmd_info is set to False, which indicates that no asynchronous command was started --- easybuild/framework/extension.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 251eed6afe..54ea39f544 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -200,6 +200,9 @@ def async_cmd_check(self): """ if self.async_cmd_info is None: raise EasyBuildError("No installation command running asynchronously for %s", self.name) + elif self.async_cmd_info is False: + self.log.info("No asynchronous command was started for extension %s", self.name) + return True else: self.log.debug("Checking on installation of extension %s...", self.name) # use small read size, to avoid waiting for a long time until sufficient output is produced @@ -207,6 +210,7 @@ def async_cmd_check(self): self.async_cmd_output += res['output'] if res['done']: self.log.info("Installation of extension %s completed!", self.name) + self.async_cmd_info = None else: self.async_cmd_check_cnt += 1 self.log.debug("Installation of extension %s still running (checked %d times)", From 0dd9061e65edbe7611e5b0d9a32aaf5637c942ac Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Oct 2021 20:27:04 +0200 Subject: [PATCH 19/19] add test for installing extensions in parallel --- .../easyblocks/generic/toy_extension.py | 54 ++++++++++++++++--- .../sandbox/easybuild/easyblocks/t/toy.py | 54 +++++++++++++++---- test/framework/toy_build.py | 39 ++++++++++++++ 3 files changed, 131 insertions(+), 16 deletions(-) diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index bbb792e7ee..603346efe0 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -30,7 +30,8 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock -from easybuild.easyblocks.toy import EB_toy +from easybuild.easyblocks.toy import EB_toy, compose_toy_build_cmd +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.run import run_cmd @@ -45,20 +46,59 @@ def extra_options(): } return ExtensionEasyBlock.extra_options(extra_vars=extra_vars) - def run(self): - """Build toy extension.""" + @property + def required_deps(self): + """Return list of required dependencies for this extension.""" + deps = { + 'bar': [], + 'barbar': ['bar'], + 'ls': [], + } + if self.name in deps: + return deps[self.name] + else: + raise EasyBuildError("Dependencies for %s are unknown!", self.name) + + def run(self, *args, **kwargs): + """ + Install toy extension. + """ if self.src: - super(Toy_Extension, self).run(unpack_src=True) - EB_toy.configure_step(self.master, name=self.name) EB_toy.build_step(self.master, name=self.name, buildopts=self.cfg['buildopts']) if self.cfg['toy_ext_param']: run_cmd(self.cfg['toy_ext_param']) - EB_toy.install_step(self.master, name=self.name) - return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper(), self.name) + def prerun(self): + """ + Prepare installation of toy extension. + """ + super(Toy_Extension, self).prerun() + + if self.src: + super(Toy_Extension, self).run(unpack_src=True) + EB_toy.configure_step(self.master, name=self.name) + + def run_async(self): + """ + Install toy extension asynchronously. + """ + if self.src: + cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) + self.async_cmd_start(cmd) + else: + self.async_cmd_info = False + + def postrun(self): + """ + Wrap up installation of toy extension. + """ + super(Toy_Extension, self).postrun() + + EB_toy.install_step(self.master, name=self.name) + def sanity_check_step(self, *args, **kwargs): """Custom sanity check for toy extensions.""" self.log.info("Loaded modules: %s", self.modules_tool.list()) diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index f9a9f7a8c5..bec0e7fe42 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -41,6 +41,19 @@ from easybuild.tools.run import run_cmd +def compose_toy_build_cmd(cfg, name, prebuildopts, buildopts): + """ + Compose command to build toy. + """ + + cmd = "%(prebuildopts)s gcc %(name)s.c -o %(name)s %(buildopts)s" % { + 'name': name, + 'prebuildopts': prebuildopts, + 'buildopts': buildopts, + } + return cmd + + class EB_toy(ExtensionEasyBlock): """Support for building/installing toy.""" @@ -92,17 +105,13 @@ def configure_step(self, name=None): def build_step(self, name=None, buildopts=None): """Build toy.""" - if buildopts is None: buildopts = self.cfg['buildopts'] - if name is None: name = self.name - run_cmd('%(prebuildopts)s gcc %(name)s.c -o %(name)s %(buildopts)s' % { - 'name': name, - 'prebuildopts': self.cfg['prebuildopts'], - 'buildopts': buildopts, - }) + + cmd = compose_toy_build_cmd(self.cfg, name, self.cfg['prebuildopts'], buildopts) + run_cmd(cmd) def install_step(self, name=None): """Install toy.""" @@ -118,11 +127,38 @@ def install_step(self, name=None): mkdir(libdir, parents=True) write_file(os.path.join(libdir, 'lib%s.a' % name), name.upper()) - def run(self): - """Install toy as extension.""" + @property + def required_deps(self): + """Return list of required dependencies for this extension.""" + if self.name == 'toy': + return ['bar', 'barbar'] + else: + raise EasyBuildError("Dependencies for %s are unknown!", self.name) + + def prerun(self): + """ + Prepare installation of toy as extension. + """ super(EB_toy, self).run(unpack_src=True) self.configure_step() + + def run(self): + """ + Install toy as extension. + """ self.build_step() + + def run_async(self): + """ + Asynchronous installation of toy as extension. + """ + cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) + self.async_cmd_start(cmd) + + def postrun(self): + """ + Wrap up installation of toy as extension. + """ self.install_step() def make_module_step(self, fake=False): diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 1a68de2dcb..5b75116ee1 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1777,6 +1777,45 @@ def test_module_only_extensions(self): self.eb_main([test_ec, '--module-only', '--force'], do_build=True, raise_error=True) self.assertTrue(os.path.exists(toy_mod)) + def test_toy_exts_parallel(self): + """ + Test parallel installation of extensions (--parallel-extensions-install) + """ + topdir = os.path.abspath(os.path.dirname(__file__)) + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_mod += '.lua' + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\n' + '\n'.join([ + "exts_list = [", + " ('ls'),", + " ('bar', '0.0'),", + " ('barbar', '0.0', {", + " 'start_dir': 'src',", + " }),", + " ('toy', '0.0'),", + "]", + "sanity_check_commands = ['barbar', 'toy']", + "sanity_check_paths = {'files': ['bin/barbar', 'bin/toy'], 'dirs': ['bin']}", + ]) + write_file(test_ec, test_ec_txt) + + args = ['--parallel-extensions-install', '--experimental', '--force'] + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + self.assertEqual(stderr, '') + expected_stdout = '\n'.join([ + "== 0 out of 4 extensions installed (2 queued, 2 running: ls, bar)", + "== 2 out of 4 extensions installed (1 queued, 1 running: barbar)", + "== 3 out of 4 extensions installed (0 queued, 1 running: toy)", + "== 4 out of 4 extensions installed (0 queued, 0 running: )", + '', + ]) + self.assertEqual(stdout, expected_stdout) + def test_backup_modules(self): """Test use of backing up of modules with --module-only."""