diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 2ec0ec7a7a..58f4303aa3 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -91,7 +91,7 @@ from easybuild.tools.modules import Lmod, curr_module_paths, invalidate_module_caches_for, get_software_root from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ALL, PROGRESS_BAR_EASYCONFIG, PROGRESS_BAR_EXTENSIONS -from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar +from easybuild.tools.output import show_progress_bars, start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.package.utilities import package from easybuild.tools.py2vs3 import extract_method_name, string_type from easybuild.tools.repository.repository import init_repository @@ -1616,14 +1616,18 @@ def skip_extensions(self): - use this to detect existing extensions and to remove them from self.ext_instances - based on initial R version """ + self.update_exts_progress_bar("skipping installed extensions") + # obtaining untemplated reference value is required here to support legacy string templates like name/version exts_filter = self.cfg.get_ref('exts_filter') if not exts_filter or len(exts_filter) == 0: raise EasyBuildError("Skipping of extensions, but no exts_filter set in easyconfig") + exts_cnt = len(self.ext_instances) + res = [] - for ext_inst in self.ext_instances: + for idx, ext_inst in enumerate(self.ext_instances): cmd, stdin = resolve_exts_filter_template(exts_filter, ext_inst) (cmdstdouterr, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, regexp=False) self.log.info("exts_filter result %s %s", cmdstdouterr, ec) @@ -1634,8 +1638,204 @@ def skip_extensions(self): else: print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log) + self.update_exts_progress_bar("skipping installed extensions (%d/%d checked)" % (idx + 1, exts_cnt)) + self.ext_instances = res + def install_extensions(self, install=True): + """ + Install extensions. + + :param install: actually install extensions, don't just prepare environment for installing + + """ + self.log.debug("List of loaded modules: %s", self.modules_tool.list()) + + if build_option('parallel_extensions_install'): + self.log.experimental("installing extensions in parallel") + self.install_extensions_parallel(install=install) + else: + self.install_extensions_sequential(install=install) + + def install_extensions_sequential(self, install=True): + """ + Install extensions sequentially. + + :param install: actually install extensions, don't just prepare environment for installing + """ + self.log.info("Installing extensions sequentially...") + + exts_cnt = len(self.ext_instances) + + for idx, ext in enumerate(self.ext_instances): + + self.log.info("Starting extension %s", ext.name) + + # always go back to original work dir to avoid running stuff from a dir that no longer exists + change_dir(self.orig_workdir) + + progress_info = "Installing '%s' extension (%s/%s)" % (ext.name, idx + 1, exts_cnt) + self.update_exts_progress_bar(progress_info) + + tup = (ext.name, ext.version or '', idx + 1, exts_cnt) + print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent, log=self.log) + start_time = datetime.now() + + if self.dry_run: + tup = (ext.name, ext.version, ext.__class__.__name__) + msg = "\n* installing extension %s %s using '%s' easyblock\n" % tup + self.dry_run_msg(msg) + + self.log.debug("List of loaded modules: %s", self.modules_tool.list()) + + # prepare toolchain build environment, but only when not doing a dry run + # since in that case the build environment is the same as for the parent + if self.dry_run: + self.dry_run_msg("defining build environment based on toolchain (options) and dependencies...") + else: + # don't reload modules for toolchain, there is no need since they will be loaded already; + # the (fake) module for the parent software gets loaded before installing extensions + ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, + rpath_filter_dirs=self.rpath_filter_dirs) + + # real work + if install: + try: + ext.prerun() + with self.module_generator.start_module_creation(): + txt = ext.run() + if txt: + self.module_extra_extensions += txt + ext.postrun() + finally: + if not self.dry_run: + ext_duration = datetime.now() - start_time + if ext_duration.total_seconds() >= 1: + print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent) + elif self.logdebug or build_option('trace'): + print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent) + + self.update_exts_progress_bar(progress_info, progress_size=1) + + def install_extensions_parallel(self, install=True): + """ + Install extensions in parallel. + + :param install: actually install extensions, don't just prepare environment for installing + """ + self.log.info("Installing extensions in parallel...") + + running_exts = [] + installed_ext_names = [] + + all_ext_names = [x['name'] for x in self.exts_all] + self.log.debug("List of names of all extensions: %s", all_ext_names) + + # take into account that some extensions may be installed already + to_install_ext_names = [x.name for x in self.ext_instances] + installed_ext_names = [n for n in all_ext_names if n not in to_install_ext_names] + + exts_cnt = len(all_ext_names) + exts_queue = self.ext_instances[:] + + def update_exts_progress_bar_helper(running_exts, progress_size): + """Helper function to update extensions progress bar.""" + running_exts_cnt = len(running_exts) + if running_exts_cnt > 1: + progress_info = "Installing %d extensions" % running_exts_cnt + elif running_exts_cnt == 1: + progress_info = "Installing extension " + else: + progress_info = "Not installing extensions (yet)" + + if running_exts_cnt: + progress_info += " (%d/%d done): " % (len(installed_ext_names), exts_cnt) + progress_info += ', '.join(e.name for e in running_exts) + + self.update_exts_progress_bar(progress_info, progress_size=progress_size) + + iter_id = 0 + while exts_queue or running_exts: + + iter_id += 1 + + # always go back to original work dir to avoid running stuff from a dir that no longer exists + change_dir(self.orig_workdir) + + # check for extension installations that have completed + if running_exts: + self.log.info("Checking for completed extension installations (%d running)...", len(running_exts)) + for ext in running_exts[:]: + if self.dry_run or ext.async_cmd_check(): + self.log.info("Installation of %s completed!", ext.name) + ext.postrun() + running_exts.remove(ext) + installed_ext_names.append(ext.name) + update_exts_progress_bar_helper(running_exts, 1) + else: + self.log.debug("Installation of %s is still running...", ext.name) + + # try to start as many extension installations as we can, taking into account number of available cores, + # but only consider first 100 extensions still in the queue + max_iter = min(100, len(exts_queue)) + + for _ in range(max_iter): + + if not (exts_queue and len(running_exts) < self.cfg['parallel']): + break + + # check whether extension at top of the queue is ready to install + ext = exts_queue.pop(0) + + pending_deps = [x for x in ext.required_deps if x not in installed_ext_names] + + if self.dry_run: + tup = (ext.name, ext.version, ext.__class__.__name__) + msg = "\n* installing extension %s %s using '%s' easyblock\n" % tup + self.dry_run_msg(msg) + running_exts.append(ext) + + # if some of the required dependencies are not installed yet, requeue this extension + elif pending_deps: + + # make sure all required dependencies are actually going to be installed, + # to avoid getting stuck in an infinite loop! + missing_deps = [x for x in ext.required_deps if x not in all_ext_names] + if missing_deps: + raise EasyBuildError("Missing required dependencies for %s are not going to be installed: %s", + ext.name, ', '.join(missing_deps)) + else: + self.log.info("Required dependencies missing for extension %s (%s), adding it back to queue...", + ext.name, ', '.join(pending_deps)) + # purposely adding extension back in the queue at Nth place rather than at the end, + # since we assume that the required dependencies will be installed soon... + exts_queue.insert(max_iter, ext) + + else: + tup = (ext.name, ext.version or '') + print_msg("starting installation of extension %s %s..." % tup, silent=self.silent, log=self.log) + + # don't reload modules for toolchain, there is no need since they will be loaded already; + # the (fake) module for the parent software gets loaded before installing extensions + ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, + rpath_filter_dirs=self.rpath_filter_dirs) + if install: + ext.prerun() + ext.run_async() + running_exts.append(ext) + self.log.info("Started installation of extension %s in the background...", ext.name) + update_exts_progress_bar_helper(running_exts, 0) + + # print progress info after every iteration (unless that info is already shown via progress bar) + if not show_progress_bars(): + msg = "%d out of %d extensions installed (%d queued, %d running: %s)" + installed_cnt, queued_cnt, running_cnt = len(installed_ext_names), len(exts_queue), len(running_exts) + if running_cnt <= 3: + running_ext_names = ', '.join(x.name for x in running_exts) + else: + running_ext_names = ', '.join(x.name for x in running_exts[:3]) + ", ..." + print_msg(msg % (installed_cnt, exts_cnt, queued_cnt, running_cnt, running_ext_names), log=self.log) + # # MISCELLANEOUS UTILITY FUNCTIONS # @@ -2307,7 +2507,11 @@ def init_ext_instances(self): error_msg = "Improper default extension class specification, should be string: %s (%s)" raise EasyBuildError(error_msg, exts_defaultclass, type(exts_defaultclass)) - for ext in self.exts: + exts_cnt = len(self.exts) + + self.update_exts_progress_bar("creating internal datastructures for extensions") + + for idx, ext in enumerate(self.exts): ext_name = ext['name'] self.log.debug("Creating class instance for extension %s...", ext_name) @@ -2368,6 +2572,15 @@ def init_ext_instances(self): self.log.debug("Installing extension %s with class %s (from %s)", ext_name, class_name, mod_path) self.ext_instances.append(inst) + pbar_label = "creating internal datastructures for extensions " + pbar_label += "(%d/%d done)" % (idx + 1, exts_cnt) + self.update_exts_progress_bar(pbar_label) + + def update_exts_progress_bar(self, info, progress_size=0): + """ + Update extensions progress bar with specified info and amount of progress made + """ + update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=info, progress_size=progress_size) def extensions_step(self, fetch=False, install=True): """ @@ -2390,9 +2603,12 @@ def extensions_step(self, fetch=False, install=True): fake_mod_data = self.load_fake_module(purge=True, extra_modules=build_dep_mods) + start_progress_bar(PROGRESS_BAR_EXTENSIONS, len(self.cfg['exts_list'])) + self.prepare_for_extensions() if fetch: + self.update_exts_progress_bar("fetching extension sources/patches") self.exts = self.collect_exts_file_info(fetch_files=True) self.exts_all = self.exts[:] # retain a copy of all extensions, regardless of filtering/skipping @@ -2411,65 +2627,14 @@ def extensions_step(self, fetch=False, install=True): if self.skip: self.skip_extensions() - exts_cnt = len(self.ext_instances) - - start_progress_bar(PROGRESS_BAR_EXTENSIONS, exts_cnt) - - for idx, ext in enumerate(self.ext_instances): - - self.log.debug("Starting extension %s" % ext.name) - - # always go back to original work dir to avoid running stuff from a dir that no longer exists - change_dir(self.orig_workdir) - - progress_label = "Installing '%s' extension" % ext.name - update_progress_bar(PROGRESS_BAR_EXTENSIONS, label=progress_label) - - tup = (ext.name, ext.version or '', idx + 1, exts_cnt) - print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent) - - start_time = datetime.now() - - if self.dry_run: - tup = (ext.name, ext.version, ext.__class__.__name__) - msg = "\n* installing extension %s %s using '%s' easyblock\n" % tup - self.dry_run_msg(msg) - - self.log.debug("List of loaded modules: %s", self.modules_tool.list()) - - # prepare toolchain build environment, but only when not doing a dry run - # since in that case the build environment is the same as for the parent - if self.dry_run: - self.dry_run_msg("defining build environment based on toolchain (options) and dependencies...") - else: - # don't reload modules for toolchain, there is no need since they will be loaded already; - # the (fake) module for the parent software gets loaded before installing extensions - ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, - rpath_filter_dirs=self.rpath_filter_dirs) - - # real work - if install: - try: - ext.prerun() - with self.module_generator.start_module_creation(): - txt = ext.run() - if txt: - self.module_extra_extensions += txt - ext.postrun() - finally: - if not self.dry_run: - ext_duration = datetime.now() - start_time - if ext_duration.total_seconds() >= 1: - print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent) - elif self.logdebug or build_option('trace'): - print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent) - - stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False) + self.install_extensions(install=install) # cleanup (unload fake module, remove fake module dir) if fake_mod_data: self.clean_up_fake_module(fake_mod_data) + stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False) + def package_step(self): """Package installed software (e.g., into an RPM), if requested, using selected package tool.""" diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index fcd2cc9596..54ea39f544 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -40,7 +40,7 @@ from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.tools.build_log import EasyBuildError, raise_nosupport from easybuild.tools.filetools import change_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import check_async_cmd, run_cmd from easybuild.tools.py2vs3 import string_type @@ -139,6 +139,12 @@ def __init__(self, mself, ext, extra_params=None): key, name, version, value) self.sanity_check_fail_msgs = [] + self.async_cmd_info = None + self.async_cmd_output = None + self.async_cmd_check_cnt = None + # initial read size should be relatively small, + # to avoid hanging for a long time until desired output is available in async_cmd_check + self.async_cmd_read_size = 1024 @property def name(self): @@ -160,18 +166,67 @@ def prerun(self): """ pass - def run(self): + def run(self, *args, **kwargs): """ - Actual installation of a extension. + Actual installation of an extension. """ pass + def run_async(self, *args, **kwargs): + """ + Asynchronous installation of an extension. + """ + raise NotImplementedError + def postrun(self): """ Stuff to do after installing a extension. """ self.master.run_post_install_commands(commands=self.cfg.get('postinstallcmds', [])) + def async_cmd_start(self, cmd, inp=None): + """ + Start installation asynchronously using specified command. + """ + self.async_cmd_output = '' + self.async_cmd_check_cnt = 0 + self.async_cmd_info = run_cmd(cmd, log_all=True, simple=False, inp=inp, regexp=False, asynchronous=True) + + def async_cmd_check(self): + """ + Check progress of installation command that was started asynchronously. + + :return: True if command completed, False otherwise + """ + if self.async_cmd_info is None: + raise EasyBuildError("No installation command running asynchronously for %s", self.name) + elif self.async_cmd_info is False: + self.log.info("No asynchronous command was started for extension %s", self.name) + return True + else: + self.log.debug("Checking on installation of extension %s...", self.name) + # use small read size, to avoid waiting for a long time until sufficient output is produced + res = check_async_cmd(*self.async_cmd_info, output_read_size=self.async_cmd_read_size) + self.async_cmd_output += res['output'] + if res['done']: + self.log.info("Installation of extension %s completed!", self.name) + self.async_cmd_info = None + else: + self.async_cmd_check_cnt += 1 + self.log.debug("Installation of extension %s still running (checked %d times)", + self.name, self.async_cmd_check_cnt) + # increase read size after sufficient checks, + # to avoid that installation hangs due to output buffer filling up... + if self.async_cmd_check_cnt % 10 == 0 and self.async_cmd_read_size < (1024 ** 2): + self.async_cmd_read_size *= 2 + + return res['done'] + + @property + def required_deps(self): + """Return list of required dependencies for this extension.""" + raise NotImplementedError("Don't know how to determine required dependencies for %s" % self.name) + @property def toolchain(self): """ diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 7b669764f4..c5c7711d81 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -270,6 +270,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'module_extensions', 'module_only', 'package', + 'parallel_extensions_install', 'read_only_installdir', 'remove_ghost_install_dirs', 'rebuild', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 5ef43c952c..2142a50f4a 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -456,6 +456,8 @@ def override_options(self): 'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES), 'parallel': ("Specify (maximum) level of parallellism used during build procedure", 'int', 'store', None), + 'parallel-extensions-install': ("Install list of extensions in parallel (if supported)", + None, 'store_true', False), 'pre-create-installdir': ("Create installation directory before submitting build jobs", None, 'store_true', True), 'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"), @@ -890,6 +892,10 @@ def postprocess(self): # set tmpdir self.tmpdir = set_tmpdir(self.options.tmpdir) + # early check for opt-in to installing extensions in parallel (experimental feature) + if self.options.parallel_extensions_install: + self.log.experimental("installing extensions in parallel") + # take --include options into account (unless instructed otherwise) if self.with_include: self._postprocess_include() diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index e408962338..6882af3c6b 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -242,7 +242,7 @@ def extensions_progress_bar(): Get progress bar to show progress for installing extensions. """ progress_bar = Progress( - TextColumn("[bold blue]{task.description} ({task.completed}/{task.total})"), + TextColumn("[bold blue]{task.description}"), BarColumn(), TimeElapsedColumn(), ) diff --git a/test/framework/run.py b/test/framework/run.py index a6a88b638c..da4448d741 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -611,11 +611,15 @@ def test_run_cmd_async(self): # check asynchronous running of failing command error_test_cmd = "echo 'FAIL!' >&2; exit 123" cmd_info = run_cmd(error_test_cmd, asynchronous=True) + time.sleep(1) error_pattern = 'cmd ".*" exited with exit code 123' self.assertErrorRegex(EasyBuildError, error_pattern, check_async_cmd, *cmd_info) cmd_info = run_cmd(error_test_cmd, asynchronous=True) res = check_async_cmd(*cmd_info, fail_on_error=False) + # keep checking until command is fully done + while not res['done']: + res = check_async_cmd(*cmd_info, fail_on_error=False) self.assertEqual(res, {'done': True, 'exit_code': 123, 'output': "FAIL!\n"}) # also test with a command that produces a lot of output, diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index bbb792e7ee..603346efe0 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -30,7 +30,8 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock -from easybuild.easyblocks.toy import EB_toy +from easybuild.easyblocks.toy import EB_toy, compose_toy_build_cmd +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.run import run_cmd @@ -45,20 +46,59 @@ def extra_options(): } return ExtensionEasyBlock.extra_options(extra_vars=extra_vars) - def run(self): - """Build toy extension.""" + @property + def required_deps(self): + """Return list of required dependencies for this extension.""" + deps = { + 'bar': [], + 'barbar': ['bar'], + 'ls': [], + } + if self.name in deps: + return deps[self.name] + else: + raise EasyBuildError("Dependencies for %s are unknown!", self.name) + + def run(self, *args, **kwargs): + """ + Install toy extension. + """ if self.src: - super(Toy_Extension, self).run(unpack_src=True) - EB_toy.configure_step(self.master, name=self.name) EB_toy.build_step(self.master, name=self.name, buildopts=self.cfg['buildopts']) if self.cfg['toy_ext_param']: run_cmd(self.cfg['toy_ext_param']) - EB_toy.install_step(self.master, name=self.name) - return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper(), self.name) + def prerun(self): + """ + Prepare installation of toy extension. + """ + super(Toy_Extension, self).prerun() + + if self.src: + super(Toy_Extension, self).run(unpack_src=True) + EB_toy.configure_step(self.master, name=self.name) + + def run_async(self): + """ + Install toy extension asynchronously. + """ + if self.src: + cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) + self.async_cmd_start(cmd) + else: + self.async_cmd_info = False + + def postrun(self): + """ + Wrap up installation of toy extension. + """ + super(Toy_Extension, self).postrun() + + EB_toy.install_step(self.master, name=self.name) + def sanity_check_step(self, *args, **kwargs): """Custom sanity check for toy extensions.""" self.log.info("Loaded modules: %s", self.modules_tool.list()) diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index f9a9f7a8c5..bec0e7fe42 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -41,6 +41,19 @@ from easybuild.tools.run import run_cmd +def compose_toy_build_cmd(cfg, name, prebuildopts, buildopts): + """ + Compose command to build toy. + """ + + cmd = "%(prebuildopts)s gcc %(name)s.c -o %(name)s %(buildopts)s" % { + 'name': name, + 'prebuildopts': prebuildopts, + 'buildopts': buildopts, + } + return cmd + + class EB_toy(ExtensionEasyBlock): """Support for building/installing toy.""" @@ -92,17 +105,13 @@ def configure_step(self, name=None): def build_step(self, name=None, buildopts=None): """Build toy.""" - if buildopts is None: buildopts = self.cfg['buildopts'] - if name is None: name = self.name - run_cmd('%(prebuildopts)s gcc %(name)s.c -o %(name)s %(buildopts)s' % { - 'name': name, - 'prebuildopts': self.cfg['prebuildopts'], - 'buildopts': buildopts, - }) + + cmd = compose_toy_build_cmd(self.cfg, name, self.cfg['prebuildopts'], buildopts) + run_cmd(cmd) def install_step(self, name=None): """Install toy.""" @@ -118,11 +127,38 @@ def install_step(self, name=None): mkdir(libdir, parents=True) write_file(os.path.join(libdir, 'lib%s.a' % name), name.upper()) - def run(self): - """Install toy as extension.""" + @property + def required_deps(self): + """Return list of required dependencies for this extension.""" + if self.name == 'toy': + return ['bar', 'barbar'] + else: + raise EasyBuildError("Dependencies for %s are unknown!", self.name) + + def prerun(self): + """ + Prepare installation of toy as extension. + """ super(EB_toy, self).run(unpack_src=True) self.configure_step() + + def run(self): + """ + Install toy as extension. + """ self.build_step() + + def run_async(self): + """ + Asynchronous installation of toy as extension. + """ + cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) + self.async_cmd_start(cmd) + + def postrun(self): + """ + Wrap up installation of toy as extension. + """ self.install_step() def make_module_step(self, fake=False): diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 1a68de2dcb..5b75116ee1 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1777,6 +1777,45 @@ def test_module_only_extensions(self): self.eb_main([test_ec, '--module-only', '--force'], do_build=True, raise_error=True) self.assertTrue(os.path.exists(toy_mod)) + def test_toy_exts_parallel(self): + """ + Test parallel installation of extensions (--parallel-extensions-install) + """ + topdir = os.path.abspath(os.path.dirname(__file__)) + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_mod += '.lua' + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\n' + '\n'.join([ + "exts_list = [", + " ('ls'),", + " ('bar', '0.0'),", + " ('barbar', '0.0', {", + " 'start_dir': 'src',", + " }),", + " ('toy', '0.0'),", + "]", + "sanity_check_commands = ['barbar', 'toy']", + "sanity_check_paths = {'files': ['bin/barbar', 'bin/toy'], 'dirs': ['bin']}", + ]) + write_file(test_ec, test_ec_txt) + + args = ['--parallel-extensions-install', '--experimental', '--force'] + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + self.assertEqual(stderr, '') + expected_stdout = '\n'.join([ + "== 0 out of 4 extensions installed (2 queued, 2 running: ls, bar)", + "== 2 out of 4 extensions installed (1 queued, 1 running: barbar)", + "== 3 out of 4 extensions installed (0 queued, 1 running: toy)", + "== 4 out of 4 extensions installed (0 queued, 0 running: )", + '', + ]) + self.assertEqual(stdout, expected_stdout) + def test_backup_modules(self): """Test use of backing up of modules with --module-only."""