From 51df040c93e3ef94d8f0a79d21b9a3ade717bc69 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 16 Sep 2021 14:56:00 +0200 Subject: [PATCH 01/16] Add script to find dependencies of Python packages Example usage with EasyBuild: ``` eb TensorFlow-2.3.4.eb --dump-env source TensorFlow-2.3.4.env findPythonDeps.py tensorflow==2.3.4 ``` --- easybuild/scripts/findPythonDeps.py | 109 ++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100755 easybuild/scripts/findPythonDeps.py diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py new file mode 100755 index 0000000000..2609700d88 --- /dev/null +++ b/easybuild/scripts/findPythonDeps.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python + +import argparse +import json +import os +import pkg_resources +import re +import subprocess +import sys +import tempfile +from pprint import pprint + + +def extract_pkg_name(package_spec): + return re.split('<|>|=|~', args.package, 1)[0] + + +def run_cmd(arguments, action_desc, **kwargs): + """Run the command and return the return code and output""" + extra_args = kwargs or {} + if sys.version_info[0] >= 3: + extra_args['universal_newlines'] = True + p = subprocess.Popen(arguments, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **extra_args) + out, _ = p.communicate() + if p.returncode != 0: + raise RuntimeError('Failed to %s: %s' % (action_desc, out)) + return out + + +def run_in_venv(cmd, venv_path, action_desc): + """Run the givven command in the virtualenv at the given path""" + cmd = 'source %s/bin/activate && %s' % (venv_path, cmd) + return run_cmd(cmd, action_desc, shell=True, executable='/bin/bash') + + +def get_dep_tree(package_spec): + """Get the dep-tree for installing the given Python package spec""" + package_name = extract_pkg_name(package_spec) + with tempfile.TemporaryDirectory(suffix=package_name + '-deps') as tmp_dir: + # prevent pip from (ab)using $HOME/.cache/pip + os.environ['XDG_CACHE_HOME'] = os.path.join(tmp_dir, 'pip-cache') + venv_dir = os.path.join(tmp_dir, 'venv') + # create virtualenv, install package in it + run_cmd(['virtualenv', '--system-site-packages', venv_dir], action_desc='create virtualenv') + run_in_venv('pip install "%s"' % package_spec, venv_dir, action_desc='install ' + package_spec) + # install pipdeptree, figure out dependency tree for installed package + run_in_venv('pip install pipdeptree', venv_dir, action_desc='install pipdeptree') + dep_tree = run_in_venv('pipdeptree -j -p "%s"' % package_name, + venv_dir, action_desc='collect dependencies') + return json.loads(dep_tree) + + +def find_deps(pkgs, dep_tree): + """Recursively resolve dependencies of the given package(s) and return them""" + res = [] + for pkg in pkgs: + matching_entries = [entry for entry in dep_tree + if pkg in (entry['package']['package_name'], entry['package']['key'])] + if not matching_entries: + raise RuntimeError("Found no installed package for '%s'" % pkg) + if len(matching_entries) > 1: + raise RuntimeError("Found multiple installed packages for '%s'" % pkg) + entry = matching_entries[0] + res.append((entry['package']['package_name'], entry['package']['installed_version'])) + deps = (dep['package_name'] for dep in entry['dependencies']) + res.extend(find_deps(deps, dep_tree)) + return res + + +parser = argparse.ArgumentParser( + description='Find dependencies of Python packages by installing it in a temporary virtualenv. ', + epilog=' && '.join(['Example usage with EasyBuild: ' + 'eb TensorFlow-2.3.4.eb --dump-env', + 'source TensorFlow-2.3.4.env', + sys.argv[0] + ' tensorflow==2.3.4']) +) +parser.add_argument('package', metavar='python-pkg-spec', + help='Python package spec, e.g. tensorflow==2.3.4') +parser.add_argument('--verbose', help='Verbose output', action='store_true') +args = parser.parse_args() + +if args.verbose: + print('Getting dep tree of ' + args.package) +dep_tree = get_dep_tree(args.package) +if args.verbose: + print('Extracting dependencies of ' + args.package) +deps = find_deps([extract_pkg_name(args.package)], dep_tree) + +installed_modules = {mod.project_name for mod in pkg_resources.working_set} +if args.verbose: + print("Installed modules: %s" % installed_modules) + +# iterate over deps in reverse order, get rid of duplicates along the way +# also filter out Python packages that are already installed in current environment +res = [] +handled = set() +for dep in reversed(deps): + if dep not in handled: + handled.add(dep) + if dep[0] in installed_modules: + if args.verbose: + print("Skipping installed module '%s'" % dep[0]) + else: + res.append(dep) + +print("List of dependencies in (likely) install order:") +pprint(res, indent=4) +print("Sorted list of dependencies:") +pprint(sorted(res), indent=4) From c19f7f1e49403fc5cbd29926d5567e09328a6084 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 16 Sep 2021 15:26:01 +0200 Subject: [PATCH 02/16] Add --ec option --- easybuild/scripts/findPythonDeps.py | 92 ++++++++++++++++++++--------- 1 file changed, 63 insertions(+), 29 deletions(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index 2609700d88..bb91e58902 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -5,12 +5,24 @@ import os import pkg_resources import re +import shutil import subprocess import sys import tempfile +from contextlib import contextmanager from pprint import pprint +@contextmanager +def temporary_directory(*args, **kwargs): + """Resource wrapper over tempfile.mkdtemp""" + name = tempfile.mkdtemp(*args, **kwargs) + try: + yield name + finally: + shutil.rmtree(name) + + def extract_pkg_name(package_spec): return re.split('<|>|=|~', args.package, 1)[0] @@ -36,7 +48,7 @@ def run_in_venv(cmd, venv_path, action_desc): def get_dep_tree(package_spec): """Get the dep-tree for installing the given Python package spec""" package_name = extract_pkg_name(package_spec) - with tempfile.TemporaryDirectory(suffix=package_name + '-deps') as tmp_dir: + with temporary_directory(suffix=package_name + '-deps') as tmp_dir: # prevent pip from (ab)using $HOME/.cache/pip os.environ['XDG_CACHE_HOME'] = os.path.join(tmp_dir, 'pip-cache') venv_dir = os.path.join(tmp_dir, 'venv') @@ -67,6 +79,37 @@ def find_deps(pkgs, dep_tree): return res +def print_deps(package, verbose): + if verbose: + print('Getting dep tree of ' + package) + dep_tree = get_dep_tree(package) + if verbose: + print('Extracting dependencies of ' + package) + deps = find_deps([extract_pkg_name(package)], dep_tree) + + installed_modules = {mod.project_name for mod in pkg_resources.working_set} + if verbose: + print("Installed modules: %s" % installed_modules) + + # iterate over deps in reverse order, get rid of duplicates along the way + # also filter out Python packages that are already installed in current environment + res = [] + handled = set() + for dep in reversed(deps): + if dep not in handled: + handled.add(dep) + if dep[0] in installed_modules: + if verbose: + print("Skipping installed module '%s'" % dep[0]) + else: + res.append(dep) + + print("List of dependencies in (likely) install order:") + pprint(res, indent=4) + print("Sorted list of dependencies:") + pprint(sorted(res), indent=4) + + parser = argparse.ArgumentParser( description='Find dependencies of Python packages by installing it in a temporary virtualenv. ', epilog=' && '.join(['Example usage with EasyBuild: ' @@ -76,34 +119,25 @@ def find_deps(pkgs, dep_tree): ) parser.add_argument('package', metavar='python-pkg-spec', help='Python package spec, e.g. tensorflow==2.3.4') +parser.add_argument('--ec', metavar='easyconfig', help='EasyConfig to use as the build environment') parser.add_argument('--verbose', help='Verbose output', action='store_true') args = parser.parse_args() -if args.verbose: - print('Getting dep tree of ' + args.package) -dep_tree = get_dep_tree(args.package) -if args.verbose: - print('Extracting dependencies of ' + args.package) -deps = find_deps([extract_pkg_name(args.package)], dep_tree) - -installed_modules = {mod.project_name for mod in pkg_resources.working_set} -if args.verbose: - print("Installed modules: %s" % installed_modules) - -# iterate over deps in reverse order, get rid of duplicates along the way -# also filter out Python packages that are already installed in current environment -res = [] -handled = set() -for dep in reversed(deps): - if dep not in handled: - handled.add(dep) - if dep[0] in installed_modules: - if args.verbose: - print("Skipping installed module '%s'" % dep[0]) - else: - res.append(dep) - -print("List of dependencies in (likely) install order:") -pprint(res, indent=4) -print("Sorted list of dependencies:") -pprint(sorted(res), indent=4) +if args.ec: + with temporary_directory() as tmp_dir: + old_dir = os.getcwd() + os.chdir(tmp_dir) + if args.verbose: + print('Running EasyBuild to get build environment') + run_cmd(['eb', args.ec, '--dump-env', '--force'], action_desc='Dump build environment') + os.chdir(old_dir) + + cmd = 'source %s/*.env && %s "%s"' % (tmp_dir, sys.argv[0], args.package) + if args.verbose: + cmd += ' --verbose' + print('Restarting script in new build environment') + + out = run_cmd(cmd, action_desc='Run in new environment', shell=True, executable='/bin/bash') + print(out) +else: + print_deps(args.package, args.verbose) From 3cd35960a079b727b11be06281465b4df90d3716 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 16 Sep 2021 16:09:20 +0200 Subject: [PATCH 03/16] Import pkg_resources with error if not found --- easybuild/scripts/findPythonDeps.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index bb91e58902..a596cb30df 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -3,7 +3,6 @@ import argparse import json import os -import pkg_resources import re import shutil import subprocess @@ -11,6 +10,11 @@ import tempfile from contextlib import contextmanager from pprint import pprint +try: + import pkg_resources +except ImportError as e: + print('pkg_resources could not be imported: %s\nYou might need to install setuptools!' % e) + sys.exit(1) @contextmanager From cfa1929ce8f0b1cc31188eef8ae61234438f5d4a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 20 Sep 2021 08:55:29 +0200 Subject: [PATCH 04/16] Better output --- easybuild/scripts/findPythonDeps.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index a596cb30df..b424031b01 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -58,7 +58,8 @@ def get_dep_tree(package_spec): venv_dir = os.path.join(tmp_dir, 'venv') # create virtualenv, install package in it run_cmd(['virtualenv', '--system-site-packages', venv_dir], action_desc='create virtualenv') - run_in_venv('pip install "%s"' % package_spec, venv_dir, action_desc='install ' + package_spec) + out = run_in_venv('pip install "%s"' % package_spec, venv_dir, action_desc='install ' + package_spec) + print('%s installed: %s' % (package_spec, out)) # install pipdeptree, figure out dependency tree for installed package run_in_venv('pip install pipdeptree', venv_dir, action_desc='install pipdeptree') dep_tree = run_in_venv('pipdeptree -j -p "%s"' % package_name, @@ -73,9 +74,9 @@ def find_deps(pkgs, dep_tree): matching_entries = [entry for entry in dep_tree if pkg in (entry['package']['package_name'], entry['package']['key'])] if not matching_entries: - raise RuntimeError("Found no installed package for '%s'" % pkg) + raise RuntimeError("Found no installed package for '%s' in %s" % (pkg, dep_tree)) if len(matching_entries) > 1: - raise RuntimeError("Found multiple installed packages for '%s'" % pkg) + raise RuntimeError("Found multiple installed packages for '%s' in %s" % (pkg, dep_tree)) entry = matching_entries[0] res.append((entry['package']['package_name'], entry['package']['installed_version'])) deps = (dep['package_name'] for dep in entry['dependencies']) From 5323e82bee492193b6028f556e1a1001dd57731e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 30 Jun 2022 13:40:12 +0200 Subject: [PATCH 05/16] Fix typo --- easybuild/scripts/findPythonDeps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index b424031b01..86ac0f88c9 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -44,7 +44,7 @@ def run_cmd(arguments, action_desc, **kwargs): def run_in_venv(cmd, venv_path, action_desc): - """Run the givven command in the virtualenv at the given path""" + """Run the given command in the virtualenv at the given path""" cmd = 'source %s/bin/activate && %s' % (venv_path, cmd) return run_cmd(cmd, action_desc, shell=True, executable='/bin/bash') From 766df6f4d352b8d8d9f9f65447ea2a3803ea7bd8 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 30 Jun 2022 13:47:10 +0200 Subject: [PATCH 06/16] Check availability of EasyBuild and virtualenv Improves the error messages generated. --- easybuild/scripts/findPythonDeps.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index 86ac0f88c9..3f937786a1 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -31,6 +31,13 @@ def extract_pkg_name(package_spec): return re.split('<|>|=|~', args.package, 1)[0] +def can_run(cmd, argument): + """Check if the given cmd and argument can be run successfully""" + try: + return subprocess.call([cmd, argument]) == 0 + except (subprocess.CalledProcessError, OSError): + return False + def run_cmd(arguments, action_desc, **kwargs): """Run the command and return the return code and output""" extra_args = kwargs or {} @@ -129,6 +136,9 @@ def print_deps(package, verbose): args = parser.parse_args() if args.ec: + if not can_run('eb', '--version'): + print('EasyBuild not found or executable. Make sure it is in your $PATH when using --ec!') + sys.exit(1) with temporary_directory() as tmp_dir: old_dir = os.getcwd() os.chdir(tmp_dir) @@ -145,4 +155,8 @@ def print_deps(package, verbose): out = run_cmd(cmd, action_desc='Run in new environment', shell=True, executable='/bin/bash') print(out) else: + if not can_run('virtualenv', '--version'): + print('Virtualenv not found or executable. ' + + 'Make sure it is installed (e.g. in the currently loaded Python module)!') + sys.exit(1) print_deps(args.package, args.verbose) From aae7002c14b6e4587bb9acdca2fe34138865a9da Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 30 Jun 2022 13:57:18 +0200 Subject: [PATCH 07/16] Formatting --- easybuild/scripts/findPythonDeps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index 3f937786a1..94338dee94 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -38,6 +38,7 @@ def can_run(cmd, argument): except (subprocess.CalledProcessError, OSError): return False + def run_cmd(arguments, action_desc, **kwargs): """Run the command and return the return code and output""" extra_args = kwargs or {} From 70c2afcc287f80a84421b0f5a709c5fb73d6dfac Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 1 Jul 2022 12:05:39 +0200 Subject: [PATCH 08/16] Be a bit more verbose for --verbose --- easybuild/scripts/findPythonDeps.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index 94338dee94..cbb5f280b2 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -57,15 +57,18 @@ def run_in_venv(cmd, venv_path, action_desc): return run_cmd(cmd, action_desc, shell=True, executable='/bin/bash') -def get_dep_tree(package_spec): +def get_dep_tree(package_spec, verbose): """Get the dep-tree for installing the given Python package spec""" package_name = extract_pkg_name(package_spec) with temporary_directory(suffix=package_name + '-deps') as tmp_dir: # prevent pip from (ab)using $HOME/.cache/pip os.environ['XDG_CACHE_HOME'] = os.path.join(tmp_dir, 'pip-cache') venv_dir = os.path.join(tmp_dir, 'venv') - # create virtualenv, install package in it + if verbose: + print('Creating virtualenv at ' + venv_dir) run_cmd(['virtualenv', '--system-site-packages', venv_dir], action_desc='create virtualenv') + if verbose: + print('Installing %s into virtualenv' % package_spec) out = run_in_venv('pip install "%s"' % package_spec, venv_dir, action_desc='install ' + package_spec) print('%s installed: %s' % (package_spec, out)) # install pipdeptree, figure out dependency tree for installed package @@ -95,7 +98,7 @@ def find_deps(pkgs, dep_tree): def print_deps(package, verbose): if verbose: print('Getting dep tree of ' + package) - dep_tree = get_dep_tree(package) + dep_tree = get_dep_tree(package, verbose) if verbose: print('Extracting dependencies of ' + package) deps = find_deps([extract_pkg_name(package)], dep_tree) From 03ef15071d7573e1da7a5e358372f478aeb4272e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 1 Jul 2022 12:06:33 +0200 Subject: [PATCH 09/16] Update PIP in virtualenv first --- easybuild/scripts/findPythonDeps.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index cbb5f280b2..e3d2414708 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -67,6 +67,9 @@ def get_dep_tree(package_spec, verbose): if verbose: print('Creating virtualenv at ' + venv_dir) run_cmd(['virtualenv', '--system-site-packages', venv_dir], action_desc='create virtualenv') + if verbose: + print('Updating pip in virtualenv') + run_in_venv('pip install --upgrade pip', venv_dir, action_desc='update pip') if verbose: print('Installing %s into virtualenv' % package_spec) out = run_in_venv('pip install "%s"' % package_spec, venv_dir, action_desc='install ' + package_spec) From 5c1c503fa5d5765e7054b224031766b202a68694 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 7 Jul 2022 10:59:01 +0200 Subject: [PATCH 10/16] Improve help text for --ec --- easybuild/scripts/findPythonDeps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index e3d2414708..0c99192cd4 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -138,7 +138,8 @@ def print_deps(package, verbose): ) parser.add_argument('package', metavar='python-pkg-spec', help='Python package spec, e.g. tensorflow==2.3.4') -parser.add_argument('--ec', metavar='easyconfig', help='EasyConfig to use as the build environment') +parser.add_argument('--ec', metavar='easyconfig', help='EasyConfig to use as the build environment. ' + 'You need to have dependency modules installed already!') parser.add_argument('--verbose', help='Verbose output', action='store_true') args = parser.parse_args() From ca22095533d07089f7d27450f9db3c9295bf1303 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 7 Jul 2022 10:59:11 +0200 Subject: [PATCH 11/16] Don't assume script is executable --- easybuild/scripts/findPythonDeps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index 0c99192cd4..b792b713c8 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -155,7 +155,7 @@ def print_deps(package, verbose): run_cmd(['eb', args.ec, '--dump-env', '--force'], action_desc='Dump build environment') os.chdir(old_dir) - cmd = 'source %s/*.env && %s "%s"' % (tmp_dir, sys.argv[0], args.package) + cmd = 'source %s/*.env && %s %s "%s"' % (tmp_dir, sys.executable, sys.argv[0], args.package) if args.verbose: cmd += ' --verbose' print('Restarting script in new build environment') From 83cfffe3fcfc84418eee8f316c1d829d7b62a020 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 8 Jul 2022 11:15:32 +0200 Subject: [PATCH 12/16] Improve help message --- easybuild/scripts/findPythonDeps.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index b792b713c8..aa8c28815a 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -129,12 +129,19 @@ def print_deps(package, verbose): pprint(sorted(res), indent=4) +examples = [ + 'Example usage with EasyBuild (after installing dependency modules):', + '\t' + sys.argv[0] + ' --ec TensorFlow-2.3.4.eb tensorflow==2.3.4', + 'Which is the same as:', + '\t' + ' && '.join(['eb TensorFlow-2.3.4.eb --dump-env', + 'source TensorFlow-2.3.4.env', + sys.argv[0] + ' tensorflow==2.3.4', + ]), +] parser = argparse.ArgumentParser( description='Find dependencies of Python packages by installing it in a temporary virtualenv. ', - epilog=' && '.join(['Example usage with EasyBuild: ' - 'eb TensorFlow-2.3.4.eb --dump-env', - 'source TensorFlow-2.3.4.env', - sys.argv[0] + ' tensorflow==2.3.4']) + epilog='\n'.join(examples), + formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument('package', metavar='python-pkg-spec', help='Python package spec, e.g. tensorflow==2.3.4') From b1c6a4254838fbe4aa34b167caef43bf5886668a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 8 Jul 2022 11:32:56 +0200 Subject: [PATCH 13/16] Suppress output of call --- easybuild/scripts/findPythonDeps.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index aa8c28815a..5b3d1084d7 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -33,10 +33,11 @@ def extract_pkg_name(package_spec): def can_run(cmd, argument): """Check if the given cmd and argument can be run successfully""" - try: - return subprocess.call([cmd, argument]) == 0 - except (subprocess.CalledProcessError, OSError): - return False + with open(os.devnull, 'w') as FNULL: + try: + return subprocess.call([cmd, argument], stdout=FNULL, stderr=subprocess.STDOUT) == 0 + except (subprocess.CalledProcessError, OSError): + return False def run_cmd(arguments, action_desc, **kwargs): From b6d8423c38ec77832d8adf7552e86bc2ea246e32 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 8 Jul 2022 11:33:27 +0200 Subject: [PATCH 14/16] Check and report missing modules when using --ec --- easybuild/scripts/findPythonDeps.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index 5b3d1084d7..3f4d5cd506 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -155,6 +155,15 @@ def print_deps(package, verbose): if not can_run('eb', '--version'): print('EasyBuild not found or executable. Make sure it is in your $PATH when using --ec!') sys.exit(1) + if args.verbose: + print('Checking with EasyBuild for missing dependencies') + missing_dep_out = run_cmd(['eb', args.ec, '--missing'], action_desc='Get missing dependencies') + missing_deps = [dep for dep in missing_dep_out.split('\n') if dep.startswith('*') and '(%s)' % args.ec not in dep] + if missing_deps: + print('You need to install all modules on which %s depends first!' % args.ec) + print('\n\t'.join(['Missing:'] + missing_deps)) + sys.exit(1) + with temporary_directory() as tmp_dir: old_dir = os.getcwd() os.chdir(tmp_dir) From 9a3eb244f3beb872fd0d305aafed5e49e06a675d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 8 Jul 2022 11:37:51 +0200 Subject: [PATCH 15/16] Unset $PIP_PREFIX if set --- easybuild/scripts/findPythonDeps.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index 3f4d5cd506..95433a6c9f 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -184,4 +184,7 @@ def print_deps(package, verbose): print('Virtualenv not found or executable. ' + 'Make sure it is installed (e.g. in the currently loaded Python module)!') sys.exit(1) + if 'PIP_PREFIX' in os.environ: + print("$PIP_PREFIX is set. Unsetting it as it doesn't work well with virtualenv.") + del os.environ['PIP_PREFIX'] print_deps(args.package, args.verbose) From 7f8ab8dd4721fbcd16ca9389d4c1f79c6d457c00 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 8 Jul 2022 16:09:47 +0200 Subject: [PATCH 16/16] Avoid false positives in dependency checking Check only stdout not stderr. The latter may contain lines like > WARNING: Found one or more non-allowed loaded (EasyBuild-generated) modules in current environment: > * GCCcore/10.3.0 --- easybuild/scripts/findPythonDeps.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index 95433a6c9f..6ce11acf77 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -40,12 +40,13 @@ def can_run(cmd, argument): return False -def run_cmd(arguments, action_desc, **kwargs): +def run_cmd(arguments, action_desc, capture_stderr=True, **kwargs): """Run the command and return the return code and output""" extra_args = kwargs or {} if sys.version_info[0] >= 3: extra_args['universal_newlines'] = True - p = subprocess.Popen(arguments, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **extra_args) + stderr = subprocess.STDOUT if capture_stderr else subprocess.PIPE + p = subprocess.Popen(arguments, stdout=subprocess.PIPE, stderr=stderr, **extra_args) out, _ = p.communicate() if p.returncode != 0: raise RuntimeError('Failed to %s: %s' % (action_desc, out)) @@ -157,8 +158,13 @@ def print_deps(package, verbose): sys.exit(1) if args.verbose: print('Checking with EasyBuild for missing dependencies') - missing_dep_out = run_cmd(['eb', args.ec, '--missing'], action_desc='Get missing dependencies') - missing_deps = [dep for dep in missing_dep_out.split('\n') if dep.startswith('*') and '(%s)' % args.ec not in dep] + missing_dep_out = run_cmd(['eb', args.ec, '--missing'], + capture_stderr=False, + action_desc='Get missing dependencies' + ) + missing_deps = [dep for dep in missing_dep_out.split('\n') + if dep.startswith('*') and '(%s)' % args.ec not in dep + ] if missing_deps: print('You need to install all modules on which %s depends first!' % args.ec) print('\n\t'.join(['Missing:'] + missing_deps))