From 377e66db0e1d28255021fb6d55870ff75a67080e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 3 Oct 2024 12:03:12 +0200 Subject: [PATCH 1/4] include path to cmd.sh script in output generated by run_shell_cmd when a command fails + use colors: red for ERROR line, yellow for path to output files + cmd.sh script --- easybuild/tools/run.py | 46 ++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 8798c22fc4..6fd4d43404 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -68,6 +68,7 @@ from easybuild.tools.build_log import dry_run_msg, print_msg, time_str_since from easybuild.tools.config import build_option from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook +from easybuild.tools.output import COLOR_RED, COLOR_YELLOW, colorize from easybuild.tools.utilities import trace_msg @@ -86,7 +87,7 @@ ) RunShellCmdResult = namedtuple('RunShellCmdResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir', - 'out_file', 'err_file', 'thread_id', 'task_id')) + 'out_file', 'err_file', 'cmd_sh', 'thread_id', 'task_id')) class RunShellCmdError(BaseException): @@ -101,6 +102,7 @@ def __init__(self, cmd_result, caller_info, *args, **kwargs): self.out_file = cmd_result.out_file self.stderr = cmd_result.stderr self.err_file = cmd_result.err_file + self.cmd_sh = cmd_result.cmd_sh self.caller_info = caller_info @@ -112,31 +114,37 @@ def print(self): Report failed shell command for this RunShellCmdError instance """ - def pad_4_spaces(msg): - return ' ' * 4 + msg + def pad_4_spaces(msg, color=None): + padded_msg = ' ' * 4 + msg + if color: + return colorize(padded_msg, color) + else: + return padded_msg + + caller_file_name, caller_line_nr, caller_function_name = self.caller_info + called_from_info = f"'{caller_function_name}' function in {caller_file_name} (line {caller_line_nr})" error_info = [ '', - "ERROR: Shell command failed!", + colorize("ERROR: Shell command failed!", COLOR_RED), pad_4_spaces(f"full command -> {self.cmd}"), pad_4_spaces(f"exit code -> {self.exit_code}"), + pad_4_spaces(f"called from -> {called_from_info}"), pad_4_spaces(f"working directory -> {self.work_dir}"), ] if self.out_file is not None: # if there's no separate file for error/warnings, then out_file includes both stdout + stderr out_info_msg = "output (stdout + stderr)" if self.err_file is None else "output (stdout) " - error_info.append(pad_4_spaces(f"{out_info_msg} -> {self.out_file}")) + error_info.append(pad_4_spaces(f"{out_info_msg} -> {self.out_file}", color=COLOR_YELLOW)) if self.err_file is not None: - error_info.append(pad_4_spaces(f"error/warnings (stderr) -> {self.err_file}")) + error_info.append(pad_4_spaces(f"error/warnings (stderr) -> {self.err_file}", color=COLOR_YELLOW)) - caller_file_name, caller_line_nr, caller_function_name = self.caller_info - called_from_info = f"'{caller_function_name}' function in {caller_file_name} (line {caller_line_nr})" - error_info.extend([ - pad_4_spaces(f"called from -> {called_from_info}"), - '', - ]) + if self.cmd_sh is not None: + error_info.append(pad_4_spaces(f"interactive shell script -> {self.cmd_sh}", color=COLOR_YELLOW)) + + error_info.append('') sys.stderr.write('\n'.join(error_info) + '\n') @@ -254,6 +262,8 @@ def create_cmd_scripts(cmd_str, work_dir, env, tmpdir, out_file, err_file): ])) os.chmod(cmd_fp, 0o775) + return cmd_fp + def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): """ @@ -430,9 +440,9 @@ def to_cmd_str(cmd): else: cmd_err_fp = None - create_cmd_scripts(cmd_str, work_dir, env, tmpdir, cmd_out_fp, cmd_err_fp) + cmd_sh = create_cmd_scripts(cmd_str, work_dir, env, tmpdir, cmd_out_fp, cmd_err_fp) else: - tmpdir, cmd_out_fp, cmd_err_fp = None, None, None + tmpdir, cmd_out_fp, cmd_err_fp, cmd_sh = None, None, None, None interactive_msg = 'interactive ' if interactive else '' @@ -445,7 +455,8 @@ def to_cmd_str(cmd): dry_run_msg(msg, silent=silent) return RunShellCmdResult(cmd=cmd_str, exit_code=0, output='', stderr=None, work_dir=work_dir, - out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id, task_id=task_id) + out_file=cmd_out_fp, err_file=cmd_err_fp, cmd_sh=cmd_sh, + thread_id=thread_id, task_id=task_id) start_time = datetime.now() if not hidden: @@ -571,8 +582,9 @@ def to_cmd_str(cmd): except IOError as err: raise EasyBuildError(f"Failed to dump command output to temporary file: {err}") - res = RunShellCmdResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr, work_dir=work_dir, - out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id, task_id=task_id) + res = RunShellCmdResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr, + work_dir=work_dir, out_file=cmd_out_fp, err_file=cmd_err_fp, cmd_sh=cmd_sh, + thread_id=thread_id, task_id=task_id) # always log command output cmd_name = cmd_str.split(' ')[0] From e996c516a9f85db0f65fe95b180a6ee8483c4ca6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 3 Oct 2024 14:14:17 +0200 Subject: [PATCH 2/4] enhance/fix test_run_shell_cmd_fail, need to take into account colored output produced by run_shell_cmd for failing command --- test/framework/run.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/test/framework/run.py b/test/framework/run.py index 2928e140fd..00d22130fd 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -491,12 +491,13 @@ def handler(signum, _): # check error reporting output stderr = stderr.getvalue() patterns = [ - r"^ERROR: Shell command failed!", - r"^\s+full command\s* -> kill -9 \$\$", - r"^\s+exit code\s* -> -9", - r"^\s+working directory\s* -> " + work_dir, - r"^\s+called from\s* -> 'test_run_shell_cmd_fail' function in .*/test/.*/run.py \(line [0-9]+\)", - r"^\s+output \(stdout \+ stderr\)\s* -> .*/run-shell-cmd-output/kill-.*/out.txt", + r"ERROR: Shell command failed!", + r"\s+full command\s* -> kill -9 \$\$", + r"\s+exit code\s* -> -9", + r"\s+working directory\s* -> " + work_dir, + r"\s+called from\s* -> 'test_run_shell_cmd_fail' function in .*/test/.*/run.py \(line [0-9]+\)", + r"\s+output \(stdout \+ stderr\)\s* -> .*/run-shell-cmd-output/kill-.*/out.txt", + r"\s+interactive shell script\s* -> .*/run-shell-cmd-output/kill-.*/cmd.sh", ] for pattern in patterns: regex = re.compile(pattern, re.M) @@ -526,13 +527,14 @@ def handler(signum, _): # check error reporting output stderr = stderr.getvalue() patterns = [ - r"^ERROR: Shell command failed!", - r"^\s+full command\s+ -> kill -9 \$\$", - r"^\s+exit code\s+ -> -9", - r"^\s+working directory\s+ -> " + work_dir, - r"^\s+called from\s+ -> 'test_run_shell_cmd_fail' function in .*/test/.*/run.py \(line [0-9]+\)", - r"^\s+output \(stdout\)\s+ -> .*/run-shell-cmd-output/kill-.*/out.txt", - r"^\s+error/warnings \(stderr\)\s+ -> .*/run-shell-cmd-output/kill-.*/err.txt", + r"ERROR: Shell command failed!", + r"\s+full command\s+ -> kill -9 \$\$", + r"\s+exit code\s+ -> -9", + r"\s+working directory\s+ -> " + work_dir, + r"\s+called from\s+ -> 'test_run_shell_cmd_fail' function in .*/test/.*/run.py \(line [0-9]+\)", + r"\s+output \(stdout\)\s+ -> .*/run-shell-cmd-output/kill-.*/out.txt", + r"\s+error/warnings \(stderr\)\s+ -> .*/run-shell-cmd-output/kill-.*/err.txt", + r"\s+interactive shell script\s* -> .*/run-shell-cmd-output/kill-.*/cmd.sh", ] for pattern in patterns: regex = re.compile(pattern, re.M) From 4f630b02ae2f990710c91813f758228e09cf6d74 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 3 Oct 2024 14:23:15 +0200 Subject: [PATCH 3/4] fix test_run_shell_cmd_cache --- test/framework/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/run.py b/test/framework/run.py index 00d22130fd..9147fe4aec 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1385,7 +1385,7 @@ def test_run_shell_cmd_cache(self): with self.mocked_stdout_stderr(): cached_res = RunShellCmdResult(cmd=cmd, output="123456", exit_code=123, stderr=None, work_dir='/test_ulimit', out_file='/tmp/foo.out', err_file=None, - thread_id=None, task_id=None) + cmd_sh='/tmp/cmd.sh', thread_id=None, task_id=None) run_shell_cmd.update_cache({(cmd, None): cached_res}) res = run_shell_cmd(cmd) self.assertEqual(res.cmd, cmd) @@ -1405,7 +1405,7 @@ def test_run_shell_cmd_cache(self): with self.mocked_stdout_stderr(): cached_res = RunShellCmdResult(cmd=cmd, output="bar", exit_code=123, stderr=None, work_dir='/test_cat', out_file='/tmp/cat.out', err_file=None, - thread_id=None, task_id=None) + cmd_sh='/tmp/cmd.sh', thread_id=None, task_id=None) run_shell_cmd.update_cache({(cmd, 'foo'): cached_res}) res = run_shell_cmd(cmd, stdin='foo') self.assertEqual(res.cmd, cmd) From 9630c8d1d963f706e57b47e217de553d3723bbb6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 3 Oct 2024 14:24:02 +0200 Subject: [PATCH 4/4] fix use of RunShellCmdResult in systemtools tests --- test/framework/systemtools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index fd40d2c60a..103116c4c6 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -341,7 +341,7 @@ def mocked_run_shell_cmd(cmd, **kwargs): } if cmd in known_cmds: return RunShellCmdResult(cmd=cmd, exit_code=0, output=known_cmds[cmd], stderr=None, work_dir=os.getcwd(), - out_file=None, err_file=None, thread_id=None, task_id=None) + out_file=None, err_file=None, cmd_sh=None, thread_id=None, task_id=None) else: return run_shell_cmd(cmd, **kwargs) @@ -774,7 +774,7 @@ def test_gcc_version_darwin(self): out = "Apple LLVM version 7.0.0 (clang-700.1.76)" cwd = os.getcwd() mocked_run_res = RunShellCmdResult(cmd="gcc --version", exit_code=0, output=out, stderr=None, work_dir=cwd, - out_file=None, err_file=None, thread_id=None, task_id=None) + out_file=None, err_file=None, cmd_sh=None, thread_id=None, task_id=None) st.run_shell_cmd = lambda *args, **kwargs: mocked_run_res self.assertEqual(get_gcc_version(), None)