From 89eb0048a3c641285f113279d15b7ddf2c47a763 Mon Sep 17 00:00:00 2001 From: superstar54 Date: Wed, 18 Sep 2024 02:35:23 +0200 Subject: [PATCH 1/3] Add a helper function to prepare the inputs Move the input preparation logic into a separate function called prepare_shell_job_inputs --- src/aiida_shell/launch.py | 67 +++++++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 9 deletions(-) diff --git a/src/aiida_shell/launch.py b/src/aiida_shell/launch.py index a660f26..eb237b0 100644 --- a/src/aiida_shell/launch.py +++ b/src/aiida_shell/launch.py @@ -22,7 +22,7 @@ LOGGER = logging.getLogger('aiida_shell') -def launch_shell_job( # noqa: PLR0913 +def prepare_shell_job_inputs( command: str | AbstractCode, arguments: list[str] | str | None = None, nodes: t.Mapping[str, str | pathlib.Path | Data] | None = None, @@ -30,10 +30,9 @@ def launch_shell_job( # noqa: PLR0913 outputs: list[str] | None = None, parser: ParserFunctionType | str | None = None, metadata: dict[str, t.Any] | None = None, - submit: bool = False, resolve_command: bool = True, -) -> tuple[dict[str, Data], ProcessNode]: - """Launch a :class:`aiida_shell.ShellJob` job for the given command. +) -> dict[str, t.Any]: + """Prepare inputs for the ShellJob based on the provided parameters. :param command: The shell command to run. Should be the relative command name, e.g., ``date``. An ``AbstractCode`` instance will be automatically created for this command if it doesn't already exist. Alternatively, a pre- @@ -48,17 +47,13 @@ def launch_shell_job( # noqa: PLR0913 a complete entry point, i.e. a string of the form ``{entry_point_group}:{entry_point_name}`` pointing to such a callable. :param metadata: Optional dictionary of metadata inputs to be passed to the ``ShellJob``. - :param submit: Boolean, if ``True`` will submit the job to the daemon instead of running in current interpreter. :param resolve_command: Whether to resolve the command to the absolute path of the executable. If set to ``True``, the ``which`` command is executed on the target computer to attempt and determine the absolute path. Otherwise, the command is set as the ``filepath_executable`` attribute of the created ``AbstractCode`` instance. :raises TypeError: If the value specified for ``metadata.options.computer`` is not a ``Computer``. :raises ValueError: If ``resolve_command=True`` and the absolute path of the command on the computer could not be determined. - :returns: The tuple of results dictionary and ``ProcessNode``, or just the ``ProcessNode`` if ``submit=True``. The - results dictionary intentionally doesn't include the ``retrieved`` and ``remote_folder`` outputs as they are - generated for each ``CalcJob`` and typically are not of interest to a user running ``launch_shell_job``. In - order to not confuse them, these nodes are omitted, but they can always be accessed through the node. + :returns: A dictionary containing prepared inputs for the ShellJob. """ metadata = metadata or {} computer = metadata.get('options', {}).pop('computer', None) @@ -94,6 +89,60 @@ def launch_shell_job( # noqa: PLR0913 'metadata': metadata or {}, } + return inputs + + +def launch_shell_job( # noqa: PLR0913 + command: str | AbstractCode, + arguments: list[str] | str | None = None, + nodes: t.Mapping[str, str | pathlib.Path | Data] | None = None, + filenames: dict[str, str] | None = None, + outputs: list[str] | None = None, + parser: ParserFunctionType | str | None = None, + metadata: dict[str, t.Any] | None = None, + submit: bool = False, + resolve_command: bool = True, +) -> tuple[dict[str, Data], ProcessNode]: + """Launch a :class:`aiida_shell.ShellJob` job for the given command. + + :param command: The shell command to run. Should be the relative command name, e.g., ``date``. An ``AbstractCode`` + instance will be automatically created for this command if it doesn't already exist. Alternatively, a pre- + configured ``AbstractCode`` instance can be passed directly. + :param arguments: Optional list of command line arguments optionally containing placeholders for input nodes. The + arguments can also be specified as a single string. In this case, it will be split into separate parameters + using ``shlex.split``. + :param nodes: A dictionary of ``Data`` nodes whose content is to replace placeholders in the ``arguments`` list. + :param filenames: Optional dictionary of explicit filenames to use for the ``nodes`` to be written to ``dirpath``. + :param outputs: Optional list of relative filenames that should be captured as outputs. + :param parser: Optional callable that can implement custom parsing logic of produced output files. Alternatively, + a complete entry point, i.e. a string of the form ``{entry_point_group}:{entry_point_name}`` pointing to such a + callable. + :param metadata: Optional dictionary of metadata inputs to be passed to the ``ShellJob``. + :param submit: Boolean, if ``True`` will submit the job to the daemon instead of running in current interpreter. + :param resolve_command: Whether to resolve the command to the absolute path of the executable. If set to ``True``, + the ``which`` command is executed on the target computer to attempt and determine the absolute path. Otherwise, + the command is set as the ``filepath_executable`` attribute of the created ``AbstractCode`` instance. + :raises TypeError: If the value specified for ``metadata.options.computer`` is not a ``Computer``. + :raises ValueError: If ``resolve_command=True`` and the absolute path of the command on the computer could not be + determined. + :returns: The tuple of results dictionary and ``ProcessNode``, or just the ``ProcessNode`` if ``submit=True``. The + results dictionary intentionally doesn't include the ``retrieved`` and ``remote_folder`` outputs as they are + generated for each ``CalcJob`` and typically are not of interest to a user running ``launch_shell_job``. In + order to not confuse them, these nodes are omitted, but they can always be accessed through the node. + """ + + # Prepare inputs for the ShellJob + inputs = prepare_shell_job_inputs( + command=command, + arguments=arguments, + nodes=nodes, + filenames=filenames, + outputs=outputs, + parser=parser, + metadata=metadata, + resolve_command=resolve_command, + ) + if submit: current_process = Process.current() if current_process is not None and isinstance(current_process, WorkChain): From a0e5eba85d84ddc31e640068d610f90b8117a333 Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Wed, 18 Sep 2024 08:08:09 +0200 Subject: [PATCH 2/3] Apply suggestions from code review --- src/aiida_shell/launch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/aiida_shell/launch.py b/src/aiida_shell/launch.py index eb237b0..9244b2e 100644 --- a/src/aiida_shell/launch.py +++ b/src/aiida_shell/launch.py @@ -131,7 +131,6 @@ def launch_shell_job( # noqa: PLR0913 order to not confuse them, these nodes are omitted, but they can always be accessed through the node. """ - # Prepare inputs for the ShellJob inputs = prepare_shell_job_inputs( command=command, arguments=arguments, From 57cc3abfeaae57a4b8546fb67109e1973a5dd695 Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Wed, 18 Sep 2024 08:34:40 +0200 Subject: [PATCH 3/3] fix pre-commit --- src/aiida_shell/launch.py | 123 +++++++++++++++++++------------------- 1 file changed, 61 insertions(+), 62 deletions(-) diff --git a/src/aiida_shell/launch.py b/src/aiida_shell/launch.py index 9244b2e..8b53865 100644 --- a/src/aiida_shell/launch.py +++ b/src/aiida_shell/launch.py @@ -22,7 +22,67 @@ LOGGER = logging.getLogger('aiida_shell') -def prepare_shell_job_inputs( +def launch_shell_job( # noqa: PLR0913 + command: str | AbstractCode, + arguments: list[str] | str | None = None, + nodes: t.Mapping[str, str | pathlib.Path | Data] | None = None, + filenames: dict[str, str] | None = None, + outputs: list[str] | None = None, + parser: ParserFunctionType | str | None = None, + metadata: dict[str, t.Any] | None = None, + submit: bool = False, + resolve_command: bool = True, +) -> tuple[dict[str, Data], ProcessNode]: + """Launch a :class:`aiida_shell.ShellJob` job for the given command. + + :param command: The shell command to run. Should be the relative command name, e.g., ``date``. An ``AbstractCode`` + instance will be automatically created for this command if it doesn't already exist. Alternatively, a pre- + configured ``AbstractCode`` instance can be passed directly. + :param arguments: Optional list of command line arguments optionally containing placeholders for input nodes. The + arguments can also be specified as a single string. In this case, it will be split into separate parameters + using ``shlex.split``. + :param nodes: A dictionary of ``Data`` nodes whose content is to replace placeholders in the ``arguments`` list. + :param filenames: Optional dictionary of explicit filenames to use for the ``nodes`` to be written to ``dirpath``. + :param outputs: Optional list of relative filenames that should be captured as outputs. + :param parser: Optional callable that can implement custom parsing logic of produced output files. Alternatively, + a complete entry point, i.e. a string of the form ``{entry_point_group}:{entry_point_name}`` pointing to such a + callable. + :param metadata: Optional dictionary of metadata inputs to be passed to the ``ShellJob``. + :param submit: Boolean, if ``True`` will submit the job to the daemon instead of running in current interpreter. + :param resolve_command: Whether to resolve the command to the absolute path of the executable. If set to ``True``, + the ``which`` command is executed on the target computer to attempt and determine the absolute path. Otherwise, + the command is set as the ``filepath_executable`` attribute of the created ``AbstractCode`` instance. + :raises TypeError: If the value specified for ``metadata.options.computer`` is not a ``Computer``. + :raises ValueError: If ``resolve_command=True`` and the absolute path of the command on the computer could not be + determined. + :returns: The tuple of results dictionary and ``ProcessNode``, or just the ``ProcessNode`` if ``submit=True``. The + results dictionary intentionally doesn't include the ``retrieved`` and ``remote_folder`` outputs as they are + generated for each ``CalcJob`` and typically are not of interest to a user running ``launch_shell_job``. In + order to not confuse them, these nodes are omitted, but they can always be accessed through the node. + """ + inputs = prepare_shell_job_inputs( + command=command, + arguments=arguments, + nodes=nodes, + filenames=filenames, + outputs=outputs, + parser=parser, + metadata=metadata, + resolve_command=resolve_command, + ) + + if submit: + current_process = Process.current() + if current_process is not None and isinstance(current_process, WorkChain): + return {}, current_process.submit(ShellJob, inputs) + return {}, launch.submit(ShellJob, inputs) + + results, node = launch.run_get_node(ShellJob, inputs) + + return {label: node for label, node in results.items() if label not in ('retrieved', 'remote_folder')}, node + + +def prepare_shell_job_inputs( # noqa: PLR0913 command: str | AbstractCode, arguments: list[str] | str | None = None, nodes: t.Mapping[str, str | pathlib.Path | Data] | None = None, @@ -92,67 +152,6 @@ def prepare_shell_job_inputs( return inputs -def launch_shell_job( # noqa: PLR0913 - command: str | AbstractCode, - arguments: list[str] | str | None = None, - nodes: t.Mapping[str, str | pathlib.Path | Data] | None = None, - filenames: dict[str, str] | None = None, - outputs: list[str] | None = None, - parser: ParserFunctionType | str | None = None, - metadata: dict[str, t.Any] | None = None, - submit: bool = False, - resolve_command: bool = True, -) -> tuple[dict[str, Data], ProcessNode]: - """Launch a :class:`aiida_shell.ShellJob` job for the given command. - - :param command: The shell command to run. Should be the relative command name, e.g., ``date``. An ``AbstractCode`` - instance will be automatically created for this command if it doesn't already exist. Alternatively, a pre- - configured ``AbstractCode`` instance can be passed directly. - :param arguments: Optional list of command line arguments optionally containing placeholders for input nodes. The - arguments can also be specified as a single string. In this case, it will be split into separate parameters - using ``shlex.split``. - :param nodes: A dictionary of ``Data`` nodes whose content is to replace placeholders in the ``arguments`` list. - :param filenames: Optional dictionary of explicit filenames to use for the ``nodes`` to be written to ``dirpath``. - :param outputs: Optional list of relative filenames that should be captured as outputs. - :param parser: Optional callable that can implement custom parsing logic of produced output files. Alternatively, - a complete entry point, i.e. a string of the form ``{entry_point_group}:{entry_point_name}`` pointing to such a - callable. - :param metadata: Optional dictionary of metadata inputs to be passed to the ``ShellJob``. - :param submit: Boolean, if ``True`` will submit the job to the daemon instead of running in current interpreter. - :param resolve_command: Whether to resolve the command to the absolute path of the executable. If set to ``True``, - the ``which`` command is executed on the target computer to attempt and determine the absolute path. Otherwise, - the command is set as the ``filepath_executable`` attribute of the created ``AbstractCode`` instance. - :raises TypeError: If the value specified for ``metadata.options.computer`` is not a ``Computer``. - :raises ValueError: If ``resolve_command=True`` and the absolute path of the command on the computer could not be - determined. - :returns: The tuple of results dictionary and ``ProcessNode``, or just the ``ProcessNode`` if ``submit=True``. The - results dictionary intentionally doesn't include the ``retrieved`` and ``remote_folder`` outputs as they are - generated for each ``CalcJob`` and typically are not of interest to a user running ``launch_shell_job``. In - order to not confuse them, these nodes are omitted, but they can always be accessed through the node. - """ - - inputs = prepare_shell_job_inputs( - command=command, - arguments=arguments, - nodes=nodes, - filenames=filenames, - outputs=outputs, - parser=parser, - metadata=metadata, - resolve_command=resolve_command, - ) - - if submit: - current_process = Process.current() - if current_process is not None and isinstance(current_process, WorkChain): - return {}, current_process.submit(ShellJob, inputs) - return {}, launch.submit(ShellJob, inputs) - - results, node = launch.run_get_node(ShellJob, inputs) - - return {label: node for label, node in results.items() if label not in ('retrieved', 'remote_folder')}, node - - def prepare_code(command: str, computer: Computer | None = None, resolve_command: bool = True) -> AbstractCode: """Prepare a code for the given command and computer.