From d22d7a1b75d0262d6b0d63a30c0379637aac3046 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Thu, 11 May 2023 16:01:23 +1000 Subject: [PATCH 1/5] wrote cmd_out arg and unittest based on "ls" for shell_cmd --- pydra/mark/__init__.py | 3 +- pydra/mark/shell_commands.py | 156 ++++++++++++++++++++++++ pydra/mark/tests/test_shell_commands.py | 75 ++++++++++++ 3 files changed, 233 insertions(+), 1 deletion(-) create mode 100644 pydra/mark/shell_commands.py create mode 100644 pydra/mark/tests/test_shell_commands.py diff --git a/pydra/mark/__init__.py b/pydra/mark/__init__.py index 31e4cf832e..d4338cf621 100644 --- a/pydra/mark/__init__.py +++ b/pydra/mark/__init__.py @@ -1,3 +1,4 @@ from .functions import annotate, task +from .shell_commands import cmd_arg, cmd_out -__all__ = ("annotate", "task") +__all__ = ("annotate", "task", "cmd_arg", "cmd_out") diff --git a/pydra/mark/shell_commands.py b/pydra/mark/shell_commands.py new file mode 100644 index 0000000000..62579ce543 --- /dev/null +++ b/pydra/mark/shell_commands.py @@ -0,0 +1,156 @@ +"""Decorators and helper functions to create ShellCommandTasks used in Pydra workflows""" +from __future__ import annotations +import typing as ty +import attrs + + +def cmd_arg( + help_string: str, + default: ty.Any = attrs.NOTHING, + argstr: str = None, + position: int = None, + mandatory: bool = False, + sep: str = None, + allowed_values: list = None, + requires: list = None, + xor: list = None, + copyfile: bool = None, + container_path: bool = False, + output_file_template: str = None, + output_field_name: str = None, + keep_extension: bool = True, + readonly: bool = False, + formatter: ty.Callable = None, +): + """ + Returns an attrs field with appropriate metadata for it to be added as an argument in + a Pydra shell command task definition + + Parameters + ------------ + help_string: str + A short description of the input field. + default : Any, optional + the default value for the argument + argstr: str, optional + A flag or string that is used in the command before the value, e.g. -v or + -v {inp_field}, but it could be and empty string, “”. If … are used, e.g. -v…, + the flag is used before every element if a list is provided as a value. If no + argstr is used the field is not part of the command. + position: int, optional + Position of the field in the command, could be nonnegative or negative integer. + If nothing is provided the field will be inserted between all fields with + nonnegative positions and fields with negative positions. + mandatory: bool, optional + If True user has to provide a value for the field, by default it is False + sep: str, optional + A separator if a list is provided as a value. + allowed_values: list, optional + List of allowed values for the field. + requires: list, optional + List of field names that are required together with the field. + xor: list, optional + List of field names that are mutually exclusive with the field. + copyfile: bool, optional + If True, a hard link is created for the input file in the output directory. If + hard link not possible, the file is copied to the output directory, by default + it is False + container_path: bool, optional + If True a path will be consider as a path inside the container (and not as a + local path, by default it is False + output_file_template: str, optional + If provided, the field is treated also as an output field and it is added to + the output spec. The template can use other fields, e.g. {file1}. Used in order + to create an output specification. + output_field_name: str, optional + If provided the field is added to the output spec with changed name. Used in + order to create an output specification. Used together with output_file_template + keep_extension: bool, optional + A flag that specifies if the file extension should be removed from the field value. + Used in order to create an output specification, by default it is True + readonly: bool, optional + If True the input field can’t be provided by the user but it aggregates other + input fields (for example the fields with argstr: -o {fldA} {fldB}), by default + it is False + formatter: function, optional + If provided the argstr of the field is created using the function. This function + can for example be used to combine several inputs into one command argument. The + function can take field (this input field will be passed to the function), + inputs (entire inputs will be passed) or any input field name (a specific input + field will be sent). + """ + + metadata = { + "help_string": help_string, + "argstr": argstr, + "position": position, + "mandatory": mandatory, + "sep": sep, + "allowed_values": allowed_values, + "requires": requires, + "xor": xor, + "copyfile": copyfile, + "container_path": container_path, + "output_file_template": output_file_template, + "output_field_name": output_field_name, + "keep_extension": keep_extension, + "readonly": readonly, + "formatter": formatter, + } + + return attrs.field( + default=default, metadata={k: v for k, v in metadata.items() if v is not None} + ) + + +def cmd_out( + help_string: str, + mandatory: bool = False, + output_file_template: str = None, + output_field_name: str = None, + keep_extension: bool = True, + requires: list = None, + callable: ty.Callable = None, +): + """Returns an attrs field with appropriate metadata for it to be added as an output of + a Pydra shell command task definition + + Parameters + ---------- + help_string: str + A short description of the input field. The same as in input_spec. + mandatory: bool, default: False + If True the output file has to exist, otherwise an error will be raised. + output_file_template: str, optional + If provided the output file name (or list of file names) is created using the + template. The template can use other fields, e.g. {file1}. The same as in + input_spec. + output_field_name: str, optional + If provided the field is added to the output spec with changed name. The same as + in input_spec. Used together with output_file_template + keep_extension: bool, default: True + A flag that specifies if the file extension should be removed from the field + value. The same as in input_spec. + requires: list + List of field names that are required to create a specific output. The fields + do not have to be a part of the output_file_template and if any field from the + list is not provided in the input, a NOTHING is returned for the specific output. + This has a different meaning than the requires form the input_spec. + callable: Callable + If provided the output file name (or list of file names) is created using the + function. The function can take field (the specific output field will be passed + to the function), output_dir (task output_dir will be used), stdout, stderr + (stdout and stderr of the task will be sent) inputs (entire inputs will be + passed) or any input field name (a specific input field will be sent). + """ + metadata = { + "help_string": help_string, + "mandatory": mandatory, + "output_file_template": output_file_template, + "output_field_name": output_field_name, + "keep_extension": keep_extension, + "requires": requires, + "callable": callable, + } + + return attrs.field(metadata={k: v for k, v in metadata.items() if v is not None}) diff --git a/pydra/mark/tests/test_shell_commands.py b/pydra/mark/tests/test_shell_commands.py new file mode 100644 index 0000000000..084f4464bb --- /dev/null +++ b/pydra/mark/tests/test_shell_commands.py @@ -0,0 +1,75 @@ +import os +import tempfile +from pathlib import Path +import attrs +import pydra.engine +from pydra.mark import cmd_arg, cmd_out + + +def test_shell_cmd(): + @attrs.define(kw_only=True, slots=False) + class LsInputSpec(pydra.specs.ShellSpec): + directory: os.PathLike = cmd_arg( + help_string="the directory to list the contents of", + argstr="", + mandatory=True, + ) + hidden: bool = cmd_arg(help_string=("display hidden FS objects"), argstr="-a") + long_format: bool = cmd_arg( + help_string=( + "display properties of FS object, such as permissions, size and timestamps " + ), + argstr="-l", + ) + human_readable: bool = cmd_arg( + help_string="display file sizes in human readable form", + argstr="-h", + requires=["long_format"], + ) + complete_date: bool = cmd_arg( + help_string="Show complete date in long format", + argstr="-T", + requires=["long_format"], + xor=["date_format_str"], + ) + date_format_str: str = cmd_arg( + help_string="format string for ", + argstr="-D", + requires=["long_format"], + xor=["complete_date"], + ) + + def list_outputs(stdout): + return stdout.split("\n")[:-1] + + @attrs.define(kw_only=True, slots=False) + class LsOutputSpec(pydra.specs.ShellOutSpec): + entries: list = cmd_out( + help_string="list of entries returned by ls command", callable=list_outputs + ) + + class Ls(pydra.engine.ShellCommandTask): + """Task definition for mri_aparc2aseg.""" + + executable = "ls" + + input_spec = pydra.specs.SpecInfo( + name="LsInput", + bases=(LsInputSpec,), + ) + + output_spec = pydra.specs.SpecInfo( + name="LsOutput", + bases=(LsOutputSpec,), + ) + + tmpdir = Path(tempfile.mkdtemp()) + Path.touch(tmpdir / "a") + Path.touch(tmpdir / "b") + Path.touch(tmpdir / "c") + + ls = Ls(directory=tmpdir) + + result = ls() + + assert result.output.entries == ["a", "b", "c"] From ac15a1e2c7f0c1e070aef817b88aff077223b2bf Mon Sep 17 00:00:00 2001 From: Tom Close Date: Thu, 11 May 2023 17:06:08 +1000 Subject: [PATCH 2/5] added small note to docs --- docs/input_spec.rst | 11 ++++++----- pydra/mark/tests/test_shell_commands.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/input_spec.rst b/docs/input_spec.rst index 48d66fd814..92e4c945e5 100644 --- a/docs/input_spec.rst +++ b/docs/input_spec.rst @@ -174,8 +174,9 @@ In the example we used multiple keys in the metadata dictionary including `help_ (a specific input field will be sent). -Validators ----------- -Pydra allows for using simple validator for types and `allowev_values`. -The validators are disabled by default, but can be enabled by calling -`pydra.set_input_validator(flag=True)`. +`cmd_arg` Function +------------------ + +For convenience, there is a function in `pydra.mark` called `cmd_arg()`, which will +takes the above metadata values as arguments and inserts them into the metadata passed +to `attrs.field`. This can be especially useful when using an IDE with code-completion. diff --git a/pydra/mark/tests/test_shell_commands.py b/pydra/mark/tests/test_shell_commands.py index 084f4464bb..e6127edcde 100644 --- a/pydra/mark/tests/test_shell_commands.py +++ b/pydra/mark/tests/test_shell_commands.py @@ -49,7 +49,7 @@ class LsOutputSpec(pydra.specs.ShellOutSpec): ) class Ls(pydra.engine.ShellCommandTask): - """Task definition for mri_aparc2aseg.""" + """Task definition for the `ls` command line tool""" executable = "ls" From 4b78ab046326b74b9201d68d07aebd5bd3b726c0 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Mon, 15 May 2023 12:50:25 +1000 Subject: [PATCH 3/5] renamed cmd_arg and cmd_out to shell_arg and shell_out --- docs/input_spec.rst | 4 ++-- pydra/mark/__init__.py | 4 ++-- pydra/mark/shell_commands.py | 4 ++-- pydra/mark/tests/test_shell_commands.py | 16 ++++++++-------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/input_spec.rst b/docs/input_spec.rst index 92e4c945e5..559c2c1f66 100644 --- a/docs/input_spec.rst +++ b/docs/input_spec.rst @@ -174,9 +174,9 @@ In the example we used multiple keys in the metadata dictionary including `help_ (a specific input field will be sent). -`cmd_arg` Function +`shell_arg` Function ------------------ -For convenience, there is a function in `pydra.mark` called `cmd_arg()`, which will +For convenience, there is a function in `pydra.mark` called `shell_arg()`, which will takes the above metadata values as arguments and inserts them into the metadata passed to `attrs.field`. This can be especially useful when using an IDE with code-completion. diff --git a/pydra/mark/__init__.py b/pydra/mark/__init__.py index d4338cf621..5fae37d03d 100644 --- a/pydra/mark/__init__.py +++ b/pydra/mark/__init__.py @@ -1,4 +1,4 @@ from .functions import annotate, task -from .shell_commands import cmd_arg, cmd_out +from .shell_commands import shell_arg, shell_out -__all__ = ("annotate", "task", "cmd_arg", "cmd_out") +__all__ = ("annotate", "task", "shell_arg", "shell_out") diff --git a/pydra/mark/shell_commands.py b/pydra/mark/shell_commands.py index 62579ce543..71d4b10f78 100644 --- a/pydra/mark/shell_commands.py +++ b/pydra/mark/shell_commands.py @@ -4,7 +4,7 @@ import attrs -def cmd_arg( +def shell_arg( help_string: str, default: ty.Any = attrs.NOTHING, argstr: str = None, @@ -103,7 +103,7 @@ def cmd_arg( ) -def cmd_out( +def shell_out( help_string: str, mandatory: bool = False, output_file_template: str = None, diff --git a/pydra/mark/tests/test_shell_commands.py b/pydra/mark/tests/test_shell_commands.py index e6127edcde..5f2b428024 100644 --- a/pydra/mark/tests/test_shell_commands.py +++ b/pydra/mark/tests/test_shell_commands.py @@ -3,36 +3,36 @@ from pathlib import Path import attrs import pydra.engine -from pydra.mark import cmd_arg, cmd_out +from pydra.mark import shell_arg, shell_out def test_shell_cmd(): @attrs.define(kw_only=True, slots=False) class LsInputSpec(pydra.specs.ShellSpec): - directory: os.PathLike = cmd_arg( + directory: os.PathLike = shell_arg( help_string="the directory to list the contents of", argstr="", mandatory=True, ) - hidden: bool = cmd_arg(help_string=("display hidden FS objects"), argstr="-a") - long_format: bool = cmd_arg( + hidden: bool = shell_arg(help_string=("display hidden FS objects"), argstr="-a") + long_format: bool = shell_arg( help_string=( "display properties of FS object, such as permissions, size and timestamps " ), argstr="-l", ) - human_readable: bool = cmd_arg( + human_readable: bool = shell_arg( help_string="display file sizes in human readable form", argstr="-h", requires=["long_format"], ) - complete_date: bool = cmd_arg( + complete_date: bool = shell_arg( help_string="Show complete date in long format", argstr="-T", requires=["long_format"], xor=["date_format_str"], ) - date_format_str: str = cmd_arg( + date_format_str: str = shell_arg( help_string="format string for ", argstr="-D", requires=["long_format"], @@ -44,7 +44,7 @@ def list_outputs(stdout): @attrs.define(kw_only=True, slots=False) class LsOutputSpec(pydra.specs.ShellOutSpec): - entries: list = cmd_out( + entries: list = shell_out( help_string="list of entries returned by ls command", callable=list_outputs ) From 35039182373d8bc2cf563c62634403491d995466 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Mon, 15 May 2023 12:52:31 +1000 Subject: [PATCH 4/5] touched up docs --- docs/input_spec.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/input_spec.rst b/docs/input_spec.rst index 559c2c1f66..2940c17820 100644 --- a/docs/input_spec.rst +++ b/docs/input_spec.rst @@ -175,7 +175,7 @@ In the example we used multiple keys in the metadata dictionary including `help_ `shell_arg` Function ------------------- +-------------------- For convenience, there is a function in `pydra.mark` called `shell_arg()`, which will takes the above metadata values as arguments and inserts them into the metadata passed From ba49ad0bb56fd31ea6c9d3177f8306efcb451d97 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Mon, 15 May 2023 13:17:01 +1000 Subject: [PATCH 5/5] pass through kwargs to attrs.field in shell_arg and shell_out --- pydra/mark/shell_commands.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pydra/mark/shell_commands.py b/pydra/mark/shell_commands.py index 71d4b10f78..b9d29db21b 100644 --- a/pydra/mark/shell_commands.py +++ b/pydra/mark/shell_commands.py @@ -21,6 +21,7 @@ def shell_arg( keep_extension: bool = True, readonly: bool = False, formatter: ty.Callable = None, + **kwargs, ): """ Returns an attrs field with appropriate metadata for it to be added as an argument in @@ -78,6 +79,8 @@ def shell_arg( function can take field (this input field will be passed to the function), inputs (entire inputs will be passed) or any input field name (a specific input field will be sent). + **kwargs + remaining keyword arguments are passed onto the underlying attrs.field function """ metadata = { @@ -99,7 +102,9 @@ def shell_arg( } return attrs.field( - default=default, metadata={k: v for k, v in metadata.items() if v is not None} + default=default, + metadata={k: v for k, v in metadata.items() if v is not None}, + **kwargs, ) @@ -111,6 +116,7 @@ def shell_out( keep_extension: bool = True, requires: list = None, callable: ty.Callable = None, + **kwargs, ): """Returns an attrs field with appropriate metadata for it to be added as an output of a Pydra shell command task definition @@ -142,6 +148,8 @@ def shell_out( to the function), output_dir (task output_dir will be used), stdout, stderr (stdout and stderr of the task will be sent) inputs (entire inputs will be passed) or any input field name (a specific input field will be sent). + **kwargs + remaining keyword arguments are passed onto the underlying attrs.field function """ metadata = { "help_string": help_string, @@ -153,4 +161,6 @@ def shell_out( "callable": callable, } - return attrs.field(metadata={k: v for k, v in metadata.items() if v is not None}) + return attrs.field( + metadata={k: v for k, v in metadata.items() if v is not None}, **kwargs + )