diff --git a/.circleci/config.yml b/.circleci/config.yml index 741e44e024..39aea3517a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,12 +32,12 @@ docs_deploy: &docs name: Deploy docs to gh-pages branch command: gh-pages --no-history --dotfiles --message "doc(update) [skip ci]" --dist docs/_build/html -version: 2 +version: 2.1 jobs: build_docs: docker: - - image: python:3.7.4 + - image: cimg/python:3.11 working_directory: /tmp/gh-pages environment: - FSLOUTPUTTYPE: NIFTI diff --git a/docs/conf.py b/docs/conf.py index b3a1d23acc..fd0b69ca43 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -89,7 +89,7 @@ napoleon_custom_sections = [("Inputs", "Parameters"), ("Outputs", "Parameters")] # Intersphinx -intersphinx_mapping = {"https://docs.python.org/3/": None} +intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} # Linkcode # The following is used by sphinx.ext.linkcode to provide links to github diff --git a/pydra/__init__.py b/pydra/__init__.py index 0416c9be75..da3d278539 100644 --- a/pydra/__init__.py +++ b/pydra/__init__.py @@ -11,15 +11,28 @@ import logging -logger = logging.getLogger("pydra") +import __main__ +import attr + +from . import mark +from .engine import AuditFlag, DockerTask, ShellCommandTask, Submitter, Workflow, specs + +__all__ = ( + "Submitter", + "Workflow", + "AuditFlag", + "ShellCommandTask", + "DockerTask", + "specs", + "mark", +) try: from ._version import __version__ except ImportError: pass -from .engine import Submitter, Workflow, AuditFlag, ShellCommandTask, DockerTask, specs -from . import mark +logger = logging.getLogger("pydra") def check_latest_version(): @@ -29,8 +42,6 @@ def check_latest_version(): # Run telemetry on import for interactive sessions, such as IPython, Jupyter notebooks, Python REPL -import __main__ - if not hasattr(__main__, "__file__"): from .engine.core import TaskBase @@ -39,8 +50,6 @@ def check_latest_version(): # attr run_validators is set to False, but could be changed using use_validator -import attr - attr.set_run_validators(False) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index ece23239e6..6c35eb77eb 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -1,12 +1,16 @@ """Module to keep track of provenance information.""" import os -from pathlib import Path import json import attr from ..utils.messenger import send_message, make_message, gen_uuid, now, AuditFlag from .helpers import ensure_list, gather_runtime_info, hash_file from .specs import attr_fields, File, Directory +try: + import importlib_resources +except ImportError: + import importlib.resources as importlib_resources # type: ignore + class Audit: """Handle provenance tracking and resource utilization.""" @@ -133,9 +137,8 @@ def audit_message(self, message, flags=None): """ if self.develop: - with open( - Path(os.path.dirname(__file__)) / ".." / "schema/context.jsonld" - ) as fp: + context_file = importlib_resources.files("pydra") / "schema/context.jsonld" + with context_file.open() as fp: context = json.load(fp) else: context = { diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 31127cac99..25bb56bdf4 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -491,7 +491,7 @@ def _run(self, rerun=False, **kwargs): lockfile = self.cache_dir / (checksum + ".lock") # Eagerly retrieve cached - see scenarios in __init__() self.hooks.pre_run(self) - logger.debug(f"'%s' is attempting to acquire lock on %s", self.name, lockfile) + logger.debug("'%s' is attempting to acquire lock on %s", self.name, lockfile) with SoftFileLock(lockfile): if not (rerun or self.task_rerun): result = self.result() @@ -1105,7 +1105,7 @@ async def _run(self, submitter=None, rerun=False, **kwargs): lockfile = self.cache_dir / (checksum + ".lock") self.hooks.pre_run(self) logger.debug( - f"'%s' is attempting to acquire lock on %s with Pydra lock", + "'%s' is attempting to acquire lock on %s with Pydra lock", self.name, lockfile, ) @@ -1231,7 +1231,7 @@ def _collect_outputs(self): err_file = getattr(self, val.name).output_dir / "_error.pklz" raise ValueError( f"Task {val.name} raised an error, full crash report is here: " - f"{str(err_file)}" + f"{err_file}" ) return attr.evolve(output, **output_wf) diff --git a/pydra/engine/helpers_file.py b/pydra/engine/helpers_file.py index 5481c5d784..5a408923b7 100644 --- a/pydra/engine/helpers_file.py +++ b/pydra/engine/helpers_file.py @@ -606,7 +606,7 @@ def template_update(inputs, output_dir, state_ind=None, map_copyfiles=None): for fld in fields_templ: if fld.type not in [str, ty.Union[str, bool]]: raise Exception( - f"fields with output_file_template" + "fields with output_file_template" " has to be a string or Union[str, bool]" ) dict_mod[fld.name] = template_update_single( @@ -638,7 +638,7 @@ def template_update_single( if spec_type == "input": if field.type not in [str, ty.Union[str, bool]]: raise Exception( - f"fields with output_file_template" + "fields with output_file_template" "has to be a string or Union[str, bool]" ) inp_val_set = inputs_dict_st[field.name] diff --git a/pydra/engine/helpers_state.py b/pydra/engine/helpers_state.py index 4eb9248f85..58a9fc74e7 100644 --- a/pydra/engine/helpers_state.py +++ b/pydra/engine/helpers_state.py @@ -344,7 +344,7 @@ def _add_name(mlist, name): if "." in elem or elem.startswith("_"): pass else: - mlist[i] = "{}.{}".format(name, mlist[i]) + mlist[i] = f"{name}.{mlist[i]}" elif isinstance(elem, list): mlist[i] = _add_name(elem, name) elif isinstance(elem, tuple): diff --git a/pydra/engine/state.py b/pydra/engine/state.py index 4e532775ae..79e3d5cb34 100644 --- a/pydra/engine/state.py +++ b/pydra/engine/state.py @@ -454,13 +454,11 @@ def _remove_repeated(self, previous_splitters): f"{self.other_states}" ) - repeated = set( - [ - (el, previous_splitters.count(el)) - for el in previous_splitters - if previous_splitters.count(el) > 1 - ] - ) + repeated = { + (el, previous_splitters.count(el)) + for el in previous_splitters + if previous_splitters.count(el) > 1 + } if repeated: # assuming that I want to remove from right previous_splitters.reverse() diff --git a/pydra/engine/tests/conftest.py b/pydra/engine/tests/conftest.py new file mode 100644 index 0000000000..ff13263ca2 --- /dev/null +++ b/pydra/engine/tests/conftest.py @@ -0,0 +1,23 @@ +import pytest +from pydra import set_input_validator + +try: + import importlib_resources +except ImportError: + import importlib.resources as importlib_resources + + +@pytest.fixture(scope="package") +def data_tests_dir(): + test_nii = importlib_resources.files("pydra").joinpath( + "engine", "tests", "data_tests" + ) + with importlib_resources.as_file(test_nii) as path: + yield path + + +@pytest.fixture() +def use_validator(): + set_input_validator(flag=True) + yield None + set_input_validator(flag=False) diff --git a/pydra/engine/tests/data_tests/loading.py b/pydra/engine/tests/data_tests/loading.py index 0fe80e1b2b..8240b20798 100644 --- a/pydra/engine/tests/data_tests/loading.py +++ b/pydra/engine/tests/data_tests/loading.py @@ -1,6 +1,3 @@ -import os - - def loading(filename): with open(filename) as f: txt = f.read() diff --git a/pydra/engine/tests/data_tests/saving.py b/pydra/engine/tests/data_tests/saving.py index 1d5b67257b..f50aa82a45 100644 --- a/pydra/engine/tests/data_tests/saving.py +++ b/pydra/engine/tests/data_tests/saving.py @@ -1,6 +1,3 @@ -import os - - def saving(filename): with open(filename, "w") as f: f.write("Hello!") diff --git a/pydra/engine/tests/test_boutiques.py b/pydra/engine/tests/test_boutiques.py index 0008b78873..48f484b687 100644 --- a/pydra/engine/tests/test_boutiques.py +++ b/pydra/engine/tests/test_boutiques.py @@ -1,6 +1,5 @@ -import os, shutil +import shutil import subprocess as sp -from pathlib import Path import attr import pytest @@ -9,7 +8,6 @@ from ..submitter import Submitter from ..boutiques import BoshTask from .utils import result_no_submitter, result_submitter, no_win -from ...engine.specs import File need_bosh_docker = pytest.mark.skipif( shutil.which("docker") is None @@ -18,8 +16,6 @@ reason="requires docker and bosh", ) -Infile = Path(__file__).resolve().parent / "data_tests" / "test.nii.gz" - pytestmark = pytest.mark.skip() @@ -30,10 +26,10 @@ "maskfile", ["test_brain.nii.gz", "test_brain", "test_brain.nii"] ) @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) -def test_boutiques_1(maskfile, plugin, results_function, tmpdir): +def test_boutiques_1(maskfile, plugin, results_function, tmpdir, data_tests_dir): """simple task to run fsl.bet using BoshTask""" btask = BoshTask(name="NA", zenodo_id="1482743") - btask.inputs.infile = Infile + btask.inputs.infile = data_tests_dir / "test.nii.gz" btask.inputs.maskfile = maskfile btask.cache_dir = tmpdir res = results_function(btask, plugin) @@ -50,12 +46,12 @@ def test_boutiques_1(maskfile, plugin, results_function, tmpdir): @no_win @need_bosh_docker @pytest.mark.flaky(reruns=3) -def test_boutiques_spec_1(): +def test_boutiques_spec_1(data_tests_dir): """testing spec: providing input/output fields names""" btask = BoshTask( name="NA", zenodo_id="1482743", - infile=Infile, + infile=data_tests_dir / "test.nii.gz", maskfile="test_brain.nii.gz", input_spec_names=["infile", "maskfile"], output_spec_names=["outfile", "out_outskin_off"], @@ -75,12 +71,12 @@ def test_boutiques_spec_1(): @no_win @need_bosh_docker @pytest.mark.flaky(reruns=3) -def test_boutiques_spec_2(): +def test_boutiques_spec_2(data_tests_dir): """testing spec: providing partial input/output fields names""" btask = BoshTask( name="NA", zenodo_id="1482743", - infile=Infile, + infile=data_tests_dir / "test.nii.gz", maskfile="test_brain.nii.gz", input_spec_names=["infile"], output_spec_names=[], @@ -101,11 +97,11 @@ def test_boutiques_spec_2(): @pytest.mark.parametrize( "maskfile", ["test_brain.nii.gz", "test_brain", "test_brain.nii"] ) -def test_boutiques_wf_1(maskfile, plugin, tmpdir): +def test_boutiques_wf_1(maskfile, plugin, tmpdir, infile): """wf with one task that runs fsl.bet using BoshTask""" wf = Workflow(name="wf", input_spec=["maskfile", "infile"]) wf.inputs.maskfile = maskfile - wf.inputs.infile = Infile + wf.inputs.infile = infile wf.cache_dir = tmpdir wf.add( @@ -134,11 +130,11 @@ def test_boutiques_wf_1(maskfile, plugin, tmpdir): @pytest.mark.parametrize( "maskfile", ["test_brain.nii.gz", "test_brain", "test_brain.nii"] ) -def test_boutiques_wf_2(maskfile, plugin, tmpdir): +def test_boutiques_wf_2(maskfile, plugin, tmpdir, infile): """wf with two BoshTasks (fsl.bet and fsl.stats) and one ShellTask""" wf = Workflow(name="wf", input_spec=["maskfile", "infile"]) wf.inputs.maskfile = maskfile - wf.inputs.infile = Infile + wf.inputs.infile = infile wf.cache_dir = tmpdir wf.add( diff --git a/pydra/engine/tests/test_dockertask.py b/pydra/engine/tests/test_dockertask.py index 117d35d658..b8135209e9 100644 --- a/pydra/engine/tests/test_dockertask.py +++ b/pydra/engine/tests/test_dockertask.py @@ -1,4 +1,3 @@ -import os import pytest import attr @@ -71,9 +70,7 @@ def test_docker_1_dockerflag_exception(plugin): """using ShellComandTask with container_info=("docker"), no image provided""" cmd = "whoami" with pytest.raises(Exception) as excinfo: - shocky = ShellCommandTask( - name="shocky", executable=cmd, container_info=("docker") - ) + ShellCommandTask(name="shocky", executable=cmd, container_info=("docker")) assert "container_info has to have 2 elements" in str(excinfo.value) @@ -509,19 +506,17 @@ def test_wf_docker_1_dockerflag(plugin, tmpdir): @no_win @need_docker @pytest.mark.skip(reason="we probably don't want to support bindings as an input") -def test_wf_docker_2pre(plugin, tmpdir): +def test_wf_docker_2pre(plugin, tmpdir, data_tests_dir): """a workflow with two connected task that run python scripts the first one creates a text file and the second one reads the file """ - scripts_dir = os.path.join(os.path.dirname(__file__), "data_tests") - cmd1 = ["python", "/scripts/saving.py", "-f", "/outputs/tmp.txt"] dt = DockerTask( name="save", image="python:3.7-alpine", executable=cmd1, - bindings=[(str(tmpdir), "/outputs"), (scripts_dir, "/scripts", "ro")], + bindings=[(str(tmpdir), "/outputs"), (str(data_tests_dir), "/scripts", "ro")], strip=True, ) res = dt(plugin=plugin) @@ -531,13 +526,11 @@ def test_wf_docker_2pre(plugin, tmpdir): @no_win @need_docker @pytest.mark.skip(reason="we probably don't want to support bindings as an input") -def test_wf_docker_2(plugin, tmpdir): +def test_wf_docker_2(plugin, tmpdir, data_tests_dir): """a workflow with two connected task that run python scripts the first one creates a text file and the second one reads the file """ - scripts_dir = os.path.join(os.path.dirname(__file__), "data_tests") - wf = Workflow(name="wf", input_spec=["cmd1", "cmd2"]) wf.inputs.cmd1 = ["python", "/scripts/saving.py", "-f", "/outputs/tmp.txt"] wf.inputs.cmd2 = ["python", "/scripts/loading.py", "-f"] @@ -546,7 +539,10 @@ def test_wf_docker_2(plugin, tmpdir): name="save", image="python:3.7-alpine", executable=wf.lzin.cmd1, - bindings=[(str(tmpdir), "/outputs"), (scripts_dir, "/scripts", "ro")], + bindings=[ + (str(tmpdir), "/outputs"), + (str(data_tests_dir), "/scripts", "ro"), + ], strip=True, ) ) @@ -556,7 +552,10 @@ def test_wf_docker_2(plugin, tmpdir): image="python:3.7-alpine", executable=wf.lzin.cmd2, args=wf.save.lzout.stdout, - bindings=[(str(tmpdir), "/outputs"), (scripts_dir, "/scripts", "ro")], + bindings=[ + (str(tmpdir), "/outputs"), + (str(data_tests_dir), "/scripts", "ro"), + ], strip=True, ) ) @@ -1078,7 +1077,7 @@ def test_docker_inputspec_3a(plugin, tmpdir): ) with pytest.raises(Exception) as excinfo: - res = docky() + docky() assert "use field.metadata['container_path']=True" in str(excinfo.value) diff --git a/pydra/engine/tests/test_graph.py b/pydra/engine/tests/test_graph.py index 4d8a58a29d..403b9e6ef9 100644 --- a/pydra/engine/tests/test_graph.py +++ b/pydra/engine/tests/test_graph.py @@ -65,13 +65,13 @@ def test_edges_3(): def test_edges_ecxeption_1(): with pytest.raises(Exception) as excinfo: - graph = DiGraph(nodes=[A, B, A], edges=[(A, B)]) + DiGraph(nodes=[A, B, A], edges=[(A, B)]) assert "repeated elements" in str(excinfo.value) def test_edges_ecxeption_2(): with pytest.raises(Exception) as excinfo: - graph = DiGraph(nodes=[A, B], edges=[(A, C)]) + DiGraph(nodes=[A, B], edges=[(A, C)]) assert "can't be added" in str(excinfo.value) diff --git a/pydra/engine/tests/test_helpers.py b/pydra/engine/tests/test_helpers.py index efceeb6753..d3a995f4fe 100644 --- a/pydra/engine/tests/test_helpers.py +++ b/pydra/engine/tests/test_helpers.py @@ -47,7 +47,7 @@ def test_save(tmpdir): def test_hash_file(tmpdir): outdir = Path(tmpdir) - with open(outdir / "test.file", "wt") as fp: + with open(outdir / "test.file", "w") as fp: fp.write("test") assert ( helpers_file.hash_file(outdir / "test.file") @@ -228,9 +228,9 @@ def test_load_and_run(tmpdir): def test_load_and_run_exception_load(tmpdir): """testing raising exception and saving info in crashfile when when load_and_run""" task_pkl = Path(tmpdir.join("task_main.pkl")) - task = raise_xeq1(name="raise", x=[1, 2]).split("x") - with pytest.raises(FileNotFoundError) as excinfo: - task_0 = load_and_run(task_pkl=task_pkl, ind=0) + raise_xeq1(name="raise", x=[1, 2]).split("x") + with pytest.raises(FileNotFoundError): + load_and_run(task_pkl=task_pkl, ind=0) def test_load_and_run_exception_run(tmpdir): @@ -245,7 +245,7 @@ def test_load_and_run_exception_run(tmpdir): cp.dump(task, fp) with pytest.raises(Exception) as excinfo: - task_0 = load_and_run(task_pkl=task_pkl, ind=0) + load_and_run(task_pkl=task_pkl, ind=0) assert "i'm raising an exception!" in str(excinfo.value) # checking if the crashfile has been created assert "crash" in str(excinfo.value) diff --git a/pydra/engine/tests/test_nipype1_convert.py b/pydra/engine/tests/test_nipype1_convert.py index fd60f30da4..f65f6ae73c 100644 --- a/pydra/engine/tests/test_nipype1_convert.py +++ b/pydra/engine/tests/test_nipype1_convert.py @@ -1,15 +1,9 @@ -import attr import typing as ty -import os, sys import pytest -from pathlib import Path from ..task import ShellCommandTask -from ..submitter import Submitter -from ..core import Workflow from ..specs import ShellOutSpec, ShellSpec, SpecInfo, File -from .utils import result_no_submitter, result_submitter, use_validator interf_input_spec = SpecInfo( name="Input", fields=[("test", ty.Any, {"help_string": "test"})], bases=(ShellSpec,) diff --git a/pydra/engine/tests/test_node_task.py b/pydra/engine/tests/test_node_task.py index 95ba9c7eac..d4588ce8de 100644 --- a/pydra/engine/tests/test_node_task.py +++ b/pydra/engine/tests/test_node_task.py @@ -575,7 +575,7 @@ def test_task_nostate_cachedir(plugin_dask_opt, tmpdir): @pytest.mark.flaky(reruns=2) # when dask def test_task_nostate_cachedir_relativepath(tmpdir, plugin_dask_opt): """task with provided cache_dir as relative path""" - cwd = tmpdir.chdir() + tmpdir.chdir() cache_dir = "test_task_nostate" tmpdir.mkdir(cache_dir) @@ -1266,7 +1266,7 @@ def test_task_state_comb_singl_1(plugin, tmpdir): assert nn.state.splitter == "NA.a" assert nn.state.splitter_rpn == ["NA.a"] assert nn.state.combiner == ["NA.a"] - assert nn.state.splitter_final == None + assert nn.state.splitter_final is None assert nn.state.splitter_rpn_final == [] with Submitter(plugin=plugin) as sub: diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index cc195fe430..46d99d4163 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -19,7 +19,7 @@ MultiOutputFile, MultiInputObj, ) -from .utils import result_no_submitter, result_submitter, use_validator, no_win +from .utils import result_no_submitter, result_submitter, no_win if sys.platform.startswith("win"): pytest.skip("SLURM not available in windows", allow_module_level=True) @@ -648,9 +648,7 @@ def test_shell_cmd_inputspec_4c_exception(plugin): # separate command into exec + args with pytest.raises(Exception) as excinfo: - shelly = ShellCommandTask( - name="shelly", executable=cmd_exec, input_spec=my_input_spec - ) + ShellCommandTask(name="shelly", executable=cmd_exec, input_spec=my_input_spec) assert ( str(excinfo.value) == "default value should not be set when the field is mandatory" @@ -682,9 +680,7 @@ def test_shell_cmd_inputspec_4d_exception(plugin): # separate command into exec + args with pytest.raises(Exception) as excinfo: - shelly = ShellCommandTask( - name="shelly", executable=cmd_exec, input_spec=my_input_spec - ) + ShellCommandTask(name="shelly", executable=cmd_exec, input_spec=my_input_spec) assert ( str(excinfo.value) == "default value should not be set together with output_file_template" @@ -737,7 +733,7 @@ def test_shell_cmd_inputspec_5_nosubm(plugin, results_function, tmpdir): ) assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "ls -t" - res = results_function(shelly, plugin) + results_function(shelly, plugin) def test_shell_cmd_inputspec_5a_exception(plugin, tmpdir): @@ -834,7 +830,7 @@ def test_shell_cmd_inputspec_6(plugin, results_function, tmpdir): ) assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "ls -l -t" - res = results_function(shelly, plugin) + results_function(shelly, plugin) def test_shell_cmd_inputspec_6a_exception(plugin): @@ -923,7 +919,7 @@ def test_shell_cmd_inputspec_6b(plugin, results_function, tmpdir): shelly.inputs.opt_l = cmd_l assert shelly.inputs.executable == cmd_exec assert shelly.cmdline == "ls -l -t" - res = results_function(shelly, plugin) + results_function(shelly, plugin) @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) @@ -1554,7 +1550,7 @@ def test_shell_cmd_inputspec_10_err(tmpdir): shelly.cache_dir = tmpdir with pytest.raises(FileNotFoundError): - res = shelly() + shelly() def test_shell_cmd_inputsspec_11(): @@ -1966,9 +1962,7 @@ def test_shell_cmd_inputspec_typeval_1(use_validator): ) with pytest.raises(TypeError): - shelly = ShellCommandTask( - executable=cmd_exec, text="hello", input_spec=my_input_spec - ) + ShellCommandTask(executable=cmd_exec, text="hello", input_spec=my_input_spec) def test_shell_cmd_inputspec_typeval_2(use_validator): @@ -1984,9 +1978,7 @@ def test_shell_cmd_inputspec_typeval_2(use_validator): ) with pytest.raises(TypeError): - shelly = ShellCommandTask( - executable=cmd_exec, text="hello", input_spec=my_input_spec - ) + ShellCommandTask(executable=cmd_exec, text="hello", input_spec=my_input_spec) @pytest.mark.parametrize("results_function", [result_no_submitter, result_submitter]) @@ -3032,7 +3024,7 @@ def test_shell_cmd_outputspec_7(tmpdir, plugin, results_function): the input field used in the template is a MultiInputObj, so it can be and is a list """ file = tmpdir.join("script.sh") - file.write(f'for var in "$@"; do touch file"$var".txt; done') + file.write('for var in "$@"; do touch file"$var".txt; done') cmd = "bash" new_files_id = ["1", "2", "3"] @@ -3108,7 +3100,7 @@ def test_shell_cmd_outputspec_7a(tmpdir, plugin, results_function): the input field used in the template is a MultiInputObj, but a single element is used """ file = tmpdir.join("script.sh") - file.write(f'for var in "$@"; do touch file"$var".txt; done') + file.write('for var in "$@"; do touch file"$var".txt; done') cmd = "bash" new_files_id = "1" @@ -3303,9 +3295,9 @@ def get_lowest_directory(directory_path): resultsDir="outdir", ).split("args") - res = results_function(shelly, plugin) + results_function(shelly, plugin) for index, arg_dir in enumerate(args): - assert Path(Path(tmpdir) / Path(arg_dir)).exists() == True + assert Path(Path(tmpdir) / Path(arg_dir)).exists() assert get_lowest_directory(arg_dir) == f"/dir{index+1}" @@ -3371,7 +3363,7 @@ def get_lowest_directory(directory_path): ) res = results_function(shelly, plugin) print("Cache_dirr:", shelly.cache_dir) - assert (shelly.output_dir / Path("test")).exists() == True + assert (shelly.output_dir / Path("test")).exists() assert get_lowest_directory(res.output.resultsDir) == get_lowest_directory( shelly.output_dir / Path("test") ) @@ -4148,7 +4140,7 @@ def test_shell_cmd_inputspec_outputspec_6_except(): shelly.inputs.file1 = "new_file_1.txt" with pytest.raises(Exception, match="requires field can be"): - res = shelly() + shelly() def no_fsl(): @@ -4157,7 +4149,7 @@ def no_fsl(): @pytest.mark.skipif(no_fsl(), reason="fsl is not installed") -def test_fsl(): +def test_fsl(data_tests_dir): """mandatory field added to fields, value provided""" _xor_inputs = [ @@ -4385,7 +4377,7 @@ def change_name(file): ) # TODO: not sure why this has to be string - in_file = Path(__file__).parent / "data_tests" / "test.nii.gz" + in_file = data_tests_dir / "test.nii.gz" # separate command into exec + args shelly = ShellCommandTask( @@ -4878,7 +4870,7 @@ def formatter_5(field): # chcking if field value is accessible when None def formatter_4(field): - assert field == None + assert field is None # formatter must return a string return "" @@ -4941,7 +4933,7 @@ def formatter_1(in1, in2): shelly = ShellCommandTask( name="f", executable="executable", input_spec=input_spec, in1=in1, in2="in2" ).split("in1") - assert shelly != None + assert shelly is not None # results = shelly.cmdline # assert len(results) == 2 diff --git a/pydra/engine/tests/test_shelltask_inputspec.py b/pydra/engine/tests/test_shelltask_inputspec.py index 215c995d5f..c19722a3c3 100644 --- a/pydra/engine/tests/test_shelltask_inputspec.py +++ b/pydra/engine/tests/test_shelltask_inputspec.py @@ -1,6 +1,5 @@ import attr import typing as ty -from pathlib import Path import pytest from ..task import ShellCommandTask @@ -10,12 +9,7 @@ SpecInfo, File, MultiInputObj, - MultiInputFile, - MultiOutputFile, ) -from .utils import use_validator -from ..core import Workflow -from ..submitter import Submitter def test_shell_cmd_execargs_1(): @@ -110,7 +104,7 @@ def test_shell_cmd_inputs_1_st(): bases=(ShellSpec,), ) - shelly = ShellCommandTask( + ShellCommandTask( name="shelly", executable="executable", args="arg", @@ -729,7 +723,7 @@ def test_shell_cmd_inputs_not_given_1(): shelly.inputs.arg2 = "argument2" - assert shelly.cmdline == f"executable --arg2 argument2" + assert shelly.cmdline == "executable --arg2 argument2" def test_shell_cmd_inputs_template_1(): @@ -770,7 +764,7 @@ def test_shell_cmd_inputs_template_1(): ) # outA has argstr in the metadata fields, so it's a part of the command line # the full path will be use din the command line - assert shelly.cmdline == f"executable inpA -o {str(shelly.output_dir / 'inpA_out')}" + assert shelly.cmdline == f"executable inpA -o {shelly.output_dir / 'inpA_out'}" # checking if outA in the output fields assert shelly.output_names == ["return_code", "stdout", "stderr", "outA"] @@ -810,7 +804,7 @@ def test_shell_cmd_inputs_template_1a(): executable="executable", input_spec=my_input_spec, inpA="inpA" ) # outA has no argstr in metadata, so it's not a part of the command line - assert shelly.cmdline == f"executable inpA" + assert shelly.cmdline == "executable inpA" # TODO: after deciding how we use requires/templates @@ -922,7 +916,7 @@ def test_shell_cmd_inputs_template_3(): # using syntax from the outAB field assert ( shelly.cmdline - == f"executable inpA inpB -o {str(shelly.output_dir / 'inpA_out')} {str(shelly.output_dir / 'inpB_out')}" + == f"executable inpA inpB -o {shelly.output_dir / 'inpA_out'} {str(shelly.output_dir / 'inpB_out')}" ) # checking if outA and outB in the output fields (outAB should not be) assert shelly.output_names == ["return_code", "stdout", "stderr", "outA", "outB"] @@ -1002,7 +996,7 @@ def test_shell_cmd_inputs_template_3a(): # using syntax from the outAB field assert ( shelly.cmdline - == f"executable inpA inpB -o {str(shelly.output_dir / 'inpA_out')} {str(shelly.output_dir / 'inpB_out')}" + == f"executable inpA inpB -o {shelly.output_dir / 'inpA_out'} {str(shelly.output_dir / 'inpB_out')}" ) # checking if outA and outB in the output fields (outAB should not be) assert shelly.output_names == ["return_code", "stdout", "stderr", "outA", "outB"] @@ -1076,7 +1070,7 @@ def test_shell_cmd_inputs_template_4(): executable="executable", input_spec=my_input_spec, inpA="inpA" ) # inpB is not provided so outB not in the command line - assert shelly.cmdline == f"executable inpA -o {str(shelly.output_dir / 'inpA_out')}" + assert shelly.cmdline == f"executable inpA -o {shelly.output_dir / 'inpA_out'}" assert shelly.output_names == ["return_code", "stdout", "stderr", "outA", "outB"] @@ -1151,7 +1145,7 @@ def test_shell_cmd_inputs_template_6(): shelly = ShellCommandTask( executable="executable", input_spec=my_input_spec, inpA="inpA" ) - assert shelly.cmdline == f"executable inpA -o {str(shelly.output_dir / 'inpA_out')}" + assert shelly.cmdline == f"executable inpA -o {shelly.output_dir / 'inpA_out'}" # a string is provided for outA, so this should be used as the outA value shelly = ShellCommandTask( @@ -1163,7 +1157,7 @@ def test_shell_cmd_inputs_template_6(): shelly = ShellCommandTask( executable="executable", input_spec=my_input_spec, inpA="inpA", outA=True ) - assert shelly.cmdline == f"executable inpA -o {str(shelly.output_dir / 'inpA_out')}" + assert shelly.cmdline == f"executable inpA -o {shelly.output_dir / 'inpA_out'}" # False is provided for outA, so the outA shouldn't be used shelly = ShellCommandTask( @@ -1225,7 +1219,7 @@ def test_shell_cmd_inputs_template_6a(): shelly = ShellCommandTask( executable="executable", input_spec=my_input_spec, inpA="inpA", outA=True ) - assert shelly.cmdline == f"executable inpA -o {str(shelly.output_dir / 'inpA_out')}" + assert shelly.cmdline == f"executable inpA -o {shelly.output_dir / 'inpA_out'}" # False is provided for outA, so the outA shouldn't be used shelly = ShellCommandTask( @@ -1278,7 +1272,7 @@ def test_shell_cmd_inputs_template_7(tmpdir): # outA should be formatted in a way that that .txt goes to the end assert ( shelly.cmdline - == f"executable {tmpdir.join('a_file.txt')} {str(shelly.output_dir / 'a_file_out.txt')}" + == f"executable {tmpdir.join('a_file.txt')} {shelly.output_dir / 'a_file_out.txt'}" ) @@ -1327,7 +1321,7 @@ def test_shell_cmd_inputs_template_7a(tmpdir): # outA should be formatted in a way that that .txt goes to the end assert ( shelly.cmdline - == f"executable {tmpdir.join('a_file.txt')} {str(shelly.output_dir / 'a_file_out.txt')}" + == f"executable {tmpdir.join('a_file.txt')} {shelly.output_dir / 'a_file_out.txt'}" ) @@ -1376,7 +1370,7 @@ def test_shell_cmd_inputs_template_7b(tmpdir): # outA should be formatted in a way that that .txt goes to the end assert ( shelly.cmdline - == f"executable {tmpdir.join('a_file.txt')} {str(shelly.output_dir / 'a_file_out')}" + == f"executable {tmpdir.join('a_file.txt')} {shelly.output_dir / 'a_file_out'}" ) @@ -1422,7 +1416,7 @@ def test_shell_cmd_inputs_template_8(tmpdir): # outA should be formatted in a way that inpA extension is removed and the template extension is used assert ( shelly.cmdline - == f"executable {tmpdir.join('a_file.t')} {str(shelly.output_dir / 'a_file_out.txt')}" + == f"executable {tmpdir.join('a_file.t')} {shelly.output_dir / 'a_file_out.txt'}" ) @@ -1482,7 +1476,7 @@ def test_shell_cmd_inputs_template_9(tmpdir): assert ( shelly.cmdline - == f"executable {tmpdir.join('inpA.t')} -i 3 -o {str(shelly.output_dir / 'inpA_3_out.txt')}" + == f"executable {tmpdir.join('inpA.t')} -i 3 -o {shelly.output_dir / 'inpA_3_out.txt'}" ) # checking if outA in the output fields assert shelly.output_names == ["return_code", "stdout", "stderr", "outA"] @@ -1544,7 +1538,7 @@ def test_shell_cmd_inputs_template_9a(tmpdir): assert ( shelly.cmdline - == f"executable {tmpdir.join('inpA.t')} -i hola -o {str(shelly.output_dir / 'inpA_hola_out.txt')}" + == f"executable {tmpdir.join('inpA.t')} -i hola -o {shelly.output_dir / 'inpA_hola_out.txt'}" ) # checking if outA in the output fields assert shelly.output_names == ["return_code", "stdout", "stderr", "outA"] @@ -1714,9 +1708,7 @@ def test_shell_cmd_inputs_template_10(): ) # outA has argstr in the metadata fields, so it's a part of the command line # the full path will be use din the command line - assert ( - shelly.cmdline == f"executable 3.3 -o {str(shelly.output_dir / 'file_3.3_out')}" - ) + assert shelly.cmdline == f"executable 3.3 -o {shelly.output_dir / 'file_3.3_out'}" # checking if outA in the output fields assert shelly.output_names == ["return_code", "stdout", "stderr", "outA"] @@ -1817,7 +1809,7 @@ def template_fun(inputs): executable="executable", input_spec=my_input_spec, inpA="inpA" ) - assert shelly.cmdline == f"executable inpA -o {str(shelly.output_dir / 'inpA_out')}" + assert shelly.cmdline == f"executable inpA -o {shelly.output_dir / 'inpA_out'}" def test_shell_cmd_inputs_template_function_2(): @@ -1880,7 +1872,7 @@ def template_fun(inputs): inpB=1, ) - assert shelly.cmdline == f"executable inpA -o {str(shelly.output_dir / 'inpA_odd')}" + assert shelly.cmdline == f"executable inpA -o {shelly.output_dir / 'inpA_odd'}" def test_shell_cmd_inputs_template_1_st(): @@ -1919,7 +1911,7 @@ def test_shell_cmd_inputs_template_1_st(): ) inpA = ["inpA_1", "inpA_2"] - shelly = ShellCommandTask( + ShellCommandTask( name="f", executable="executable", input_spec=my_input_spec, @@ -1930,7 +1922,7 @@ def test_shell_cmd_inputs_template_1_st(): # assert len(cmdline_list) == 2 # for i in range(2): # path_out = Path(shelly.output_dir[i]) / f"{inpA[i]}_out" - # assert cmdline_list[i] == f"executable {inpA[i]} -o {str(path_out)}" + # assert cmdline_list[i] == f"executable {inpA[i]} -o {path_out}" # TODO: after deciding how we use requires/templates @@ -2129,7 +2121,7 @@ def test_shell_cmd_inputs_di(tmpdir, use_validator): ) assert ( shelly.cmdline - == f"DenoiseImage -i {tmpdir.join('a_file.ext')} -s 1 -p 1 -r 2 -o [{str(shelly.output_dir / 'a_file_out.ext')}]" + == f"DenoiseImage -i {tmpdir.join('a_file.ext')} -s 1 -p 1 -r 2 -o [{shelly.output_dir / 'a_file_out.ext'}]" ) # input file name, noiseImage is set to True, so template is used in the output @@ -2141,7 +2133,7 @@ def test_shell_cmd_inputs_di(tmpdir, use_validator): ) assert ( shelly.cmdline == f"DenoiseImage -i {tmpdir.join('a_file.ext')} -s 1 -p 1 -r 2 " - f"-o [{str(shelly.output_dir / 'a_file_out.ext')}, {str(shelly.output_dir / 'a_file_noise.ext')}]" + f"-o [{shelly.output_dir / 'a_file_out.ext'}, {str(shelly.output_dir / 'a_file_noise.ext')}]" ) # input file name and help_short @@ -2153,7 +2145,7 @@ def test_shell_cmd_inputs_di(tmpdir, use_validator): ) assert ( shelly.cmdline - == f"DenoiseImage -i {tmpdir.join('a_file.ext')} -s 1 -p 1 -r 2 -h -o [{str(shelly.output_dir / 'a_file_out.ext')}]" + == f"DenoiseImage -i {tmpdir.join('a_file.ext')} -s 1 -p 1 -r 2 -h -o [{shelly.output_dir / 'a_file_out.ext'}]" ) assert shelly.output_names == [ @@ -2173,7 +2165,7 @@ def test_shell_cmd_inputs_di(tmpdir, use_validator): ) assert ( shelly.cmdline - == f"DenoiseImage -d 2 -i {tmpdir.join('a_file.ext')} -s 1 -p 1 -r 2 -o [{str(shelly.output_dir / 'a_file_out.ext')}]" + == f"DenoiseImage -d 2 -i {tmpdir.join('a_file.ext')} -s 1 -p 1 -r 2 -o [{shelly.output_dir / 'a_file_out.ext'}]" ) # adding image_dimensionality that has allowed_values [2, 3, 4] and providing 5 - exception should be raised diff --git a/pydra/engine/tests/test_singularity.py b/pydra/engine/tests/test_singularity.py index 2beb7218f5..ed946c992a 100644 --- a/pydra/engine/tests/test_singularity.py +++ b/pydra/engine/tests/test_singularity.py @@ -1,4 +1,4 @@ -import os, shutil +import shutil import subprocess as sp import pytest import attr diff --git a/pydra/engine/tests/test_specs.py b/pydra/engine/tests/test_specs.py index a0af757546..e819334919 100644 --- a/pydra/engine/tests/test_specs.py +++ b/pydra/engine/tests/test_specs.py @@ -6,7 +6,6 @@ BaseSpec, SpecInfo, File, - RuntimeSpec, Runtime, Result, ShellSpec, @@ -38,7 +37,7 @@ def test_result(): assert hasattr(result, "runtime") assert hasattr(result, "output") assert hasattr(result, "errored") - assert getattr(result, "errored") == False + assert getattr(result, "errored") is False def test_shellspec(): @@ -146,7 +145,7 @@ def test_lazy_out(): def test_laxy_errorattr(): with pytest.raises(Exception) as excinfo: tn = NodeTesting() - lf = LazyField(node=tn, attr_type="out") + LazyField(node=tn, attr_type="out") assert "LazyField: Unknown attr_type:" in str(excinfo.value) @@ -168,7 +167,7 @@ def test_input_file_hash_1(tmpdir): inputs(in_file=outfile).hash == "1384a1eb11cd94a5b826a82b948313b9237a0956d406ccff59e79ec92b3c935f" ) - with open(outfile, "wt") as fp: + with open(outfile, "w") as fp: fp.write("test") fields = [("in_file", File)] input_spec = SpecInfo(name="Inputs", fields=fields, bases=(BaseSpec,)) diff --git a/pydra/engine/tests/test_state.py b/pydra/engine/tests/test_state.py index 93a71793f3..c8ef0941ca 100644 --- a/pydra/engine/tests/test_state.py +++ b/pydra/engine/tests/test_state.py @@ -97,13 +97,13 @@ def test_state_1( def test_state_2_err(): with pytest.raises(PydraStateError) as exinfo: - st = State("NA", splitter={"a"}) + State("NA", splitter={"a"}) assert "splitter has to be a string, a tuple or a list" == str(exinfo.value) def test_state_3_err(): with pytest.raises(PydraStateError) as exinfo: - st = State("NA", splitter=["a", "b"], combiner=("a", "b")) + State("NA", splitter=["a", "b"], combiner=("a", "b")) assert "combiner has to be a string or a list" == str(exinfo.value) @@ -516,7 +516,7 @@ def test_state_connect_1a(): def test_state_connect_1b_exception(): """can't provide explicitly NA.a (should be _NA)""" - st1 = State(name="NA", splitter="a", other_states={}) + State(name="NA", splitter="a", other_states={}) st2 = State(name="NB", splitter="NA.a") with pytest.raises(PydraStateError) as excinfo: st2.splitter_validation() @@ -526,7 +526,7 @@ def test_state_connect_1b_exception(): @pytest.mark.parametrize("splitter2, other_states2", [("_NA", {}), ("_N", {"NA": ()})]) def test_state_connect_1c_exception(splitter2, other_states2): """can't ask for splitter from node that is not connected""" - with pytest.raises(PydraStateError) as excinfo: + with pytest.raises(PydraStateError): st2 = State(name="NB", splitter=splitter2, other_states=other_states2) st2.splitter_validation() @@ -1136,7 +1136,7 @@ def test_state_connect_innerspl_1b(): """incorrect splitter - the current & prev-state parts in scalar splitter""" with pytest.raises(PydraStateError): st1 = State(name="NA", splitter="a") - st2 = State(name="NB", splitter=("_NA", "b"), other_states={"NA": (st1, "b")}) + State(name="NB", splitter=("_NA", "b"), other_states={"NA": (st1, "b")}) def test_state_connect_innerspl_2(): @@ -1511,7 +1511,7 @@ def test_state_combine_1(): assert st.splitter_rpn == ["NA.a"] assert st.current_combiner == st.current_combiner_all == st.combiner == ["NA.a"] assert st.prev_state_combiner == st.prev_state_combiner_all == [] - assert st.splitter_final == None + assert st.splitter_final is None assert st.splitter_rpn_final == [] st.prepare_states(inputs={"NA.a": [3, 5]}) @@ -2174,7 +2174,7 @@ def test_connect_splitters( ) def test_connect_splitters_exception_1(splitter, other_states): with pytest.raises(PydraStateError) as excinfo: - st = State(name="CN", splitter=splitter, other_states=other_states) + State(name="CN", splitter=splitter, other_states=other_states) assert "prev-state and current splitters are mixed" in str(excinfo.value) @@ -2191,7 +2191,7 @@ def test_connect_splitters_exception_2(): def test_connect_splitters_exception_3(): with pytest.raises(PydraStateError) as excinfo: - st = State( + State( name="CN", splitter="_NB", other_states=["NA", (State(name="NA", splitter="a"), "b")], diff --git a/pydra/engine/tests/test_submitter.py b/pydra/engine/tests/test_submitter.py index 49b10c2b26..197be0ca52 100644 --- a/pydra/engine/tests/test_submitter.py +++ b/pydra/engine/tests/test_submitter.py @@ -13,11 +13,10 @@ gen_basic_wf_with_threadcount_concurrent, ) from ..core import Workflow -from ..submitter import Submitter, get_runnable_tasks, _list_blocked_tasks +from ..submitter import Submitter from ... import mark from pathlib import Path from datetime import datetime -from pydra.engine.specs import LazyField @mark.task @@ -404,7 +403,6 @@ def test_sge_wf(tmpdir): assert res.output.out == 9 script_dir = tmpdir / "SGEWorker_scripts" assert script_dir.exists() - sdirs = [sd for sd in script_dir.listdir() if sd.isdir()] # ensure each task was executed with sge assert len([sd for sd in script_dir.listdir() if sd.isdir()]) == 2 @@ -521,18 +519,18 @@ def test_sge_limit_maxthreads(tmpdir): sp.run(["qacct", "-j", jobids[3]], capture_output=True).stdout.decode().strip() ) - out_job0_dict = qacct_output_to_dict(out_job0) + qacct_output_to_dict(out_job0) out_job1_dict = qacct_output_to_dict(out_job1) out_job2_dict = qacct_output_to_dict(out_job2) - out_job3_dict = qacct_output_to_dict(out_job3) + qacct_output_to_dict(out_job3) job_1_endtime = datetime.strptime( - out_job1_dict["end_time"][0], f"%a %b %d %H:%M:%S %Y" + out_job1_dict["end_time"][0], "%a %b %d %H:%M:%S %Y" ) # Running both task_1_1 and task_1_2 at once would exceed max_threads, # so task_1_2 waits for task_1_1 to complete job_2_starttime = datetime.strptime( - out_job2_dict["start_time"][0], f"%a %b %d %H:%M:%S %Y" + out_job2_dict["start_time"][0], "%a %b %d %H:%M:%S %Y" ) assert job_1_endtime < job_2_starttime @@ -562,17 +560,17 @@ def test_sge_no_limit_maxthreads(tmpdir): sp.run(["qacct", "-j", jobids[2]], capture_output=True).stdout.decode().strip() ) - out_job0_dict = qacct_output_to_dict(out_job0) + qacct_output_to_dict(out_job0) out_job1_dict = qacct_output_to_dict(out_job1) out_job2_dict = qacct_output_to_dict(out_job2) job_1_endtime = datetime.strptime( - out_job1_dict["end_time"][0], f"%a %b %d %H:%M:%S %Y" + out_job1_dict["end_time"][0], "%a %b %d %H:%M:%S %Y" ) # Running both task_1_1 and task_1_2 at once would not exceed max_threads, # so task_1_2 does not wait for task_1_1 to complete job_2_starttime = datetime.strptime( - out_job2_dict["start_time"][0], f"%a %b %d %H:%M:%S %Y" + out_job2_dict["start_time"][0], "%a %b %d %H:%M:%S %Y" ) assert job_1_endtime > job_2_starttime diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 2762bcf950..678a07eb25 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -4,14 +4,13 @@ import pytest import cloudpickle as cp from pathlib import Path -import re import json import glob as glob from ... import mark from ..core import Workflow from ..task import AuditFlag, ShellCommandTask, DockerTask, SingularityTask from ...utils.messenger import FileMessenger, PrintMessenger, collect_messages -from .utils import gen_basic_wf, use_validator, Submitter +from .utils import gen_basic_wf from ..specs import ( MultiInputObj, MultiOutputObj, @@ -43,10 +42,10 @@ def test_output(): def test_name_conflict(): """raise error if task name conflicts with a class attribute or method""" with pytest.raises(ValueError) as excinfo1: - nn = funaddtwo(name="split", a=3) + funaddtwo(name="split", a=3) assert "Cannot use names of attributes or methods" in str(excinfo1.value) with pytest.raises(ValueError) as excinfo2: - nn = funaddtwo(name="checksum", a=3) + funaddtwo(name="checksum", a=3) assert "Cannot use names of attributes or methods" in str(excinfo2.value) @@ -193,7 +192,7 @@ def testfunc(a: int): return a with pytest.raises(TypeError): - funky = testfunc(a=3.5) + testfunc(a=3.5) def test_annotated_input_func_2a(use_validator): @@ -253,7 +252,7 @@ def testfunc(a: ty.List[int]): return sum(a) with pytest.raises(TypeError): - funky = testfunc(a=[1, 3.5]) + testfunc(a=[1, 3.5]) def test_annotated_input_func_4(use_validator): @@ -286,7 +285,7 @@ def testfunc(a: ty.Dict[str, int]): return sum(a.values()) with pytest.raises(TypeError): - funky = testfunc(a={"el1": 1, "el2": 3.5}) + testfunc(a={"el1": 1, "el2": 3.5}) def test_annotated_input_func_5(use_validator): @@ -313,7 +312,7 @@ def testfunc(a: ty.Dict[str, ty.Dict[str, float]]): return sum(a["el1"]) with pytest.raises(TypeError): - funky = testfunc(a={"el1": [1, 3.5]}) + testfunc(a={"el1": [1, 3.5]}) def test_annotated_input_func_6(use_validator): @@ -339,7 +338,7 @@ def testfunc(a: ty.Dict[str, ty.Union[str, int]]): return sum(a["el1"]) with pytest.raises(TypeError): - funky = testfunc(a={"el1": 1, "el2": 3.5}) + testfunc(a={"el1": 1, "el2": 3.5}) def test_annotated_input_func_7(use_validator): @@ -366,7 +365,7 @@ def testfunc(a: int): return a with pytest.raises(TypeError): - funky = testfunc(a=[3.5, 2.1]).split("a") + testfunc(a=[3.5, 2.1]).split("a") def test_annotated_input_func_8(): @@ -614,7 +613,7 @@ def testfunc(a): bases=(FunctionSpec,), ) with pytest.raises(TypeError): - funky = testfunc(a=3.5, input_spec=my_input_spec) + testfunc(a=3.5, input_spec=my_input_spec) def test_input_spec_func_1b_except(use_validator): @@ -637,7 +636,7 @@ def testfunc(a): bases=(FunctionSpec,), ) with pytest.raises(AttributeError, match="only these keys are supported"): - funky = testfunc(a=3.5, input_spec=my_input_spec) + testfunc(a=3.5, input_spec=my_input_spec) def test_input_spec_func_1d_except(use_validator): @@ -745,7 +744,7 @@ def testfunc(a): ) with pytest.raises(ValueError, match="value of a has to be"): - funky = testfunc(a=3, input_spec=my_input_spec) + testfunc(a=3, input_spec=my_input_spec) def test_input_spec_func_4(use_validator): @@ -867,7 +866,7 @@ def testfunc(a): funky = testfunc(a=3.5, output_spec=my_output_spec) with pytest.raises(TypeError): - res = funky() + funky() def test_output_spec_func_2(use_validator): @@ -1030,7 +1029,7 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] message_path = tmpdir / funky.checksum / "messages" for file in glob(str(message_path) + "/*.jsonld"): - with open(file, "r") as f: + with open(file) as f: data = json.load(f) if "@type" in data: if "AssociatedWith" in data: @@ -1038,9 +1037,9 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] if "@type" in data: if data["@type"] == "input": - assert None == data["Label"] + assert None is data["Label"] if "AssociatedWith" in data: - assert None == data["AssociatedWith"] + assert None is data["AssociatedWith"] # assert any(json_content) @@ -1065,7 +1064,7 @@ def test_audit_shellcommandtask(tmpdir): command_content = [] for file in glob(str(message_path) + "/*.jsonld"): - with open(file, "r") as f: + with open(file) as f: data = json.load(f) if "@type" in data: @@ -1074,7 +1073,7 @@ def test_audit_shellcommandtask(tmpdir): if "@type" in data: if data["@type"] == "input": - assert data["Label"] == None + assert data["Label"] is None if "Command" in data: command_content.append(True) @@ -1148,7 +1147,7 @@ def test_audit_shellcommandtask_file(tmpdir): shelly() message_path = tmpdir / shelly.checksum / "messages" for file in glob.glob(str(message_path) + "/*.jsonld"): - with open(file, "r") as x: + with open(file) as x: data = json.load(x) if "@type" in data: if data["@type"] == "input": @@ -1184,7 +1183,7 @@ def test_audit_shellcommandtask_version(tmpdir): # go through each jsonld file in message_path and check if the label field exists version_content = [] for file in glob.glob(str(message_path) + "/*.jsonld"): - with open(file, "r") as f: + with open(file) as f: data = json.load(f) if "AssociatedWith" in data: if version_cmd in data["AssociatedWith"]: @@ -1501,7 +1500,7 @@ def myhook_postrun_task(task, result, *args): print(f"postrun task hook was called, result object is {result}") def myhook_postrun(task, result, *args): - print(f"postrun hook should not be called") + print("postrun hook should not be called") foo.hooks.post_run = myhook_postrun foo.hooks.post_run_task = myhook_postrun_task @@ -1529,7 +1528,7 @@ def fun_error(x): task = fun_error(name="error", x=[3, 4], cache_dir=tmpdir).split("x") with pytest.raises(Exception, match="from the function") as exinfo: - res = task() + task() # getting error file from the error message error_file_match = str(exinfo.value).split("here: ")[-1].split("_error.pklz")[0] @@ -1558,7 +1557,7 @@ def fun_error(x): wf.set_output([("out", wf.error.lzout.out)]) with pytest.raises(Exception, match="Task error raised an error") as exinfo: - res = wf() + wf() # getting error file from the error message error_file_match = str(exinfo.value).split("here: ")[-1].split("_error.pklz")[0] diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index df11b08a8c..5d46d1d224 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -3,7 +3,6 @@ import time import attr from pathlib import Path -import logging from .utils import ( add2, @@ -71,10 +70,10 @@ def test_wf_specinfo_input_spec(): def test_wf_name_conflict1(): """raise error when workflow name conflicts with a class attribute or method""" with pytest.raises(ValueError) as excinfo1: - wf = Workflow(name="result", input_spec=["x"]) + Workflow(name="result", input_spec=["x"]) assert "Cannot use names of attributes or methods" in str(excinfo1.value) with pytest.raises(ValueError) as excinfo2: - wf = Workflow(name="done", input_spec=["x"]) + Workflow(name="done", input_spec=["x"]) assert "Cannot use names of attributes or methods" in str(excinfo2.value) diff --git a/pydra/engine/tests/utils.py b/pydra/engine/tests/utils.py index b2f3b6652d..a2352764ec 100644 --- a/pydra/engine/tests/utils.py +++ b/pydra/engine/tests/utils.py @@ -10,7 +10,6 @@ from ..submitter import Submitter from ... import mark from ..specs import File -from ... import set_input_validator need_docker = pytest.mark.skipif( shutil.which("docker") is None or sp.call(["docker", "info"]), @@ -297,13 +296,3 @@ def gen_basic_wf_with_threadcount_concurrent(name="basic-wf-with-threadcount"): wf.add(fun_addvar(name="task2", a=wf.task1_1.lzout.out, b=2)) wf.set_output([("out1", wf.task2.lzout.out), ("out2", wf.task1_2.lzout.out)]) return wf - - -@pytest.fixture(scope="function") -def use_validator(request): - set_input_validator(flag=True) - - def fin(): - set_input_validator(flag=False) - - request.addfinalizer(fin) diff --git a/pydra/engine/workers.py b/pydra/engine/workers.py index 152a8680c4..54d78508d3 100644 --- a/pydra/engine/workers.py +++ b/pydra/engine/workers.py @@ -143,7 +143,7 @@ async def exec_serial(self, runnable, rerun=False): async def fetch_finished(self, futures): await asyncio.gather(*futures) - return set([]) + return set() # async def fetch_finished(self, futures): # return await asyncio.wait(futures) @@ -252,12 +252,12 @@ def _prepare_runscripts(self, task, interpreter="/bin/sh", rerun=False): batchscript = script_dir / f"batchscript_{uid}.sh" python_string = ( f"""'from pydra.engine.helpers import load_and_run; """ - f"""load_and_run(task_pkl="{str(task_pkl)}", ind={ind}, rerun={rerun}) '""" + f"""load_and_run(task_pkl="{task_pkl}", ind={ind}, rerun={rerun}) '""" ) bcmd = "\n".join( ( f"#!{interpreter}", - f"#SBATCH --output={str(script_dir / 'slurm-%j.out')}", + f"#SBATCH --output={script_dir / 'slurm-%j.out'}", f"{sys.executable} -c " + python_string, ) ) @@ -812,7 +812,7 @@ async def _submit_job( await asyncio.sleep(self.poll_delay) async def _poll_job(self, jobid, cache_dir): - cmd = (f"qstat", "-j", jobid) + cmd = ("qstat", "-j", jobid) logger.debug(f"Polling job {jobid}") rc, stdout, stderr = await read_and_display_async(*cmd, hide_display=True) @@ -823,7 +823,7 @@ async def _poll_job(self, jobid, cache_dir): return False async def _verify_exit_code(self, jobid): - cmd = (f"qacct", "-j", jobid) + cmd = ("qacct", "-j", jobid) rc, stdout, stderr = await read_and_display_async(*cmd, hide_display=True) if not stdout: await asyncio.sleep(10) diff --git a/pydra/mark/__init__.py b/pydra/mark/__init__.py index 020031de1a..31e4cf832e 100644 --- a/pydra/mark/__init__.py +++ b/pydra/mark/__init__.py @@ -1 +1,3 @@ from .functions import annotate, task + +__all__ = ("annotate", "task") diff --git a/pydra/utils/messenger.py b/pydra/utils/messenger.py index b1c587153c..40e1000672 100644 --- a/pydra/utils/messenger.py +++ b/pydra/utils/messenger.py @@ -196,5 +196,5 @@ def collect_messages(collected_path, message_path, ld_op="compact"): pld.jsonld.from_rdf(pld.jsonld.to_rdf(data, {})), data[0] ) records["@id"] = f"uid:{gen_uuid()}" - with open(collected_path / "messages.jsonld", "wt") as fp: + with open(collected_path / "messages.jsonld", "w") as fp: json.dump(records, fp, ensure_ascii=False, indent=2, sort_keys=False) diff --git a/pyproject.toml b/pyproject.toml index ce7eb465dc..5ac0fe6706 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "cloudpickle >=2.0.0", "etelemetry >=0.2.2", "filelock >=3.0.0", + "importlib_resources >=5.7; python_version < '3.11'", ] license = {file = "LICENSE"} authors = [