Get tests passing

src/pydoit_nb/serialization.py

@@ -120,7 +120,7 @@ def load_config_from_file(config_file: Path, target: type[T], converter: Convert
 UnstructuredArray: TypeAlias = Union[Sequence[Union[int, float]], Sequence["UnstructuredArray"]]
 
 
-def unstructure_np_array(arr: nptype.NDArray[np.float64]) -> UnstructuredArray:
+def unstructure_np_array(arr: N) -> UnstructuredArray:
     """
     Unstructure :obj:`npt.ArrayLike`
 

src/pydoit_nb/tasks_copy_source.py

@@ -544,8 +544,10 @@ def copy_zenodo_default(in_path: Path, out_path: Path, version: str) -> None:
 
 copy_file_default = swallow_output(shutil.copy2)
 
-copy_tree_default = partial(
-    shutil.copytree,
-    ignore=shutil.ignore_patterns("*.pyc", "__pycache__"),
-    dirs_exist_ok=True,
+copy_tree_default = swallow_output(
+    partial(
+        shutil.copytree,
+        ignore=shutil.ignore_patterns("*.pyc", "__pycache__"),
+        dirs_exist_ok=True,
+    )
 )

tests/integration/test_doit_integration.py

@@ -134,7 +134,7 @@ def test_task_run(example_project_dir, tmp_path_factory):
     # This may require actually parsing the stdout.
     assert f"run_id: {env['EXAMPLE_PROJECT_RUN_ID']}" in res_stdout
 
-    assert False, "Do some other tests of run here?"
+    # Other tests could go in here
 
 
 # - test execute realises that task doesn't need to be re-run

tests/test-data/example-project/README.md

@@ -0,0 +1,3 @@
+# Example project
+
+README normally contains something.

tests/test-data/example-project/dodo.py

@@ -161,6 +161,16 @@ def create_config_bundle(
         "doc": "Generate tasks for the workflow",
     }
 
+    # not worrying about full README checks here
+    def copy_readme(in_path, out_path, run_id, config_file):
+        with open(in_path) as fh:
+            raw = fh.read()
+
+        with open(out_path, "w") as fh:
+            fh.write(raw)
+            fh.write(f"Run ID: {run_id}")
+            fh.write(f"Config file: {config_file}")
+
     yield from generate_all_tasks(
         config_bundle=config_bundle,
         root_dir_raw_notebooks=root_dir_raw_notebooks,
@@ -180,7 +190,8 @@ def create_config_bundle(
             root_dir_raw_notebooks=root_dir_raw_notebooks,
             config_file_raw=configuration_file,
             other_files_to_copy=(),
-            src_dir="src",  # nothing in here for this example, but to illustrate the point
+            src_dir="src",
+            copy_readme=copy_readme,
         ),
     )
 

tests/test-data/example-project/notebooks/001_set-seed.py

@@ -45,7 +45,7 @@
 config = load_config_from_file(config_file)
 config_step = get_config_for_step_id(config=config, step=step, step_config_id=step_config_id)
 
-# %% [markdown]
-# ## Action
+# %%
+config_step.file_seed.parent.mkdir(exist_ok=True, parents=True)
 with open(config_step.file_seed, "w") as fh:
-    fh.write(config_step.seed)
+    fh.write(str(config_step.seed))

tests/test-data/example-project/notebooks/011_make-draws.py

@@ -0,0 +1,80 @@
+# ---
+# jupyter:
+#   jupytext:
+#     text_representation:
+#       extension: .py
+#       format_name: percent
+#       format_version: '1.3'
+#       jupytext_version: 1.16.1
+#   kernelspec:
+#     display_name: Python 3 (ipykernel)
+#     language: python
+#     name: python3
+# ---
+
+# %% [markdown]
+# # Make draws
+#
+# Make draws of data
+
+# %% [markdown]
+# ## Imports
+
+# %%
+import numpy as np
+from local.config import converter_yaml, load_config_from_file
+
+from pydoit_nb.config_handling import get_config_for_step_id
+
+# %% [markdown]
+# ## Define the notebook-based step this notebook belongs to
+
+# %%
+step: str = "make_draws"
+
+# %% [markdown]
+# ## Parameters
+
+# %% editable=true slideshow={"slide_type": ""} tags=["parameters"]
+config_file: str = "dev-config-absolute.yaml"  # config file
+step_config_id: str = "only"  # config ID to select for this branch
+
+# %% [markdown]
+# ## Load config
+
+# %%
+config = load_config_from_file(config_file)
+config_step = get_config_for_step_id(config=config, step=step, step_config_id=step_config_id)
+config_set_seed = get_config_for_step_id(config=config, step="set_seed", step_config_id="only")
+
+# %% [markdown]
+# ## Action
+
+# %%
+N_DRAWS: int = 25
+
+# %%
+with open(config_set_seed.file_seed) as fh:
+    seed = int(fh.read())
+
+seed
+
+# %% [markdown]
+# ### Make draws
+
+# %%
+generator = np.random.Generator(np.random.PCG64(seed))
+
+# %%
+cov = np.array([[0.9, 0.1], [0.1, 0.3]])
+draws = generator.multivariate_normal(
+    mean=np.zeros_like(np.diag(cov)),
+    cov=cov,
+    size=N_DRAWS,
+)
+
+# %%
+with open(config_step.file_draws, "w") as fh:
+    fh.write(converter_yaml.dumps(draws))
+
+config_step.file_draws

tests/test-data/example-project/notebooks/012_scale-draws.py

@@ -0,0 +1,68 @@
+# ---
+# jupyter:
+#   jupytext:
+#     text_representation:
+#       extension: .py
+#       format_name: percent
+#       format_version: '1.3'
+#       jupytext_version: 1.16.1
+#   kernelspec:
+#     display_name: Python 3 (ipykernel)
+#     language: python
+#     name: python3
+# ---
+
+# %% [markdown]
+# # Scale draws
+#
+# Scale the draws
+
+# %% [markdown]
+# ## Imports
+
+# %%
+import numpy as np
+import numpy.typing as nptype
+from local.config import converter_yaml, load_config_from_file
+
+from pydoit_nb.config_handling import get_config_for_step_id
+
+# %% [markdown]
+# ## Define the notebook-based step this notebook belongs to
+
+# %%
+step: str = "make_draws"
+
+# %% [markdown]
+# ## Parameters
+
+# %% editable=true slideshow={"slide_type": ""} tags=["parameters"]
+config_file: str = "dev-config-absolute.yaml"  # config file
+step_config_id: str = "only"  # config ID to select for this branch
+
+# %% [markdown]
+# ## Load config
+
+# %%
+config = load_config_from_file(config_file)
+config_step = get_config_for_step_id(config=config, step=step, step_config_id=step_config_id)
+
+# %% [markdown]
+# ## Action
+
+# %% [markdown]
+# ### Scale draws
+
+# %%
+with open(config_step.file_draws) as fh:
+    draws = converter_yaml.loads(fh.read(), nptype.NDArray[np.float64])
+
+# %%
+scaled = draws * config_step.factor
+
+
+# %%
+with open(config_step.file_draws_scaled, "w") as fh:
+    fh.write(converter_yaml.dumps(scaled))
+
+config_step.file_draws_scaled

tests/test-data/example-project/notebooks/020_retrieve-data.py

@@ -0,0 +1,68 @@
+# ---
+# jupyter:
+#   jupytext:
+#     text_representation:
+#       extension: .py
+#       format_name: percent
+#       format_version: '1.3'
+#       jupytext_version: 1.16.1
+#   kernelspec:
+#     display_name: Python 3 (ipykernel)
+#     language: python
+#     name: python3
+# ---
+
+# %% [markdown]
+# # Retrieve data
+#
+# Retrieve data from source
+
+# %% [markdown]
+# ## Imports
+
+# %%
+from local.config import load_config_from_file
+
+from pydoit_nb.config_handling import get_config_for_step_id
+
+# %% [markdown]
+# ## Define the notebook-based step this notebook belongs to
+
+# %%
+step: str = "retrieve_data"
+
+# %% [markdown]
+# ## Parameters
+
+# %% editable=true slideshow={"slide_type": ""} tags=["parameters"]
+config_file: str = "dev-config-absolute.yaml"  # config file
+step_config_id: str = "only"  # config ID to select for this branch
+
+# %% [markdown]
+# ## Load config
+
+# %%
+config = load_config_from_file(config_file)
+config_step = get_config_for_step_id(config=config, step=step, step_config_id=step_config_id)
+
+# %% [markdown]
+# ## Action
+
+# %% [markdown]
+# ### Retrieve raw data
+
+# %%
+config_step.file_raw_data.parent.mkdir(exist_ok=True, parents=True)
+with open(config_step.file_raw_data, "w") as fh:
+    fh.write(config_step.source)
+
+# %% [markdown]
+# ### Clean data
+
+# %%
+retrieved = f"Retrieved from {config_step.source}"
+
+# %%
+config_step.file_clean_data.parent.mkdir(exist_ok=True, parents=True)
+with open(config_step.file_clean_data, "w") as fh:
+    fh.write(retrieved)

tests/test-data/example-project/notebooks/110_make-plots.py

@@ -0,0 +1,93 @@
+# ---
+# jupyter:
+#   jupytext:
+#     text_representation:
+#       extension: .py
+#       format_name: percent
+#       format_version: '1.3'
+#       jupytext_version: 1.16.1
+#   kernelspec:
+#     display_name: Python 3 (ipykernel)
+#     language: python
+#     name: python3
+# ---
+
+# %% [markdown]
+# # Plot
+#
+# Plot the data
+
+# %% [markdown]
+# ## Imports
+
+# %%
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+import numpy.typing as nptype
+from local.config import converter_yaml, load_config_from_file
+
+from pydoit_nb.config_handling import get_config_for_step_id
+
+# %% [markdown]
+# ## Define the notebook-based step this notebook belongs to
+
+# %%
+step: str = "plot"
+
+# %% [markdown]
+# ## Parameters
+
+# %% editable=true slideshow={"slide_type": ""} tags=["parameters"]
+config_file: str = "dev-config-absolute.yaml"  # config file
+step_config_id: str = "only"  # config ID to select for this branch
+
+# %% [markdown]
+# ## Load config
+
+# %%
+config = load_config_from_file(config_file)
+config_step = get_config_for_step_id(config=config, step=step, step_config_id=step_config_id)
+
+# %% [markdown]
+# ## Action
+
+# %% [markdown]
+# ### Load draws
+
+
+# %%
+def load_file_to_np(fin: Path) -> nptype.NDArray[np.float64]:
+    with open(fin) as fh:
+        read = converter_yaml.loads(fh.read(), nptype.NDArray[np.float64])
+
+    return read
+
+
+loaded = {}
+for md in config.make_draws:
+    loaded[md.step_config_id] = {}
+    loaded[md.step_config_id]["draws"] = load_file_to_np(md.file_draws)
+    loaded[md.step_config_id]["scaled"] = load_file_to_np(md.file_draws_scaled)
+
+# %% [markdown]
+# ### Plot
+
+# %%
+for idx, axis in ((0, "x"), (1, "y")):
+    for k, v in loaded.items():
+        for sk in ["draws", "scaled"]:
+            plt.hist(v[sk][:, idx], label=f"{k} {sk} {axis}", alpha=0.4)
+
+    plt.legend()
+    plt.show()
+
+# %%
+for k, v in loaded.items():
+    for sk in ["draws", "scaled"]:
+        plt.scatter(v[sk][:, 0], v[sk][:, 1], label=f"{k} {sk}", alpha=0.4, color=config_step.colour)
+
+plt.legend()
+config_step.file_plot.parent.mkdir(exist_ok=True, parents=True)
+plt.savefig(config_step.file_plot)

tests/test-data/example-project/pyproject.toml

@@ -11,3 +11,4 @@ packages = [{ include = "local", from = "src" }]
 python = ">=3.9"
 doit = "^0.36.0"
 notebook = "^7.0.0"
+matplotlib = "^3.8.2"