Move to creating venv in tests

.gitignore

@@ -1,6 +1,6 @@
 # Test example output and DB
 tests/test-data/example-project/output-bundles
-tests/test-data/example-project/test-task-display.json
+tests/test-data/example-project/*.json
 
 # Notebooks
 *.ipynb

pyproject.toml

@@ -16,7 +16,6 @@ numpy = "^1.26.3"
 cattrs = "^23.2.3"
 
 [tool.poetry.extras]
-extras = ["numpy"]
 notebooks = ["notebook"]
 
 [tool.poetry.group.tests.dependencies]

tests/integration/test_doit_integration.py

@@ -11,6 +11,8 @@
 
 import pytest
 
+ROOT_DIR = Path(__file__).parent.parent.parent
+
 T = TypeVar("T")
 
 
@@ -27,11 +29,72 @@ def setup_doit_env_vars(test_name: str, env: T) -> T:
     return env
 
 
+def assert_venv_dir_in_output(venv_dir_exp, output):
+    assert str(venv_dir_exp / ".venv") in output, "venv set incorrectly"
+
+
+def assert_tool_in_venv(tool, venv_dir, env):
+    res_subprocess = subprocess.run(
+        ("poetry", "run", "which", tool),  # noqa: S603
+        cwd=venv_dir,
+        env=env,
+        stdout=subprocess.PIPE,
+        check=False,
+    )
+
+    assert_venv_dir_in_output(venv_dir, res_subprocess.stdout.decode())
+
+
+def assert_using_expected_venv(venv_dir_exp, env):
+    res_venv = subprocess.run(
+        ("poetry", "env", "info"),  # noqa: S603
+        cwd=venv_dir_exp,
+        env=env,
+        stdout=subprocess.PIPE,
+        check=True,
+    )
+    assert_venv_dir_in_output(venv_dir_exp, res_venv.stdout.decode())
+
+
+def setup_venv(venv_dir, env, remove_lock_if_added: bool = True):
+    try:
+        del env["VIRTUAL_ENV"]
+    except KeyError:
+        pass
+
+    lock_file = venv_dir / "poetry.lock"
+    lock_exists = lock_file.exists()
+    subprocess.run(("poetry", "config", "virtualenvs.in-project", "true"), cwd=venv_dir, env=env, check=True)  # noqa: S603
+    subprocess.run(("poetry", "install", "--all-extras"), cwd=venv_dir, env=env, check=True)  # noqa: S603
+
+    assert_tool_in_venv("pip", venv_dir, env)
+
+    subprocess.run(
+        ("poetry", "run", "pip", "install", "-e", str(ROOT_DIR)),  # noqa: S603
+        cwd=venv_dir,
+        env=env,
+        stdout=subprocess.PIPE,
+        check=False,
+    )
+
+    assert_using_expected_venv(venv_dir, env)
+
+    if not lock_exists and remove_lock_if_added:
+        # remove the lock file which was made by creating the virtual environment
+        lock_file.unlink()
+
+    return env
+
+
 def test_task_display(example_project_dir, tmp_path_factory, file_regression):
-    env = setup_doit_env_vars("test-task-display", copy.deepcopy(os.environ))
+    env = copy.deepcopy(os.environ)
+    env = setup_venv(example_project_dir, env)
+    env = setup_doit_env_vars("test-task-display", env)
+
+    assert_tool_in_venv("doit", example_project_dir, env)
 
     res = subprocess.run(
-        ("doit", "list", "--all", "--status"),  # noqa: S603
+        ("poetry", "run", "doit", "list", "--all", "--status"),  # noqa: S603
         cwd=example_project_dir,
         env=env,  # may need virtual env setup here too
         stdout=subprocess.PIPE,
@@ -50,10 +113,14 @@ def test_task_display(example_project_dir, tmp_path_factory, file_regression):
 
 def test_task_run(example_project_dir, tmp_path_factory):
     # No regression test here at the moment. Maybe we'll at that in if it seems helpful.
-    env = setup_doit_env_vars("test-task-display", copy.deepcopy(os.environ))
+    env = copy.deepcopy(os.environ)
+    env = setup_venv(example_project_dir, env)
+    env = setup_doit_env_vars("test-task-run", env)
+
+    assert_tool_in_venv("doit", example_project_dir, env)
 
     res = subprocess.run(
-        ("doit", "run", "--verbosity=2"),  # noqa: S603
+        ("poetry", "run", "doit", "run", "--verbosity=2"),  # noqa: S603
         cwd=example_project_dir,
         env=env,  # may need virtual env setup here too
         stdout=subprocess.PIPE,

tests/test-data/example-project/dodo.py

@@ -44,13 +44,8 @@
 from functools import partial
 from pathlib import Path
 
-import local_make_draws
-import local_plot
-import local_retrieve_data
-
-# Not great as package isn't installed, but fine for testing
-import local_set_seed
-from local_config import Config, ConfigBundle, converter_yaml, load_config_from_file
+from local.config import Config, ConfigBundle, converter_yaml, load_config_from_file
+from local.notebook_steps import make_draws, plot, retrieve_data, set_seed
 
 from pydoit_nb.config_handling import load_hydrate_write_config_bundle
 from pydoit_nb.display import gen_show_configuration_task
@@ -172,10 +167,10 @@ def create_config_bundle(
         converter=converter_yaml,
         # Lots of control given through these bits
         step_defining_modules=[
-            local_set_seed,
-            local_make_draws,
-            local_retrieve_data,
-            local_plot,
+            set_seed,
+            make_draws,
+            retrieve_data,
+            plot,
         ],
         gen_zenodo_bundle_task=partial(
             gen_copy_source_into_output_tasks,

tests/test-data/example-project/notebooks/001_set-seed.py

@@ -22,7 +22,8 @@
 
 # %%
 from local.config import load_config_from_file
-from local.pydoit_nb.config_handling import get_config_for_step_id
+
+from pydoit_nb.config_handling import get_config_for_step_id
 
 # %% [markdown]
 # ## Define the notebook-based step this notebook belongs to

tests/test-data/example-project/pyproject.toml

@@ -0,0 +1,13 @@
+[tool.poetry]
+name = "local"
+version = "0.0.0"
+description = "Local code for example notebook running workflow"
+authors = [
+    "Zebedee Nicholls <[email protected]>",
+]
+packages = [{ include = "local", from = "src" }]
+
+[tool.poetry.dependencies]
+python = ">=3.9"
+doit = "^0.36.0"
+notebook = "^7.0.0"

tests/test-data/example-project/src/local/notebook_steps/__init__.py

@@ -0,0 +1,6 @@
+"""
+Notebook steps
+
+I have typically set these up in modules, but could be done other ways of
+course.
+"""