Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Renamed failing test, added prospective use case for NWBI warning. #1162

Merged
merged 5 commits into from
Nov 24, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 18 additions & 7 deletions dandi/cli/tests/test_cmd_validate.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,21 @@ def test_validate_bids_error(bids_error_examples, dataset):
assert key in r.output


def test_validate_nwb_error(simple3_nwb):
"""
Do we fail on critical NWB validation errors?
"""

r = CliRunner().invoke(validate, [simple3_nwb])
# does it fail? as per:
# https://github.com/dandi/dandi-cli/pull/1157#issuecomment-1312546812
assert r.exit_code != 0


def test_validate_bids_grouping_error(bids_error_examples, dataset="invalid_asl003"):
"""
This is currently a placeholder test, and should be updated once we have paths with
multiple errors.
multiple errors for which grouping functionality can actually be tested.
"""

dataset = os.path.join(bids_error_examples, dataset)
Expand All @@ -39,18 +50,18 @@ def test_validate_bids_grouping_error(bids_error_examples, dataset="invalid_asl0
assert dataset in r.output


def test_validate_nwb_path_grouping(simple3_nwb):
def test_validate_nwb_path_grouping(simple4_nwb):
"""
This is currently a placeholder test, and should be updated once we have paths with
multiple errors.
multiple errors for which grouping functionality can actually be tested.
"""

r = CliRunner().invoke(validate, ["--grouping=path", simple3_nwb])
assert r.exit_code != 0
r = CliRunner().invoke(validate, ["--grouping=path", simple4_nwb])
assert r.exit_code == 0

# Does it give required warnings for required path?
assert simple3_nwb in r.output
assert "NWBI.check_subject_id_exists" in r.output
assert simple4_nwb in r.output
assert "NWBI.check_data_orientation" in r.output


def test_validate_bids_error_grouping_notification(
Expand Down
10 changes: 5 additions & 5 deletions dandi/files/bases.py
Original file line number Diff line number Diff line change
Expand Up @@ -505,12 +505,12 @@ def get_validation_errors(
nwbfile_path=self.filepath,
skip_validate=True,
config=load_config(filepath_or_keyword="dandi"),
importance_threshold=Importance.CRITICAL,
importance_threshold=Importance.BEST_PRACTICE_VIOLATION,
# we might want to switch to a lower threshold once nwbinspector
# upstream reporting issues are clarified:
# https://github.com/dandi/dandi-cli/pull/1162#issuecomment-1322238896
# importance_threshold=Importance.BEST_PRACTICE_SUGGESTION,
):
# unfortunate name collision; the InspectorMessage.severity is used only for
# minor ordering of final report - will have to remove the
# `importance_threshold` currently used to filter DANDI-level CRITICAL
# evaluations as errors vs. validation results
severity = NWBI_IMPORTANCE_TO_DANDI_SEVERITY[error.importance.name]
kw: Any = {}
if error.location:
Expand Down
41 changes: 40 additions & 1 deletion dandi/tests/fixtures.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from __future__ import annotations

from dataclasses import dataclass, field
from datetime import datetime
from datetime import datetime, timezone
import logging
import os
from pathlib import Path
Expand Down Expand Up @@ -150,6 +150,45 @@ def simple3_nwb(
)


@pytest.fixture(scope="session")
def simple4_nwb(
simple1_nwb_metadata: Dict[str, Any], tmp_path_factory: pytest.TempPathFactory
) -> str:
"""
With, subject, subject_id, species, but including data orientation ambiguity,
the only currently non-critical issue in the dandi schema for nwbinspector validation:
NWBI.check_data_orientation
https://github.com/NeurodataWithoutBorders/nwbinspector/blob/
54ac2bc7cdcb92802b9251e29f249f155fb1ff52
/src/nwbinspector/internal_configs/dandi.inspector_config.yaml#L10
"""

start_time = datetime(2017, 4, 3, 11, tzinfo=timezone.utc)
time_series = pynwb.TimeSeries(
name="test_time_series",
unit="test_units",
data=np.zeros(shape=(2, 100)),
rate=1.0,
)

nwbfile = NWBFile(
session_description="some session",
identifier="NWBE4",
session_start_time=start_time,
subject=Subject(
subject_id="mouse001",
age="P1D/",
sex="O",
species="Mus musculus",
),
)
nwbfile.add_acquisition(time_series)
filename = str(tmp_path_factory.mktemp("simple4") / "simple4.nwb")
with pynwb.NWBHDF5IO(filename, "w") as io:
io.write(nwbfile, cache_spec=False)
return filename


@pytest.fixture(scope="session")
def organized_nwb_dir(
simple2_nwb: str, tmp_path_factory: pytest.TempPathFactory
Expand Down