[MRG] FIX: Support commas in BrainVision channel names, event types, and event descriptions

doc/changes/latest.inc

@@ -105,6 +105,8 @@ Bugs
 
 - :meth:`mne.Report.parse_folder` will now correctly handle split FIFF files by `Richard Höchenberger`_ (:gh:`8486`, :gh:`8491`)
 
+- Fix bug where BrainVision channel names, event types, and event descriptions containing commas were incorrectly parsed (:gh:`8492` by `Stefan Appelhoff`_)
+
 API changes
 ~~~~~~~~~~~
 

mne/io/brainvision/__init__.py

@@ -1,6 +1,7 @@
-"""Brainvision module for conversion to FIF."""
+"""BrainVision module for conversion to FIF."""
 
 # Author: Teon Brooks <[email protected]>
+#         Stefan Appelhoff <[email protected]>
 #
 # License: BSD (3-clause)
 

mne/io/brainvision/brainvision.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-"""Conversion tool from Brain Vision EEG to FIF."""
+"""Conversion tool from BrainVision EEG to FIF."""
 # Authors: Teon Brooks <[email protected]>
 #          Christian Brodbeck <[email protected]>
 #          Eric Larson <[email protected]>
@@ -220,6 +220,9 @@ def _read_vmrk(fname):
     for info in items:
         info_data = info.split(',')
         mtype, mdesc, this_onset, this_duration = info_data[:4]
+        # commas in mtype and mdesc are handled as "\1". convert back to comma
+        mtype = mtype.replace(r'\1', ',')
+        mdesc = mdesc.replace(r'\1', ',')
         if date_str == '' and len(info_data) == 5 and mtype == 'New Segment':
             # to handle the origin of time and handle the presence of multiple
             # New Segment annotations. We only keep the first one that is
@@ -510,6 +513,8 @@ def _get_vhdr_info(vhdr_fname, eog, misc, scale):
             props[3] = 'µV'
 
         name, _, resolution, unit = props[:4]
+        # in BrainVision, commas in channel names are encoded as "\1"
+        name = name.replace(r'\1', ',')
         ch_dict[chan] = name
         ch_names[n] = name
         if resolution == "":

mne/io/brainvision/tests/data/test.vmrk

@@ -22,4 +22,4 @@ Mk10=Response,R255,6000,1,0
 Mk11=Event,254,6620,1,0
 Mk12=Stimulus,S255,6630,1,0
 Mk13=SyncStatus,Sync On,7630,1,0
-Mk14=Optic,O  1,7700,1,0
+Mk14=Optic,O  1,7700,1,0

mne/io/brainvision/tests/test_brainvision.py

@@ -5,6 +5,7 @@
 #
 # License: BSD (3-clause)
 import os.path as op
+import re
 import shutil
 
 import numpy as np
@@ -272,6 +273,46 @@ def test_ascii(tmpdir):
     assert_allclose(times_new, times)
 
 
+def test_ch_names_comma(tmpdir):
+    """Test that channel names containing commas are properly read."""
+    # commas in BV are encoded as \1
+    replace_dict = {
+        r"^Ch4=F4,": r"Ch4=F4\\1foo,",
+        r"^4\s\s\s\s\sF4": "4     F4,foo ",
+    }
+
+    # Copy existing vhdr file to tmpdir and manipulate to contain
+    # a channel with comma
+    for src, dest in zip((vhdr_path, vmrk_path, eeg_path),
+                         ('test.vhdr', 'test.vmrk', 'test.eeg')):
+        shutil.copyfile(src, tmpdir / dest)
+
+    comma_vhdr = tmpdir / 'test.vhdr'
+    with open(comma_vhdr, 'r') as fin:
+        lines = fin.readlines()
+
+    new_lines = []
+    nperformed_replacements = 0
+    for line in lines:
+        for to_replace, replacement in replace_dict.items():
+            match = re.search(to_replace, line)
+            if match is not None:
+                new = re.sub(to_replace, replacement, line)
+                new_lines.append(new)
+                nperformed_replacements += 1
+                break
+        else:
+            new_lines.append(line)
+    assert nperformed_replacements == len(replace_dict)
+
+    with open(comma_vhdr, 'w') as fout:
+        fout.writelines(new_lines)
+
+    # Read the line containing a "comma channel name"
+    raw = read_raw_brainvision(comma_vhdr)
+    assert "F4,foo" in raw.ch_names
+
+
 def test_brainvision_data_highpass_filters():
     """Test reading raw Brain Vision files with amplifier filter settings."""
     # Homogeneous highpass in seconds (default measurement unit)
@@ -577,32 +618,41 @@ def test_read_vmrk_annotations(tmpdir):
 
 
 @testing.requires_testing_data
-def test_read_vhdr_annotations_and_events():
+def test_read_vhdr_annotations_and_events(tmpdir):
     """Test load brainvision annotations and parse them to events."""
+    # First we add a custom event that contains a comma in its description
+    for src, dest in zip((vhdr_path, vmrk_path, eeg_path),
+                         ('test.vhdr', 'test.vmrk', 'test.eeg')):
+        shutil.copyfile(src, tmpdir / dest)
+
+    # Commas are encoded as "\1"
+    with open(tmpdir / 'test.vmrk', 'a') as fout:
+        fout.write(r"Mk15=Comma\1Type,CommaValue\11,7800,1,0\n")
+
     sfreq = 1000.0
     expected_orig_time = _stamp_to_dt((1384359243, 794232))
     expected_onset_latency = np.array(
         [0, 486., 496., 1769., 1779., 3252., 3262., 4935., 4945., 5999., 6619.,
-         6629., 7629., 7699.]
+         6629., 7629., 7699., 7799.]
     )
     expected_annot_description = [
         'New Segment/', 'Stimulus/S253', 'Stimulus/S255', 'Event/254',
         'Stimulus/S255', 'Event/254', 'Stimulus/S255', 'Stimulus/S253',
         'Stimulus/S255', 'Response/R255', 'Event/254', 'Stimulus/S255',
-        'SyncStatus/Sync On', 'Optic/O  1'
+        'SyncStatus/Sync On', 'Optic/O  1', 'Comma,Type/CommaValue,1'
     ]
     expected_events = np.stack([
         expected_onset_latency,
         np.zeros_like(expected_onset_latency),
         [99999, 253, 255, 254, 255, 254, 255, 253, 255, 1255, 254, 255, 99998,
-         2001],
+         2001, 10001],
     ]).astype('int64').T
     expected_event_id = {'New Segment/': 99999, 'Stimulus/S253': 253,
                          'Stimulus/S255': 255, 'Event/254': 254,
                          'Response/R255': 1255, 'SyncStatus/Sync On': 99998,
-                         'Optic/O  1': 2001}
+                         'Optic/O  1': 2001, 'Comma,Type/CommaValue,1': 10001}
 
-    raw = read_raw_brainvision(vhdr_path, eog=eog)
+    raw = read_raw_brainvision(tmpdir / 'test.vhdr', eog=eog)
 
     # validate annotations
     assert raw.annotations.orig_time == expected_orig_time
@@ -623,7 +673,9 @@ def test_read_vhdr_annotations_and_events():
     # Add some custom ones, plus a 2-digit one
     s_10 = 'Stimulus/S 10'
     raw.annotations.append([1, 2, 3], 10, ['ZZZ', s_10, 'YYY'])
-    expected_event_id.update(YYY=10001, ZZZ=10002)  # others starting at 10001
+    # others starting at 10001 ...
+    # we already have "Comma,Type/CommaValue,1" as 10001
+    expected_event_id.update(YYY=10002, ZZZ=10003)
     expected_event_id[s_10] = 10
     _, event_id = events_from_annotations(raw)
     assert event_id == expected_event_id
@@ -646,7 +698,7 @@ def test_automatic_vmrk_sfreq_recovery():
 @testing.requires_testing_data
 def test_event_id_stability_when_save_and_fif_reload(tmpdir):
     """Test load events from brainvision annotations when read_raw_fif."""
-    fname = op.join(str(tmpdir), 'bv-raw.fif')
+    fname = tmpdir / 'bv-raw.fif'
     raw = read_raw_brainvision(vhdr_path, eog=eog)
     original_events, original_event_id = events_from_annotations(raw)