Always update first capacitors, add more tests

src/ctapipe_io_lst/__init__.py

@@ -52,7 +52,7 @@
     PixelStatus, TriggerBits,
 )
 
-from .evb_preprocessing import get_processings_for_trigger_bits, EVBPreprocessing, EVBPreprocessingFlag
+from .evb_preprocessing import get_processings_for_trigger_bits, EVBPreprocessingFlag
 
 
 __all__ = [
@@ -416,7 +416,7 @@ def scheduling_blocks(self):
     @property
     def datalevels(self):
         if self.cta_r1:
-            if EVBPreprocessing.PE_CALIBRATION in self.evb_preprocessing[TriggerBits.MONO]:
+            if EVBPreprocessingFlag.PE_CALIBRATION in self.evb_preprocessing[TriggerBits.MONO]:
                 return (DataLevel.R1, )
 
         if self.r0_r1_calibrator.calibration_path is not None:
@@ -617,6 +617,8 @@ def _generator(self):
                 self.log.warning('Event with event_id=0 found, skipping')
                 continue
 
+
+
             # container for LST data
             array_event = LSTArrayEventContainer(
                 count=count,
@@ -651,6 +653,7 @@ def _generator(self):
                 self.fill_pointing_info(array_event)
 
             # apply low level corrections
+            self.r0_r1_calibrator.update_first_capacitors(array_event)
             tdp_action = array_event.lst.tel[self.tel_id].evt.tdp_action
             is_calibrated = False
             if tdp_action is not None:

src/ctapipe_io_lst/calibration.py

@@ -222,7 +222,6 @@ def __init__(self, subarray, config=None, parent=None, **kwargs):
             self.mon_data = self._read_calibration_file(self.calibration_path)
 
     def apply_drs4_corrections(self, event: LSTArrayEventContainer):
-        self.update_first_capacitors(event)
 
         for tel_id in event.trigger.tels_with_trigger:
             r1 = event.r1.tel[tel_id]

src/ctapipe_io_lst/tests/test_calib.py

@@ -1,12 +1,14 @@
-import pytest
-import pickle
-from ctapipe_io_lst.constants import HIGH_GAIN
 import os
+from importlib_resources import files, as_file
 from pathlib import Path
-from traitlets.config import Config
+import pickle
+
 import numpy as np
+import pytest
 import tables
-from importlib_resources import files, as_file
+from traitlets.config import Config
+
+from ctapipe_io_lst.constants import HIGH_GAIN
 
 
 resource_dir = files('ctapipe_io_lst') / 'tests/resources'
@@ -299,3 +301,43 @@ def test_spike_positions():
 
     for key, pos in positions.items():
         assert sorted(pos) == sorted(expected_positions[key])
+
+
+def test_calibrate_precalibrated():
+    from ctapipe_io_lst import LSTEventSource
+
+    # file with pe calibration applied by EVB
+    test_file = "20231219/LST-1.1.Run16255.0000_first50.fits.fz"
+
+    path = test_data / "real/R0" / test_file
+    config = Config({
+        'LSTEventSource': {
+            'pointing_information': False,
+            'apply_drs4_corrections': True,
+            'LSTR0Corrections': {
+                'drs4_pedestal_path': test_drs4_pedestal_path,
+                'drs4_time_calibration_path': test_time_calib_path,
+                'calibration_path': test_calib_path,
+            },
+        },
+    })
+
+
+    previous = None
+    with LSTEventSource(path, config=config) as source:
+        n_read = 0
+        for event in source:
+            n_read += 1
+
+            time_shift = event.calibration.tel[1].dl1.time_shift
+            # test we filled the timeshift although the data is pre-calibrated
+            assert time_shift is not None
+            assert np.any(time_shift != 0)
+
+            # check that time_shift varies from event to event due to drs4 time shift
+            # regression test for the bug of not updating first_capacitors
+            if previous is not None and time_shift.shape == previous.shape:
+                assert np.any(time_shift != previous)
+            previous = time_shift
+
+        assert n_read == 200

src/ctapipe_io_lst/tests/test_cta_r1.py

@@ -1,25 +1,28 @@
 import os
+import socket
 from pathlib import Path
 from contextlib import ExitStack
+
 import pytest
 import numpy as np
-from ctapipe.io import EventSource
 from astropy.time import Time
 import astropy.units as u
+from traitlets.config import Config
+
+from ctapipe.io import EventSource
+from ctapipe.image.toymodel import WaveformModel, Gaussian
+from ctapipe.containers import EventType
 
 import protozfits
 from protozfits.R1v1_pb2 import CameraConfiguration, Event, TelescopeDataStream
 from protozfits.R1v1_debug_pb2 import DebugEvent, DebugCameraConfiguration
 from protozfits.CoreMessages_pb2 import AnyArray
-from traitlets.config import Config
+
 from ctapipe_io_lst import LSTEventSource
 from ctapipe_io_lst.anyarray_dtypes import CDTS_AFTER_37201_DTYPE, TIB_DTYPE
-from ctapipe_io_lst.constants import CLOCK_FREQUENCY_KHZ
-from ctapipe.image.toymodel import WaveformModel, Gaussian
-from ctapipe.containers import EventType
-import socket
-
+from ctapipe_io_lst.constants import CLOCK_FREQUENCY_KHZ, TriggerBits
 from ctapipe_io_lst.event_time import time_to_cta_high
+from ctapipe_io_lst.evb_preprocessing import EVBPreprocessingFlag
 
 
 test_data = Path(os.getenv('LSTCHAIN_TEST_DATA', 'test_data'))
@@ -126,6 +129,30 @@ def dummy_cta_r1(dummy_cta_r1_dir):
     ucts_address = np.frombuffer(socket.inet_pton(socket.AF_INET, "10.0.100.123"), dtype=np.uint32)[0]
 
     run_start = Time("2023-05-16T16:06:31.123")
+
+    # trigger dependent processing definition
+    # tdp_type is a list of TriggerBits
+    # tdp_action is the bit pattern of pre-processing steps to apply
+    # to the corresponding tdp_type. 
+    tdp_type = np.zeros(15, dtype=np.uint16)
+    tdp_action = np.zeros(16, dtype=np.uint16)
+
+    # tdp_action[0] is the default for unknown trigger bits 
+    tdp_action[0] = EVBPreprocessingFlag.BASELINE_SUBTRACTION | EVBPreprocessingFlag.GAIN_SELECTION
+    # tdp_action[i] corresponds to tdp_type[i - 1]
+    # physics (mono for now)
+    tdp_type[0] = TriggerBits.MONO
+    tdp_action[1] = EVBPreprocessingFlag.BASELINE_SUBTRACTION | EVBPreprocessingFlag.GAIN_SELECTION
+    # flat field events
+    tdp_type[1] = TriggerBits.CALIBRATION
+    tdp_action[2] = EVBPreprocessingFlag.BASELINE_SUBTRACTION
+    tdp_type[2] = TriggerBits.CALIBRATION | TriggerBits.MONO
+    tdp_action[3] = EVBPreprocessingFlag.BASELINE_SUBTRACTION
+    # pedestal events
+    tdp_type[3] = TriggerBits.PEDESTAL
+    tdp_action[4] = EVBPreprocessingFlag.BASELINE_SUBTRACTION
+
+
     camera_config = CameraConfiguration(
         tel_id=1,
         local_run_id=10000,
@@ -138,12 +165,15 @@ def dummy_cta_r1(dummy_cta_r1_dir):
         num_samples_nominal=num_samples,
         pixel_id_map=to_anyarray(np.arange(num_pixels).astype(np.uint16)),
         module_id_map=to_anyarray(np.arange(num_modules).astype(np.uint16)),
+
         debug=DebugCameraConfiguration(
             cs_serial="???",
             evb_version="evb-dummy",
             cdhs_version="evb-dummy",
-            tdp_type=to_anyarray(np.zeros(15, dtype=np.uint16)),
+            tdp_type=to_anyarray(tdp_type),
             tdp_action=to_anyarray(np.zeros(16, dtype=np.uint16)),
+            # at the moment, the tdp_action and ttype_pattern fields are mixed up in EVB
+            ttype_pattern=to_anyarray(tdp_action),
         )
     )
 
@@ -304,3 +334,27 @@ def test_drs4_calibration(dummy_cta_r1):
 
             n_events += 1
         assert n_events == 100
+
+
+test_files = [
+    "20231214/LST-1.1.Run16102.0000_first50.fits.fz",  # has only baseline enabled
+    "20231218/LST-1.1.Run16231.0000_first50.fits.fz",  # baseline + gain selection for physics
+    "20231219/LST-1.1.Run16255.0000_first50.fits.fz",  # all corrections + gain selection for physics
+]
+
+
+@pytest.mark.parametrize("test_file", test_files)
+def test_read_real_files(test_file):
+    config = Config({
+        'LSTEventSource': {
+            'pointing_information': False,
+            'apply_drs4_corrections': False,
+        },
+    })
+
+    with EventSource(test_data / "real/R0" / test_file, config=config) as source:
+        n_read = 0
+        for _ in source:
+            n_read += 1
+
+        assert n_read == 200