Ctapipe upgrade to 0.19

pyproject.toml

@@ -18,11 +18,11 @@ classifiers = [
 ]
 requires-python = "~=3.8"
 dependencies = [
-    "astropy~=4.2",
-    "ctapipe~=0.12",
+    "astropy>=5.0",
+    "ctapipe=0.19",
     "ctapipe-io-nectarcam",
-    "numpy~=1.22",
-    "protozfits~=2.0",
+    "numpy>=1.22",
+    "protozfits>=2.0",
     "tables>=3.7",
 ]
 dynamic = ["version"]

src/nectarchain/calibration/container/charge.py

@@ -34,6 +34,7 @@
 from numba import guvectorize, float64, int64, bool_
 
 from .waveforms import WaveformsContainer,WaveformsContainers
+from .utils import CtaPipeExtractor
 
 
 
@@ -131,6 +132,7 @@ def __init__(self,charge_hg,charge_lg,peak_hg,peak_lg,run_number,pixels_id,neven
         self.event_id = np.zeros((self.nevents),dtype = np.uint16)
         self.trig_pattern_all = np.zeros((self.nevents,self.CAMERA.n_pixels,4),dtype = bool)
 
+
     @classmethod
     def from_waveforms(cls,waveformContainer : WaveformsContainer,method : str = "FullWaveformSum",**kwargs) : 
         """ create a new ChargeContainer from a WaveformsContainer
@@ -299,11 +301,12 @@ def compute_charge(waveformContainer : WaveformsContainer,channel : int,method :
 
         log.debug(f"Extracting charges with method {method} and extractor_kwargs {extractor_kwargs}")
         ImageExtractor = eval(method)(waveformContainer.subarray,**extractor_kwargs)
+
         if channel == constants.HIGH_GAIN:
-            out = np.array([ImageExtractor(waveformContainer.wfs_hg[i],waveformContainer.TEL_ID,channel) for i in range(len(waveformContainer.wfs_hg))]).transpose(1,0,2)
+            out = np.array([CtaPipeExtractor.get_image_peak_time(ImageExtractor(waveformContainer.wfs_hg[i],waveformContainer.TEL_ID,channel,waveformContainer.broken_pixels_hg)) for i in range(len(waveformContainer.wfs_hg))]).transpose(1,0,2)
             return out[0],out[1]
         elif channel == constants.LOW_GAIN:
-            out = np.array([ImageExtractor(waveformContainer.wfs_lg[i],waveformContainer.TEL_ID,channel) for i in range(len(waveformContainer.wfs_lg))]).transpose(1,0,2)
+            out = np.array([CtaPipeExtractor.get_image_peak_time(ImageExtractor(waveformContainer.wfs_lg[i],waveformContainer.TEL_ID,channel,waveformContainer.broken_pixels_lg)) for i in range(len(waveformContainer.wfs_lg))]).transpose(1,0,2)
             return out[0],out[1]
         else :
             raise ArgumentError(f"channel must be {constants.LOW_GAIN} or {constants.HIGH_GAIN}")

src/nectarchain/calibration/container/utils.py

@@ -10,6 +10,8 @@
 from pathlib import Path
 from typing import List,Tuple
 
+from ctapipe.containers import DL1CameraContainer 
+
 
 import logging
 logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s')
@@ -180,3 +182,6 @@ def chainEventSource(list : list,max_events : int = None) : #useless with ctapip
         else :
             return ChainGenerator.chain(list[0],ChainGenerator.chainEventSource(list[1:]))
 
+class CtaPipeExtractor():
+    def get_image_peak_time(cameraContainer : DL1CameraContainer) : 
+        return cameraContainer.image, cameraContainer.peak_time

src/nectarchain/calibration/container/waveforms.py

@@ -40,7 +40,7 @@ def __new__(cls,*args,**kwargs) :
         obj = object.__new__(cls)
         return obj
 
-    def __init__(self,run_number : int,max_events : int = None,nevents : int = -1,run_file = None):
+    def __init__(self,run_number : int,max_events : int = None,nevents : int = -1,run_file = None, init_arrays : bool = False):
         """construtor
 
         Args:
@@ -52,7 +52,9 @@ def __init__(self,run_number : int,max_events : int = None,nevents : int = -1,ru
         """
 
         self.__run_number = run_number
-        #gerer ici le fait de traiter plusieurs fichiers ou simplement 1 par 1
+        self.__run_file = run_file
+        self.__max_events = max_events
+
         self.__reader = WaveformsContainer.load_run(run_number,max_events,run_file = run_file)
 
         #from reader members
@@ -68,12 +70,20 @@ def __init__(self,run_number : int,max_events : int = None,nevents : int = -1,ru
         #run properties
         if nevents != -1 :
             self.__nevents = nevents if max_events is None else min(max_events,nevents) #self.check_events()
+            self.__reader = None
         else :
             self.__nevents = self.check_events()
-            #reload file (bc check_events has drained reader generator)
-            self.__reader = WaveformsContainer.load_run(run_number,max_events,run_file = run_file)
+
         log.info(f"N_events : {self.nevents}")
 
+        if init_arrays : 
+            self.__init_arrays()
+
+    def __init_arrays(self,**kwargs) : 
+        log.debug('creation of the EventSource reader')
+        self.__reader = WaveformsContainer.load_run(self.__run_number,self.__max_events,run_file = self.__run_file)
+
+        log.debug("create wfs, ucts, event properties and triger pattern arrays")
         #define zeros members which will be filled therafter
         self.wfs_hg = np.zeros((self.nevents,self.npixels,self.nsamples),dtype = np.uint16)
         self.wfs_lg = np.zeros((self.nevents,self.npixels,self.nsamples),dtype = np.uint16)
@@ -86,6 +96,14 @@ def __init__(self,run_number : int,max_events : int = None,nevents : int = -1,ru
         #self.trig_pattern = np.zeros((self.nevents,self.npixels),dtype = bool)
         #self.multiplicity = np.zeros((self.nevents,self.npixels),dtype = np.uint16)
 
+        self.__broken_pixels_hg = np.zeros((self.npixels),dtype = bool)
+        self.__broken_pixels_lg = np.zeros((self.npixels),dtype = bool)
+
+
+    def __compute_broken_pixels(self,**kwargs) : 
+        log.warning("computation of broken pixels is not yet implemented")
+        self.__broken_pixels_hg = np.zeros((self.npixels),dtype = bool)
+        self.__broken_pixels_lg = np.zeros((self.npixels),dtype = bool)
 
     @staticmethod
     def load_run(run_number : int,max_events : int = None, run_file = None) : 
@@ -98,8 +116,8 @@ def load_run(run_number : int,max_events : int = None, run_file = None) :
         Returns:
             List[ctapipe_io_nectarcam.NectarCAMEventSource]: List of EventSource for each run files
         """
-        generic_filename,filenames = DataManagement.findrun(run_number)
         if run_file is None : 
+            generic_filename,_ = DataManagement.findrun(run_number)
             log.info(f"{str(generic_filename)} will be loaded")
             eventsource = NectarCAMEventSource(input_url=generic_filename,max_events=max_events)
         else :  
@@ -135,6 +153,9 @@ def load_wfs(self,compute_trigger_patern = False):
         Args:
             compute_trigger_patern (bool, optional): To recompute on our side the trigger patern. Defaults to False.
         """
+        if not(hasattr(self, "wfs_hg")) : 
+            self.__init_arrays()
+
         wfs_hg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16)
         wfs_lg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16)
 
@@ -159,7 +180,7 @@ def load_wfs(self,compute_trigger_patern = False):
             self.wfs_hg[i] = wfs_hg_tmp
             self.wfs_lg[i] = wfs_lg_tmp
 
-
+        self.__compute_broken_pixels()
 
         #if compute_trigger_patern and np.max(self.trig_pattern) == 0:
         #    self.compute_trigger_patern()
@@ -262,6 +283,8 @@ def load(path : str) :
             table_trigger = hdul[4].data
             cls.trig_pattern_all = table_trigger["trig_pattern_all"]
 
+        cls.__compute_broken_pixels()
+
         return cls
 
 
@@ -345,6 +368,13 @@ def select_waveforms_lg(self,pixel_id : np.ndarray) :
         res = res.transpose(res.shape[1],res.shape[0],res.shape[2])
         return res
 
+
+    @property 
+    def _run_file(self) : return self.__run_file 
+
+    @property
+    def _max_events(self) : return self.__max_events
+
     @property
     def reader(self) : return self.__reader
 
@@ -369,7 +399,12 @@ def nevents(self) : return self.__nevents
     @property
     def run_number(self) : return self.__run_number
 
+    @property
+    def broken_pixels_hg(self) : return self.__broken_pixels_hg
 
+    @property
+    def broken_pixels_lg(self) : return self.__broken_pixels_lg
+
     #physical properties
     @property
     def multiplicity(self) :  return np.uint16(np.count_nonzero(self.trig_pattern,axis = 1))
@@ -395,7 +430,7 @@ def __new__(cls,*args,**kwargs) :
         obj = object.__new__(cls)
         return obj
 
-    def __init__(self,run_number : int,max_events : int = None) :
+    def __init__(self,run_number : int,max_events : int = None, init_arrays : bool = False) :
         """initialize the waveformsContainer list inside the main object
 
         Args:
@@ -407,7 +442,7 @@ def __init__(self,run_number : int,max_events : int = None) :
         self.waveformsContainer = []
         self.__nWaveformsContainer = 0
         for i,file in enumerate(filenames) : 
-            self.waveformsContainer.append(WaveformsContainer(run_number,max_events=max_events,run_file=file))
+            self.waveformsContainer.append(WaveformsContainer(run_number,max_events=max_events,run_file=file, init_arrays= init_arrays))
             self.__nWaveformsContainer += 1
             if not(max_events is None) : max_events -= self.waveformsContainer[i].nevents
             log.info(f'WaveformsContainer number {i} is created')

src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py

@@ -165,7 +165,7 @@ def load_wfs_charge_split(i,runs_list,max_events,overwrite,charge_childpath,extr
 
     log.info("splitting wafevorms extraction with raw data list files")
     log.debug(f"creation of the WaveformsContainers")
-    wfs = WaveformsContainers(runs_list[i],max_events = max_events[i])
+    wfs = WaveformsContainers(runs_list[i],max_events = max_events[i],init_arrays = False)
     log.info(f"computation of charge with {charge_childpath}")
     log.info("splitting charge computation with raw data list files")
     charge = ChargeContainers()