Thermal tests + CI in DQM (#122)

* Adding thermal tests scripts / starting CI for DQM

* Adding script for processing laser thermal data

* adding CI tests for all DQM modules - initial + cleaning code

* fixing the path to the test run file for the CI tests

* adding more CI tests to the pixel_particiapation module

* removing prints

* Adding more CI tests for the remaining DQM modules

---------

Co-authored-by: Jean-Philippe Lenain <[email protected]>

src/nectarchain/dqm/__init__.py

@@ -1,4 +1,5 @@
 from .db_utils import DQMDB
 from .dqm_summary_processor import DQMSummary
+from .pixel_participation import PixelParticipationHighLowGain
 
-__all__ = ["DQMDB", "DQMSummary"]
+__all__ = ["DQMDB", "DQMSummary", "PixelParticipationHighLowGain"]

src/nectarchain/dqm/camera_monitoring.py

@@ -4,11 +4,11 @@
 import numpy as np
 from astropy import time as astropytime
 from ctapipe.coordinates import EngineeringCameraFrame
-from ctapipe.instrument import CameraGeometry
 from ctapipe.visualization import CameraDisplay
-from dqm_summary_processor import DQMSummary
 from matplotlib import pyplot as plt
 
+from nectarchain.dqm.dqm_summary_processor import DQMSummary
+
 
 class CameraMonitoring(DQMSummary):
     def __init__(self, gaink):
@@ -139,7 +139,6 @@ def GetResults(self):
         return self.CameraMonitoring_Results_Dict
 
     def PlotResults(self, name, FigPath):
-
         try:
             fig, disp = plt.subplots()
             disp = CameraDisplay(self.camera)

src/nectarchain/dqm/charge_integration.py

@@ -2,12 +2,12 @@
 import numpy as np
 from ctapipe.coordinates import EngineeringCameraFrame
 from ctapipe.image import LocalPeakWindowSum
-from ctapipe.instrument import CameraGeometry
 from ctapipe.visualization import CameraDisplay
-from dqm_summary_processor import DQMSummary
 from matplotlib import pyplot as plt
 from traitlets.config.loader import Config
 
+from nectarchain.dqm.dqm_summary_processor import DQMSummary
+
 
 class ChargeIntegrationHighLowGain(DQMSummary):
     def __init__(self, gaink):
@@ -20,21 +20,31 @@ def __init__(self, gaink):
         self.peakpos_all = []
         self.image_ped = []
         self.peakpos_ped = []
-        self.camera  = None
+        self.ped_all = []
+        self.ped_ped = []
+        self.camera = None
         self.cmap = None
-        self.subarray  = None
+        self.subarray = None
         self.integrator = None
-        self.pixelBAD  = None
+        self.pixelBAD = None
         self.image_all = []
         self.image_all_median = None
         self.image_all_average = None
         self.image_all_std = None
         self.image_all_rms = None
+        self.ped_all_median = None
+        self.ped_all_average = None
+        self.ped_all_std = None
+        self.ped_all_rms = None
         self.image_ped = []
         self.image_ped_median = None
         self.image_ped_average = None
         self.image_ped_std = None
         self.image_ped_rms = None
+        self.ped_ped_median = None
+        self.ped_ped_average = None
+        self.ped_ped_std = None
+        self.ped_ped_rms = None
         self.ChargeInt_Results_Dict = {}
         self.ChargeInt_Figures_Dict = {}
         self.ChargeInt_Figures_Names_Dict = {}
@@ -50,7 +60,7 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1):
         self.camera = Reader1.subarray.tel[0].camera.geometry.transform_to(
             EngineeringCameraFrame()
         )
-        
+
         self.cmap = "gnuplot2"
 
         self.subarray = Reader1.subarray
@@ -64,8 +74,6 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1):
 
         self.integrator = LocalPeakWindowSum(subarray, config=config)
 
-
-
     def ProcessEvent(self, evt, noped):
         self.pixelBAD = evt.mon.tel[0].pixel_status.hardware_failing_pixels
         pixel = evt.nectarcam.tel[0].svc.pixel_ids
@@ -80,8 +88,9 @@ def ProcessEvent(self, evt, noped):
 
         waveform = evt.r0.tel[0].waveform[self.k]
 
+        ped = np.mean(waveform[:, 20])
+
         if noped:
-            ped = np.mean(waveform[:, 20])
             w_noped = waveform - ped
             output = self.integrator(
                 w_noped, 0, np.zeros(self.Pix, dtype=int), self.pixelBAD
@@ -104,10 +113,12 @@ def ProcessEvent(self, evt, noped):
             self.counter_ped += 1
             self.image_ped.append(image)
             self.peakpos_ped.append(peakpos)
+            self.ped_ped.append(ped)
         else:
             self.counter_evt += 1
             self.image_all.append(image)
             self.peakpos_all.append(peakpos)
+            self.ped_all.append(ped)
 
     def FinishRun(self):
         self.peakpos_all = np.array(self.peakpos_all, dtype=float)
@@ -121,15 +132,26 @@ def FinishRun(self):
         self.image_all_std = np.std(self.image_all, axis=0)
         self.image_all_rms = np.sqrt(np.sum(self.image_all**2, axis=0))
 
+        self.ped_all = np.array(self.ped_all, dtype=float)
+        self.ped_all_average = np.mean(self.ped_all, axis=0)
+        self.ped_all_median = np.median(self.ped_all, axis=0)
+        self.ped_all_std = np.std(self.ped_all, axis=0)
+        self.ped_all_rms = np.sqrt(np.sum(self.ped_all**2, axis=0))
+
         if self.counter_ped > 0:
             self.image_ped = np.array(self.image_ped, dtype=float)
             self.image_ped_median = np.median(self.image_ped, axis=0)
             self.image_ped_average = np.mean(self.image_ped, axis=0)
             self.image_ped_std = np.std(self.image_ped, axis=0)
             self.image_ped_rms = np.sqrt(np.sum(self.image_ped**2, axis=0))
 
-    def GetResults(self):
+            self.ped_ped = np.array(self.ped_ped, dtype=float)
+            self.ped_ped_average = np.mean(self.ped_ped, axis=0)
+            self.ped_ped_median = np.median(self.ped_ped, axis=0)
+            self.ped_ped_std = np.std(self.ped_ped, axis=0)
+            self.ped_ped_rms = np.sqrt(np.sum(self.ped_ped**2, axis=0))
 
+    def GetResults(self):
         if self.k == 0:
             self.ChargeInt_Results_Dict[
                 "CHARGE-INTEGRATION-IMAGE-ALL-AVERAGE-HIGH-GAIN"
@@ -144,6 +166,19 @@ def GetResults(self):
                 "CHARGE-INTEGRATION-IMAGE-ALL-STD-HIGH-GAIN"
             ] = self.image_all_std
 
+            self.ChargeInt_Results_Dict[
+                "PED-INTEGRATION-IMAGE-ALL-AVERAGE-HIGH-GAIN"
+            ] = self.ped_all_average
+            self.ChargeInt_Results_Dict[
+                "PED-INTEGRATION-IMAGE-ALL-MEDIAN-HIGH-GAIN"
+            ] = self.ped_all_median
+            self.ChargeInt_Results_Dict[
+                "PED-INTEGRATION-IMAGE-ALL-RMS-HIGH-GAIN"
+            ] = self.ped_all_rms
+            self.ChargeInt_Results_Dict[
+                "PED-INTEGRATION-IMAGE-ALL-STD-HIGH-GAIN"
+            ] = self.ped_all_std
+
             if self.counter_ped > 0:
                 self.ChargeInt_Results_Dict[
                     "CHARGE-INTEGRATION-PED-ALL-AVERAGE-HIGH-GAIN"
@@ -158,6 +193,19 @@ def GetResults(self):
                     "CHARGE-INTEGRATION-PED-ALL-STD-HIGH-GAIN"
                 ] = self.image_ped_std
 
+                self.ChargeInt_Results_Dict[
+                    "PED-INTEGRATION-PED-ALL-AVERAGE-HIGH-GAIN"
+                ] = self.ped_ped_average
+                self.ChargeInt_Results_Dict[
+                    "PED-INTEGRATION-PED-ALL-MEDIAN-HIGH-GAIN"
+                ] = self.ped_ped_median
+                self.ChargeInt_Results_Dict[
+                    "PED-INTEGRATION-PED-ALL-RMS-HIGH-GAIN"
+                ] = self.ped_ped_rms
+                self.ChargeInt_Results_Dict[
+                    "PED-INTEGRATION-PED-ALL-STD-HIGH-GAIN"
+                ] = self.ped_ped_std
+
         if self.k == 1:
             self.ChargeInt_Results_Dict[
                 "CHARGE-INTEGRATION-IMAGE-ALL-AVERAGE-LOW-GAIN"
@@ -172,6 +220,19 @@ def GetResults(self):
                 "CHARGE-INTEGRATION-IMAGE-ALL-STD-LOW-GAIN"
             ] = self.image_all_std
 
+            self.ChargeInt_Results_Dict[
+                "PED-INTEGRATION-IMAGE-ALL-AVERAGE-LOW-GAIN"
+            ] = self.ped_all_average
+            self.ChargeInt_Results_Dict[
+                "PED-INTEGRATION-IMAGE-ALL-MEDIAN-LOW-GAIN"
+            ] = self.ped_all_median
+            self.ChargeInt_Results_Dict[
+                "PED-INTEGRATION-IMAGE-ALL-RMS-LOW-GAIN"
+            ] = self.ped_all_rms
+            self.ChargeInt_Results_Dict[
+                "PED-INTEGRATION-IMAGE-ALL-STD-LOW-GAIN"
+            ] = self.ped_all_std
+
             if self.counter_ped > 0:
                 self.ChargeInt_Results_Dict[
                     "CHARGE-INTEGRATION-PED-ALL-AVERAGE-LOW-GAIN"
@@ -186,10 +247,22 @@ def GetResults(self):
                     "CHARGE-INTEGRATION-PED-ALL-STD-LOW-GAIN"
                 ] = self.image_ped_std
 
+                self.ChargeInt_Results_Dict[
+                    "PED-INTEGRATION-PED-ALL-AVERAGE-LOW-GAIN"
+                ] = self.ped_ped_average
+                self.ChargeInt_Results_Dict[
+                    "PED-INTEGRATION-PED-ALL-MEDIAN-LOW-GAIN"
+                ] = self.ped_ped_median
+                self.ChargeInt_Results_Dict[
+                    "PED-INTEGRATION-PED-ALL-RMS-LOW-GAIN"
+                ] = self.ped_ped_rms
+                self.ChargeInt_Results_Dict[
+                    "PED-INTEGRATION-PED-ALL-STD-LOW-GAIN"
+                ] = self.ped_ped_std
+
         return self.ChargeInt_Results_Dict
 
     def PlotResults(self, name, FigPath):
-
         # titles = ['All', 'Pedestals']
         if self.k == 0:
             gain_c = "High"

src/nectarchain/dqm/mean_camera_display.py

@@ -1,11 +1,12 @@
 import numpy as np
 from ctapipe.coordinates import EngineeringCameraFrame
 from ctapipe.visualization import CameraDisplay
-from dqm_summary_processor import DQMSummary
 from matplotlib import pyplot as plt
 
+from nectarchain.dqm.dqm_summary_processor import DQMSummary
 
-class MeanCameraDisplay_HighLowGain(DQMSummary):
+
+class MeanCameraDisplayHighLowGain(DQMSummary):
     def __init__(self, gaink):
         self.k = gaink
         self.Pix = None
@@ -22,7 +23,7 @@ def __init__(self, gaink):
         self.CameraAverage_ped1 = []
         self.CameraAverage_overEvents = None
         self.CameraAverage_overEvents_overSamp = None
-        self.CameraAverage_ped_overEvents  = None
+        self.CameraAverage_ped_overEvents = None
         self.CameraAverage_ped_overEvents_overSamp = None
         self.MeanCameraDisplay_Results_Dict = {}
         self.MeanCameraDisplay_Figures_Dict = {}
@@ -42,7 +43,6 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1):
 
         self.cmap = "gnuplot2"
 
-
     def ProcessEvent(self, evt, noped):
         self.pixelBAD = evt.mon.tel[0].pixel_status.hardware_failing_pixels
         pixel = evt.nectarcam.tel[0].svc.pixel_ids
@@ -86,7 +86,6 @@ def FinishRun(self):
             )
 
     def GetResults(self):
-
         # ASSIGN RESUTLS TO DICT
         if self.k == 0:
             if self.counter_evt > 0:
@@ -125,7 +124,6 @@ def GetResults(self):
         return self.MeanCameraDisplay_Results_Dict
 
     def PlotResults(self, name, FigPath):
-
         # titles = ['All', 'Pedestals']
         if self.k == 0:
             gain_c = "High"

src/nectarchain/dqm/mean_waveforms.py

@@ -1,7 +1,8 @@
 import numpy as np
-from dqm_summary_processor import DQMSummary
 from matplotlib import pyplot as plt
 
+from nectarchain.dqm.dqm_summary_processor import DQMSummary
+
 
 class MeanWaveFormsHighLowGain(DQMSummary):
     def __init__(self, gaink):
@@ -70,7 +71,6 @@ def FinishRun(self):
         return None
 
     def GetResults(self):
-
         # ASSIGN RESUTLS TO DICT
         if self.k == 0:
             self.MeanWaveForms_Results_Dict[
@@ -93,7 +93,6 @@ def GetResults(self):
         return self.MeanWaveForms_Results_Dict
 
     def PlotResults(self, name, FigPath):
-
         wf_list = np.array(self.wf_list_plot)
 
         counter_fig = 0

src/nectarchain/dqm/pixel_participation.py

@@ -1,9 +1,12 @@
 import numpy as np
 from ctapipe.coordinates import EngineeringCameraFrame
 from ctapipe.visualization import CameraDisplay
-from dqm_summary_processor import DQMSummary
 from matplotlib import pyplot as plt
 
+from nectarchain.dqm.dqm_summary_processor import DQMSummary
+
+__all__ = ["PixelParticipationHighLowGain"]
+
 
 class PixelParticipationHighLowGain(DQMSummary):
     def __init__(self, gaink):

src/nectarchain/dqm/pixel_timeline.py

@@ -1,7 +1,8 @@
 import numpy as np
-from dqm_summary_processor import DQMSummary
 from matplotlib import pyplot as plt
 
+from nectarchain.dqm.dqm_summary_processor import DQMSummary
+
 
 class PixelTimelineHighLowGain(DQMSummary):
     def __init__(self, gaink):

src/nectarchain/dqm/start_dqm.py

@@ -9,7 +9,7 @@
 from ctapipe_io_nectarcam.constants import HIGH_GAIN, LOW_GAIN
 from db_utils import DQMDB
 from matplotlib import pyplot as plt
-from mean_camera_display import MeanCameraDisplay_HighLowGain
+from mean_camera_display import MeanCameraDisplayHighLowGain
 from mean_waveforms import MeanWaveFormsHighLowGain
 from pixel_participation import PixelParticipationHighLowGain
 from pixel_timeline import PixelTimelineHighLowGain
@@ -145,8 +145,8 @@ def CreateFigFolder(name, type):
 a = TriggerStatistics(HIGH_GAIN)
 b = MeanWaveFormsHighLowGain(HIGH_GAIN)
 c = MeanWaveFormsHighLowGain(LOW_GAIN)
-d = MeanCameraDisplay_HighLowGain(HIGH_GAIN)
-e = MeanCameraDisplay_HighLowGain(LOW_GAIN)
+d = MeanCameraDisplayHighLowGain(HIGH_GAIN)
+e = MeanCameraDisplayHighLowGain(LOW_GAIN)
 f = ChargeIntegrationHighLowGain(HIGH_GAIN)
 g = ChargeIntegrationHighLowGain(LOW_GAIN)
 h = CameraMonitoring(HIGH_GAIN)

src/nectarchain/dqm/tests/test_camera_monitoring.py

@@ -0,0 +1,46 @@
+from astropy import time as astropytime
+from ctapipe.io import EventSource
+from ctapipe.utils import get_dataset_path
+from ctapipe_io_nectarcam.constants import HIGH_GAIN
+from tqdm import tqdm
+from traitlets.config import Config
+
+from nectarchain.dqm.camera_monitoring import CameraMonitoring
+
+
+class TestCameraMonitoring:
+    run_number = 3798
+    max_events = 1
+
+    def test_camera_monitoring(self):
+        # run_number = 3938
+        path = get_dataset_path("NectarCAM.Run3938.30events.fits.fz")
+
+        config = None
+
+        config = Config(
+            dict(
+                NectarCAMEventSource=dict(
+                    NectarCAMR0Corrections=dict(
+                        calibration_path=None,
+                        apply_flatfield=False,
+                        select_gain=False,
+                    )
+                )
+            )
+        )
+        print(path)
+
+        reader1 = EventSource(input_url=path, config=config, max_events=1)
+
+        Pix, Samp = CameraMonitoring(HIGH_GAIN).DefineForRun(reader1)
+
+        CameraMonitoring(HIGH_GAIN).ConfigureForRun(path, Pix, Samp, reader1)
+
+        for evt in tqdm(reader1, total=1):
+            run_start1 = evt.nectarcam.tel[0].svc.date
+            SqlFileDate = astropytime.Time(run_start1, format="unix").iso.split(" ")[0]
+        # print("SqlFileDate", SqlFileDate)
+        # CameraMonitoring(HIGH_GAIN).FinishRun()
+        assert Pix + Samp == 1915
+        assert SqlFileDate == "2023-01-23"