From 7554a70f8e36d3a3a393a7c561637a41e18e2735 Mon Sep 17 00:00:00 2001 From: hashkar <43830561+hashkar@users.noreply.github.com> Date: Thu, 22 Aug 2024 23:16:44 +0200 Subject: [PATCH] Thermal tests + CI in DQM (#122) * Adding thermal tests scripts / starting CI for DQM * Adding script for processing laser thermal data * adding CI tests for all DQM modules - initial + cleaning code * fixing the path to the test run file for the CI tests * adding more CI tests to the pixel_particiapation module * removing prints * Adding more CI tests for the remaining DQM modules --------- Co-authored-by: Jean-Philippe Lenain --- src/nectarchain/dqm/__init__.py | 3 +- src/nectarchain/dqm/camera_monitoring.py | 5 +- src/nectarchain/dqm/charge_integration.py | 95 +++++++++++-- src/nectarchain/dqm/mean_camera_display.py | 10 +- src/nectarchain/dqm/mean_waveforms.py | 5 +- src/nectarchain/dqm/pixel_participation.py | 5 +- src/nectarchain/dqm/pixel_timeline.py | 3 +- ...nectarcam_monitoring_db_2022-02-24.sqlite} | 0 src/nectarchain/dqm/start_dqm.py | 6 +- .../dqm/tests/test_camera_monitoring.py | 46 +++++++ .../dqm/tests/test_charge_integration.py | 50 +++++++ .../dqm/tests/test_mean_camera_display.py | 47 +++++++ .../dqm/tests/test_mean_waveforms.py | 45 ++++++ .../dqm/tests/test_pixel_participation.py | 48 +++++++ .../dqm/tests/test_pixel_timeline.py | 46 +++++++ .../dqm/tests/test_trigger_statistics.py | 45 ++++++ src/nectarchain/dqm/trigger_statistics.py | 4 +- .../process_thermal_test_data_laser.py | 130 ++++++++++++++++++ 18 files changed, 562 insertions(+), 31 deletions(-) rename src/nectarchain/dqm/{tests/__init__.py => runs/nectarcam_monitoring_db_2022-02-24.sqlite} (100%) create mode 100644 src/nectarchain/dqm/tests/test_camera_monitoring.py create mode 100644 src/nectarchain/dqm/tests/test_charge_integration.py create mode 100644 src/nectarchain/dqm/tests/test_mean_camera_display.py create mode 100644 src/nectarchain/dqm/tests/test_mean_waveforms.py create mode 100644 src/nectarchain/dqm/tests/test_pixel_participation.py create mode 100644 src/nectarchain/dqm/tests/test_pixel_timeline.py create mode 100644 src/nectarchain/dqm/tests/test_trigger_statistics.py create mode 100644 src/nectarchain/user_scripts/hashkar/process_thermal_test_data_laser.py diff --git a/src/nectarchain/dqm/__init__.py b/src/nectarchain/dqm/__init__.py index cb255dcd..74050196 100644 --- a/src/nectarchain/dqm/__init__.py +++ b/src/nectarchain/dqm/__init__.py @@ -1,4 +1,5 @@ from .db_utils import DQMDB from .dqm_summary_processor import DQMSummary +from .pixel_participation import PixelParticipationHighLowGain -__all__ = ["DQMDB", "DQMSummary"] +__all__ = ["DQMDB", "DQMSummary", "PixelParticipationHighLowGain"] diff --git a/src/nectarchain/dqm/camera_monitoring.py b/src/nectarchain/dqm/camera_monitoring.py index 57af29e9..686fb870 100644 --- a/src/nectarchain/dqm/camera_monitoring.py +++ b/src/nectarchain/dqm/camera_monitoring.py @@ -4,11 +4,11 @@ import numpy as np from astropy import time as astropytime from ctapipe.coordinates import EngineeringCameraFrame -from ctapipe.instrument import CameraGeometry from ctapipe.visualization import CameraDisplay -from dqm_summary_processor import DQMSummary from matplotlib import pyplot as plt +from nectarchain.dqm.dqm_summary_processor import DQMSummary + class CameraMonitoring(DQMSummary): def __init__(self, gaink): @@ -139,7 +139,6 @@ def GetResults(self): return self.CameraMonitoring_Results_Dict def PlotResults(self, name, FigPath): - try: fig, disp = plt.subplots() disp = CameraDisplay(self.camera) diff --git a/src/nectarchain/dqm/charge_integration.py b/src/nectarchain/dqm/charge_integration.py index 6417f04e..d3fa673b 100644 --- a/src/nectarchain/dqm/charge_integration.py +++ b/src/nectarchain/dqm/charge_integration.py @@ -2,12 +2,12 @@ import numpy as np from ctapipe.coordinates import EngineeringCameraFrame from ctapipe.image import LocalPeakWindowSum -from ctapipe.instrument import CameraGeometry from ctapipe.visualization import CameraDisplay -from dqm_summary_processor import DQMSummary from matplotlib import pyplot as plt from traitlets.config.loader import Config +from nectarchain.dqm.dqm_summary_processor import DQMSummary + class ChargeIntegrationHighLowGain(DQMSummary): def __init__(self, gaink): @@ -20,21 +20,31 @@ def __init__(self, gaink): self.peakpos_all = [] self.image_ped = [] self.peakpos_ped = [] - self.camera = None + self.ped_all = [] + self.ped_ped = [] + self.camera = None self.cmap = None - self.subarray = None + self.subarray = None self.integrator = None - self.pixelBAD = None + self.pixelBAD = None self.image_all = [] self.image_all_median = None self.image_all_average = None self.image_all_std = None self.image_all_rms = None + self.ped_all_median = None + self.ped_all_average = None + self.ped_all_std = None + self.ped_all_rms = None self.image_ped = [] self.image_ped_median = None self.image_ped_average = None self.image_ped_std = None self.image_ped_rms = None + self.ped_ped_median = None + self.ped_ped_average = None + self.ped_ped_std = None + self.ped_ped_rms = None self.ChargeInt_Results_Dict = {} self.ChargeInt_Figures_Dict = {} self.ChargeInt_Figures_Names_Dict = {} @@ -50,7 +60,7 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1): self.camera = Reader1.subarray.tel[0].camera.geometry.transform_to( EngineeringCameraFrame() ) - + self.cmap = "gnuplot2" self.subarray = Reader1.subarray @@ -64,8 +74,6 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1): self.integrator = LocalPeakWindowSum(subarray, config=config) - - def ProcessEvent(self, evt, noped): self.pixelBAD = evt.mon.tel[0].pixel_status.hardware_failing_pixels pixel = evt.nectarcam.tel[0].svc.pixel_ids @@ -80,8 +88,9 @@ def ProcessEvent(self, evt, noped): waveform = evt.r0.tel[0].waveform[self.k] + ped = np.mean(waveform[:, 20]) + if noped: - ped = np.mean(waveform[:, 20]) w_noped = waveform - ped output = self.integrator( w_noped, 0, np.zeros(self.Pix, dtype=int), self.pixelBAD @@ -104,10 +113,12 @@ def ProcessEvent(self, evt, noped): self.counter_ped += 1 self.image_ped.append(image) self.peakpos_ped.append(peakpos) + self.ped_ped.append(ped) else: self.counter_evt += 1 self.image_all.append(image) self.peakpos_all.append(peakpos) + self.ped_all.append(ped) def FinishRun(self): self.peakpos_all = np.array(self.peakpos_all, dtype=float) @@ -121,6 +132,12 @@ def FinishRun(self): self.image_all_std = np.std(self.image_all, axis=0) self.image_all_rms = np.sqrt(np.sum(self.image_all**2, axis=0)) + self.ped_all = np.array(self.ped_all, dtype=float) + self.ped_all_average = np.mean(self.ped_all, axis=0) + self.ped_all_median = np.median(self.ped_all, axis=0) + self.ped_all_std = np.std(self.ped_all, axis=0) + self.ped_all_rms = np.sqrt(np.sum(self.ped_all**2, axis=0)) + if self.counter_ped > 0: self.image_ped = np.array(self.image_ped, dtype=float) self.image_ped_median = np.median(self.image_ped, axis=0) @@ -128,8 +145,13 @@ def FinishRun(self): self.image_ped_std = np.std(self.image_ped, axis=0) self.image_ped_rms = np.sqrt(np.sum(self.image_ped**2, axis=0)) - def GetResults(self): + self.ped_ped = np.array(self.ped_ped, dtype=float) + self.ped_ped_average = np.mean(self.ped_ped, axis=0) + self.ped_ped_median = np.median(self.ped_ped, axis=0) + self.ped_ped_std = np.std(self.ped_ped, axis=0) + self.ped_ped_rms = np.sqrt(np.sum(self.ped_ped**2, axis=0)) + def GetResults(self): if self.k == 0: self.ChargeInt_Results_Dict[ "CHARGE-INTEGRATION-IMAGE-ALL-AVERAGE-HIGH-GAIN" @@ -144,6 +166,19 @@ def GetResults(self): "CHARGE-INTEGRATION-IMAGE-ALL-STD-HIGH-GAIN" ] = self.image_all_std + self.ChargeInt_Results_Dict[ + "PED-INTEGRATION-IMAGE-ALL-AVERAGE-HIGH-GAIN" + ] = self.ped_all_average + self.ChargeInt_Results_Dict[ + "PED-INTEGRATION-IMAGE-ALL-MEDIAN-HIGH-GAIN" + ] = self.ped_all_median + self.ChargeInt_Results_Dict[ + "PED-INTEGRATION-IMAGE-ALL-RMS-HIGH-GAIN" + ] = self.ped_all_rms + self.ChargeInt_Results_Dict[ + "PED-INTEGRATION-IMAGE-ALL-STD-HIGH-GAIN" + ] = self.ped_all_std + if self.counter_ped > 0: self.ChargeInt_Results_Dict[ "CHARGE-INTEGRATION-PED-ALL-AVERAGE-HIGH-GAIN" @@ -158,6 +193,19 @@ def GetResults(self): "CHARGE-INTEGRATION-PED-ALL-STD-HIGH-GAIN" ] = self.image_ped_std + self.ChargeInt_Results_Dict[ + "PED-INTEGRATION-PED-ALL-AVERAGE-HIGH-GAIN" + ] = self.ped_ped_average + self.ChargeInt_Results_Dict[ + "PED-INTEGRATION-PED-ALL-MEDIAN-HIGH-GAIN" + ] = self.ped_ped_median + self.ChargeInt_Results_Dict[ + "PED-INTEGRATION-PED-ALL-RMS-HIGH-GAIN" + ] = self.ped_ped_rms + self.ChargeInt_Results_Dict[ + "PED-INTEGRATION-PED-ALL-STD-HIGH-GAIN" + ] = self.ped_ped_std + if self.k == 1: self.ChargeInt_Results_Dict[ "CHARGE-INTEGRATION-IMAGE-ALL-AVERAGE-LOW-GAIN" @@ -172,6 +220,19 @@ def GetResults(self): "CHARGE-INTEGRATION-IMAGE-ALL-STD-LOW-GAIN" ] = self.image_all_std + self.ChargeInt_Results_Dict[ + "PED-INTEGRATION-IMAGE-ALL-AVERAGE-LOW-GAIN" + ] = self.ped_all_average + self.ChargeInt_Results_Dict[ + "PED-INTEGRATION-IMAGE-ALL-MEDIAN-LOW-GAIN" + ] = self.ped_all_median + self.ChargeInt_Results_Dict[ + "PED-INTEGRATION-IMAGE-ALL-RMS-LOW-GAIN" + ] = self.ped_all_rms + self.ChargeInt_Results_Dict[ + "PED-INTEGRATION-IMAGE-ALL-STD-LOW-GAIN" + ] = self.ped_all_std + if self.counter_ped > 0: self.ChargeInt_Results_Dict[ "CHARGE-INTEGRATION-PED-ALL-AVERAGE-LOW-GAIN" @@ -186,10 +247,22 @@ def GetResults(self): "CHARGE-INTEGRATION-PED-ALL-STD-LOW-GAIN" ] = self.image_ped_std + self.ChargeInt_Results_Dict[ + "PED-INTEGRATION-PED-ALL-AVERAGE-LOW-GAIN" + ] = self.ped_ped_average + self.ChargeInt_Results_Dict[ + "PED-INTEGRATION-PED-ALL-MEDIAN-LOW-GAIN" + ] = self.ped_ped_median + self.ChargeInt_Results_Dict[ + "PED-INTEGRATION-PED-ALL-RMS-LOW-GAIN" + ] = self.ped_ped_rms + self.ChargeInt_Results_Dict[ + "PED-INTEGRATION-PED-ALL-STD-LOW-GAIN" + ] = self.ped_ped_std + return self.ChargeInt_Results_Dict def PlotResults(self, name, FigPath): - # titles = ['All', 'Pedestals'] if self.k == 0: gain_c = "High" diff --git a/src/nectarchain/dqm/mean_camera_display.py b/src/nectarchain/dqm/mean_camera_display.py index 43baf4b7..dcad59b5 100644 --- a/src/nectarchain/dqm/mean_camera_display.py +++ b/src/nectarchain/dqm/mean_camera_display.py @@ -1,11 +1,12 @@ import numpy as np from ctapipe.coordinates import EngineeringCameraFrame from ctapipe.visualization import CameraDisplay -from dqm_summary_processor import DQMSummary from matplotlib import pyplot as plt +from nectarchain.dqm.dqm_summary_processor import DQMSummary -class MeanCameraDisplay_HighLowGain(DQMSummary): + +class MeanCameraDisplayHighLowGain(DQMSummary): def __init__(self, gaink): self.k = gaink self.Pix = None @@ -22,7 +23,7 @@ def __init__(self, gaink): self.CameraAverage_ped1 = [] self.CameraAverage_overEvents = None self.CameraAverage_overEvents_overSamp = None - self.CameraAverage_ped_overEvents = None + self.CameraAverage_ped_overEvents = None self.CameraAverage_ped_overEvents_overSamp = None self.MeanCameraDisplay_Results_Dict = {} self.MeanCameraDisplay_Figures_Dict = {} @@ -42,7 +43,6 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1): self.cmap = "gnuplot2" - def ProcessEvent(self, evt, noped): self.pixelBAD = evt.mon.tel[0].pixel_status.hardware_failing_pixels pixel = evt.nectarcam.tel[0].svc.pixel_ids @@ -86,7 +86,6 @@ def FinishRun(self): ) def GetResults(self): - # ASSIGN RESUTLS TO DICT if self.k == 0: if self.counter_evt > 0: @@ -125,7 +124,6 @@ def GetResults(self): return self.MeanCameraDisplay_Results_Dict def PlotResults(self, name, FigPath): - # titles = ['All', 'Pedestals'] if self.k == 0: gain_c = "High" diff --git a/src/nectarchain/dqm/mean_waveforms.py b/src/nectarchain/dqm/mean_waveforms.py index 7e17b2cb..bd02cdb3 100644 --- a/src/nectarchain/dqm/mean_waveforms.py +++ b/src/nectarchain/dqm/mean_waveforms.py @@ -1,7 +1,8 @@ import numpy as np -from dqm_summary_processor import DQMSummary from matplotlib import pyplot as plt +from nectarchain.dqm.dqm_summary_processor import DQMSummary + class MeanWaveFormsHighLowGain(DQMSummary): def __init__(self, gaink): @@ -70,7 +71,6 @@ def FinishRun(self): return None def GetResults(self): - # ASSIGN RESUTLS TO DICT if self.k == 0: self.MeanWaveForms_Results_Dict[ @@ -93,7 +93,6 @@ def GetResults(self): return self.MeanWaveForms_Results_Dict def PlotResults(self, name, FigPath): - wf_list = np.array(self.wf_list_plot) counter_fig = 0 diff --git a/src/nectarchain/dqm/pixel_participation.py b/src/nectarchain/dqm/pixel_participation.py index b58b2aed..7eaad8a9 100644 --- a/src/nectarchain/dqm/pixel_participation.py +++ b/src/nectarchain/dqm/pixel_participation.py @@ -1,9 +1,12 @@ import numpy as np from ctapipe.coordinates import EngineeringCameraFrame from ctapipe.visualization import CameraDisplay -from dqm_summary_processor import DQMSummary from matplotlib import pyplot as plt +from nectarchain.dqm.dqm_summary_processor import DQMSummary + +__all__ = ["PixelParticipationHighLowGain"] + class PixelParticipationHighLowGain(DQMSummary): def __init__(self, gaink): diff --git a/src/nectarchain/dqm/pixel_timeline.py b/src/nectarchain/dqm/pixel_timeline.py index e120f561..5437ff42 100644 --- a/src/nectarchain/dqm/pixel_timeline.py +++ b/src/nectarchain/dqm/pixel_timeline.py @@ -1,7 +1,8 @@ import numpy as np -from dqm_summary_processor import DQMSummary from matplotlib import pyplot as plt +from nectarchain.dqm.dqm_summary_processor import DQMSummary + class PixelTimelineHighLowGain(DQMSummary): def __init__(self, gaink): diff --git a/src/nectarchain/dqm/tests/__init__.py b/src/nectarchain/dqm/runs/nectarcam_monitoring_db_2022-02-24.sqlite similarity index 100% rename from src/nectarchain/dqm/tests/__init__.py rename to src/nectarchain/dqm/runs/nectarcam_monitoring_db_2022-02-24.sqlite diff --git a/src/nectarchain/dqm/start_dqm.py b/src/nectarchain/dqm/start_dqm.py index f6c91701..1b70e310 100644 --- a/src/nectarchain/dqm/start_dqm.py +++ b/src/nectarchain/dqm/start_dqm.py @@ -9,7 +9,7 @@ from ctapipe_io_nectarcam.constants import HIGH_GAIN, LOW_GAIN from db_utils import DQMDB from matplotlib import pyplot as plt -from mean_camera_display import MeanCameraDisplay_HighLowGain +from mean_camera_display import MeanCameraDisplayHighLowGain from mean_waveforms import MeanWaveFormsHighLowGain from pixel_participation import PixelParticipationHighLowGain from pixel_timeline import PixelTimelineHighLowGain @@ -145,8 +145,8 @@ def CreateFigFolder(name, type): a = TriggerStatistics(HIGH_GAIN) b = MeanWaveFormsHighLowGain(HIGH_GAIN) c = MeanWaveFormsHighLowGain(LOW_GAIN) -d = MeanCameraDisplay_HighLowGain(HIGH_GAIN) -e = MeanCameraDisplay_HighLowGain(LOW_GAIN) +d = MeanCameraDisplayHighLowGain(HIGH_GAIN) +e = MeanCameraDisplayHighLowGain(LOW_GAIN) f = ChargeIntegrationHighLowGain(HIGH_GAIN) g = ChargeIntegrationHighLowGain(LOW_GAIN) h = CameraMonitoring(HIGH_GAIN) diff --git a/src/nectarchain/dqm/tests/test_camera_monitoring.py b/src/nectarchain/dqm/tests/test_camera_monitoring.py new file mode 100644 index 00000000..a3a99318 --- /dev/null +++ b/src/nectarchain/dqm/tests/test_camera_monitoring.py @@ -0,0 +1,46 @@ +from astropy import time as astropytime +from ctapipe.io import EventSource +from ctapipe.utils import get_dataset_path +from ctapipe_io_nectarcam.constants import HIGH_GAIN +from tqdm import tqdm +from traitlets.config import Config + +from nectarchain.dqm.camera_monitoring import CameraMonitoring + + +class TestCameraMonitoring: + run_number = 3798 + max_events = 1 + + def test_camera_monitoring(self): + # run_number = 3938 + path = get_dataset_path("NectarCAM.Run3938.30events.fits.fz") + + config = None + + config = Config( + dict( + NectarCAMEventSource=dict( + NectarCAMR0Corrections=dict( + calibration_path=None, + apply_flatfield=False, + select_gain=False, + ) + ) + ) + ) + print(path) + + reader1 = EventSource(input_url=path, config=config, max_events=1) + + Pix, Samp = CameraMonitoring(HIGH_GAIN).DefineForRun(reader1) + + CameraMonitoring(HIGH_GAIN).ConfigureForRun(path, Pix, Samp, reader1) + + for evt in tqdm(reader1, total=1): + run_start1 = evt.nectarcam.tel[0].svc.date + SqlFileDate = astropytime.Time(run_start1, format="unix").iso.split(" ")[0] + # print("SqlFileDate", SqlFileDate) + # CameraMonitoring(HIGH_GAIN).FinishRun() + assert Pix + Samp == 1915 + assert SqlFileDate == "2023-01-23" diff --git a/src/nectarchain/dqm/tests/test_charge_integration.py b/src/nectarchain/dqm/tests/test_charge_integration.py new file mode 100644 index 00000000..0e5ec1f9 --- /dev/null +++ b/src/nectarchain/dqm/tests/test_charge_integration.py @@ -0,0 +1,50 @@ +import numpy as np +from ctapipe.image import LocalPeakWindowSum +from ctapipe.io import EventSource +from ctapipe.utils import get_dataset_path +from ctapipe_io_nectarcam.constants import HIGH_GAIN +from tqdm import tqdm +from traitlets.config import Config + +from nectarchain.dqm.charge_integration import ChargeIntegrationHighLowGain + + +class TestChargeIntegrationHighLowGain: + run_number = 3938 + max_events = 1 + + def test_charge_integration(self): + # run_number = 3938 + path = get_dataset_path("NectarCAM.Run3938.30events.fits.fz") + + config = None + + config = Config( + dict( + NectarCAMEventSource=dict( + NectarCAMR0Corrections=dict( + calibration_path=None, + apply_flatfield=False, + select_gain=False, + ) + ) + ) + ) + reader1 = EventSource(input_url=path, config=config, max_events=1) + subarray = reader1.subarray + self.integrator = LocalPeakWindowSum(subarray, config=config) + + Pix, Samp = ChargeIntegrationHighLowGain(HIGH_GAIN).DefineForRun(reader1) + + ChargeIntegrationHighLowGain(HIGH_GAIN).ConfigureForRun( + path, Pix, Samp, reader1 + ) + + for evt in tqdm(reader1, total=1): + self.pixelBAD = evt.mon.tel[0].pixel_status.hardware_failing_pixels + waveform = evt.r0.tel[0].waveform[HIGH_GAIN] + ped = np.mean(waveform[:, 20]) + # ChargeIntegrationHighLowGain(HIGH_GAIN).ProcessEvent(evt, noped = False) + # ChargeIntegrationHighLowGain(HIGH_GAIN).FinishRun() + assert Pix + Samp == 1915 + assert np.sum(ped) == 985.8636118598383 diff --git a/src/nectarchain/dqm/tests/test_mean_camera_display.py b/src/nectarchain/dqm/tests/test_mean_camera_display.py new file mode 100644 index 00000000..c747d9cc --- /dev/null +++ b/src/nectarchain/dqm/tests/test_mean_camera_display.py @@ -0,0 +1,47 @@ +from ctapipe.io import EventSource +from ctapipe.utils import get_dataset_path +from ctapipe_io_nectarcam.constants import HIGH_GAIN +from tqdm import tqdm +from traitlets.config import Config + +from nectarchain.dqm.mean_camera_display import MeanCameraDisplayHighLowGain + + +class TestMeanCameraDisplayHighLowGain: + run_number = 3798 + max_events = 1 + + def test_mean_camera_display(self): + # run_number = 3938 + path = get_dataset_path("NectarCAM.Run3938.30events.fits.fz") + + config = None + + config = Config( + dict( + NectarCAMEventSource=dict( + NectarCAMR0Corrections=dict( + calibration_path=None, + apply_flatfield=False, + select_gain=False, + ) + ) + ) + ) + print(path) + + reader1 = EventSource(input_url=path, config=config, max_events=1) + + Pix, Samp = MeanCameraDisplayHighLowGain(HIGH_GAIN).DefineForRun(reader1) + + MeanCameraDisplayHighLowGain(HIGH_GAIN).ConfigureForRun( + path, Pix, Samp, reader1 + ) + + for evt in tqdm(reader1, total=1): + # MeanCameraDisplayHighLowGain(HIGH_GAIN).ProcessEvent(evt, noped = False) + CameraAverage = evt.r0.tel[0].waveform[HIGH_GAIN].sum(axis=1) + + # MeanCameraDisplayHighLowGain(HIGH_GAIN).FinishRun() + assert Pix + Samp == 1915 + assert CameraAverage.sum(axis=0) == 109723121 diff --git a/src/nectarchain/dqm/tests/test_mean_waveforms.py b/src/nectarchain/dqm/tests/test_mean_waveforms.py new file mode 100644 index 00000000..6c16e502 --- /dev/null +++ b/src/nectarchain/dqm/tests/test_mean_waveforms.py @@ -0,0 +1,45 @@ +import numpy as np +from ctapipe.io import EventSource +from ctapipe.utils import get_dataset_path +from ctapipe_io_nectarcam.constants import HIGH_GAIN +from tqdm import tqdm +from traitlets.config import Config + +from nectarchain.dqm.mean_waveforms import MeanWaveFormsHighLowGain + + +class TestMeanWaveForms: + run_number = 3798 + max_events = 1 + + def test_mean_waveforms(self): + # run_number = 3938 + path = get_dataset_path("NectarCAM.Run3938.30events.fits.fz") + + config = None + + config = Config( + dict( + NectarCAMEventSource=dict( + NectarCAMR0Corrections=dict( + calibration_path=None, + apply_flatfield=False, + select_gain=False, + ) + ) + ) + ) + + reader1 = EventSource(input_url=path, config=config, max_events=1) + + Pix, Samp = MeanWaveFormsHighLowGain(HIGH_GAIN).DefineForRun(reader1) + + MeanWaveFormsHighLowGain(HIGH_GAIN).ConfigureForRun(path, Pix, Samp, reader1) + + for evt in tqdm(reader1, total=1): + self.pixelBAD = evt.mon.tel[0].pixel_status.hardware_failing_pixels + # MeanWaveFormsHighLowGain(HIGH_GAIN).ProcessEvent(evt, noped = False) + # MeanWaveFormsHighLowGain(HIGH_GAIN).FinishRun() + assert Pix + Samp == 1915 + + assert np.sum(evt.r0.tel[0].waveform[HIGH_GAIN][0]) == 3932100 diff --git a/src/nectarchain/dqm/tests/test_pixel_participation.py b/src/nectarchain/dqm/tests/test_pixel_participation.py new file mode 100644 index 00000000..961e85bb --- /dev/null +++ b/src/nectarchain/dqm/tests/test_pixel_participation.py @@ -0,0 +1,48 @@ +import numpy as np +from ctapipe.io import EventSource +from ctapipe.utils import get_dataset_path +from ctapipe_io_nectarcam.constants import HIGH_GAIN +from tqdm import tqdm +from traitlets.config import Config + +from nectarchain.dqm.pixel_participation import PixelParticipationHighLowGain + + +class TestPixelParticipation: + run_number = 3798 + max_events = 1 + + def test_pixel_participation(self): + # run_number = 3938 + path = get_dataset_path("NectarCAM.Run3938.30events.fits.fz") + + config = None + + config = Config( + dict( + NectarCAMEventSource=dict( + NectarCAMR0Corrections=dict( + calibration_path=None, + apply_flatfield=False, + select_gain=False, + ) + ) + ) + ) + + reader1 = EventSource(input_url=path, config=config, max_events=1) + + Pix, Samp = PixelParticipationHighLowGain(HIGH_GAIN).DefineForRun(reader1) + + PixelParticipationHighLowGain(HIGH_GAIN).ConfigureForRun( + path, Pix, Samp, reader1 + ) + + for evt in tqdm(reader1, total=1): + self.pixelBAD = evt.mon.tel[0].pixel_status.hardware_failing_pixels + # PixelParticipationHighLowGain(HIGH_GAIN).ProcessEvent(evt, noped = False) + + PixelParticipationHighLowGain(HIGH_GAIN).FinishRun() + + assert Pix + Samp == 1915 + assert np.sum(evt.nectarcam.tel[0].svc.pixel_ids) == 1719375 diff --git a/src/nectarchain/dqm/tests/test_pixel_timeline.py b/src/nectarchain/dqm/tests/test_pixel_timeline.py new file mode 100644 index 00000000..db358aa1 --- /dev/null +++ b/src/nectarchain/dqm/tests/test_pixel_timeline.py @@ -0,0 +1,46 @@ +import numpy as np +from ctapipe.io import EventSource +from ctapipe.utils import get_dataset_path +from ctapipe_io_nectarcam.constants import HIGH_GAIN +from tqdm import tqdm +from traitlets.config import Config + +from nectarchain.dqm.pixel_timeline import PixelTimelineHighLowGain + + +class TestPixelTimeline: + run_number = 3798 + max_events = 1 + + def test_pixel_timeline(self): + # run_number = 3938 + path = get_dataset_path("NectarCAM.Run3938.30events.fits.fz") + + config = None + + config = Config( + dict( + NectarCAMEventSource=dict( + NectarCAMR0Corrections=dict( + calibration_path=None, + apply_flatfield=False, + select_gain=False, + ) + ) + ) + ) + + reader1 = EventSource(input_url=path, config=config, max_events=1) + + Pix, Samp = PixelTimelineHighLowGain(HIGH_GAIN).DefineForRun(reader1) + + PixelTimelineHighLowGain(HIGH_GAIN).ConfigureForRun(path, Pix, Samp, reader1) + + for evt in tqdm(reader1, total=1): + self.pixelBAD = evt.mon.tel[0].pixel_status.hardware_failing_pixels + # PixelTimelineHighLowGain(HIGH_GAIN).ProcessEvent(evt, noped = False) + + PixelTimelineHighLowGain(HIGH_GAIN).FinishRun() + + assert Pix + Samp == 1915 + assert np.sum(evt.nectarcam.tel[0].svc.pixel_ids) == 1719375 diff --git a/src/nectarchain/dqm/tests/test_trigger_statistics.py b/src/nectarchain/dqm/tests/test_trigger_statistics.py new file mode 100644 index 00000000..a4711de5 --- /dev/null +++ b/src/nectarchain/dqm/tests/test_trigger_statistics.py @@ -0,0 +1,45 @@ +from ctapipe.io import EventSource +from ctapipe.utils import get_dataset_path +from ctapipe_io_nectarcam.constants import HIGH_GAIN +from tqdm import tqdm +from traitlets.config import Config + +from nectarchain.dqm.trigger_statistics import TriggerStatistics + + +class TestTriggerStatistics: + run_number = 3798 + max_events = 1 + + def test_trigger_statistics(self): + # run_number = 3938 + path = get_dataset_path("NectarCAM.Run3938.30events.fits.fz") + + config = None + + config = Config( + dict( + NectarCAMEventSource=dict( + NectarCAMR0Corrections=dict( + calibration_path=None, + apply_flatfield=False, + select_gain=False, + ) + ) + ) + ) + print(path) + + reader1 = EventSource(input_url=path, config=config, max_events=1) + + Pix, Samp = TriggerStatistics(HIGH_GAIN).DefineForRun(reader1) + + TriggerStatistics(HIGH_GAIN).ConfigureForRun(path, Pix, Samp, reader1) + + for evt in tqdm(reader1, total=1): + time = evt.trigger.time.value + TriggerStatistics(HIGH_GAIN).ProcessEvent(evt, noped=False) + + # TriggerStatistics(HIGH_GAIN).FinishRun() + assert Pix + Samp == 1915 + assert time == 1674462932.6398556 diff --git a/src/nectarchain/dqm/trigger_statistics.py b/src/nectarchain/dqm/trigger_statistics.py index fe429223..a33fe0e0 100644 --- a/src/nectarchain/dqm/trigger_statistics.py +++ b/src/nectarchain/dqm/trigger_statistics.py @@ -2,9 +2,10 @@ import numpy as np from astropy import time as astropytime -from dqm_summary_processor import DQMSummary from matplotlib import pyplot as plt +from nectarchain.dqm.dqm_summary_processor import DQMSummary + class TriggerStatistics(DQMSummary): def __init__(self, gaink): @@ -29,7 +30,6 @@ def __init__(self, gaink): self.TriggerStat_Figures_Dict = {} self.TriggerStat_Figures_Names_Dict = {} - def ConfigureForRun(self, path, Pix, Samp, Reader1): # define number of pixels and samples self.Pix = Pix diff --git a/src/nectarchain/user_scripts/hashkar/process_thermal_test_data_laser.py b/src/nectarchain/user_scripts/hashkar/process_thermal_test_data_laser.py new file mode 100644 index 00000000..c8ba5b5f --- /dev/null +++ b/src/nectarchain/user_scripts/hashkar/process_thermal_test_data_laser.py @@ -0,0 +1,130 @@ +import matplotlib.pyplot as plt +import numpy as np +from astropy.io import fits +from ctapipe.coordinates import EngineeringCameraFrame +from ctapipe.instrument import CameraGeometry +from ctapipe.visualization import CameraDisplay + +################################################################ +# OPTIONS ARE 2: PED/CHARGE/PEDSTD .... and 1:AVERAGE/STD -> if 2:PEDSTD change name of files to _ped +name_entity2 = "Charge" +name_entity = "average" +# entity = "PED-INTEGRATION-PED-ALL-AVERAGE-HIGH-GAIN" #what i WANT TO STUDY +entity = "CHARGE-INTEGRATION-PED-ALL-AVERAGE-HIGH-GAIN" # what i WANT TO STUDY + +# 2 1 + +intensity = [13, 15, 16.5, 18, 20, 22, 23.5, 25, 30, 35] +temperature = [15, 5, 0, -5] # should be taken from logs + +divide = [58, 58, 58, 58, 58, 58, 58, 58, 58, 58] +divide = np.array(divide) + + +deg15 = [] +deg15std = [] +deg5 = [] +deg5std = [] +deg0 = [] +deg0std = [] +deg_5 = [] +deg_5std = [] + +temperature = [15, 5, 0, -5] + +runs_neg5 = [3798, 3797, 3796, 3795, 3794, 3793, 3792, 3791, 3790, 3789] +runs_0 = [3764, 3763, 3762, 3761, 3760, 3759, 3758, 3757, 3756, 3755] +runs_5 = [3714, 3713, 3712, 3711, 3710, 3709, 3708, 3707, 3706, 3705] +runs_15 = [3630, 3631, 3629, 3628, 3627, 3626, 3625, 3624, 3622, 3623] + +path = "./" + +################################################ + + +# open the figure +fig = plt.figure(figsize=(10, 10 / 1.61)) +axis_font = {"fontname": "Arial", "size": "16"} + +# Open the fits file +for i in runs_15: + with fits.open("%sNectarCAM_Run%s_Results.fits" % (path, i)) as hdulist1: + table_data1 = hdulist1["Camera"].data + print(table_data1[entity][21:]) + charge_av = np.mean(table_data1[entity][21:]) + print(charge_av) + charge_std = np.std(table_data1[entity][21:]) + deg15.append(charge_av) + print(deg15) + deg15std.append(charge_std) + +plt.errorbar( + intensity, + deg15 / divide, + yerr=np.array(deg15std / np.sqrt(len(table_data1[entity][21:]))), + label="%s deg" % temperature[0], +) +# plt.errorbar(intensity, deg15/(divide), yerr = deg15std/(divide * np.sqrt(1855))) + +for i in runs_5: + with fits.open("%sNectarCAM_Run%s_Results.fits" % (path, i)) as hdulist1: + table_data1 = hdulist1["Camera"].data + charge_av = np.mean(table_data1[entity][21:]) + charge_std = np.std(table_data1[entity][21:]) + deg5.append(charge_av) + deg5std.append(charge_std) + +plt.errorbar( + intensity, + deg5 / divide, + yerr=np.array(deg5std / np.sqrt(len(table_data1[entity][21:]))), + label="%s deg" % temperature[1], +) +# plt.errorbar(intensity, deg5/(divide), yerr = deg5std/(divide* np.sqrt(1855))) + +for i in runs_0: + with fits.open("%sNectarCAM_Run%s_Results.fits" % (path, i)) as hdulist1: + table_data1 = hdulist1["Camera"].data + charge_av = np.mean(table_data1[entity][21:]) + charge_std = np.std(table_data1[entity][21:]) + deg0.append(charge_av) + deg0std.append(charge_std) + +plt.errorbar( + intensity, + deg0 / divide, + yerr=np.array(deg0std / np.sqrt(len(table_data1[entity][21:]))), + label="%s deg" % temperature[2], +) +# plt.errorbar(intensity, deg0/(divide), yerr = deg0std/(divide* np.sqrt(1855))) + +for i in runs_neg5: + with fits.open("%sNectarCAM_Run%s_Results.fits" % (path, i)) as hdulist1: + table_data1 = hdulist1["Camera"].data + charge_av = np.mean(table_data1[entity][21:]) + charge_std = np.std(table_data1[entity][21:]) + deg_5.append(charge_av) + deg_5std.append(charge_std) + +plt.errorbar( + intensity, + deg_5 / divide, + yerr=np.array(deg_5std / np.sqrt(len(table_data1[entity][21:]))), + label="%s deg" % temperature[3], +) +# plt.errorbar(intensity, deg_5/(divide), yerr = deg_5std/(divide* np.sqrt(1855))) + + +################################################################ +plt.yscale("log") +plt.xlabel("Laser intensity (%)", **axis_font) +plt.ylabel("%s %s (p.e.)" % (name_entity2, name_entity), **axis_font) +plt.legend(prop={"size": 16}) +plt.grid() +fig.savefig( + "temp_%s_%s_all_intensity.png" % (name_entity2, name_entity), + dpi=300, + bbox_inches="tight", +) +plt.show() +############################################################################################################