Add Cat-B calibration onsite scripts

lstchain/calib/camera/calibration_calculator.py

@@ -76,8 +76,8 @@ class CalibrationCalculator(Component):
     ).tag(config=True)
 
     hg_lg_ratio = traits.Float(
-        17.4,
-        help='HG/LG ratio applied if use_scaled_low_gain is True'
+        1.,
+        help='HG/LG ratio applied if use_scaled_low_gain is True. In case of calibrated data the ratio should be 1.'
     ).tag(config=True)
 
     classes = (

lstchain/data/catB_camera_calibration_param.json

@@ -8,7 +8,6 @@
   "LSTCalibrationCalculator": {
     "squared_excess_noise_factor": 1.222,
     "use_scaled_low_gain": true,
-    "hg_lg_ratio": 17.4,
     "npe_median_cut_outliers": [-5,5],
     "flatfield_product": "FlasherFlatFieldCalculator",
     "pedestal_product": "PedestalIntegrator"

lstchain/io/io.py

@@ -77,6 +77,10 @@
 dl1_params_tel_mon_ped_key = "/dl1/event/telescope/monitoring/pedestal"
 dl1_params_tel_mon_cal_key = "/dl1/event/telescope/monitoring/calibration"
 dl1_params_tel_mon_flat_key = "/dl1/event/telescope/monitoring/flatfield"
+dl1_params_tel_mon_CatA_ped_key = "/dl1/monitoring/telescope/CatA/pedestal"
+dl1_params_tel_mon_CatA_flat_key = "/dl1/monitoring/telescope/CatA/flatfield"
+dl1_params_tel_mon_CatA_cal_key = "/dl1/monitoring/telescope/CatA/calibration"
+
 dl1_params_lstcam_key = "/dl1/event/telescope/parameters/LST_LSTCam"
 dl1_images_lstcam_key = "/dl1/event/telescope/image/LST_LSTCam"
 dl2_params_lstcam_key = "/dl2/event/telescope/parameters/LST_LSTCam"
@@ -815,7 +819,7 @@ def recursive_copy_node(src_file, dir_file, path):
             recursive_path = os.path.join(recursive_path, p)
 
 
-def write_calibration_data(writer, mon_index, mon_event, new_ped=False, new_ff=False):
+def write_calibration_data(writer, table_group, mon_index, mon_event, new_ped=False, new_ff=False):
     mon_event.pedestal.prefix = ''
     mon_event.flatfield.prefix = ''
     mon_event.calibration.prefix = ''
@@ -831,21 +835,22 @@ def write_calibration_data(writer, mon_index, mon_event, new_ped=False, new_ff=F
 
     if new_ped:
         # write ped container
+        print(f"{table_group}/pedestal")
         writer.write(
-            table_name="telescope/monitoring/pedestal",
+            table_name=f"{table_group}/pedestal",
             containers=[mon_index, mon_event.pedestal],
         )
 
     if new_ff:
         # write calibration container
         writer.write(
-            table_name="telescope/monitoring/flatfield",
+            table_name=f"{table_group}/flatfield",
             containers=[mon_index, mon_event.flatfield],
         )
 
         # write ff container
         writer.write(
-            table_name="telescope/monitoring/calibration",
+            table_name=f"{table_group}/calibration",
             containers=[mon_index, mon_event.calibration],
         )
 

lstchain/onsite.py

@@ -3,17 +3,25 @@
 from pkg_resources import resource_filename
 from datetime import datetime
 import tempfile
+from astropy.time import Time
+import pymongo
 
 from .paths import parse_calibration_name
 
 DEFAULT_BASE_PATH = Path('/fefs/aswg/data/real')
 DEFAULT_R0_PATH = DEFAULT_BASE_PATH / 'R0'
-LEVEL_A_PIXEL_DIR = 'monitoring/PixelCalibration/Cat-A'
+DEFAULT_DL1_PATH = DEFAULT_BASE_PATH / 'DL1'
+CAT_A_PIXEL_DIR = 'monitoring/PixelCalibration/Cat-A'
+CAT_B_PIXEL_DIR = 'monitoring/PixelCalibration/Cat-B'
 
 DEFAULT_CONFIG = Path(resource_filename(
     'lstchain',
     "data/onsite_camera_calibration_param.json",
 ))
+DEFAULT_CONFIG_CAT_B_CALIB = Path(resource_filename(
+    'lstchain',
+    "data/catB_camera_calibration_param.json",
+))
 
 
 def is_date(s):
@@ -88,7 +96,7 @@
 
 def find_pedestal_file(pro, pedestal_run=None, date=None, base_dir=DEFAULT_BASE_PATH):
     # pedestal base dir
-    ped_dir = Path(base_dir) / LEVEL_A_PIXEL_DIR / "drs4_baseline"
+    ped_dir = Path(base_dir) / CAT_A_PIXEL_DIR / "drs4_baseline"
 
     if pedestal_run is None and date is None:
         raise ValueError('Must give at least `date` or `run`')
@@ -123,7 +131,7 @@
 def find_time_calibration_file(pro, run, time_run=None, base_dir=DEFAULT_BASE_PATH):
     '''Find a time calibration file for given run
     '''
-    time_dir = Path(base_dir) / LEVEL_A_PIXEL_DIR / "drs4_time_sampling_from_FF"
+    time_dir = Path(base_dir) / CAT_A_PIXEL_DIR / "drs4_time_sampling_from_FF"
 
 
     # search the last time run before or equal to the calibration run
@@ -155,7 +163,7 @@
 
 
 def find_systematics_correction_file(pro, date, sys_date=None, base_dir=DEFAULT_BASE_PATH):
-    sys_dir = Path(base_dir) / LEVEL_A_PIXEL_DIR / "ffactor_systematics"
+    sys_dir = Path(base_dir) / CAT_A_PIXEL_DIR / "ffactor_systematics"
 
     if sys_date is not None:
         path =  (sys_dir / sys_date / pro / f"ffactor_systematics_{sys_date}.h5").resolve()
@@ -171,3 +179,100 @@
     selected_date = next((day for day in sys_date_list if day <= date), sys_date_list[-1])
 
     return (sys_dir / selected_date / pro / f"ffactor_systematics_{selected_date}.h5").resolve()
+
+def find_calibration_file(pro, calibration_run=None, date=None, category='Cat_A',base_dir=DEFAULT_BASE_PATH):
+    # pedestal base dir
+
+    if category == 'Cat_A':
+        cal_dir = Path(base_dir) / CAT_A_PIXEL_DIR / "calibration"
+    elif category == 'Cat_B':
+        cal_dir = Path(base_dir) / CAT_B_PIXEL_DIR / "calibration"
+    else:
+        raise ValueError(f'Argument \'category\' can be only \'Cat_A\' or \'Cat_B\', not {category}')    
+
+    if calibration_run is None and date is None:
+        raise ValueError('Must give at least `date` or `run`')
+
+    if calibration_run is not None:
+        # search a specific calibration run
+        file_list = sorted(cal_dir.rglob(f'{pro}/calibration*.Run{calibration_run:05d}.0000.h5'))
+
+        if len(file_list) == 0:
+            raise IOError(f"Calibration file from run {calibration_run} not found\n")
+
+        return file_list[0].resolve()
+
+    # search for a unique calibration file from the same date
+    file_list = sorted((cal_dir / date / pro).glob('calibration*.0000.h5'))
+    if len(file_list) == 0:
+        raise IOError(f"No calibration file found for date {date}")
+
+    if len(file_list) > 1:
+        raise IOError(f"Too many calibration files found for date {date}: {file_list}, choose one run\n")
+
+    return file_list[0].resolve()
+
+def find_DL1_subrun(run, sub_run, dl1_dir=DEFAULT_DL1_PATH):
+    '''
+    Find the given subrun DL1 file (i.e. globbing for the date part)
+    '''
+    file_list = rglob_symlinks(dl1_dir, f'dl1_LST-1.1.Run{run:05d}.{sub_run:04d}*.h5')
+    # ignore directories that are not a date, e.g. "Trash"
+    file_list = [p for p in file_list if is_date(p.parent.name)]
+
+    if len(file_list) == 0:
+        raise IOError(f"Run {run} not found\n")
+
+    if len(file_list) > 1:
+        raise IOError(f"Found more than one file for run {run}.{sub_run}: {file_list}")
+
+    return file_list[0]
+
+def find_interleaved_subruns(run, dl1_dir=DEFAULT_DL1_PATH):
+    '''
+    Find the given subrun interleaved file (i.e. globbing for the date part)
+    '''
+    file_list = rglob_symlinks(dl1_dir, f'interleaved_LST-1.Run{run:05d}.*.h5')
+    # ignore directories that are not a date, e.g. "Trash"
+    file_list = [p for p in file_list if is_date(p.parent.name)]
+
+    if len(file_list) == 0:
+        raise IOError(f"Run {run} not found\n")
+
+    return file_list
+
+
+def find_filter_wheels(run, database_url):
+    """read the employed filters from mongodb"""
+
+    # there was a change of Mongo DB data names on 5/12/2022
+    NEW_DB_NAMES_DATE = Time("2022-12-04T00:00:00")
+
+    filters = None
+    try:
+
+        myclient = pymongo.MongoClient(database_url)
+
+        mydb = myclient["CACO"]
+        mycol = mydb["RUN_INFORMATION"]
+        mydoc = mycol.find({"run_number": {"$eq": run}})
+        for x in mydoc:
+            date =  Time(x["start_time"])
+            if date < NEW_DB_NAMES_DATE:
+                w1 = int(x["cbox"]["wheel1 position"])
+                w2 = int(x["cbox"]["wheel2 position"])
+            else:
+                w1 = int(x["cbox"]["CBOX_WheelPosition1"])
+                w2 = int(x["cbox"]["CBOX_WheelPosition2"])
+
+            filters = f"{w1:1d}{w2:1d}"
+
+    except Exception as e:
+        print(f"\n >>> Exception: {e}")
+        raise IOError(
+            "--> No mongo DB filter information."
+            " You must pass the filters by argument: -f [filters]"
+        )
+
+    return filters
+

lstchain/reco/r0_to_dl1.py

@@ -507,7 +507,9 @@ def r0_to_dl1(
                         add_config_metadata(container, config)
 
                     # write the first calibration event (initialized from calibration h5 file)
+                    # these data a supposed to change table_path with "dl1/monitoring/telescope/CatA" in short future
                     write_calibration_data(writer,
+                                           'telescope/monitoring',
                                            calibration_index,
                                            event.mon.tel[tel_id],
                                            new_ped=True, new_ff=True)
@@ -520,8 +522,10 @@ def r0_to_dl1(
                     new_ped_event, new_ff_event = calibration_calculator.process_interleaved(event)
 
                     # write monitoring containers if updated
+                    # these data a supposed to be replaced by the Cat_B data in a short future
                     if new_ped_event or new_ff_event:
                         write_calibration_data(writer,
+                                           'telescope/monitoring',
                                            calibration_index,
                                            event.mon.tel[tel_id],
                                            new_ped=new_ped_event, new_ff=new_ff_event)
@@ -756,7 +760,8 @@ def r0_to_dl1(
             # at the end of event loop ask calculation of remaining interleaved statistics
             new_ped, new_ff = calibration_calculator.output_interleaved_results(event)
             # write monitoring events
-            write_calibration_data(writer,
+            # these data a supposed to be replaced by the Cat_B data in a short future
+            write_calibration_data(writer,'telescope/monitoring',
                                    calibration_index,
                                    event.mon.tel[tel_id],
                                    new_ped=new_ped, new_ff=new_ff)

lstchain/scripts/onsite/onsite_create_calibration_file.py

@@ -13,8 +13,6 @@
 import subprocess
 import sys
 from pathlib import Path
-from astropy.time import Time
-import pymongo
 
 import lstchain
 import lstchain.visualization.plot_calib as calib
@@ -23,13 +21,14 @@
 from lstchain.onsite import (
     DEFAULT_BASE_PATH,
     DEFAULT_CONFIG,
-    LEVEL_A_PIXEL_DIR,
+    CAT_A_PIXEL_DIR,
     create_pro_symlink,
     find_r0_subrun,
     find_pedestal_file,
     find_run_summary,
     find_systematics_correction_file,
     find_time_calibration_file,
+    find_filter_wheels,
 )
 
 # parse arguments
@@ -118,7 +117,7 @@
 
     # looks for the filter values in the database if not given
     if args.filters is None:
-        filters = search_filter(run, args.mongodb)
+        filters = find_filter_wheels(run, args.mongodb)
     else:
         filters = args.filters
 
@@ -146,7 +145,7 @@
     print(f"\n--> Input file: {input_file}")
 
     # verify output dir
-    calib_dir = args.base_dir / LEVEL_A_PIXEL_DIR
+    calib_dir = args.base_dir / CAT_A_PIXEL_DIR
     output_dir = calib_dir / "calibration" / date / prod_id
     if not output_dir.exists():
         print(f"--> Create directory {output_dir}")
@@ -253,41 +252,6 @@
     print("\n--> END")
 
 
-def search_filter(run, database_url):
-    """read the employed filters form mongodb"""
-
-    # there was a change of Mongo DB data names on 5/12/2022
-    NEW_DB_NAMES_DATE = Time("2022-12-04T00:00:00")
-
-    filters = None
-    try:
-
-        myclient = pymongo.MongoClient(database_url)
-
-        mydb = myclient["CACO"]
-        mycol = mydb["RUN_INFORMATION"]
-        mydoc = mycol.find({"run_number": {"$eq": run}})
-        for x in mydoc:
-            date =  Time(x["start_time"])
-            if date < NEW_DB_NAMES_DATE:
-                w1 = int(x["cbox"]["wheel1 position"])
-                w2 = int(x["cbox"]["wheel2 position"])
-            else:
-                w1 = int(x["cbox"]["CBOX_WheelPosition1"])
-                w2 = int(x["cbox"]["CBOX_WheelPosition2"])
-
-            filters = f"{w1:1d}{w2:1d}"
-
-    except Exception as e:
-        print(f"\n >>> Exception: {e}")
-        raise IOError(
-            "--> No mongo DB filter information."
-            " You must pass the filters by argument: -f [filters]"
-        )
-
-    return filters
-
-
 def define_FF_selection_range(filters):
     """ return the range of charges to select the FF events """
 

lstchain/scripts/onsite/onsite_create_calibration_files_with_batch.py

@@ -18,7 +18,7 @@
 import lstchain
 from lstchain.onsite import (
     DEFAULT_BASE_PATH,
-    LEVEL_A_PIXEL_DIR,
+    CAT_A_PIXEL_DIR,
     find_r0_subrun,
     DEFAULT_CONFIG,
 )
@@ -111,7 +111,7 @@
 
     output_base_name = args.output_base_name
 
-    calib_dir = base_dir / LEVEL_A_PIXEL_DIR
+    calib_dir = base_dir / CAT_A_PIXEL_DIR
 
     if shutil.which('srun') is None:
         sys.exit(">>> This script needs a slurm batch system. Stop")
@@ -178,7 +178,6 @@
                     f"--sub_run={sub_run}",
                     f"-b {base_dir}",
                     f"-s {stat_events}",
-                    f"--r0-dir={r0_dir}",
                     f"--output_base_name={output_base_name}",
                     f"--config={config_file}",
                 ]
@@ -207,6 +206,9 @@
                 if args.use_flatfield_heuristic is False:
                     cmd.append("--no-flatfield-heuristic")
 
+                if args.no_pro_symlink is True:
+                    cmd.append("--no_pro_symlink")
+
                 cmd.extend(remaining_args)
 
                 # join command together with newline, line continuation and indentation