From fa6d509a9d0bc19b937dec30cd8c08738346301f Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Tue, 9 Mar 2021 21:40:30 +0100 Subject: [PATCH 1/8] update training tests --- .../aux/example_config_files/analysis.yaml | 10 ++-- protopipe/pipeline/temp.py | 60 +++++++++++++++++++ protopipe/scripts/tests/test_dataTraining.py | 45 +++++++++----- 3 files changed, 95 insertions(+), 20 deletions(-) diff --git a/protopipe/aux/example_config_files/analysis.yaml b/protopipe/aux/example_config_files/analysis.yaml index 00b3d250..fedebd55 100644 --- a/protopipe/aux/example_config_files/analysis.yaml +++ b/protopipe/aux/example_config_files/analysis.yaml @@ -2,7 +2,7 @@ # NOTE: only Prod3b simulations are currently supported. General: config_name: 'v0.4.0_dev1' - site: 'south' # 'north' or 'south' + site: 'north' # 'north' or 'south' # array can be either # - 'subarray_LSTs', 'subarray_MSTs', 'subarray_SSTs' or 'full_array' # - a custom list of telescope IDs @@ -18,8 +18,8 @@ ImageCleaning: biggest: tail: # thresholds: # picture, boundary - - LSTCam: [6.61, 3.30] # TBC - - NectarCam: [5.75, 2.88] # TBC + - LSTCam: [6.61, 3.30] + - NectarCam: [5.75, 2.88] - FlashCam: [4,2] # dummy values for reliable unit-testing - ASTRICam: [4,2] # dummy values for reliable unit-testing - DigiCam: [0,0] # values left unset for future studies @@ -52,8 +52,8 @@ ImageCleaning: extended: tail: # thresholds: # picture, boundary - - LSTCam: [6.61, 3.30] # TBC - - NectarCam: [5.75, 2.88] # TBC + - LSTCam: [6.61, 3.30] + - NectarCam: [5.75, 2.88] - FlashCam: [4,2] # dummy values for reliable unit-testing - ASTRICam: [4,2] # dummy values for reliable unit-testing - DigiCam: [0,0] # values left unset for future studies diff --git a/protopipe/pipeline/temp.py b/protopipe/pipeline/temp.py index 1bf9034f..3bda3507 100644 --- a/protopipe/pipeline/temp.py +++ b/protopipe/pipeline/temp.py @@ -10,6 +10,10 @@ """ +import os +import logging +from pathlib import Path + import numpy as np from numpy import nan from scipy.sparse import lil_matrix, csr_matrix @@ -22,6 +26,7 @@ from ctapipe.reco import HillasReconstructor from ctapipe.reco.HillasReconstructor import HillasPlane +logger = logging.getLogger(__name__) class HillasParametersTelescopeFrameContainer(Container): container_prefix = "hillas" @@ -199,3 +204,58 @@ def initialize_hillas_planes( weight=moments.intensity * (moments.length / moments.width), ) self.hillas_planes[tel_id] = circle + + +try: + import ctapipe_resources + + has_resources = True +except ImportError: + has_resources = False + +DEFAULT_URL = "http://cccta-dataserver.in2p3.fr/data/ctapipe-extra/v0.3.3/" +def get_dataset_path(filename, url=DEFAULT_URL): + """ + Returns the full file path to an auxiliary dataset needed by + ctapipe, given the dataset's full name (filename with no directory). + This will first search for the file in directories listed in + tne environment variable CTAPIPE_SVC_PATH (if set), and if not found, + will look in the ctapipe_resources module + (if installed with the ctapipe-extra package), which contains the defaults. + Parameters + ---------- + filename: str + name of dataset to fetch + Returns + ------- + string with full path to the given dataset + """ + + searchpath = os.getenv("CTAPIPE_SVC_PATH") + + if searchpath: + filepath = find_in_path(filename=filename, searchpath=searchpath, url=url) + + if filepath: + return filepath + + if has_resources: + logger.debug( + "Resource '{}' not found in CTAPIPE_SVC_PATH, looking in " + "ctapipe_resources...".format(filename) + ) + + return Path(ctapipe_resources.get(filename)) + + # last, try downloading the data + try: + return download_file_cached(filename, default_url=url, progress=True) + except HTTPError as e: + # let 404 raise the FileNotFoundError instead of HTTPError + if e.response.status_code != 404: + raise + + raise FileNotFoundError( + f"Couldn't find resource: '{filename}'," + " You might want to install ctapipe_resources" + ) \ No newline at end of file diff --git a/protopipe/scripts/tests/test_dataTraining.py b/protopipe/scripts/tests/test_dataTraining.py index 382eb6eb..1ed147ed 100644 --- a/protopipe/scripts/tests/test_dataTraining.py +++ b/protopipe/scripts/tests/test_dataTraining.py @@ -1,21 +1,35 @@ -"""Test the data training script in a variety of conditions.""" +"""Test the data training script. + +TODO +---- + +- test only diffuse data (more general case) +- add Paranal diffuse test file +- add Prod5 test files + +""" from os import path, system from pkg_resources import resource_filename from protopipe.scripts import data_training -from ctapipe.utils import get_dataset_path - -# TEST FILES -# 110 events, 98 telescopes at Paranal. -# Instruments tested: LST_LST_LSTCam, MST_MST_FlashCam, SST_ASTRI_ASTRICam -GAMMA_TEST_LARGE = get_dataset_path("gamma_test_large.simtel.gz") -# WARNING: absolutely not sufficient! -# This is just the only file easily usable without external resources. -# Later on, we will need a sub-simtel file from each of the -# MC productions expected to be analyzed with protopipe. +from protopipe.pipeline.temp import get_dataset_path # configuration files ana_config = resource_filename("protopipe", "aux/example_config_files/analysis.yaml") +# TEST FILES + +DEFAULT_URL = "http://cccta-dataserver.in2p3.fr/data/ctapipe-extra/v0.3.3/" + +# Prod 2 + +# CTA_SOUTH = get_dataset_path("gamma_test_large.simtel.gz") + +# PROD 3b + +CTA_NORTH = get_dataset_path("gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz") + + + def test_dataTraining_noImages(): """Very bare test to see if the script reaches the end correctly. @@ -30,8 +44,9 @@ def test_dataTraining_noImages(): --config_file {ana_config}\ -o test_training_noImages.h5\ -m 10\ - -i {path.dirname(GAMMA_TEST_LARGE)}\ - -f {path.basename(GAMMA_TEST_LARGE)}" + --debug\ + -i {path.dirname(CTA_NORTH)}\ + -f {path.basename(CTA_NORTH)}" ) assert exit_status == 0 @@ -50,7 +65,7 @@ def test_dataTraining_withImages(): -o test_training_withImages.h5\ -m 10\ --save_images\ - -i {path.dirname(GAMMA_TEST_LARGE)}\ - -f {path.basename(GAMMA_TEST_LARGE)}" + -i {path.dirname(CTA_NORTH)}\ + -f {path.basename(CTA_NORTH)}" ) assert exit_status == 0 From 39ef1bc434addbce9bea3379a23f75cc11b957d0 Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Tue, 9 Mar 2021 22:00:27 +0100 Subject: [PATCH 2/8] remove default URL from test code --- protopipe/scripts/tests/test_dataTraining.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/protopipe/scripts/tests/test_dataTraining.py b/protopipe/scripts/tests/test_dataTraining.py index 1ed147ed..aca38da3 100644 --- a/protopipe/scripts/tests/test_dataTraining.py +++ b/protopipe/scripts/tests/test_dataTraining.py @@ -18,8 +18,6 @@ # TEST FILES -DEFAULT_URL = "http://cccta-dataserver.in2p3.fr/data/ctapipe-extra/v0.3.3/" - # Prod 2 # CTA_SOUTH = get_dataset_path("gamma_test_large.simtel.gz") From f692f3e38346c347b2b1480e4302ad2b25460dd2 Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Tue, 9 Mar 2021 22:02:43 +0100 Subject: [PATCH 3/8] remove degub flag added during tests --- protopipe/scripts/tests/test_dataTraining.py | 1 - 1 file changed, 1 deletion(-) diff --git a/protopipe/scripts/tests/test_dataTraining.py b/protopipe/scripts/tests/test_dataTraining.py index aca38da3..41343837 100644 --- a/protopipe/scripts/tests/test_dataTraining.py +++ b/protopipe/scripts/tests/test_dataTraining.py @@ -42,7 +42,6 @@ def test_dataTraining_noImages(): --config_file {ana_config}\ -o test_training_noImages.h5\ -m 10\ - --debug\ -i {path.dirname(CTA_NORTH)}\ -f {path.basename(CTA_NORTH)}" ) From 55546c656f3967b819e77903ce8ee46b802ead2d Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Tue, 9 Mar 2021 22:32:01 +0100 Subject: [PATCH 4/8] test on multiple prod3b files --- protopipe/scripts/tests/test_dataTraining.py | 39 ++++++++++++-------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/protopipe/scripts/tests/test_dataTraining.py b/protopipe/scripts/tests/test_dataTraining.py index 41343837..a72974bf 100644 --- a/protopipe/scripts/tests/test_dataTraining.py +++ b/protopipe/scripts/tests/test_dataTraining.py @@ -10,26 +10,21 @@ """ from os import path, system from pkg_resources import resource_filename + +import pytest + from protopipe.scripts import data_training from protopipe.pipeline.temp import get_dataset_path -# configuration files -ana_config = resource_filename("protopipe", "aux/example_config_files/analysis.yaml") - # TEST FILES -# Prod 2 - -# CTA_SOUTH = get_dataset_path("gamma_test_large.simtel.gz") - # PROD 3b -CTA_NORTH = get_dataset_path("gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz") - +PROD3B_CTA_NORTH = get_dataset_path("gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz") +PROD3B_CTA_SOUTH = get_dataset_path("gamma_Paranal_baseline_20Zd_180Az_prod3_test.simtel.gz") - - -def test_dataTraining_noImages(): +@pytest.mark.parametrize("input_file",[PROD3B_CTA_NORTH, PROD3B_CTA_SOUTH]) +def test_dataTraining_noImages(input_file): """Very bare test to see if the script reaches the end correctly. WARNING: some of the cuts in the example config file are not optimized for @@ -37,17 +32,24 @@ def test_dataTraining_noImages(): In any case, it is expected that in absence of fatal bugs, the script ends successfully. """ + + # the difference is only the 'site' key as a check for the user + if "Paranal" in str(input_file): + ana_config = resource_filename("protopipe", "scripts/tests/test_config_analysis_south.yaml") + else: + ana_config = resource_filename("protopipe", "scripts/tests/test_config_analysis_north.yaml") + exit_status = system( f"python {data_training.__file__}\ --config_file {ana_config}\ -o test_training_noImages.h5\ -m 10\ - -i {path.dirname(CTA_NORTH)}\ - -f {path.basename(CTA_NORTH)}" + -i {path.dirname(input_file)}\ + -f {path.basename(input_file)}" ) assert exit_status == 0 - +@pytest.mark.parametrize("input_file",[PROD3B_CTA_NORTH, PROD3B_CTA_SOUTH]) def test_dataTraining_withImages(): """Very bare test to see if the script reaches the end correctly. @@ -56,6 +58,13 @@ def test_dataTraining_withImages(): In any case, it is expected that in absence of fatal bugs, the script ends successfully. """ + + # the difference is only the 'site' key as a check for the user + if "Paranal" in str(input_file): + ana_config = resource_filename("protopipe", "scripts/tests/test_config_analysis_south.yaml") + else: + ana_config = resource_filename("protopipe", "scripts/tests/test_config_analysis_north.yaml") + exit_status = system( f"python {data_training.__file__}\ --config_file {ana_config}\ From 232503c312dba49bf46d1b944e76f479633e1935 Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Tue, 9 Mar 2021 22:33:44 +0100 Subject: [PATCH 5/8] Add config files for TRAINING integration tests --- .../tests/test_config_analysis_north.yaml | 106 ++++++++++++++++++ .../tests/test_config_analysis_south.yaml | 106 ++++++++++++++++++ 2 files changed, 212 insertions(+) create mode 100644 protopipe/scripts/tests/test_config_analysis_north.yaml create mode 100644 protopipe/scripts/tests/test_config_analysis_south.yaml diff --git a/protopipe/scripts/tests/test_config_analysis_north.yaml b/protopipe/scripts/tests/test_config_analysis_north.yaml new file mode 100644 index 00000000..fedebd55 --- /dev/null +++ b/protopipe/scripts/tests/test_config_analysis_north.yaml @@ -0,0 +1,106 @@ +# General informations +# NOTE: only Prod3b simulations are currently supported. +General: + config_name: 'v0.4.0_dev1' + site: 'north' # 'north' or 'south' + # array can be either + # - 'subarray_LSTs', 'subarray_MSTs', 'subarray_SSTs' or 'full_array' + # - a custom list of telescope IDs + # WARNING: for simulations containing multiple copies of the telescopes, + # only 'full_array' or custom list are supported options! + array: full_array + cam_id_list : ['LSTCam', 'NectarCam'] # Selected cameras (disabled option) + +# Cleaning for reconstruction +ImageCleaning: + + # Cleaning for reconstruction + biggest: + tail: # + thresholds: # picture, boundary + - LSTCam: [6.61, 3.30] + - NectarCam: [5.75, 2.88] + - FlashCam: [4,2] # dummy values for reliable unit-testing + - ASTRICam: [4,2] # dummy values for reliable unit-testing + - DigiCam: [0,0] # values left unset for future studies + - CHEC: [0,0] # values left unset for future studies + - SCTCam: [0,0] # values left unset for future studies + keep_isolated_pixels: False + min_number_picture_neighbors: 1 + + wave: + # Directory to write temporary files + #tmp_files_directory: '/dev/shm/' + tmp_files_directory: './' + options: + LSTCam: + type_of_filtering: 'hard_filtering' + filter_thresholds: [3, 0.2] + last_scale_treatment: 'drop' + kill_isolated_pixels: True + detect_only_positive_structures: False + clusters_threshold: 0 + NectarCam: # TBC + type_of_filtering: 'hard_filtering' + filter_thresholds: [3, 0.2] + last_scale_treatment: 'drop' + kill_isolated_pixels: True + detect_only_positive_structures: False + clusters_threshold: 0 + + # Cleaning for energy/score estimation + extended: + tail: # + thresholds: # picture, boundary + - LSTCam: [6.61, 3.30] + - NectarCam: [5.75, 2.88] + - FlashCam: [4,2] # dummy values for reliable unit-testing + - ASTRICam: [4,2] # dummy values for reliable unit-testing + - DigiCam: [0,0] # values left unset for future studies + - CHEC: [0,0] # values left unset for future studies + - SCTCam: [0,0] # values left unset for future studies + keep_isolated_pixels: False + min_number_picture_neighbors: 1 + + wave: + # Directory to write temporary files + #tmp_files_directory: '/dev/shm/' + tmp_files_directory: './' + options: + LSTCam: + type_of_filtering: 'hard_filtering' + filter_thresholds: [3, 0.2] + last_scale_treatment: 'posmask' + kill_isolated_pixels: True + detect_only_positive_structures: False + clusters_threshold: 0 + NectarCam: # TBC + type_of_filtering: 'hard_filtering' + filter_thresholds: [3, 0.2] + last_scale_treatment: 'posmask' + kill_isolated_pixels: True + detect_only_positive_structures: False + clusters_threshold: 0 + +# Cut for image selection +ImageSelection: + charge: [50., 1e10] + pixel: [3, 1e10] + ellipticity: [0.1, 0.6] + nominal_distance: [0., 0.8] # in camera radius + +# Minimal number of telescopes to consider events +Reconstruction: + min_tel: 2 + +# Parameters for energy estimation +EnergyRegressor: + # Name of the regression method (e.g. AdaBoostRegressor, etc.) + method_name: 'AdaBoostRegressor' + +# Parameters for g/h separation +GammaHadronClassifier: + # Name of the classification method (e.g. AdaBoostRegressor, etc.) + method_name: 'RandomForestClassifier' + # Use probability output or score + use_proba: True diff --git a/protopipe/scripts/tests/test_config_analysis_south.yaml b/protopipe/scripts/tests/test_config_analysis_south.yaml new file mode 100644 index 00000000..6a9de344 --- /dev/null +++ b/protopipe/scripts/tests/test_config_analysis_south.yaml @@ -0,0 +1,106 @@ +# General informations +# NOTE: only Prod3b simulations are currently supported. +General: + config_name: 'v0.4.0_dev1' + site: 'south' # 'north' or 'south' + # array can be either + # - 'subarray_LSTs', 'subarray_MSTs', 'subarray_SSTs' or 'full_array' + # - a custom list of telescope IDs + # WARNING: for simulations containing multiple copies of the telescopes, + # only 'full_array' or custom list are supported options! + array: full_array + cam_id_list : ['LSTCam', 'NectarCam'] # Selected cameras (disabled option) + +# Cleaning for reconstruction +ImageCleaning: + + # Cleaning for reconstruction + biggest: + tail: # + thresholds: # picture, boundary + - LSTCam: [6.61, 3.30] + - NectarCam: [5.75, 2.88] + - FlashCam: [4,2] # dummy values for reliable unit-testing + - ASTRICam: [4,2] # dummy values for reliable unit-testing + - DigiCam: [0,0] # values left unset for future studies + - CHEC: [0,0] # values left unset for future studies + - SCTCam: [0,0] # values left unset for future studies + keep_isolated_pixels: False + min_number_picture_neighbors: 1 + + wave: + # Directory to write temporary files + #tmp_files_directory: '/dev/shm/' + tmp_files_directory: './' + options: + LSTCam: + type_of_filtering: 'hard_filtering' + filter_thresholds: [3, 0.2] + last_scale_treatment: 'drop' + kill_isolated_pixels: True + detect_only_positive_structures: False + clusters_threshold: 0 + NectarCam: # TBC + type_of_filtering: 'hard_filtering' + filter_thresholds: [3, 0.2] + last_scale_treatment: 'drop' + kill_isolated_pixels: True + detect_only_positive_structures: False + clusters_threshold: 0 + + # Cleaning for energy/score estimation + extended: + tail: # + thresholds: # picture, boundary + - LSTCam: [6.61, 3.30] + - NectarCam: [5.75, 2.88] + - FlashCam: [4,2] # dummy values for reliable unit-testing + - ASTRICam: [4,2] # dummy values for reliable unit-testing + - DigiCam: [0,0] # values left unset for future studies + - CHEC: [0,0] # values left unset for future studies + - SCTCam: [0,0] # values left unset for future studies + keep_isolated_pixels: False + min_number_picture_neighbors: 1 + + wave: + # Directory to write temporary files + #tmp_files_directory: '/dev/shm/' + tmp_files_directory: './' + options: + LSTCam: + type_of_filtering: 'hard_filtering' + filter_thresholds: [3, 0.2] + last_scale_treatment: 'posmask' + kill_isolated_pixels: True + detect_only_positive_structures: False + clusters_threshold: 0 + NectarCam: # TBC + type_of_filtering: 'hard_filtering' + filter_thresholds: [3, 0.2] + last_scale_treatment: 'posmask' + kill_isolated_pixels: True + detect_only_positive_structures: False + clusters_threshold: 0 + +# Cut for image selection +ImageSelection: + charge: [50., 1e10] + pixel: [3, 1e10] + ellipticity: [0.1, 0.6] + nominal_distance: [0., 0.8] # in camera radius + +# Minimal number of telescopes to consider events +Reconstruction: + min_tel: 2 + +# Parameters for energy estimation +EnergyRegressor: + # Name of the regression method (e.g. AdaBoostRegressor, etc.) + method_name: 'AdaBoostRegressor' + +# Parameters for g/h separation +GammaHadronClassifier: + # Name of the classification method (e.g. AdaBoostRegressor, etc.) + method_name: 'RandomForestClassifier' + # Use probability output or score + use_proba: True From 2476477a35a2f9ca0bb034215d04656274c17854 Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Tue, 9 Mar 2021 22:38:43 +0100 Subject: [PATCH 6/8] add missing input_file argument in no images test --- protopipe/scripts/tests/test_dataTraining.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/protopipe/scripts/tests/test_dataTraining.py b/protopipe/scripts/tests/test_dataTraining.py index a72974bf..77b61928 100644 --- a/protopipe/scripts/tests/test_dataTraining.py +++ b/protopipe/scripts/tests/test_dataTraining.py @@ -71,7 +71,7 @@ def test_dataTraining_withImages(): -o test_training_withImages.h5\ -m 10\ --save_images\ - -i {path.dirname(CTA_NORTH)}\ - -f {path.basename(CTA_NORTH)}" + -i {path.dirname(input_file)}\ + -f {path.basename(input_file)}" ) assert exit_status == 0 From 19b8956fad07235f4caa7580ecc9fe1b4b79df55 Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Tue, 9 Mar 2021 22:40:51 +0100 Subject: [PATCH 7/8] small fix + format --- protopipe/scripts/tests/test_dataTraining.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/protopipe/scripts/tests/test_dataTraining.py b/protopipe/scripts/tests/test_dataTraining.py index 77b61928..3a55d5c2 100644 --- a/protopipe/scripts/tests/test_dataTraining.py +++ b/protopipe/scripts/tests/test_dataTraining.py @@ -23,7 +23,8 @@ PROD3B_CTA_NORTH = get_dataset_path("gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz") PROD3B_CTA_SOUTH = get_dataset_path("gamma_Paranal_baseline_20Zd_180Az_prod3_test.simtel.gz") -@pytest.mark.parametrize("input_file",[PROD3B_CTA_NORTH, PROD3B_CTA_SOUTH]) + +@pytest.mark.parametrize("input_file", [PROD3B_CTA_NORTH, PROD3B_CTA_SOUTH]) def test_dataTraining_noImages(input_file): """Very bare test to see if the script reaches the end correctly. @@ -32,13 +33,13 @@ def test_dataTraining_noImages(input_file): In any case, it is expected that in absence of fatal bugs, the script ends successfully. """ - + # the difference is only the 'site' key as a check for the user if "Paranal" in str(input_file): ana_config = resource_filename("protopipe", "scripts/tests/test_config_analysis_south.yaml") else: ana_config = resource_filename("protopipe", "scripts/tests/test_config_analysis_north.yaml") - + exit_status = system( f"python {data_training.__file__}\ --config_file {ana_config}\ @@ -49,8 +50,9 @@ def test_dataTraining_noImages(input_file): ) assert exit_status == 0 -@pytest.mark.parametrize("input_file",[PROD3B_CTA_NORTH, PROD3B_CTA_SOUTH]) -def test_dataTraining_withImages(): + +@pytest.mark.parametrize("input_file", [PROD3B_CTA_NORTH, PROD3B_CTA_SOUTH]) +def test_dataTraining_withImages(input_file): """Very bare test to see if the script reaches the end correctly. WARNING: some of the cuts in the example config file are not optimized for @@ -58,13 +60,13 @@ def test_dataTraining_withImages(): In any case, it is expected that in absence of fatal bugs, the script ends successfully. """ - + # the difference is only the 'site' key as a check for the user if "Paranal" in str(input_file): ana_config = resource_filename("protopipe", "scripts/tests/test_config_analysis_south.yaml") else: ana_config = resource_filename("protopipe", "scripts/tests/test_config_analysis_north.yaml") - + exit_status = system( f"python {data_training.__file__}\ --config_file {ana_config}\ From d864ddda3a65f4908ca0ba8618a0e3a69b3322c2 Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Wed, 10 Mar 2021 14:16:09 +0100 Subject: [PATCH 8/8] add check on file size --- protopipe/scripts/tests/test_dataTraining.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/protopipe/scripts/tests/test_dataTraining.py b/protopipe/scripts/tests/test_dataTraining.py index 3a55d5c2..29d10812 100644 --- a/protopipe/scripts/tests/test_dataTraining.py +++ b/protopipe/scripts/tests/test_dataTraining.py @@ -11,6 +11,7 @@ from os import path, system from pkg_resources import resource_filename +import tables import pytest from protopipe.scripts import data_training @@ -48,8 +49,14 @@ def test_dataTraining_noImages(input_file): -i {path.dirname(input_file)}\ -f {path.basename(input_file)}" ) + + # check that the script ends without crashing assert exit_status == 0 + # check that the produced HDF5 file is non-empty + with tables.open_file("test_training_noImages.h5") as file: + assert file.get_filesize() > 0 + @pytest.mark.parametrize("input_file", [PROD3B_CTA_NORTH, PROD3B_CTA_SOUTH]) def test_dataTraining_withImages(input_file): @@ -76,4 +83,10 @@ def test_dataTraining_withImages(input_file): -i {path.dirname(input_file)}\ -f {path.basename(input_file)}" ) + + # check that the script ends without crashing assert exit_status == 0 + + # check that the produced HDF5 file is non-empty + with tables.open_file("test_training_noImages.h5") as file: + assert file.get_filesize() > 0