adding if condition to remove keys

environment.yml

@@ -12,6 +12,7 @@ dependencies:
   - ipython
   - jupyter
   - notebook
+  - joblib
   - pip:
       - pytest_runner
       - pytest-ordering

lstchain/reco/dl1_to_dl2.py

@@ -10,7 +10,7 @@
 import numpy as np
 import pandas as pd
 from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
-from sklearn.externals import joblib
+import joblib
 from sklearn.model_selection import train_test_split
 import os
 from . import utils

lstchain/scripts/lstchain_dl1_to_dl2.py

@@ -21,7 +21,7 @@
 import pandas as pd
 
 from tables import open_file
-from sklearn.externals import joblib
+import joblib
 from lstchain.reco.utils import filter_events, impute_pointing
 from lstchain.reco import dl1_to_dl2
 from lstchain.io import (
@@ -119,8 +119,10 @@ def main():
         raise IOError(output_file + ' exists, exiting.')
 
     dl1_keys = get_dataset_keys(args.input_file)
-    dl1_keys.remove(dl1_images_lstcam_key)
-    dl1_keys.remove(dl1_params_lstcam_key)
+    if dl1_images_lstcam_key in dl1_keys:
+        dl1_keys.remove(dl1_images_lstcam_key)
+    if dl1_params_lstcam_key in dl1_keys:
+        dl1_keys.remove(dl1_params_lstcam_key)
 
     if dl1_params_src_dep_lstcam_key in dl1_keys:
         dl1_keys.remove(dl1_params_src_dep_lstcam_key)

lstchain/scripts/tests/test_lstchain_scripts.py

@@ -7,10 +7,12 @@
 from lstchain.io.io import dl1_params_src_dep_lstcam_key
 import subprocess as sp
 import pkg_resources
+import shutil
 
 
 output_dir = os.path.join(test_dir, 'scripts')
 dl1_file = os.path.join(output_dir, 'dl1_gamma_test_large.h5')
+merged_dl1_file = os.path.join(output_dir, 'script_merged_dl1.h5')
 dl2_file = os.path.join(output_dir, 'dl2_gamma_test_large.h5')
 file_model_energy = os.path.join(output_dir, 'reg_energy.sav')
 file_model_disp = os.path.join(output_dir, 'reg_disp_vector.sav')
@@ -103,6 +105,29 @@ def test_lstchain_mc_rfperformance():
     assert os.path.exists(file_model_energy)
 
 
+@pytest.mark.run(after='test_lstchain_mc_r0_to_dl1')
+def test_lstchain_merge_dl1_hdf5_files():
+    shutil.copy(dl1_file, os.path.join(output_dir, 'dl1_copy.h5'))
+    run_program('lstchain_merge_hdf5_files',
+                '-d', output_dir,
+                '-o', merged_dl1_file,
+                '--no-image', 'True',
+                )
+    assert os.path.exists(merged_dl1_file)
+
+
+@pytest.mark.run(after='test_lstchain_merge_dl1_hdf5_files')
+def test_lstchain_merged_dl1_to_dl2():
+    output_file = merged_dl1_file.replace('dl1', 'dl2')
+    run_program(
+        'lstchain_dl1_to_dl2',
+        '-f', merged_dl1_file,
+        '-p', output_dir,
+        '-o', output_dir,
+    )
+    assert os.path.exists(output_file)
+
+
 @pytest.mark.run(after='test_lstchain_trainpipe')
 def test_lstchain_dl1_to_dl2():
     run_program(

lstchain/tests/test_lstchain.py

@@ -110,7 +110,7 @@ def test_build_models():
 
     reg_energy, reg_disp, cls_gh = build_models(infile, infile, custom_config=custom_config, save_models=False)
 
-    from sklearn.externals import joblib
+    import joblib
     joblib.dump(reg_energy, file_model_energy)
     joblib.dump(reg_disp, file_model_disp)
     joblib.dump(cls_gh, file_model_gh_sep)
@@ -119,7 +119,7 @@ def test_build_models():
 @pytest.mark.run(order=3)
 def test_apply_models():
     from lstchain.reco.dl1_to_dl2 import apply_models
-    from sklearn.externals import joblib
+    import joblib
 
     dl1 = pd.read_hdf(dl1_file, key=dl1_params_lstcam_key)
     dl1 = filter_events(dl1, filters=custom_config["events_filters"])

notebooks/lstchain.ipynb

@@ -66,7 +66,7 @@
     }
    ],
    "source": [
-    "from sklearn.externals import joblib\n",
+    "import joblib\n",
     "from ctapipe.utils import get_dataset_path\n",
     "from lstchain.io.config import get_standard_config\n",
     "from copy import copy\n",

notebooks/rf-performance.ipynb

@@ -52,7 +52,7 @@
     "import numpy as np                                                                                            \n",
     "import pandas as pd                                                                                                                                                                                      \n",
     "import matplotlib.pyplot as plt                                                                               \n",
-    "from sklearn.externals import joblib                                                                               \n",
+    "import joblib                                                                               \n",
     "from lstchain.reco import dl1_to_dl2     \n",
     "from lstchain.visualization import plot_dl2                                                                   \n",
     "from lstchain.reco import utils\n",

setup.py

@@ -57,6 +57,7 @@ def find_scripts(script_dir, prefix):
         'scikit-learn',
         'tables',
         'traitlets',
+        'joblib',
     ],
     package_data={
         'lstchain': ['data/lstchain_standard_config.json'],