From bbc2cc4c2c0213d888921c560411e11f6846bf4f Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Tue, 11 May 2021 10:20:13 +0200 Subject: [PATCH 1/4] Update setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5c9b8283..ed0b6b1c 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ "gammapy", "pytest", "numpy", - "ctapipe==0.10.5", + "ctapipe==0.11.0", "pyirf", ], "tests": [ @@ -43,7 +43,7 @@ packages=find_packages(), package_data={"protopipe": ["aux/example_config_files/analysis.yaml"]}, include_package_data=True, - install_requires=["ctapipe==0.10.5", "pyirf"], + install_requires=["ctapipe==0.11.0", "pyirf"], zip_safe=False, use_scm_version={"write_to": os.path.join("protopipe", "_version.py")}, tests_require=extras_require["tests"], From f19b3db6396695da4e2766ccb09a3aa601a30521 Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Tue, 11 May 2021 10:20:35 +0200 Subject: [PATCH 2/4] Update conda environment file for development version --- environment_development.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment_development.yml b/environment_development.yml index 0bf09877..201cf945 100644 --- a/environment_development.yml +++ b/environment_development.yml @@ -4,7 +4,7 @@ channels: dependencies: - python >=3.7,<3.9 - pip - - conda-forge::ctapipe=0.10.5 + - conda-forge::ctapipe=0.11.0 - conda-forge::gammapy - astropy>=4.0.1 - h5py=2 From 1c96a1fff6958518deda86e95470d1a1d8bbd775 Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Tue, 11 May 2021 11:02:04 +0200 Subject: [PATCH 3/4] Remove old test code now imported from ctapipe and update some functions --- protopipe/pipeline/event_preparer.py | 2 +- protopipe/pipeline/temp.py | 199 +---------------------- protopipe/pipeline/utils.py | 4 +- protopipe/scripts/data_training.py | 2 +- protopipe/scripts/tests/test_pipeline.py | 2 +- protopipe/scripts/write_dl2.py | 2 +- 6 files changed, 8 insertions(+), 203 deletions(-) diff --git a/protopipe/pipeline/event_preparer.py b/protopipe/pipeline/event_preparer.py index 217782e0..d5535100 100644 --- a/protopipe/pipeline/event_preparer.py +++ b/protopipe/pipeline/event_preparer.py @@ -15,7 +15,7 @@ number_of_islands, largest_island, concentration_parameters) -from ctapipe.utils.CutFlow import CutFlow +from ctapipe.utils import CutFlow from ctapipe.coordinates import GroundFrame, TelescopeFrame, CameraFrame # from ctapipe.image.timing_parameters import timing_parameters diff --git a/protopipe/pipeline/temp.py b/protopipe/pipeline/temp.py index 8d9a4d1e..8a6bc5d8 100644 --- a/protopipe/pipeline/temp.py +++ b/protopipe/pipeline/temp.py @@ -10,25 +10,18 @@ """ -import os -import time import logging -from pathlib import Path -import requests -from tqdm import tqdm -from urllib.parse import urlparse import numpy as np from numpy import nan import astropy.units as u from astropy.coordinates import SkyCoord -from requests.exceptions import HTTPError from ctapipe.core import Container, Field from ctapipe.coordinates import CameraFrame, TelescopeFrame from ctapipe.instrument import CameraGeometry -from ctapipe.reco import HillasReconstructor -from ctapipe.reco.HillasReconstructor import HillasPlane +from ctapipe.reco.hillas_reconstructor import HillasReconstructor +from ctapipe.reco.hillas_reconstructor import HillasPlane logger = logging.getLogger(__name__) @@ -209,191 +202,3 @@ def initialize_hillas_planes( weight=moments.intensity * (moments.length / moments.width), ) self.hillas_planes[tel_id] = circle - - -try: - import ctapipe_resources - - has_resources = True -except ImportError: - has_resources = False - - -def download_file(url, path, auth=None, chunk_size=10240, progress=False): - """ - Download a file. Will write to ``path + '.part'`` while downloading - and rename after successful download to the final name. - Parameters - ---------- - url: str or url - The URL to download - path: pathlib.Path or str - Where to store the downloaded data. - auth: None or tuple of (username, password) or a request.AuthBase instance. - chunk_size: int - Chunk size for writing the data file, 10 kB by default. - """ - logger.info(f"Downloading {url} to {path}") - name = urlparse(url).path.split("/")[-1] - path = Path(path) - - with requests.get(url, stream=True, auth=auth, timeout=5) as r: - # make sure the request is successful - r.raise_for_status() - - total = float(r.headers.get("Content-Length", float("inf"))) - pbar = tqdm( - total=total, - disable=not progress, - unit="B", - unit_scale=True, - desc=f"Downloading {name}", - ) - - try: - # open a .part file to avoid creating - # a broken file at the intended location - part_file = path.with_suffix(path.suffix + ".part") - - part_file.parent.mkdir(parents=True, exist_ok=True) - with part_file.open("wb") as f: - for chunk in r.iter_content(chunk_size=chunk_size): - f.write(chunk) - pbar.update(len(chunk)) - except: # we really want to catch everythin here - # cleanup part file if something goes wrong - if part_file.is_file(): - part_file.unlink() - raise - - # when successful, move to intended location - part_file.rename(path) - - -def get_cache_path(url, cache_name="ctapipe", env_override="CTAPIPE_CACHE"): - if os.getenv(env_override): - base = Path(os.environ["CTAPIPE_CACHE"]) - else: - base = Path(os.environ["HOME"]) / ".cache" / cache_name - - url = urlparse(url) - - path = os.path.join(url.netloc.rstrip("/"), url.path.lstrip("/")) - path = base / path - path.parent.mkdir(parents=True, exist_ok=True) - return path - - -def download_file_cached( - name, - cache_name="ctapipe", - auth=None, - env_prefix="CTAPIPE_DATA_", - default_url="http://cccta-dataserver.in2p3.fr/data/", - progress=False, -): - """ - Downloads a file from a dataserver and caches the result locally - in ``$HOME/.cache/``. - If the file is found in the cache, no new download is performed. - Parameters - ---------- - name: str or pathlib.Path - the name of the file, relative to the data server url - cache_name: str - What name to use for the cache directory - env_prefix: str - Prefix for the environt variables used for overriding the URL, - and providing username and password in case authentication is required. - auth: True, None or tuple of (username, password) - Authentication data for the request. Will be passed to ``requests.get``. - If ``True``, read username and password for the request from - the env variables ``env_prefix + 'USER'`` and ``env_prefix + PASSWORD`` - default_url: str - The default url from which to download ``name``, can be overriden - by setting the env variable ``env_prefix + URL`` - Returns - ------- - path: pathlib.Path - the full path to the downloaded data. - """ - logger.debug(f"File {name} is not available in cache, downloading.") - - base_url = os.environ.get(env_prefix + "URL", default_url).rstrip("/") - url = base_url + "/" + str(name).lstrip("/") - - path = get_cache_path(url, cache_name=cache_name) - part_file = path.with_suffix(path.suffix + ".part") - - if part_file.is_file(): - logger.warning("Another download for this file is already running, waiting.") - while part_file.is_file(): - time.sleep(1) - - # if we already dowloaded the file, just use it - if path.is_file(): - logger.debug(f"File {name} is available in cache.") - return path - - if auth is True: - try: - auth = ( - os.environ[env_prefix + "USER"], - os.environ[env_prefix + "PASSWORD"], - ) - except KeyError: - raise KeyError( - f'You need to set the env variables "{env_prefix}USER"' - f' and "{env_prefix}PASSWORD" to download test files.' - ) from None - - download_file(url=url, path=path, auth=auth, progress=progress) - return path - - -DEFAULT_URL = "http://cccta-dataserver.in2p3.fr/data/ctapipe-extra/v0.3.3/" -def get_dataset_path(filename, url=DEFAULT_URL): - """ - Returns the full file path to an auxiliary dataset needed by - ctapipe, given the dataset's full name (filename with no directory). - This will first search for the file in directories listed in - tne environment variable CTAPIPE_SVC_PATH (if set), and if not found, - will look in the ctapipe_resources module - (if installed with the ctapipe-extra package), which contains the defaults. - Parameters - ---------- - filename: str - name of dataset to fetch - Returns - ------- - string with full path to the given dataset - """ - - searchpath = os.getenv("CTAPIPE_SVC_PATH") - - if searchpath: - filepath = find_in_path(filename=filename, searchpath=searchpath, url=url) - - if filepath: - return filepath - - if has_resources and (url is DEFAULT_URL): - logger.debug( - "Resource '{}' not found in CTAPIPE_SVC_PATH, looking in " - "ctapipe_resources...".format(filename) - ) - - return Path(ctapipe_resources.get(filename)) - - # last, try downloading the data - try: - return download_file_cached(filename, default_url=url, progress=True) - except HTTPError as e: - # let 404 raise the FileNotFoundError instead of HTTPError - if e.response.status_code != 404: - raise - - raise FileNotFoundError( - f"Couldn't find resource: '{filename}'," - " You might want to install ctapipe_resources" - ) \ No newline at end of file diff --git a/protopipe/pipeline/utils.py b/protopipe/pipeline/utils.py index 6d2e5675..36525e25 100644 --- a/protopipe/pipeline/utils.py +++ b/protopipe/pipeline/utils.py @@ -195,9 +195,9 @@ def final_array_to_use(sim_array, array, subarrays=None): """ if subarrays: tel_ids = subarrays[array] - subarray = sim_array.select_subarray("", tel_ids) + subarray = sim_array.select_subarray(tel_ids, name="selected_subarray") else: - subarray = sim_array.select_subarray("", array) + subarray = sim_array.select_subarray(array, name="selected_subarray") tel_ids = subarray.tel_ids tel_types = subarray.telescope_types cams_and_foclens = { diff --git a/protopipe/scripts/data_training.py b/protopipe/scripts/data_training.py index 3c8ee0a8..4505a220 100755 --- a/protopipe/scripts/data_training.py +++ b/protopipe/scripts/data_training.py @@ -10,7 +10,7 @@ import tables as tb import pandas as pd -from ctapipe.utils.CutFlow import CutFlow +from ctapipe.utils import CutFlow from ctapipe.io import EventSource from protopipe.pipeline import EventPreparer diff --git a/protopipe/scripts/tests/test_pipeline.py b/protopipe/scripts/tests/test_pipeline.py index 1e7d1387..dd13f069 100644 --- a/protopipe/scripts/tests/test_pipeline.py +++ b/protopipe/scripts/tests/test_pipeline.py @@ -4,7 +4,7 @@ import tables import pytest -from protopipe.pipeline.temp import get_dataset_path +from ctapipe.utils.datasets import get_dataset_path from protopipe.scripts import data_training, build_model, write_dl2 diff --git a/protopipe/scripts/write_dl2.py b/protopipe/scripts/write_dl2.py index a4ae66b0..850a59d9 100755 --- a/protopipe/scripts/write_dl2.py +++ b/protopipe/scripts/write_dl2.py @@ -11,7 +11,7 @@ # ctapipe from ctapipe.io import EventSource -from ctapipe.utils.CutFlow import CutFlow +from ctapipe.utils import CutFlow # Utilities from protopipe.pipeline import EventPreparer From 3507475605f83d7123b60d9838ab6c1710c8e73b Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Tue, 11 May 2021 13:51:10 +0200 Subject: [PATCH 4/4] add pandas to docs requirements --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index ed0b6b1c..9616c44a 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ "numpy", "ctapipe==0.11.0", "pyirf", + "pandas" ], "tests": [ "pytest",