diff --git a/configs/bundle_config.yaml b/configs/bundle_config.yaml index 73f143eea..74b536bbf 100644 --- a/configs/bundle_config.yaml +++ b/configs/bundle_config.yaml @@ -36,6 +36,7 @@ databundles: category: data destination: "data" urls: + zenodo: https://sandbox.zenodo.org/record/1183583/files/tutorial_data_NGBJ.zip?download=1 gdrive: https://drive.google.com/file/d/1BSxmm3AG5TTOxrLG0ki8IdlL2n3hEviq/view output: - data/gebco/GEBCO_2021_TID.nc @@ -48,6 +49,7 @@ databundles: category: data destination: "data" urls: + zenodo: https://sandbox.zenodo.org/record/1183583/files/tutorial_data_BW.zip?download=1 gdrive: https://drive.google.com/file/d/1aXqbCxEpe0aNI6EYuCLtFS6CbdXurTn7/view output: - data/gebco/GEBCO_2021_TID.nc @@ -60,6 +62,7 @@ databundles: category: data destination: "data" urls: + zenodo: https://sandbox.zenodo.org/record/1183583/files/tutorial_data_MA.zip?download=1 gdrive: https://drive.google.com/file/d/1zY3WnKqwBF0derS4ncRIPpdTdNn978xw/view output: - data/gebco/GEBCO_2021_TID.nc @@ -72,6 +75,7 @@ databundles: category: cutouts destination: "cutouts" urls: + zenodo: https://sandbox.zenodo.org/record/1183583/files/cutouts_NGBJ.zip?download=1 gdrive: https://drive.google.com/file/d/1N6wJb-dBTqWZX1Tlf5y979lEPtqcVEfQ/view output: [cutouts/africa-2013-era5-tutorial.nc] disable_by_opt: @@ -84,6 +88,7 @@ databundles: category: cutouts destination: "cutouts" urls: + zenodo: https://sandbox.zenodo.org/record/1183583/files/cutouts_BW.zip?download=1 gdrive: https://drive.google.com/file/d/1vtckYOZeSugDXrE0PGqxFwsVkdHNimhV/view output: [cutouts/africa-2013-era5-tutorial.nc] disable_by_opt: @@ -96,6 +101,7 @@ databundles: category: cutouts destination: "cutouts" urls: + zenodo: https://sandbox.zenodo.org/record/1183583/files/cutouts_MA.zip?download=1 gdrive: https://drive.google.com/file/d/1ouIhGQ4e2M2FLzPUZgcKu5GSpRPZotqw/view output: [cutouts/africa-2013-era5-tutorial.nc] disable_by_opt: @@ -108,6 +114,7 @@ databundles: category: common destination: "data" urls: + zenodo: https://sandbox.zenodo.org/record/1183583/files/tutorial_data_general.zip?download=1 gdrive: https://drive.google.com/file/d/1nRLrs_kP0qVl-IHC4BFLjpoKa3HLk2Py/view output: - data/costs.csv @@ -124,7 +131,7 @@ databundles: category: common destination: "data" urls: - # zenodo: https://sandbox.zenodo.org/record/1016540/files/common_data.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/data_v0.1.zip?download=1 gdrive: https://drive.google.com/file/d/1jN5iV_iaRa5lc1XOaEtletKPUgPNOVuf/view output: - data/costs.csv @@ -142,7 +149,7 @@ databundles: category: natura destination: "data" urls: - # zenodo: https://sandbox.zenodo.org/record/1016878/files/resources_africa.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/natura_global.zip?download=1 gdrive: https://drive.google.com/file/d/1WmAAwY0TAcHg8YZyAGof0bPewFacm7my/view output: - data/natura.tiff @@ -153,7 +160,7 @@ databundles: category: landcover destination: "data/landcover" urls: - zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_africa.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_africa.zip?download=1 gdrive: https://drive.google.com/file/d/1qnJ0HAWZ4z_9q3PtDEUrFmZWBZYg-WNW/view output: [data/landcover/*] @@ -163,7 +170,7 @@ databundles: category: landcover destination: "data/landcover" urls: - zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_asiapacific.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_asiapacific.zip?download=1 gdrive: https://drive.google.com/file/d/163Uc9F1DPs2pRgDzkTivnblhXCw4nfd5/view output: [data/landcover/*] @@ -173,7 +180,7 @@ databundles: category: landcover destination: "data/landcover" urls: - zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_latinamerica_caribbean.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_latinamerica_caribbean.zip?download=1 gdrive: https://drive.google.com/file/d/1d3l9qxPnM9XbXODWkP4kWxK_t5cDcEIZ/view output: [data/landcover/*] @@ -183,7 +190,7 @@ databundles: category: landcover destination: "data/landcover" urls: - zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_westasia.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_westasia.zip?download=1 gdrive: https://drive.google.com/file/d/1XsZVZDORUFMKyHfAkzqxDVBENiuy4jEM/view output: [data/landcover/*] @@ -193,7 +200,7 @@ databundles: category: landcover destination: "data/landcover" urls: - zenodo: https://sandbox.zenodo.org/record/1018640/files/landcover_europe.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_europe.zip?download=1 gdrive: https://drive.google.com/file/d/1MfT3cwHJbNTtTjRLzW2T19f9qBNAxdhw/view output: [data/landcover/*] @@ -203,7 +210,7 @@ databundles: category: landcover destination: "data/landcover" urls: - zenodo: https://sandbox.zenodo.org/record/1018640/files/landcover_northamerica.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_northamerica.zip?download=1 gdrive: https://drive.google.com/file/d/1Dc11J2BZQQeJWxwCC6GiiPd_GWMCN0uF/view output: [data/landcover/*] @@ -213,7 +220,7 @@ databundles: category: landcover destination: "data/landcover" urls: - zenodo: https://sandbox.zenodo.org/record/1018640/files/landcover_polar.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_polar.zip?download=1 gdrive: https://drive.google.com/file/d/18uODyONQZJGoL15P-URPl0lJs7sphQhX/view output: [data/landcover/*] @@ -266,7 +273,7 @@ databundles: category: cutouts destination: "cutouts" urls: - zenodo: https://sandbox.zenodo.org/record/1016042/files/cutouts_africa.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/cutouts_africa.zip?download=1 gdrive: https://drive.google.com/file/d/1uCn7S5EHnPuaZHG5sWEVx2lT6Ve2r5dk/view output: [cutouts/africa-2013-era5.nc] disable_by_opt: diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 5b14eebd4..0e8127fb1 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -88,6 +88,8 @@ Upcoming Release * Fix append bug in build_powerplants rule `PR #686 `__ +* Add *zenodo_handler.py* to update and upload files via code `PR #688 `__ + PyPSA-Earth 0.1.0 ================= diff --git a/scripts/non_workflow/__init__.py b/scripts/non_workflow/__init__.py new file mode 100644 index 000000000..a635e74eb --- /dev/null +++ b/scripts/non_workflow/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +# +# SPDX-License-Identifier: GPL-3.0-or-later diff --git a/scripts/non_workflow/zenodo_handler.py b/scripts/non_workflow/zenodo_handler.py new file mode 100644 index 000000000..e382ad431 --- /dev/null +++ b/scripts/non_workflow/zenodo_handler.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +# -*- coding: utf-8 -*- +""" +Uploads local files to Zenodo or sandbox.Zenodo. + +Requirements +------------ +- Install zenodopy: pip git+https://github.com/pz-max/zenodopy@patch-4 + +- Setup zenodopy (token): https://github.com/pz-max/zenodopy/tree/patch-4#using-the-package + +- Be aware of Zenodo REST API e.g. for metadata modifications: https://developers.zenodo.org/#introduction + +Relevant Settings +----------------- +""" +from pathlib import Path + +import zenodopy + +###################### +# INPUTS AND OPTIONS # +###################### +SANDBOX_BOOL = True # sandbox should be used for testing +NEW_PROJECT = False # if False, use existing project ID +EXISTING_PROJECT_ID = 1183583 +TYPE = "upload" # 'delete' or 'upload' files +PUBLISH_BOOL = False # publish repos CANNOT be deleted +ROOT = "/home/max/Downloads" # example "/home/max/Downloads" +UPLOAD_PATHS = [ + "tutorial_data_MA.zip", + "cutouts_NGBJ.zip", +] # list of files from root e.g. "README.md" in /home/max/README.md, each component will receive a download link +DELETE_PATHS = [ + "README.md", +] # list of files from root e.g. "README.md" in /home/max/README.md, +METADATA = { + "title": "PyPSA-Earth (Dataset)", + "upload_type": "other", + "description": "Used for model https://github.com/pypsa-meets-earth/pypsa-earth. Multiple data licenses apply https://pypsa-earth.readthedocs.io/en/latest/introduction.html#license", + "creators": [ + {"name": "PyPSA-Earth Authors", "affiliation": "PyPSA meets Earth"}, + ], + "access_right": "open", + "license": {"id": "cc-by-4.0"}, + "keywords": ["Macro Energy Systems", "Power Systems"], +} # more opton visisble at Zenodo REST API https://developers.zenodo.org/#introduction + + +############# +# EXECUTION # +############# +zeno = zenodopy.Client(sandbox=SANDBOX_BOOL) # test is API key is set +zeno.list_projects + + +if NEW_PROJECT == True: + zeno.create_project(title=METADATA["title"]) + zeno.change_metadata( + dep_id=zeno.deposition_id, + metadata=METADATA, + ) + for path in UPLOAD_PATHS: + path = Path.joinpath(Path(ROOT), path) + zeno.upload_zip(source_dir=str(path)) + + +if NEW_PROJECT == False: + zeno.set_project(dep_id=EXISTING_PROJECT_ID) + zeno.change_metadata( + dep_id=zeno.deposition_id, + metadata=METADATA, + ) + for path in UPLOAD_PATHS: + path = Path.joinpath(Path(ROOT), path) + if path.exists(): + if TYPE == "upload": + if path.exists(): + try: + if path.is_file(): + zeno.upload_file(str(path)) + elif path.is_dir(): + zeno.upload_zip(str(path)) + except: + zeno.update(path) + continue + + if TYPE == "delete": + try: + zeno.delete_file(str(path)) + except: + print(f"Cannot delete {path}. Repo needs to be in edit mode.") + else: + raise FileNotFoundError(f"{path} does not exist") + + +if PUBLISH_BOOL == True: + zeno.publish() + + +zeno.list_projects diff --git a/scripts/zipfolders_googledrive.py b/scripts/non_workflow/zip_folder.py similarity index 100% rename from scripts/zipfolders_googledrive.py rename to scripts/non_workflow/zip_folder.py