add zenodo handler

configs/bundle_config.yaml

@@ -36,6 +36,7 @@ databundles:
     category: data
     destination: "data"
     urls:
+      zenodo: https://sandbox.zenodo.org/record/1183583/files/tutorial_data_NGBJ.zip?download=1
       gdrive: https://drive.google.com/file/d/1BSxmm3AG5TTOxrLG0ki8IdlL2n3hEviq/view
     output:
     - data/gebco/GEBCO_2021_TID.nc
@@ -48,6 +49,7 @@ databundles:
     category: data
     destination: "data"
     urls:
+      zenodo: https://sandbox.zenodo.org/record/1183583/files/tutorial_data_BW.zip?download=1
       gdrive: https://drive.google.com/file/d/1aXqbCxEpe0aNI6EYuCLtFS6CbdXurTn7/view
     output:
     - data/gebco/GEBCO_2021_TID.nc
@@ -60,6 +62,7 @@ databundles:
     category: data
     destination: "data"
     urls:
+      zenodo: https://sandbox.zenodo.org/record/1183583/files/tutorial_data_MA.zip?download=1
       gdrive: https://drive.google.com/file/d/1zY3WnKqwBF0derS4ncRIPpdTdNn978xw/view
     output:
     - data/gebco/GEBCO_2021_TID.nc
@@ -72,6 +75,7 @@ databundles:
     category: cutouts
     destination: "cutouts"
     urls:
+      zenodo: https://sandbox.zenodo.org/record/1183583/files/cutouts_NGBJ.zip?download=1
       gdrive: https://drive.google.com/file/d/1N6wJb-dBTqWZX1Tlf5y979lEPtqcVEfQ/view
     output: [cutouts/africa-2013-era5-tutorial.nc]
     disable_by_opt:
@@ -84,6 +88,7 @@ databundles:
     category: cutouts
     destination: "cutouts"
     urls:
+      zenodo: https://sandbox.zenodo.org/record/1183583/files/cutouts_BW.zip?download=1
       gdrive: https://drive.google.com/file/d/1vtckYOZeSugDXrE0PGqxFwsVkdHNimhV/view
     output: [cutouts/africa-2013-era5-tutorial.nc]
     disable_by_opt:
@@ -96,6 +101,7 @@ databundles:
     category: cutouts
     destination: "cutouts"
     urls:
+      zenodo: https://sandbox.zenodo.org/record/1183583/files/cutouts_MA.zip?download=1
       gdrive: https://drive.google.com/file/d/1ouIhGQ4e2M2FLzPUZgcKu5GSpRPZotqw/view
     output: [cutouts/africa-2013-era5-tutorial.nc]
     disable_by_opt:
@@ -108,6 +114,7 @@ databundles:
     category: common
     destination: "data"
     urls:
+      zenodo: https://sandbox.zenodo.org/record/1183583/files/tutorial_data_general.zip?download=1
       gdrive: https://drive.google.com/file/d/1nRLrs_kP0qVl-IHC4BFLjpoKa3HLk2Py/view
     output:
     - data/costs.csv
@@ -124,7 +131,7 @@ databundles:
     category: common
     destination: "data"
     urls:
-      # zenodo: https://sandbox.zenodo.org/record/1016540/files/common_data.zip?download=1
+      zenodo: https://sandbox.zenodo.org/record/1183583/files/data_v0.1.zip?download=1
       gdrive: https://drive.google.com/file/d/1jN5iV_iaRa5lc1XOaEtletKPUgPNOVuf/view
     output:
     - data/costs.csv
@@ -142,7 +149,7 @@ databundles:
     category: natura
     destination: "data"
     urls:
-      # zenodo: https://sandbox.zenodo.org/record/1016878/files/resources_africa.zip?download=1
+      zenodo: https://sandbox.zenodo.org/record/1183583/files/natura_global.zip?download=1
       gdrive: https://drive.google.com/file/d/1WmAAwY0TAcHg8YZyAGof0bPewFacm7my/view
     output:
     - data/natura.tiff
@@ -153,7 +160,7 @@ databundles:
     category: landcover
     destination: "data/landcover"
     urls:
-      zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_africa.zip?download=1
+      zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_africa.zip?download=1
       gdrive: https://drive.google.com/file/d/1qnJ0HAWZ4z_9q3PtDEUrFmZWBZYg-WNW/view
     output: [data/landcover/*]
 
@@ -163,7 +170,7 @@ databundles:
     category: landcover
     destination: "data/landcover"
     urls:
-      zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_asiapacific.zip?download=1
+      zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_asiapacific.zip?download=1
       gdrive: https://drive.google.com/file/d/163Uc9F1DPs2pRgDzkTivnblhXCw4nfd5/view
     output: [data/landcover/*]
 
@@ -173,7 +180,7 @@ databundles:
     category: landcover
     destination: "data/landcover"
     urls:
-      zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_latinamerica_caribbean.zip?download=1
+      zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_latinamerica_caribbean.zip?download=1
       gdrive: https://drive.google.com/file/d/1d3l9qxPnM9XbXODWkP4kWxK_t5cDcEIZ/view
     output: [data/landcover/*]
 
@@ -183,7 +190,7 @@ databundles:
     category: landcover
     destination: "data/landcover"
     urls:
-      zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_westasia.zip?download=1
+      zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_westasia.zip?download=1
       gdrive: https://drive.google.com/file/d/1XsZVZDORUFMKyHfAkzqxDVBENiuy4jEM/view
     output: [data/landcover/*]
 
@@ -193,7 +200,7 @@ databundles:
     category: landcover
     destination: "data/landcover"
     urls:
-      zenodo: https://sandbox.zenodo.org/record/1018640/files/landcover_europe.zip?download=1
+      zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_europe.zip?download=1
       gdrive: https://drive.google.com/file/d/1MfT3cwHJbNTtTjRLzW2T19f9qBNAxdhw/view
     output: [data/landcover/*]
 
@@ -203,7 +210,7 @@ databundles:
     category: landcover
     destination: "data/landcover"
     urls:
-      zenodo: https://sandbox.zenodo.org/record/1018640/files/landcover_northamerica.zip?download=1
+      zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_northamerica.zip?download=1
       gdrive: https://drive.google.com/file/d/1Dc11J2BZQQeJWxwCC6GiiPd_GWMCN0uF/view
     output: [data/landcover/*]
 
@@ -213,7 +220,7 @@ databundles:
     category: landcover
     destination: "data/landcover"
     urls:
-      zenodo: https://sandbox.zenodo.org/record/1018640/files/landcover_polar.zip?download=1
+      zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_polar.zip?download=1
       gdrive: https://drive.google.com/file/d/18uODyONQZJGoL15P-URPl0lJs7sphQhX/view
     output: [data/landcover/*]
 
@@ -266,7 +273,7 @@ databundles:
     category: cutouts
     destination: "cutouts"
     urls:
-      zenodo: https://sandbox.zenodo.org/record/1016042/files/cutouts_africa.zip?download=1
+      zenodo: https://sandbox.zenodo.org/record/1183583/files/cutouts_africa.zip?download=1
       gdrive: https://drive.google.com/file/d/1uCn7S5EHnPuaZHG5sWEVx2lT6Ve2r5dk/view
     output: [cutouts/africa-2013-era5.nc]
     disable_by_opt:

doc/release_notes.rst

@@ -88,6 +88,8 @@ Upcoming Release
 
 * Fix append bug in build_powerplants rule `PR #686 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/686>`__
 
+* Add *zenodo_handler.py* to update and upload files via code `PR #688 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/688>`__
+
 PyPSA-Earth 0.1.0
 =================
 

scripts/non_workflow/__init__.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText:  PyPSA-Earth and PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: GPL-3.0-or-later

scripts/non_workflow/zenodo_handler.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText:  PyPSA-Earth and PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# -*- coding: utf-8 -*-
+"""
+Uploads local files to Zenodo or sandbox.Zenodo.
+
+Requirements
+------------
+- Install zenodopy: pip git+https://github.com/pz-max/zenodopy@patch-4 
+
+- Setup zenodopy (token): https://github.com/pz-max/zenodopy/tree/patch-4#using-the-package
+
+- Be aware of Zenodo REST API e.g. for metadata modifications: https://developers.zenodo.org/#introduction
+
+Relevant Settings
+-----------------
+"""
+from pathlib import Path
+
+import zenodopy
+
+######################
+# INPUTS AND OPTIONS #
+######################
+SANDBOX_BOOL = True  # sandbox should be used for testing
+NEW_PROJECT = False  # if False, use existing project ID
+EXISTING_PROJECT_ID = 1183583
+TYPE = "upload"  # 'delete' or 'upload' files
+PUBLISH_BOOL = False  # publish repos CANNOT be deleted
+ROOT = "/home/max/Downloads"  # example "/home/max/Downloads"
+UPLOAD_PATHS = [
+    "tutorial_data_MA.zip",
+    "cutouts_NGBJ.zip",
+]  # list of files from root e.g. "README.md" in /home/max/README.md, each component will receive a download link
+DELETE_PATHS = [
+    "README.md",
+]  # list of files from root e.g. "README.md" in /home/max/README.md,
+METADATA = {
+    "title": "PyPSA-Earth (Dataset)",
+    "upload_type": "other",
+    "description": "Used for model https://github.com/pypsa-meets-earth/pypsa-earth. Multiple data licenses apply https://pypsa-earth.readthedocs.io/en/latest/introduction.html#license",
+    "creators": [
+        {"name": "PyPSA-Earth Authors", "affiliation": "PyPSA meets Earth"},
+    ],
+    "access_right": "open",
+    "license": {"id": "cc-by-4.0"},
+    "keywords": ["Macro Energy Systems", "Power Systems"],
+}  # more opton visisble at Zenodo REST API https://developers.zenodo.org/#introduction
+
+
+#############
+# EXECUTION #
+#############
+zeno = zenodopy.Client(sandbox=SANDBOX_BOOL)  # test is API key is set
+zeno.list_projects
+
+
+if NEW_PROJECT == True:
+    zeno.create_project(title=METADATA["title"])
+    zeno.change_metadata(
+        dep_id=zeno.deposition_id,
+        metadata=METADATA,
+    )
+    for path in UPLOAD_PATHS:
+        path = Path.joinpath(Path(ROOT), path)
+        zeno.upload_zip(source_dir=str(path))
+
+
+if NEW_PROJECT == False:
+    zeno.set_project(dep_id=EXISTING_PROJECT_ID)
+    zeno.change_metadata(
+        dep_id=zeno.deposition_id,
+        metadata=METADATA,
+    )
+    for path in UPLOAD_PATHS:
+        path = Path.joinpath(Path(ROOT), path)
+        if path.exists():
+            if TYPE == "upload":
+                if path.exists():
+                    try:
+                        if path.is_file():
+                            zeno.upload_file(str(path))
+                        elif path.is_dir():
+                            zeno.upload_zip(str(path))
+                    except:
+                        zeno.update(path)
+                        continue
+
+            if TYPE == "delete":
+                try:
+                    zeno.delete_file(str(path))
+                except:
+                    print(f"Cannot delete {path}. Repo needs to be in edit mode.")
+        else:
+            raise FileNotFoundError(f"{path} does not exist")
+
+
+if PUBLISH_BOOL == True:
+    zeno.publish()
+
+
+zeno.list_projects