Revise methodology to calculate databundle outputs (#529)

pypsa-meets-earth · Dec 8, 2022 · 7193c2a · 7193c2a
1 parent b6fd5e0
commit 7193c2a
Show file tree

Hide file tree

Showing 5 changed files with 148 additions and 91 deletions.
diff --git a/Snakefile b/Snakefile
@@ -13,6 +13,7 @@ from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider
 
 from scripts.download_osm_data import create_country_list
 from scripts.add_electricity import get_load_paths_gegis
+from scripts.retrieve_databundle_light import datafiles_retrivedatabundle
 
 HTTP = HTTPRemoteProvider()
 
@@ -106,39 +107,6 @@ rule plot_all_summaries:
         ),
 
 
-def datafiles_retrivedatabundle(config):
-    listoutputs = [
-        dvalue["output"]
-        for (dname, dvalue) in config["databundles"].items()
-        if config.get("tutorial", False) == dvalue.get("tutorial", False)
-    ]
-    unique_outputs = set(
-        (
-            [
-                inneroutput
-                for output in listoutputs
-                for inneroutput in output
-                if "*" not in inneroutput
-                or inneroutput.endswith("/")  # exclude directories
-            ]
-        )
-    )
-
-    # when option build_natura_raster is enabled, remove natura.tiff from the outputs
-    if config["enable"].get("build_natura_raster", False):
-        unique_outputs = [
-            output for output in unique_outputs if "natura.tiff" not in output
-        ]
-
-    # when option build_cutout is enabled, remove cutouts from the outputs
-    if config["enable"].get("build_cutout", False):
-        unique_outputs = [
-            output for output in unique_outputs if "cutouts/" not in output
-        ]
-
-    return unique_outputs
-
-
 if config["enable"].get("retrieve_databundle", True):
 
     rule retrieve_databundle_light:

diff --git a/configs/bundle_config.yaml b/configs/bundle_config.yaml
@@ -36,8 +36,7 @@ databundles:
     category: data
     destination: "data"
     urls:
-      # zenodo: https://sandbox.zenodo.org/record/1016522/files/tutorial_data.zip?download=1
-      gdrive: https://drive.google.com/file/d/1yg1D5o7FN19ypICQKOuKgKbcGqrOmbia/view?usp=share_link
+      gdrive: https://drive.google.com/file/d/1BSxmm3AG5TTOxrLG0ki8IdlL2n3hEviq/view
     output:
     - data/gebco/GEBCO_2021_TID.nc
     - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif
@@ -49,7 +48,7 @@ databundles:
     category: data
     destination: "data"
     urls:
-      gdrive: https://drive.google.com/file/d/1yJ8JYGFVtMPpsEBeXzDR5Ixa9n8gGu5n/view?usp=share_link
+      gdrive: https://drive.google.com/file/d/1aXqbCxEpe0aNI6EYuCLtFS6CbdXurTn7/view
     output:
     - data/gebco/GEBCO_2021_TID.nc
     - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif
@@ -61,7 +60,7 @@ databundles:
     category: data
     destination: "data"
     urls:
-      gdrive: https://drive.google.com/file/d/1Qjf0xVBuOSTKTPRD_NANruDDG94CF-Ju/view?usp=share_link
+      gdrive: https://drive.google.com/file/d/1zY3WnKqwBF0derS4ncRIPpdTdNn978xw/view
     output:
     - data/gebco/GEBCO_2021_TID.nc
     - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif
@@ -73,8 +72,7 @@ databundles:
     category: cutouts
     destination: "cutouts"
     urls:
-      zenodo: https://sandbox.zenodo.org/record/1016015/files/tutorial_cutouts.zip?download=1
-      gdrive: https://drive.google.com/file/d/1No_BNKRO4UODdR5rVtwkfZjfVGUQKJ_K/view?usp=share_link
+      gdrive: https://drive.google.com/file/d/1N6wJb-dBTqWZX1Tlf5y979lEPtqcVEfQ/view
     output: [cutouts/africa-2013-era5-tutorial.nc]
     disable_by_opt:
       build_cutout: [all]
@@ -86,19 +84,19 @@ databundles:
     category: cutouts
     destination: "cutouts"
     urls:
-      gdrive: https://drive.google.com/file/d/19-vq48OKAeJ0C_1S8kx2HVQkCMGKzuTF/view?usp=share_link
+      gdrive: https://drive.google.com/file/d/1vtckYOZeSugDXrE0PGqxFwsVkdHNimhV/view
     output: [cutouts/africa-2013-era5-tutorial.nc]
     disable_by_opt:
       build_cutout: [all]
 
   # tutorial bundle specific for Morocco only
-  bundle_cutouts_tutorial_ma:
+  bundle_cutouts_tutorial_MA:
     countries: [MA]
     tutorial: true
     category: cutouts
     destination: "cutouts"
     urls:
-      gdrive: https://drive.google.com/file/d/14mgef15MSwOTUxDtXcnPBJbC_M7zxalQ/view?usp=share_link
+      gdrive: https://drive.google.com/file/d/1ouIhGQ4e2M2FLzPUZgcKu5GSpRPZotqw/view
     output: [cutouts/africa-2013-era5-tutorial.nc]
     disable_by_opt:
       build_cutout: [all]
@@ -110,7 +108,7 @@ databundles:
     category: common
     destination: "data"
     urls:
-      gdrive: https://drive.google.com/file/d/1yWxR7noWXZP74iRLqQGCatXssRGcuEc2/view?usp=share_link
+      gdrive: https://drive.google.com/file/d/1nRLrs_kP0qVl-IHC4BFLjpoKa3HLk2Py/view
     output:
     - data/costs.csv
     - data/hydro_capacities.csv
@@ -127,7 +125,7 @@ databundles:
     destination: "data"
     urls:
       # zenodo: https://sandbox.zenodo.org/record/1016540/files/common_data.zip?download=1
-      gdrive: https://drive.google.com/file/d/1MdZ4Eb80a2ntsBRBqWMC0Hodvu4e0jSm/view?usp=sharing
+      gdrive: https://drive.google.com/file/d/1jN5iV_iaRa5lc1XOaEtletKPUgPNOVuf/view
     output:
     - data/costs.csv
     - data/hydro_capacities.csv
@@ -145,7 +143,7 @@ databundles:
     destination: "data"
     urls:
       # zenodo: https://sandbox.zenodo.org/record/1016878/files/resources_africa.zip?download=1
-      gdrive: https://drive.google.com/file/d/1WmAAwY0TAcHg8YZyAGof0bPewFacm7my/view?usp=share_link
+      gdrive: https://drive.google.com/file/d/1WmAAwY0TAcHg8YZyAGof0bPewFacm7my/view
     output:
     - data/natura.tiff
 
@@ -156,7 +154,7 @@ databundles:
     destination: "data/landcover"
     urls:
       zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_africa.zip?download=1
-      gdrive: https://drive.google.com/file/d/1l6Xgr3PXyvgqWaXUbto9zvvMS_OgSjvG/view?usp=sharing
+      gdrive: https://drive.google.com/file/d/1qnJ0HAWZ4z_9q3PtDEUrFmZWBZYg-WNW/view
     output: [data/landcover/*]
 
   # data bundle of the data folder for the african continent
@@ -166,7 +164,7 @@ databundles:
     destination: "data/landcover"
     urls:
       zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_asiapacific.zip?download=1
-      gdrive: https://drive.google.com/file/d/1HZGpxZdTB2Ai_VN_8SPivCdgt3oz0wkX/view?usp=sharing
+      gdrive: https://drive.google.com/file/d/163Uc9F1DPs2pRgDzkTivnblhXCw4nfd5/view
     output: [data/landcover/*]
 
   # data bundle of the data folder for the african continent
@@ -176,7 +174,7 @@ databundles:
     destination: "data/landcover"
     urls:
       zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_latinamerica_caribbean.zip?download=1
-      gdrive: https://drive.google.com/file/d/1lmBFPTHyzVmSx9qgrGh8D6uHiSyGaH_F/view?usp=sharing
+      gdrive: https://drive.google.com/file/d/1d3l9qxPnM9XbXODWkP4kWxK_t5cDcEIZ/view
     output: [data/landcover/*]
 
   # data bundle of the data folder for the african continent
@@ -186,7 +184,7 @@ databundles:
     destination: "data/landcover"
     urls:
       zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_westasia.zip?download=1
-      gdrive: https://drive.google.com/file/d/1s9JRWJaeet4syInGap6FyqDBJIR80UHr/view?usp=sharing
+      gdrive: https://drive.google.com/file/d/1XsZVZDORUFMKyHfAkzqxDVBENiuy4jEM/view
     output: [data/landcover/*]
 
   # data bundle of the data folder for europe
@@ -196,7 +194,7 @@ databundles:
     destination: "data/landcover"
     urls:
       zenodo: https://sandbox.zenodo.org/record/1018640/files/landcover_europe.zip?download=1
-      gdrive: https://drive.google.com/file/d/1zQuKnZFp6ASYD_e-dbWbWXYg330E0occ/view?usp=sharing
+      gdrive: https://drive.google.com/file/d/1MfT3cwHJbNTtTjRLzW2T19f9qBNAxdhw/view
     output: [data/landcover/*]
 
   # data bundle of the data folder for the poles
@@ -206,7 +204,7 @@ databundles:
     destination: "data/landcover"
     urls:
       zenodo: https://sandbox.zenodo.org/record/1018640/files/landcover_northamerica.zip?download=1
-      gdrive: https://drive.google.com/file/d/1gu7PH08ZEpL4cBQa_fMUSq2m6LY1_UJo/view?usp=sharing
+      gdrive: https://drive.google.com/file/d/1Dc11J2BZQQeJWxwCC6GiiPd_GWMCN0uF/view
     output: [data/landcover/*]
 
   # data bundle of the data folder for the poles
@@ -216,7 +214,7 @@ databundles:
     destination: "data/landcover"
     urls:
       zenodo: https://sandbox.zenodo.org/record/1018640/files/landcover_polar.zip?download=1
-      gdrive: https://drive.google.com/file/d/1cUDcE6wpQVZNf3K9ffV6pNXfxq6sA65E/view?usp=sharing
+      gdrive: https://drive.google.com/file/d/18uODyONQZJGoL15P-URPl0lJs7sphQhX/view
     output: [data/landcover/*]
 
   # # data bundle of the data folder for the african continent
@@ -269,7 +267,7 @@ databundles:
     destination: "cutouts"
     urls:
       zenodo: https://sandbox.zenodo.org/record/1016042/files/cutouts_africa.zip?download=1
-      gdrive: https://drive.google.com/file/d/1-Njs7BqG0YE5QwBHj0zgkdicb5IQvQCh/view
+      gdrive: https://drive.google.com/file/d/1uCn7S5EHnPuaZHG5sWEVx2lT6Ve2r5dk/view
     output: [cutouts/africa-2013-era5.nc]
     disable_by_opt:
       build_cutout: [all]
diff --git a/doc/release_notes.rst b/doc/release_notes.rst
@@ -39,6 +39,8 @@ Upcoming Release
 
 * Adapt dependencies of powerplantmatching to the PyPSA main branch `PR #527 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/527>`__
 
+* Calculate the outputs of retrieve_databundle dynamically depending on settings `PR #529 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/529>`__
+
 PyPSA-Earth 0.1.0
 =================
 

diff --git a/scripts/download_osm_data.py b/scripts/download_osm_data.py
@@ -693,7 +693,7 @@ def filter_codes(c_list, iso_coding=True):
         full_codes_list.extend(codes_list)
 
     # Removing duplicates and filter outputs by coding
-    full_codes_list = filter_codes(set(full_codes_list), iso_coding=iso_coding)
+    full_codes_list = filter_codes(list(set(full_codes_list)), iso_coding=iso_coding)
 
     return full_codes_list