enable add_hydro feature and parallel downloading of osm data

Snakefile

@@ -239,8 +239,7 @@ rule add_electricity:
         powerplants='resources/powerplants.csv',
         load='resources/ssp2-2.6/2030/era5_2013/Africa.nc',
         gadm_shapes='resources/gadm_shapes.geojson',
-        # hydro_capacities='data/bundle/hydro_capacities.csv',
-        # geth_hydro_capacities='data/geth2015_hydro_capacities.csv',
+        hydro_capacities='data/hydro_capacities.csv',
         **{f"profile_{tech}": f"resources/profile_{tech}.nc"
             for tech in config['renewable']}
     output: "networks/elec.nc"

config.default.yaml

@@ -34,6 +34,7 @@ enable:
   # requires cds API key https://cds.climate.copernicus.eu/api-how-to
   # More information https://atlite.readthedocs.io/en/latest/introduction.html#datasets
 
+# download_osm_data_nprocesses: 10  # (optional) number of threads used to download osm data
 
 augmented_line_connection:
   add_to_snakefile: True  # If True, includes this rule to the workflow
@@ -228,7 +229,7 @@ costs:
 solving:
   options:
     formulation: kirchhoff
-    load_shedding: false
+    load_shedding: true
     noisy_costs: true
     min_iterations: 4
     max_iterations: 6
@@ -314,6 +315,7 @@ plotting:
     "HVDC links": "#8a1caf"
     "DC-DC": "#8a1caf"
     "DC link": "#8a1caf"
+    "load": "#FF0000"
   nice_names:
     OCGT: "Open-Cycle Gas"
     CCGT: "Combined-Cycle Gas"

config.tutorial.yaml

@@ -258,7 +258,7 @@ costs:
 solving:
   options:
     formulation: kirchhoff
-    load_shedding: false
+    load_shedding: true
     noisy_costs: true
     min_iterations: 4
     max_iterations: 6
@@ -336,6 +336,7 @@ plotting:
     "HVDC links": "#8a1caf"
     "DC-DC": "#8a1caf"
     "DC link": "#8a1caf"
+    "load": "#FF0000"
   nice_names:
     OCGT: "Open-Cycle Gas"
     CCGT: "Combined-Cycle Gas"

data/hydro_capacities.csv

@@ -0,0 +1,52 @@
+country_name,Country,p_nom_discharge[GW],p_nom_store[GW],E_store[TWh],InflowHourlyAvg[GWh]
+Algeria,DZ,237.900,0.000,0.267,3.65331931
+Angola,AO,2414.800,0.000,3.892,53.32168268
+Benin,BJ,32.800,0.000,0.243,3.325925824
+Botswana,BW,-,-,-,-
+Burkina Faso,BF,32.000,0.000,1.096,15.00818418
+Burundi,BI,56.000,0.000,0.000,0
+Cameroon,CM,987.000,0.000,0.950,13.01468008
+Central African Republic,CF,18.000,0.000,0.013,0.174394165
+Chad,TD,-,-,-,-
+Congo,CG,236.000,0.000,0.052,0.711041947
+Cote d'Ivoire,CI,880.000,0.000,4.470,61.23522422
+Democratic Republic of Congo,CD,2593.000,0.000,-,-
+Djibouti,DJ,-,-,-,-
+Egypt,EG,2842.000,0.000,49.823,682.5030197
+Equatorial Guinea,GQ,128.000,0.000,0.005,0.064628888
+Eritrea,ER,-,-,-,-
+Eswatini,SZ,-,-,-,-
+Ethiopia,ET,4073.500,0.000,-,-
+Gabon,GA,330.590,0.000,0.084,1.155464484
+Gambia,GM,-,-,-,-
+Ghana,GH,1605.800,0.000,56.423,772.9224202
+Guinea,GN,367.150,0.000,0.361,4.950213441
+Guinea-Bissau,GW,-,-,-,-
+Kenya,KE,836.480,0.000,-,-
+Lesotho,LS,72.000,0.000,0.985,13.49333942
+Liberia,LR,93.000,0.000,0.012,0.169565364
+Libya,LY,-,-,-,-
+Madagascar,MG,-,-,-,-
+Malawi,MW,348.750,0.000,0.000,0
+Mali,ML,356.478,0.000,2.317,31.74441348
+Mauritania,MR,49.000,0.000,0.002,0.027472925
+Morocco,MA,1922.972,466.000,2.999,41.08339401
+Mozambique,MZ,2207.110,0.000,24.754,339.0949095
+Namibia,NA,353.000,0.000,0.951,13.03140528
+Niger,NE0
+Nigeria,NG,2058.340,0.000,6.020,82.46639066
+Rwanda,RW,140.350,0.000,-,-
+Senegal,SN,150.000,0.000,0.002,0.027472925
+Sierra Leone,SL,61.200,0.000,0.107,1.464723968
+Somalia,SO,-,-,-,-
+South Africa,ZA,3624.754,2922.400,2.311,31.66170131
+South Sudan,SS,5.000,0.000,0.000,0
+Sudan,SD,1903.800,0.000,4.650,63.69230241
+Swaziland,SZ,88.490,0.000,0.119,1.633063617
+Tanzania,TZ,572.000,0.000,0.507,6.946068164
+Togo,TG,32.800,0.000,0.243,3.325925824
+Tunisia,TN,62.325,0.000,0.106,1.45312886
+Uganda,UG,741.200,0.000,6.553,89.76871852
+Western Sahara,EH,-,-,-,-
+Zambia,ZM,2325.550,0.000,34.136,467.6157751
+Zimbabwe,ZW,1001.790,0.000,32.853,450.0456825

doc/release_notes.rst

@@ -11,6 +11,9 @@ Release Notes
 Upcoming Release
 ================
 
+* Attach hydro enabled with all hydro types
+* Parallel download of osm data
+
 
 PyPSA-Africa 0.0.1 (24th December 2021)
 =====================================

scripts/build_bus_regions.py

@@ -69,7 +69,7 @@ def save_to_geojson(df, fn):
         df.to_file(fn, driver="GeoJSON", schema=schema)
     else:
         # create empty file to avoid issues with snakemake
-        with os.open(fn, "w") as fp:
+        with open(fn, "w") as fp:
             pass
 
 
@@ -147,6 +147,7 @@ def custom_voronoi_partition_pts(points,
 
 
 def get_gadm_shape(onshore_locs, gadm_shapes):
+
     def locate_bus(coords):
         try:
             return gadm_shapes[gadm_shapes.contains(

scripts/download_osm_data.py

@@ -13,6 +13,7 @@
 import hashlib
 import json
 import logging
+import multiprocessing as mp
 import os
 import pickle
 import shutil
@@ -41,6 +42,7 @@
 from shapely.geometry import LineString
 from shapely.geometry import Point
 from shapely.geometry import Polygon
+from tqdm import tqdm
 
 # esy.osm filter: https://gitlab.com/dlr-ve-esy/esy-osmfilter/-/tree/master/
 
@@ -426,18 +428,74 @@ def output_csv_geojson(output_files, country_code, df_all_feature,
                         driver="GeoJSON")  # Generate GeoJson
 
 
+# Auxiliary function to initialize the parallel data download
+def _init_process_pop(update_, verify_):
+    global update, verify
+    update, verify = update_, verify_
+
+
+# Auxiliary function to download the data
+def _process_func_pop(c_code):
+    download_pbf(c_code, update, verify)
+
+
+def parallel_download_pbf(country_list,
+                          nprocesses,
+                          update=False,
+                          verify=False):
+    """
+    Function to download pbf data in parallel
+
+    Parameters
+    ----------
+    country_list : str
+        List of geofabrik country codes to download
+    nprocesses : int
+        Number of parallel processes
+    update : bool
+        If true, existing pbf files are updated. Default: False
+    verify : bool
+        If true, checks the md5 of the file. Default: False
+    """
+
+    # argument for the parallel processing
+    kwargs = {
+        "initializer": _init_process_pop,
+        "initargs": (update, verify),
+        "maxtasksperchild": 20,
+        "processes": nprocesses,
+    }
+
+    # execute the parallel download with tqdm progressbar
+    with mp.get_context("spawn").Pool(**kwargs) as pool:
+        for _ in tqdm(
+                pool.imap(_process_func_pop, country_list),
+                ascii=False,
+                unit=" countries",
+                total=len(country_list),
+                desc="Download pbf ",
+        ):
+            pass
+
+
 def process_data(
     feature_list,
     country_list,
     output_files,
     iso_coding=True,
     update=False,
     verify=False,
+    nprocesses=1,
 ):
     """
     Download the features in feature_list for each country of the country_list
     """
 
+    # parallel download of data if parallel download is enabled
+    if nprocesses > 1:
+        _logger.info(f"Parallel pbf download with {nprocesses} threads")
+        parallel_download_pbf(country_list, nprocesses, update, verify)
+
     # loop the request for each feature
 
     for feature in feature_list:  # feature dataframe
@@ -511,6 +569,7 @@ def create_country_list(input, iso_coding=True):
     full_codes_list : list
         Example ["NG","ZA"]
     """
+
     def filter_codes(c_list, iso_coding=True):
         """
         Filter list according to the specified coding.
@@ -568,8 +627,10 @@ def filter_codes(c_list, iso_coding=True):
     # ["substation", "generator", "line", "cable", "tower"]
     feature_list = ["substation", "generator", "line", "cable"]
 
-    input = snakemake.config["countries"]
-    output_files = snakemake.output
+    input = snakemake.config["countries"]  # country list or region
+    output_files = snakemake.output  # output snakemake
+    nprocesses = snakemake.config.get("download_osm_data_nprocesses",
+                                      1)  # number of threads
 
     country_list = create_country_list(input)
 
@@ -581,4 +642,5 @@ def filter_codes(c_list, iso_coding=True):
         iso_coding=True,
         update=False,
         verify=False,
+        nprocesses=nprocesses,
     )

scripts/solve_network.py

@@ -109,7 +109,7 @@ def prepare_network(n, solve_opts):
             bus=buses_i,
             carrier="load",
             sign=1e-3,  # Adjust sign to measure p and p_nom in kW instead of MW
-            marginal_cost=1e2,  # Eur/kWh
+            marginal_cost=8e3,  # Eur/kWh
             # intersect between macroeconomic and surveybased
             # willingness to pay
             # http://journal.frontiersin.org/article/10.3389/fenrg.2015.00055/full

test/config.test1.yaml

@@ -83,6 +83,8 @@ enable:
   build_natura_raster: true  # If True, then build_natura_raster can be run
   # custom_busmap: false
 
+download_osm_data_nprocesses: 4  # (optional) number of threads used to download osm data
+
 electricity:
   voltages: [220., 300., 380.]
   co2limit: 7.75e+7 # 0.05 * 3.1e9*0.5
@@ -258,7 +260,7 @@ costs:
 solving:
   options:
     formulation: kirchhoff
-    load_shedding: false
+    load_shedding: true
     noisy_costs: true
     min_iterations: 4
     max_iterations: 6
@@ -336,6 +338,7 @@ plotting:
     "HVDC links": "#8a1caf"
     "DC-DC": "#8a1caf"
     "DC link": "#8a1caf"
+    "load": "#FF0000"
   nice_names:
     OCGT: "Open-Cycle Gas"
     CCGT: "Combined-Cycle Gas"