Restyle Decouple iso coding from geofabrik - download osm data for the world

Snakefile

@@ -2,11 +2,16 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+import sys
+sys.path.append('./scripts')
+
 from os.path import normpath, exists, isdir
 from shutil import copyfile
 
 from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider
 
+from scripts.download_osm_data import create_country_list
+
 HTTP = HTTPRemoteProvider()
 
 if not exists("config.yaml"):
@@ -15,6 +20,8 @@ if not exists("config.yaml"):
 
 configfile: "config.yaml"
 
+# convert country list according to the desired region
+config["countries"] = create_country_list(config["countries"])
 
 COSTS = "data/costs.csv"
 ATLITE_NPROCESSES = config["atlite"].get("nprocesses", 20)

notebooks/0_osm_data_url_acsess.ipynb

@@ -0,0 +1,250 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "45072a1b",
+   "metadata": {},
+   "source": [
+    "# Acsess to the Open Street Maps Data Server\n",
+    "\n",
+    "This notebook shows how we reach the raw OpenStreetMaps data."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "98cecfff",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "This is the repository path:  /Users/ekatef/Documents/_github_/pypsa-africa\n",
+      "Had to go 0 folder(s) up.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import sys\n",
+    "sys.path.append('../')  # to import helpers\n",
+    "\n",
+    "from scripts._helpers import _sets_path_to_root\n",
+    "_sets_path_to_root(\"pypsa-africa\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "id": "9efb61cd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import logging\n",
+    "import sys\n",
+    "import pandas as pd\n",
+    "import requests\n",
+    "import urllib3\n",
+    "import time\n",
+    "\n",
+    "pd.set_option('display.max_columns', None)\n",
+    "pd.set_option('display.max_colwidth', 70)\n",
+    "\n",
+    "logger = logging.getLogger(__name__)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "id": "fd7c4412",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from scripts.config_osm_data import continent_regions\n",
+    "from scripts.config_osm_data import continents\n",
+    "from scripts.config_osm_data import iso_to_geofk_dict\n",
+    "from scripts.config_osm_data import world\n",
+    "from scripts.config_osm_data import world_geofk"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "id": "0a09e736",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def list_word_geofk_countries():\n",
+    "    countries_list = []\n",
+    "\n",
+    "    for continent in world_geofk:\n",
+    "        country = world_geofk[continent]\n",
+    "        countries_list.append(list(country.keys()))\n",
+    "        \n",
+    "    return countries_list "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "id": "3f8fea67",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "['DZ', 'AO', 'BJ', 'BW', 'BF', 'BI', 'CM', 'CF', 'TD', 'CG', 'CD', 'DJ', 'EG', 'GQ', 'ER', 'ET', 'GA', 'GH', 'GW', 'GN', 'CI', 'KE', 'LS', 'LR', 'LY', 'MG', 'MW', 'ML', 'MR', 'MA', 'MZ', 'NA', 'NE', 'NG', 'RW', 'SNGM', 'SL', 'SO', 'ZA', 'SS', 'SD', 'SZ', 'TZ', 'TG', 'TN', 'UG', 'ZM', 'ZW', 'AF', 'AM', 'AZ', 'BD', 'BT', 'KH', 'CN', 'GCC', 'IN', 'ID', 'IR', 'IQ', 'IL-PL', 'JP', 'JO', 'KZ', 'KP', 'KR', 'KG', 'LA', 'LB', 'MY-SG-BN', 'MV', 'MN', 'MM', 'NP', 'PK', 'PH', 'LK', 'SY', 'TW', 'TJ', 'TH', 'TM', 'UZ', 'VN', 'YE', 'AU', 'FJ', 'NC', 'NZ', 'PG', 'AL', 'AD', 'AT', 'BY', 'BE', 'BA', 'BG', 'HR', 'CZ', 'CY', 'DK', 'EE', 'FI', 'FR', 'GE', 'DE', 'GR', 'HU', 'IS', 'IE', 'IT', 'LV', 'LI', 'LT', 'LU', 'MK', 'MT', 'MD', 'MC', 'ME', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SK', 'SI', 'ES', 'SE', 'CH', 'UA', 'GB', 'TR', 'CEFD', 'FEFD', 'NCDF', 'NWDF', 'SBFD', 'SOFD', 'URDF', 'VOFD', 'CA', 'GL', 'MX', 'US', 'AR', 'BO', 'BR', 'CL', 'CO', 'EC', 'PE', 'SR', 'UY', 'VE', 'BZ', 'GT', 'SV', 'HN', 'NI', 'CR']\n"
+     ]
+    }
+   ],
+   "source": [
+    "world_geofk_countries = []\n",
+    "for sublist in list_word_geofk_countries():\n",
+    "    for item in sublist:\n",
+    "        world_geofk_countries.append(item)\n",
+    "\n",
+    "print(world_geofk_countries)        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "id": "88a5e6e1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def getGeofkContinentCountry(code):\n",
+    "    for continent in world_geofk:\n",
+    "        country = world_geofk[continent].get(code, 0)\n",
+    "        if country:\n",
+    "            return continent, country\n",
+    "    return continent, country\n",
+    "\n",
+    "def build_url(country_code, update, verify):\n",
+    "    continent, country_name = getGeofkContinentCountry(country_code)\n",
+    "    geofabrik_filename = f\"{country_name}-latest.osm.pbf\"\n",
+    "    geofabrik_url = f\"https://download.geofabrik.de/{continent}/{geofabrik_filename}\"\n",
+    "    return geofabrik_url\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "984f4606",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "('africa', 'algeria')\n",
+      "URL 'https://download.geofabrik.de/africa/algeria-latest.osm.pbf' is working\n",
+      "('africa', 'angola')\n",
+      "URL 'https://download.geofabrik.de/africa/angola-latest.osm.pbf' is working\n",
+      "('africa', 'benin')\n",
+      "URL 'https://download.geofabrik.de/africa/benin-latest.osm.pbf' is working\n",
+      "('africa', 'botswana')\n",
+      "URL 'https://download.geofabrik.de/africa/botswana-latest.osm.pbf' is working\n",
+      "('africa', 'burkina-faso')\n",
+      "URL 'https://download.geofabrik.de/africa/burkina-faso-latest.osm.pbf' is working\n",
+      "('africa', 'burundi')\n",
+      "URL 'https://download.geofabrik.de/africa/burundi-latest.osm.pbf' is working\n",
+      "('africa', 'cameroon')\n",
+      "URL 'https://download.geofabrik.de/africa/cameroon-latest.osm.pbf' is working\n",
+      "('africa', 'central-african-republic')\n",
+      "URL 'https://download.geofabrik.de/africa/central-african-republic-latest.osm.pbf' is working\n",
+      "('africa', 'chad')\n",
+      "URL 'https://download.geofabrik.de/africa/chad-latest.osm.pbf' is working\n",
+      "('africa', 'congo-brazzaville')\n",
+      "URL 'https://download.geofabrik.de/africa/congo-brazzaville-latest.osm.pbf' is working\n",
+      "('africa', 'congo-democratic-republic')\n",
+      "URL 'https://download.geofabrik.de/africa/congo-democratic-republic-latest.osm.pbf' is working\n",
+      "('africa', 'djibouti')\n",
+      "URL 'https://download.geofabrik.de/africa/djibouti-latest.osm.pbf' is working\n",
+      "('africa', 'egypt')\n",
+      "URL 'https://download.geofabrik.de/africa/egypt-latest.osm.pbf' is working\n",
+      "('africa', 'equatorial-guinea')\n",
+      "URL 'https://download.geofabrik.de/africa/equatorial-guinea-latest.osm.pbf' is working\n",
+      "('africa', 'eritrea')\n"
+     ]
+    }
+   ],
+   "source": [
+    "problem_urls = []\n",
+    "problem_codes = []\n",
+    "problem_domain = []\n",
+    "# for cnt in country_geofk_list:\n",
+    "for cnt in world_geofk_countries:    \n",
+    "    print(getGeofkContinentCountry(cnt))\n",
+    "    url = build_url(country_code=cnt, update=False, verify=False)\n",
+    "    time.sleep(0.01)\n",
+    "    try:\n",
+    "        request = requests.head(url)\n",
+    "        if request.status_code == 200:\n",
+    "            print(\"URL '\" + url + \"' is working\")\n",
+    "        else:\n",
+    "            problem_urls.append(url)\n",
+    "            problem_codes.append(cnt)\n",
+    "            problem_domain.append(getGeofkContinentCountry(cnt))\n",
+    "            print(\"URL '\" + url + \"' is silent\")\n",
+    "    except requests.exceptions.ConnectionError:\n",
+    "        print(f\"URL {url} not reachable\")\n",
+    "\n",
+    "print(\"There are troubles in reaching following urls:\") \n",
+    "print(problem_urls) \n",
+    "print(\"Country codes to be checked\")\n",
+    "print(problem_codes) \n",
+    "print(problem_domain)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "33ea8008",
+   "metadata": {},
+   "source": [
+    "\n",
+    "# Acknowledgments\n",
+    "\n",
+    "The project relies on the (OpenStreetMap)[https://www.geofabrik.de/] data provided via Geofabrik service. Many thanks to all the service contributors."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "be4bfbed",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "45bb68b4",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

scripts/add_electricity.py

@@ -92,7 +92,6 @@
 import xarray as xr
 from _helpers import configure_logging
 from _helpers import update_p_nom_max
-from download_osm_data import create_country_list
 from powerplantmatching.export import map_country_bus
 from shapely.validation import make_valid
 from vresutils import transfer as vtransfer
@@ -704,7 +703,7 @@ def add_nice_carrier_names(n, config=None):
     # Snakemake imports:
     regions = snakemake.input.regions
     load = snakemake.input.load
-    countries = create_country_list(snakemake.config["countries"])
+    countries = snakemake.config["countries"]
     scale = snakemake.config["load_options"]["scale"]
     admin_shapes = snakemake.input.gadm_shapes
 

scripts/base_network.py

@@ -68,7 +68,6 @@
 import yaml
 from _helpers import _read_csv_nafix
 from _helpers import configure_logging
-from download_osm_data import create_country_list
 from scipy.sparse import csgraph
 from shapely.geometry import LineString
 from shapely.geometry import Point
@@ -137,15 +136,19 @@ def _load_buses_from_osm():
 
     return buses
 
+
 def _set_links_underwater_fraction(n):
-    if n.links.empty: return
+    if n.links.empty:
+        return
 
-    if not hasattr(n.links, 'geometry'):
-        n.links['underwater_fraction'] = 0.
+    if not hasattr(n.links, "geometry"):
+        n.links["underwater_fraction"] = 0.0
     else:
-        offshore_shape = gpd.read_file(snakemake.input.offshore_shapes).unary_union
+        offshore_shape = gpd.read_file(
+            snakemake.input.offshore_shapes).unary_union
         links = gpd.GeoSeries(n.links.geometry.dropna().map(shapely.wkt.loads))
-        n.links['underwater_fraction'] = links.intersection(offshore_shape).length / links.length
+        n.links["underwater_fraction"] = (
+            links.intersection(offshore_shape).length / links.length)
 
 
 def _load_lines_from_osm(buses):
@@ -198,7 +201,7 @@ def _set_countries_and_substations(n):
 
     buses = n.buses
 
-    countries = create_country_list(snakemake.config["countries"])
+    countries = snakemake.config["countries"]
     country_shapes = (gpd.read_file(snakemake.input.country_shapes).set_index(
         "name")["geometry"].set_crs(4326))
     offshore_shapes = unary_union(

scripts/build_bus_regions.py

@@ -47,7 +47,6 @@
 import pandas as pd
 import pypsa
 from _helpers import configure_logging
-from download_osm_data import create_country_list
 from shapely.geometry import Point
 from shapely.geometry import Polygon
 from vresutils.graph import voronoi_partition_pts
@@ -183,7 +182,7 @@ def get_id(coords):
         snakemake = mock_snakemake("build_bus_regions")
     configure_logging(snakemake)
 
-    countries = create_country_list(snakemake.config["countries"])
+    countries = snakemake.config["countries"]
 
     n = pypsa.Network(snakemake.input.base_network)
 

scripts/build_osm_network.py

@@ -11,7 +11,6 @@
 from _helpers import _sets_path_to_root
 from _helpers import _to_csv_nafix
 from _helpers import configure_logging
-from download_osm_data import create_country_list
 from shapely.geometry import LineString
 from shapely.geometry import Point
 from shapely.ops import linemerge
@@ -531,7 +530,7 @@ def built_network(inputs, outputs):
     country_shapes = (gpd.read_file(country_shapes_fn).set_index("name")
                       ["geometry"].set_crs(4326))
     input = snakemake.config["countries"]
-    country_list = create_country_list(input)
+    country_list = input
     bus_country_list = buses["country"].unique().tolist()
 
     if len(bus_country_list) != len(country_list):

scripts/build_shapes.py

@@ -21,7 +21,6 @@
 from _helpers import _two_2_three_digits_country
 from _helpers import _two_digits_2_name_country
 from _helpers import configure_logging
-from download_osm_data import create_country_list
 from rasterio.mask import mask
 from shapely.geometry import LineString
 from shapely.geometry import MultiPolygon
@@ -702,7 +701,7 @@ def gadm(countries,
 
     out = snakemake.output
 
-    countries_list = create_country_list(snakemake.config["countries"])
+    countries_list = snakemake.config["countries"]
     layer_id = snakemake.config["build_shape_options"]["gadm_layer_id"]
     update = snakemake.config["build_shape_options"]["update_file"]
     out_logging = snakemake.config["build_shape_options"]["out_logging"]