Merge pull request #371 from GbotemiB/download_fix

Fix for failed data retrieval
pypsa-meets-earth · Sep 11, 2024 · 88b4d5f · 88b4d5f
2 parents 1bcf1e5 + d4d44a9
commit 88b4d5f
Show file tree

Hide file tree

Showing 4 changed files with 45 additions and 19 deletions.
diff --git a/data/temp_hard_coded/gem_ggit_gas_pipelines_december_2022.xlsx b/data/temp_hard_coded/gem_ggit_gas_pipelines_december_2022.xlsx
diff --git a/data/temp_hard_coded/global_steel_plant_tracker_2023_03.xlsx b/data/temp_hard_coded/global_steel_plant_tracker_2023_03.xlsx
diff --git a/scripts/build_industrial_database.py b/scripts/build_industrial_database.py
@@ -75,13 +75,26 @@ def create_steel_db():
 
     fn = "https://globalenergymonitor.org/wp-content/uploads/2023/03/Global-Steel-Plant-Tracker-2023-03.xlsx"
     storage_options = {"User-Agent": "Mozilla/5.0"}
-    steel_orig = pd.read_excel(
-        fn,
-        index_col=0,
-        storage_options=storage_options,
-        sheet_name="Steel Plants",
-        header=0,
-    )
+
+    try:
+        steel_orig = pd.read_excel(
+            fn,
+            index_col=0,
+            storage_options=storage_options,
+            sheet_name="Steel Plants",
+            header=0,
+        )
+    except Exception as e:
+        steel_plant_path = os.path.join(
+            os.getcwd(),
+            "data/temp_hard_coded/global_steel_plant_tracker_2023_03.xlsx",
+        )
+        steel_orig = pd.read_excel(
+            steel_plant_path,
+            index_col=0,
+            sheet_name="Steel Plants",
+            header=0,
+        )
 
     df_steel = steel_orig.copy()
     df_steel = df_steel[
@@ -506,7 +519,7 @@ def create_paper_df():
         snakemake = mock_snakemake(
             "build_industrial_database",
             simpl="",
-            clusters="4",
+            clusters="10",
             ll="c1.0",
             opts="Co2L",
             planning_horizons="2030",

diff --git a/scripts/prepare_gas_network.py b/scripts/prepare_gas_network.py
@@ -4,9 +4,6 @@
 """
 
 import logging
-
-logger = logging.getLogger(__name__)
-
 import os
 import zipfile
 from pathlib import Path
@@ -28,6 +25,9 @@
 from shapely.ops import unary_union
 from shapely.validation import make_valid
 
+logger = logging.getLogger(__name__)
+
+
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from helpers import mock_snakemake, sets_path_to_root
@@ -82,13 +82,26 @@ def download_GGIT_gas_network():
     """
     fn = "https://globalenergymonitor.org/wp-content/uploads/2022/12/GEM-GGIT-Gas-Pipelines-December-2022.xlsx"
     storage_options = {"User-Agent": "Mozilla/5.0"}
-    GGIT_gas_pipeline = pd.read_excel(
-        fn,
-        index_col=0,
-        storage_options=storage_options,
-        sheet_name="Gas Pipelines 2022-12-16",
-        header=0,
-    )
+    try:
+        GGIT_gas_pipeline = pd.read_excel(
+            fn,
+            index_col=0,
+            storage_options=storage_options,
+            sheet_name="Gas Pipelines 2022-12-16",
+            header=0,
+        )
+    except Exception as e:
+        ggit_gas_pipeline_datapath = os.path.join(
+            os.getcwd(),
+            "data/temp_hard_coded/gem_ggit_gas_pipelines_december_2022.xlsx",
+        )
+        GGIT_gas_pipeline = pd.read_excel(
+            ggit_gas_pipeline_datapath,
+        )
+        #     index_col=0,
+        #     sheet_name="Gas Pipelines 2022-12-16",
+        #     header=0,
+        # )
 
     return GGIT_gas_pipeline
 
@@ -185,7 +198,7 @@ def prepare_GGIT_data(GGIT_gas_pipeline):
         .apply(correct_Diameter_col)
         .apply(
             lambda d: inch_to_mm(float(d))
-        )  #         .apply(lambda ds: pd.Series(ds).apply(lambda d: inch_to_mm(float(d))))
+        )  # .apply(lambda ds: pd.Series(ds).apply(lambda d: inch_to_mm(float(d))))
     )
 
     # Convert Bcm/y to MW