Add run scenarios

.github/workflows/ci-linux.yaml

@@ -72,8 +72,9 @@ jobs:
 
     - name: Test custom workflow
       run: |
-        cp test/tmp/config.custom_tmp.yaml config.yaml
-        snakemake --cores all solve_all_networks --forceall
+        mkdir configs/scenarios
+        cp test/config.custom.yaml configs/scenarios/config.custom.yaml
+        snakemake --cores 1 run_all_scenarios --forceall
 
     - name: Test monte-carlo workflow
       run: |

Snakefile

@@ -14,6 +14,7 @@ from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider
 from scripts._helpers import create_country_list
 from scripts.add_electricity import get_load_paths_gegis
 from scripts.retrieve_databundle_light import datafiles_retrivedatabundle
+from pathlib import Path
 
 HTTP = HTTPRemoteProvider()
 
@@ -835,3 +836,41 @@ rule build_test_configs:
         ],
     script:
         "scripts/build_test_configs.py"
+
+
+rule run_scenario:
+    input:
+        default_config="config.default.yaml",
+        diff_config="configs/scenarios/config.{scenario_name}.yaml",
+    output:
+        touch("results/{scenario_name}/scenario.done"),
+        copyconfig="results/{scenario_name}/config.yaml",
+    threads: 1
+    resources:
+        mem_mb=5000,
+    run:
+        from scripts.build_test_configs import create_test_config
+
+        # Ensure the scenario name matches the name of the configuration
+        create_test_config(
+            input.diff_config,
+            {"run": {"name": wildcards.scenario_name}},
+            input.diff_config,
+        )
+        # merge the default config file with the difference
+        create_test_config(input.default_config, input.diff_config, "config.yaml")
+        os.system(
+            "snakemake -j all solve_all_networks --forceall --rerun-incomplete"
+        )
+        copyfile("config.yaml", input.copyconfig)
+
+
+rule run_all_scenarios:
+    input:
+        expand(
+            "results/{scenario_name}/scenario.done",
+            scenario_name=[
+                c.replace("config.", "").replace(".yaml", "")
+                for c in Path("configs/scenarios").glob("config.*.yaml")
+            ],
+        ),

scripts/build_test_configs.py

@@ -26,6 +26,60 @@ def update(d, u):
     return d
 
 
+def _parse_inputconfig(input_config, yaml):
+    """Utility function to parse input config into a dictionary"""
+    if isinstance(input_config, dict):
+        return dict
+
+    if isinstance(input_config, str):
+        input_config = Path(Path.cwd(), input_config)
+
+    with open(input_config) as fp:
+        return yaml.load(fp)
+
+
+def create_test_config(default_config, diff_config, output_path):
+    """
+    This function takes as input a default dictionary-like object
+    and a difference dictionary-like object, merges the changes of the latter into the former,
+    and saves the output in the desired output path.
+
+    Inputs
+    ------
+    default_config : dict or path-like
+        Default dictionray-like object provided as
+        a dictionary or a path to a yaml file
+    diff_config : dict or path-like
+        Difference dictionray-like object provided as
+        a dictionary or a path to a yaml file
+    output_path : path-like
+        Output path where the merged dictionary is saved
+
+    Outputs
+    -------
+    - merged dictionary
+
+    """
+
+    # Load yaml files
+    yaml = YAML()
+
+    default_config = _parse_inputconfig(default_config, yaml)
+    diff_config = _parse_inputconfig(diff_config, yaml)
+
+    # create updated yaml
+    merged_config = update(copy.deepcopy(default_config), diff_config)
+
+    # Output path
+    if isinstance(output_path, str):
+        output_path = Path(Path.cwd(), output_path)
+
+    # Save file
+    yaml.dump(merged_config, output_path)
+
+    return merged_config
+
+
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
@@ -34,27 +88,12 @@ def update(d, u):
         snakemake = mock_snakemake("build_test_configs")
 
     # Input paths
-    fp_baseconfig = Path(Path.cwd(), snakemake.input.base_config)
-    fp_update_file_list = [
-        Path(Path.cwd(), i) for i in snakemake.input.update_file_list
-    ]
+    fp_baseconfig = snakemake.input.base_config
+    fp_update_file_list = snakemake.input.update_file_list
+    fp_output_file_list = snakemake.output.tmp_test_configs
 
-    # Load yaml files
-    yaml = YAML()
-    with open(fp_baseconfig) as fp:
-        base_config = yaml.load(fp)
-
-    for c in fp_update_file_list:
-        # Load update yaml
-        with open(c) as fp:
-            update_config = yaml.load(fp)
-        # create updated yaml
-        test_config = update(copy.deepcopy(base_config), update_config)
-        # Output path
-        list_no = fp_update_file_list.index(c)
-        fp = Path(Path.cwd(), snakemake.output[list_no])
-        # Save file
-        yaml.dump(test_config, fp)
+    for (finput, foutput) in zip(fp_update_file_list, fp_output_file_list):
+        create_test_config(fp_baseconfig, finput, foutput)
 
     # Manual output in terminal
     # import sys