From fba027a4ea21596984d4cd760eceebcec12eaf42 Mon Sep 17 00:00:00 2001 From: davide-f Date: Thu, 22 Dec 2022 22:51:07 +0100 Subject: [PATCH 01/13] Revise test config style --- scripts/build_test_configs.py | 46 ++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/scripts/build_test_configs.py b/scripts/build_test_configs.py index 68bdb8dd3..d14cf1d9d 100644 --- a/scripts/build_test_configs.py +++ b/scripts/build_test_configs.py @@ -26,6 +26,27 @@ def update(d, u): return d +def create_test_config(base_path, changes_path, output_path): + # Input paths + fp_baseconfig = Path(Path.cwd(), base_path) + fp_update_file = Path(Path.cwd(), changes_path) + + # Load yaml files + yaml = YAML() + with open(fp_baseconfig) as fp: + base_config = yaml.load(fp) + + # Load update yaml + with open(fp_update_file) as fp: + update_config = yaml.load(fp) + # create updated yaml + test_config = update(copy.deepcopy(base_config), update_config) + # Output path + fp = Path(Path.cwd(), output_path) + # Save file + yaml.dump(test_config, fp) + + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -34,27 +55,12 @@ def update(d, u): snakemake = mock_snakemake("build_test_configs") # Input paths - fp_baseconfig = Path(Path.cwd(), snakemake.input.base_config) - fp_update_file_list = [ - Path(Path.cwd(), i) for i in snakemake.input.update_file_list - ] - - # Load yaml files - yaml = YAML() - with open(fp_baseconfig) as fp: - base_config = yaml.load(fp) + fp_baseconfig = snakemake.input.base_config + fp_update_file_list = snakemake.input.update_file_list + fp_output_file_list = snakemake.output.tmp_test_configs - for c in fp_update_file_list: - # Load update yaml - with open(c) as fp: - update_config = yaml.load(fp) - # create updated yaml - test_config = update(copy.deepcopy(base_config), update_config) - # Output path - list_no = fp_update_file_list.index(c) - fp = Path(Path.cwd(), snakemake.output[list_no]) - # Save file - yaml.dump(test_config, fp) + for (finput, foutput) in zip(fp_update_file_list, fp_output_file_list): + create_test_config(fp_baseconfig, finput, foutput) # Manual output in terminal # import sys From 35122a3f963cd48be59a931974b76a2c548bc260 Mon Sep 17 00:00:00 2001 From: davide-f Date: Thu, 22 Dec 2022 23:28:35 +0100 Subject: [PATCH 02/13] Revise test_config to enhance script capabilities --- scripts/build_test_configs.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/build_test_configs.py b/scripts/build_test_configs.py index d14cf1d9d..f4c2aaf7f 100644 --- a/scripts/build_test_configs.py +++ b/scripts/build_test_configs.py @@ -26,19 +26,22 @@ def update(d, u): return d -def create_test_config(base_path, changes_path, output_path): +def create_test_config(base_path, update_config, output_path): # Input paths fp_baseconfig = Path(Path.cwd(), base_path) - fp_update_file = Path(Path.cwd(), changes_path) # Load yaml files yaml = YAML() + + # get update + if isinstance(update_config, str): + fp_update_file = Path(Path.cwd(), update_config) + # Load update yaml + with open(fp_update_file) as fp: + update_config = yaml.load(fp) with open(fp_baseconfig) as fp: base_config = yaml.load(fp) - # Load update yaml - with open(fp_update_file) as fp: - update_config = yaml.load(fp) # create updated yaml test_config = update(copy.deepcopy(base_config), update_config) # Output path From de6efc1f477244b38f059d6ce39894133723948b Mon Sep 17 00:00:00 2001 From: davide-f Date: Wed, 25 Jan 2023 11:14:12 +0100 Subject: [PATCH 03/13] Add multi-scenario management --- Snakefile | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/Snakefile b/Snakefile index b4e468974..3151a1c73 100644 --- a/Snakefile +++ b/Snakefile @@ -835,3 +835,36 @@ rule build_test_configs: ], script: "scripts/build_test_configs.py" + + +rule run_scenario: + input: + base_config="config.default.yaml", + diff_config="configs/scenarios/config.{scenario_name}.yaml", + output: + touch("results/{scenario_name}/scenario.done"), + threads: 1 + resources: + mem_mb=5000, + run: + from scripts.build_test_configs import create_test_config + + create_test_config( + diff_config, {"run": {"name": scenario_name}}, diff_config + ) + create_test_config(base_config, diff_config, "config.yaml") + os.system( + "snakemake -j all solve_all_networks --forceall --rerun-incomplete" + ) + + +rule run_all_scenarios: + input: + expand( + "results/{scenario_name}/scenario.done", + scenario_name=[ + c.replace("config.", "").replace(".yaml", "") + for c in os.listdir("configs/scenarios") + if c.startswith("config.") and c.endswith(".yaml") + ], + ), From 9f28eeb82e1a4cae2643e779d4350e1b55255f8e Mon Sep 17 00:00:00 2001 From: davide-f Date: Wed, 25 Jan 2023 11:34:51 +0100 Subject: [PATCH 04/13] Finalize rule --- Snakefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Snakefile b/Snakefile index 3151a1c73..ee90a342e 100644 --- a/Snakefile +++ b/Snakefile @@ -850,9 +850,11 @@ rule run_scenario: from scripts.build_test_configs import create_test_config create_test_config( - diff_config, {"run": {"name": scenario_name}}, diff_config + input.diff_config, + {"run": {"name": wildcards.scenario_name}}, + input.diff_config, ) - create_test_config(base_config, diff_config, "config.yaml") + create_test_config(input.base_config, input.diff_config, "config.yaml") os.system( "snakemake -j all solve_all_networks --forceall --rerun-incomplete" ) From 4a7f81b585d3f6d529daa38bd4554cc913c49c5e Mon Sep 17 00:00:00 2001 From: davide-f Date: Wed, 25 Jan 2023 11:38:48 +0100 Subject: [PATCH 05/13] Test custom workflow with multi scenario management --- .github/workflows/ci-linux.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-linux.yaml b/.github/workflows/ci-linux.yaml index 3f2bf15d9..13a2bfdd6 100644 --- a/.github/workflows/ci-linux.yaml +++ b/.github/workflows/ci-linux.yaml @@ -72,8 +72,9 @@ jobs: - name: Test custom workflow run: | - cp test/tmp/config.custom_tmp.yaml config.yaml - snakemake --cores all solve_all_networks --forceall + mkdir configs/scenarios + cp test/config.custom.yaml configs/scenarios/config.custom.yaml + snakemake --cores all run_all_scenarios --forceall - name: Test monte-carlo workflow run: | From 09e078b27749cfda97628ef413696dd4623585b0 Mon Sep 17 00:00:00 2001 From: davide-f Date: Wed, 25 Jan 2023 12:06:03 +0100 Subject: [PATCH 06/13] Minor revision and bug fixing --- .github/workflows/ci-linux.yaml | 2 +- Snakefile | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-linux.yaml b/.github/workflows/ci-linux.yaml index 13a2bfdd6..2ccf7b46f 100644 --- a/.github/workflows/ci-linux.yaml +++ b/.github/workflows/ci-linux.yaml @@ -74,7 +74,7 @@ jobs: run: | mkdir configs/scenarios cp test/config.custom.yaml configs/scenarios/config.custom.yaml - snakemake --cores all run_all_scenarios --forceall + snakemake --cores 1 run_all_scenarios --forceall - name: Test monte-carlo workflow run: | diff --git a/Snakefile b/Snakefile index ee90a342e..9b06434d6 100644 --- a/Snakefile +++ b/Snakefile @@ -866,7 +866,11 @@ rule run_all_scenarios: "results/{scenario_name}/scenario.done", scenario_name=[ c.replace("config.", "").replace(".yaml", "") - for c in os.listdir("configs/scenarios") + for c in ( + os.listdir("configs/scenarios") + if isdir("configs/scenarios") + else [] + ) if c.startswith("config.") and c.endswith(".yaml") ], ), From 92737a126a80635ae63192114fe61f2801f21308 Mon Sep 17 00:00:00 2001 From: davide-f Date: Thu, 26 Jan 2023 14:04:29 +0100 Subject: [PATCH 07/13] Finalize revision --- Snakefile | 16 +++++----- scripts/build_test_configs.py | 58 ++++++++++++++++++++++++++--------- 2 files changed, 52 insertions(+), 22 deletions(-) diff --git a/Snakefile b/Snakefile index 9b06434d6..93b25fb40 100644 --- a/Snakefile +++ b/Snakefile @@ -14,6 +14,7 @@ from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider from scripts._helpers import create_country_list from scripts.add_electricity import get_load_paths_gegis from scripts.retrieve_databundle_light import datafiles_retrivedatabundle +from pathlib import Path HTTP = HTTPRemoteProvider() @@ -839,25 +840,29 @@ rule build_test_configs: rule run_scenario: input: - base_config="config.default.yaml", + default_config="config.default.yaml", diff_config="configs/scenarios/config.{scenario_name}.yaml", output: touch("results/{scenario_name}/scenario.done"), + copyconfig="results/" + RDIR + "config.yaml", threads: 1 resources: mem_mb=5000, run: from scripts.build_test_configs import create_test_config + # Ensure the scenario name matches the name of the configuration create_test_config( input.diff_config, {"run": {"name": wildcards.scenario_name}}, input.diff_config, ) - create_test_config(input.base_config, input.diff_config, "config.yaml") + # merge the default config file with the difference + create_test_config(input.default_config, input.diff_config, "config.yaml") os.system( "snakemake -j all solve_all_networks --forceall --rerun-incomplete" ) + copyfile("config.yaml", input.copyconfig) rule run_all_scenarios: @@ -866,11 +871,6 @@ rule run_all_scenarios: "results/{scenario_name}/scenario.done", scenario_name=[ c.replace("config.", "").replace(".yaml", "") - for c in ( - os.listdir("configs/scenarios") - if isdir("configs/scenarios") - else [] - ) - if c.startswith("config.") and c.endswith(".yaml") + for c in Path("configs/scenarios").glob("config.*.yaml") ], ), diff --git a/scripts/build_test_configs.py b/scripts/build_test_configs.py index f4c2aaf7f..82e9d5b3b 100644 --- a/scripts/build_test_configs.py +++ b/scripts/build_test_configs.py @@ -26,28 +26,58 @@ def update(d, u): return d -def create_test_config(base_path, update_config, output_path): - # Input paths - fp_baseconfig = Path(Path.cwd(), base_path) +def _parse_inputconfig(input_config, yaml): + """Utility function to parse input config into a dictionary""" + if isinstance(input_config, dict): + return dict + + if isinstance(input_config, str): + input_config = Path(Path.cwd(), input_config) + + with open(input_config) as fp: + return yaml.load(fp) + + +def create_test_config(default_config, diff_config, output_path): + """ + This function takes as input a default dictionary-like object + and a difference dictionary-like object, merges the changes of the latter into the former, + and saves the output in the desired output path. + + Inputs + ------ + default_config : dict or path-like + Default dictionray-like object provided as + a dictionary or a path to a yaml file + diff_config : dict or path-like + Difference dictionray-like object provided as + a dictionary or a path to a yaml file + output_path : path-like + Output path where the merged dictionary is saved + + Outputs + ------- + - merged dictionary + + """ # Load yaml files yaml = YAML() - # get update - if isinstance(update_config, str): - fp_update_file = Path(Path.cwd(), update_config) - # Load update yaml - with open(fp_update_file) as fp: - update_config = yaml.load(fp) - with open(fp_baseconfig) as fp: - base_config = yaml.load(fp) + default_config = _parse_inputconfig(default_config, yaml) + diff_config = _parse_inputconfig(diff_config, yaml) # create updated yaml - test_config = update(copy.deepcopy(base_config), update_config) + merged_config = update(copy.deepcopy(default_config), diff_config) + # Output path - fp = Path(Path.cwd(), output_path) + if isinstance(output_path, str): + output_path = Path(Path.cwd(), output_path) + # Save file - yaml.dump(test_config, fp) + yaml.dump(merged_config, output_path) + + return merged_config if __name__ == "__main__": From ba489240388d8aa5f6dfad37039594abd3a62098 Mon Sep 17 00:00:00 2001 From: davide-f Date: Thu, 26 Jan 2023 14:07:25 +0100 Subject: [PATCH 08/13] Refuse --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 93b25fb40..e1f938298 100644 --- a/Snakefile +++ b/Snakefile @@ -844,7 +844,7 @@ rule run_scenario: diff_config="configs/scenarios/config.{scenario_name}.yaml", output: touch("results/{scenario_name}/scenario.done"), - copyconfig="results/" + RDIR + "config.yaml", + copyconfig="results/{scenario_name}/config.yaml", threads: 1 resources: mem_mb=5000, From 2fa12b89a67b1b35abb6623bddc6ae3cad5f5b03 Mon Sep 17 00:00:00 2001 From: davide-f Date: Thu, 26 Jan 2023 14:31:09 +0100 Subject: [PATCH 09/13] Get stem from path in run_all_scenarios --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index e1f938298..0412f4c4c 100644 --- a/Snakefile +++ b/Snakefile @@ -870,7 +870,7 @@ rule run_all_scenarios: expand( "results/{scenario_name}/scenario.done", scenario_name=[ - c.replace("config.", "").replace(".yaml", "") + c.stem.replace("config.", "") for c in Path("configs/scenarios").glob("config.*.yaml") ], ), From 399820d06c746f452433d604fdd8339eab41773d Mon Sep 17 00:00:00 2001 From: davide-f Date: Thu, 26 Jan 2023 14:36:49 +0100 Subject: [PATCH 10/13] Change config file for custom scenario analysis --- .github/workflows/ci-linux.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-linux.yaml b/.github/workflows/ci-linux.yaml index 2ccf7b46f..8667f70f2 100644 --- a/.github/workflows/ci-linux.yaml +++ b/.github/workflows/ci-linux.yaml @@ -73,7 +73,7 @@ jobs: - name: Test custom workflow run: | mkdir configs/scenarios - cp test/config.custom.yaml configs/scenarios/config.custom.yaml + cp test/tmp/config.custom_tmp.yaml configs/scenarios/config.custom.yaml snakemake --cores 1 run_all_scenarios --forceall - name: Test monte-carlo workflow From 7da5708d6a2cc907354633b4573a128d56d5ffa3 Mon Sep 17 00:00:00 2001 From: davide-f Date: Thu, 26 Jan 2023 14:41:05 +0100 Subject: [PATCH 11/13] Revise default test config depending on tutorial status --- .github/workflows/ci-linux.yaml | 2 +- Snakefile | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-linux.yaml b/.github/workflows/ci-linux.yaml index 8667f70f2..2ccf7b46f 100644 --- a/.github/workflows/ci-linux.yaml +++ b/.github/workflows/ci-linux.yaml @@ -73,7 +73,7 @@ jobs: - name: Test custom workflow run: | mkdir configs/scenarios - cp test/tmp/config.custom_tmp.yaml configs/scenarios/config.custom.yaml + cp test/config.custom.yaml configs/scenarios/config.custom.yaml snakemake --cores 1 run_all_scenarios --forceall - name: Test monte-carlo workflow diff --git a/Snakefile b/Snakefile index 0412f4c4c..80cebcb07 100644 --- a/Snakefile +++ b/Snakefile @@ -28,6 +28,9 @@ if "config" not in globals() or not config: # skip when used as sub-workflow configfile: "configs/bundle_config.yaml" +DEFAULT_CONFIG = "config.tutorial.yaml" if config["tutorial"] else "config.default.yaml" + + # convert country list according to the desired region config["countries"] = create_country_list(config["countries"]) @@ -840,7 +843,7 @@ rule build_test_configs: rule run_scenario: input: - default_config="config.default.yaml", + default_config=DEFAULT_CONFIG, diff_config="configs/scenarios/config.{scenario_name}.yaml", output: touch("results/{scenario_name}/scenario.done"), From 82a44ee73d504dd175ff5ef79917f204e83216db Mon Sep 17 00:00:00 2001 From: davide-f Date: Thu, 26 Jan 2023 15:14:24 +0100 Subject: [PATCH 12/13] Fix typo --- scripts/build_test_configs.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/build_test_configs.py b/scripts/build_test_configs.py index 82e9d5b3b..faf83a9fa 100644 --- a/scripts/build_test_configs.py +++ b/scripts/build_test_configs.py @@ -29,7 +29,7 @@ def update(d, u): def _parse_inputconfig(input_config, yaml): """Utility function to parse input config into a dictionary""" if isinstance(input_config, dict): - return dict + return input_config if isinstance(input_config, str): input_config = Path(Path.cwd(), input_config) @@ -94,7 +94,3 @@ def create_test_config(default_config, diff_config, output_path): for (finput, foutput) in zip(fp_update_file_list, fp_output_file_list): create_test_config(fp_baseconfig, finput, foutput) - - # Manual output in terminal - # import sys - # yaml.dump(data, sys.stdout) From 38eb81b9ad482f68af6825d3f4c6fcac95897ef8 Mon Sep 17 00:00:00 2001 From: davide-f Date: Thu, 26 Jan 2023 17:24:47 +0100 Subject: [PATCH 13/13] Fix backup of config file in scenario managemetn --- Snakefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Snakefile b/Snakefile index 80cebcb07..39ccc5bfd 100644 --- a/Snakefile +++ b/Snakefile @@ -846,7 +846,7 @@ rule run_scenario: default_config=DEFAULT_CONFIG, diff_config="configs/scenarios/config.{scenario_name}.yaml", output: - touch("results/{scenario_name}/scenario.done"), + touchfile=touch("results/{scenario_name}/scenario.done"), copyconfig="results/{scenario_name}/config.yaml", threads: 1 resources: @@ -865,7 +865,7 @@ rule run_scenario: os.system( "snakemake -j all solve_all_networks --forceall --rerun-incomplete" ) - copyfile("config.yaml", input.copyconfig) + copyfile("config.yaml", output.copyconfig) rule run_all_scenarios: