From affd68272a14ebe295050876306227aca891c8a2 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Thu, 27 Jun 2024 17:06:04 +0200 Subject: [PATCH 01/11] set up public config --- config/config.public.yaml | 34 ++++++++++++++++++++++++++++++++++ workflow/Snakefile | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 config/config.public.yaml diff --git a/config/config.public.yaml b/config/config.public.yaml new file mode 100644 index 00000000..8e68b1f6 --- /dev/null +++ b/config/config.public.yaml @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC0-1.0 + +run: + prefix: 20240627public_db + name: + - 8Gt_Bal_v3 + # - 8Gt_Elec_v3 + # - 8Gt_H2_v3 + scenarios: + enable: true + manual_file: config/scenarios.public.yaml + file: config/scenarios.automated.yaml + shared_resources: + policy: base #stops recalculating + exclude: + - existing_heating.csv # specify files which should not be shared between scenarios + - costs + - retrieve_cost # This is necessary to save retrieve_cost_data_{year}.log in the correct folder + +iiasa_database: + db_name: ariadne + leitmodelle: + general: REMIND-EU v1.1 + buildings: REMod v1.0 + transport: DLR-DEMO + industry: FORECAST v1.0 + scenarios: + - 8Gt_Bal_v3 + - 8Gt_Elec_v3 + - 8Gt_H2_v3 + reference_scenario: 8Gt_Bal_v3 + region: Deutschland \ No newline at end of file diff --git a/workflow/Snakefile b/workflow/Snakefile index 507f458e..9b5f3e10 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -419,7 +419,7 @@ rule build_scenarios: scenarios=config_provider("run", "name"), input: ariadne_database=resources("ariadne_database.csv"), - scenario_yaml="config/scenarios.manual.yaml", + scenario_yaml=config["run"]["scenarios"]["manual_file"], output: scenario_yaml=config["run"]["scenarios"]["file"], log: From 9e473305a5824b507b350e76503dd794cdd57e27 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Thu, 27 Jun 2024 17:06:28 +0200 Subject: [PATCH 02/11] define sectoral leitmodelle --- config/config.yaml | 9 +++++---- workflow/scripts/retrieve_ariadne_database.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index 507a1809..3d5d951b 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -16,6 +16,7 @@ run: # - KN2045minus_SupplyFocus scenarios: enable: true + manual_file: config/scenarios.public.yaml file: config/scenarios.automated.yaml shared_resources: policy: base #stops recalculating @@ -31,10 +32,10 @@ run: iiasa_database: db_name: ariadne2_intern leitmodelle: - - REMIND-EU v1.1 - - REMod v1.0 - - Aladin v1 - - FORECAST v1.0 + general: REMIND-EU v1.1 + buildings: REMod v1.0 + transport: Aladin v1 + industry: FORECAST v1.0 scenarios: - CurrentPolicies - KN2045_Elec_v4 diff --git a/workflow/scripts/retrieve_ariadne_database.py b/workflow/scripts/retrieve_ariadne_database.py index 94730749..6ae278b2 100644 --- a/workflow/scripts/retrieve_ariadne_database.py +++ b/workflow/scripts/retrieve_ariadne_database.py @@ -13,7 +13,7 @@ db = pyam.read_iiasa( snakemake.params.db_name, - model=snakemake.params.leitmodelle, + model=snakemake.params.leitmodelle.values(), scenario=snakemake.params.scenarios, # Download only the most recent iterations of scenarios ) From 8b0a4315afb3f5c932d835b00e012575b7b33115 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Fri, 28 Jun 2024 12:45:37 +0200 Subject: [PATCH 03/11] scenario file for public db --- config/config.yaml | 2 +- config/scenarios.public.yaml | 174 +++++++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 config/scenarios.public.yaml diff --git a/config/config.yaml b/config/config.yaml index 3d5d951b..db2e2294 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -16,7 +16,7 @@ run: # - KN2045minus_SupplyFocus scenarios: enable: true - manual_file: config/scenarios.public.yaml + manual_file: config/scenarios.manual.yaml file: config/scenarios.automated.yaml shared_resources: policy: base #stops recalculating diff --git a/config/scenarios.public.yaml b/config/scenarios.public.yaml new file mode 100644 index 00000000..f2128ab5 --- /dev/null +++ b/config/scenarios.public.yaml @@ -0,0 +1,174 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + + +8Gt_Bal_v3: + iiasa_database: + reference_scenario: 8Gt_Bal_v3 + fallback_reference_scenario: 8Gt_Bal_v3 + co2_budget_DE_source: KSG + + costs: + horizon: "mean" + NEP: 2023 + transmission: "overhead" # either overhead line ("overhead") or underground cable ("underground") + + # boundary condition of maximum volumes + limits_volume_max: + # constrain electricity import in TWh + electricity_import: + DE: + 2020: -20 + 2025: 0 + 2030: 0 + 2035: 40 + 2040: 80 + 2045: 125 + electrolysis: + DE: + 2020: 0 + 2025: 5 + 2030: 45 + 2035: 130 + 2040: 215 + 2045: 300 + h2_derivate_import: + DE: + 2020: 0 + 2025: 0 + 2030: 10 + 2035: 105 + 2040: 200 + 2045: 300 + h2_import: + DE: + 2020: 0 + 2025: 5 + 2030: 15 + 2035: 115 + 2040: 220 + 2045: 325 + limits_volume_min: + electrolysis: + DE: + 2020: 0 + 2025: 0 + 2030: 0 + 2035: 0 + 2040: 0 + 2045: 0 + +8Gt_Elec_v3: + iiasa_database: + reference_scenario: 8Gt_Elec_v3 + fallback_reference_scenario: 8Gt_Elec_v3 + co2_budget_DE_source: KSG + + costs: + horizon: "mean" + NEP: 2023 + transmission: "overhead" # either overhead line ("overhead") or underground cable ("underground") + + limits_volume_max: + # constrain electricity import in TWh + electricity_import: + DE: + 2020: -20 + 2025: 0 + 2030: 0 + 2035: 50 + 2040: 100 + 2045: 150 + electrolysis: + DE: + 2020: 0 + 2025: 5 + 2030: 45 + 2035: 95 + 2040: 145 + 2045: 200 + h2_derivate_import: + DE: + 2020: 0 + 2025: 0 + 2030: 10 + 2035: 70 + 2040: 130 + 2045: 200 + h2_import: + DE: + 2020: 0 + 2025: 5 + 2030: 10 + 2035: 90 + 2040: 170 + 2045: 250 + limits_volume_min: + electrolysis: + DE: + 2020: 0 + 2025: 0 + 2030: 0 + 2035: 0 + 2040: 0 + 2045: 0 + +8Gt_H2_v3: + iiasa_database: + reference_scenario: 8Gt_H2_v3 + fallback_reference_scenario: 8Gt_H2_v3 + co2_budget_DE_source: KSG + + costs: + horizon: "mean" + NEP: 2023 + transmission: "overhead" # either overhead line ("overhead") or underground cable ("underground") + + limits_volume_max: + # constrain electricity import in TWh + electricity_import: + DE: + 2020: -20 + 2025: 0 + 2030: 0 + 2035: 30 + 2040: 70 + 2045: 100 # scenario guidelines + + # constrain hydrogen import in TWh + h2_import: + DE: + 2020: 0 + 2025: 5 + 2030: 45 # scenario guidelines + 2035: 155 + 2040: 265 + 2045: 400 # scenario guidelines + # import of h2 derivatives in TWh + h2_derivate_import: + DE: + 2020: 0 + 2025: 0 + 2030: 10 # scenario guidelines + 2035: 140 + 2040: 270 + 2045: 400 # scenario guidelines + electrolysis: + DE: + 2020: 0 + 2025: 5 + 2030: 45 # scenario guidelines + 2035: 160 + 2040: 275 + 2045: 400 # scenario guidelines + + limits_volume_min: + electrolysis: + DE: + 2025: 0 + 2030: 0 + 2035: 0 + 2040: 0 + 2045: 200 From 2b8cf22cda074ddf267347a2c89ec349d4fdbe86 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Fri, 28 Jun 2024 12:46:17 +0200 Subject: [PATCH 04/11] simplifying non-co2 computation --- workflow/scripts/build_scenarios.py | 31 +++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/workflow/scripts/build_scenarios.py b/workflow/scripts/build_scenarios.py index 8172340b..31f0b3c1 100644 --- a/workflow/scripts/build_scenarios.py +++ b/workflow/scripts/build_scenarios.py @@ -14,8 +14,12 @@ def get_transport_growth(df, planning_horizons): # Aviation growth factor - using REMIND-EU v1.1 since Aladin v1 does not include bunkers - aviation_model = "REMIND-EU v1.1" - aviation = df.loc[aviation_model,"Final Energy|Bunkers|Aviation", "PJ/yr"] + aviation_model = snakemake.params.leitmodelle["general"] + try: + aviation = df.loc[aviation_model,"Final Energy|Bunkers|Aviation", "PJ/yr"] + except KeyError: + aviation = df.loc[aviation_model,"Final Energy|Bunkers|Aviation", "TWh/yr"] * 3.6 # TWh to PJ + aviation_growth_factor = aviation / aviation[2020] return aviation_growth_factor[planning_horizons] @@ -23,14 +27,14 @@ def get_transport_growth(df, planning_horizons): def get_primary_steel_share(df, planning_horizons): # Get share of primary steel production - model = "FORECAST v1.0" + model = snakemake.params.leitmodelle["industry"] total_steel = df.loc[model, "Production|Steel"] primary_steel = df.loc[model, "Production|Steel|Primary"] primary_steel_share = primary_steel / total_steel primary_steel_share = primary_steel_share[planning_horizons] - if model == "FORECAST v1.0" and planning_horizons[0] == 2020: + if model == "FORECAST v1.0" and (planning_horizons[0] == 2020) and snakemake.params.db_name == "ariadne2_intern": logger.warning("FORECAST v1.0 does not have data for 2020. Using 2021 data for Production|Steel instead.") primary_steel_share[2020] = primary_steel[2021] / total_steel[2021] @@ -70,16 +74,23 @@ def get_co2_budget(df, source): raise ValueError("Invalid source for CO2 budget.") ## Compute nonco2 from Ariadne-Leitmodell (REMIND) + # co2 = ( + # df.loc["Emissions|CO2 incl Bunkers","Mt CO2/yr"] + # - df.loc["Emissions|CO2|Land-Use Change","Mt CO2-equiv/yr"] + # - df.loc["Emissions|CO2|Energy|Demand|Bunkers","Mt CO2/yr"] + # ) + # ghg = ( + # df.loc["Emissions|Kyoto Gases","Mt CO2-equiv/yr"] + # - df.loc["Emissions|Kyoto Gases|Land-Use Change","Mt CO2-equiv/yr"] + # # No Kyoto Gas emissions for Bunkers recorded in Ariadne DB + # ) + co2 = ( - df.loc["Emissions|CO2 incl Bunkers","Mt CO2/yr"] - - df.loc["Emissions|CO2|Land-Use Change","Mt CO2-equiv/yr"] - - df.loc["Emissions|CO2|Energy|Demand|Bunkers","Mt CO2/yr"] + df.loc["Emissions|CO2","Mt CO2/yr"] ) ghg = ( df.loc["Emissions|Kyoto Gases","Mt CO2-equiv/yr"] - - df.loc["Emissions|Kyoto Gases|Land-Use Change","Mt CO2-equiv/yr"] - # No Kyoto Gas emissions for Bunkers recorded in Ariadne DB ) nonco2 = ghg - co2 @@ -112,7 +123,7 @@ def write_to_scenario_yaml( co2_budget_source = config[scenario]["co2_budget_DE_source"] co2_budget_fractions = get_co2_budget( - df.loc["REMIND-EU v1.1", fallback_reference_scenario], + df.loc[snakemake.params.leitmodelle["general"], fallback_reference_scenario], co2_budget_source ) From c9498f1b22b84e695e20d0a04a2afa13781dcf49 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Fri, 28 Jun 2024 12:46:30 +0200 Subject: [PATCH 05/11] adjustments for old db --- config/config.public.yaml | 3 +- workflow/Snakefile | 5 +++ workflow/scripts/build_mobility_demand.py | 50 +++++++++++++--------- workflow/scripts/modify_industry_demand.py | 2 +- 4 files changed, 38 insertions(+), 22 deletions(-) diff --git a/config/config.public.yaml b/config/config.public.yaml index 8e68b1f6..878d657e 100644 --- a/config/config.public.yaml +++ b/config/config.public.yaml @@ -24,7 +24,8 @@ iiasa_database: leitmodelle: general: REMIND-EU v1.1 buildings: REMod v1.0 - transport: DLR-DEMO + transport: DLR + transport_stock: DLR-VECTOR21 industry: FORECAST v1.0 scenarios: - 8Gt_Bal_v3 diff --git a/workflow/Snakefile b/workflow/Snakefile index 9b5f3e10..c76f1ab2 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -132,8 +132,10 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_cost_data", T rule build_mobility_demand: params: + db_name=config_provider("iiasa_database", "db_name"), reference_scenario=config_provider("iiasa_database", "reference_scenario"), planning_horizons=config_provider("scenario", "planning_horizons"), + leitmodelle=config_provider("iiasa_database", "leitmodelle"), input: ariadne=resources("ariadne_database.csv"), clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), @@ -246,6 +248,7 @@ use rule build_existing_heating_distribution from pypsaeur with: rule modify_industry_demand: input: + db_name=config_provider("iiasa_database", "db_name"), ariadne=resources("ariadne_database.csv"), industrial_production_per_country_tomorrow=resources("industrial_production_per_country_tomorrow_{planning_horizons}.csv"), output: @@ -417,6 +420,8 @@ rule ariadne_all: rule build_scenarios: params: scenarios=config_provider("run", "name"), + db_name=config_provider("iiasa_database","db_name"), + leitmodelle=config_provider("iiasa_database", "leitmodelle"), input: ariadne_database=resources("ariadne_database.csv"), scenario_yaml=config["run"]["scenarios"]["manual_file"], diff --git a/workflow/scripts/build_mobility_demand.py b/workflow/scripts/build_mobility_demand.py index ff5b3466..f766b9ee 100644 --- a/workflow/scripts/build_mobility_demand.py +++ b/workflow/scripts/build_mobility_demand.py @@ -3,35 +3,35 @@ logger = logging.getLogger(__name__) -def get_aladin_data(): +def get_transport_data(db): """ - Retrieve the German mobility demand from the Aladin model. + Retrieve the German mobility demand from the transport_data model. Sum over the subsectors Bus, LDV, Rail, and Truck for the fuels electricity, hydrogen, and synthetic fuels. """ - # get aladin data - db = pd.read_csv( - snakemake.input.ariadne, - index_col=["model", "scenario", "region", "variable", "unit"] - ).loc[ - "Aladin v1", - snakemake.params.reference_scenario, - "Deutschland", - :, - :,] - year = snakemake.wildcards.planning_horizons + # get transport_data data + + df = db.loc[snakemake.params.leitmodelle["transport"]] subsectors = ["Bus", "LDV", "Rail", "Truck"] fuels = ["Electricity", "Hydrogen", "Liquids"] - transport_demand = pd.Series(0, index=fuels) + transport_demand = pd.Series(0.0, index=fuels) for fuel in fuels: for subsector in subsectors: key = f"Final Energy|Transportation|{subsector}|{fuel}" - transport_demand.loc[fuel] += db.loc[key, year].iloc[0] + if snakemake.params.db_name == "ariadne": + transport_demand.loc[fuel] += df.get((key, "TWh/yr"), 0.0) * 3.6 + else: + transport_demand.loc[fuel] += df.loc[key]["PJ/yr"] + transport_demand = transport_demand.div(3.6e-6) # convert PJ to MWh - transport_demand["number_of_cars"] = db.loc["Stock|Transportation|LDV|BEV", year].iloc[0] + + if "transport_stock" in snakemake.params.leitmodelle: + df = db.loc[snakemake.params.leitmodelle["transport_stock"]] + + transport_demand["number_of_cars"] = df.loc["Stock|Transportation|LDV|BEV", "million"] return transport_demand @@ -55,15 +55,25 @@ def get_aladin_data(): run="KN2045_Bal_v4" ) - logger.info("Retrieving German mobility demand from Aladin model.") - # get aladin data - aladin = get_aladin_data() + db = pd.read_csv( + snakemake.input.ariadne, + index_col=["model", "scenario", "region", "variable", "unit"] + ).loc[ + :, + snakemake.params.reference_scenario, + "Deutschland", + :, + :,][snakemake.wildcards.planning_horizons] + + logger.info("Retrieving German mobility demand from transport_data model.") + # get transport_data data + transport_data = get_transport_data(db) # get German mobility weighting pop_layout = pd.read_csv(snakemake.input.clustered_pop_layout, index_col=0) # only get German data pop_layout = pop_layout[pop_layout.ct == "DE"].fraction - mobility_demand = pd.DataFrame(pop_layout.values[:, None] * aladin.values, index=pop_layout.index, columns=aladin.index) + mobility_demand = pd.DataFrame(pop_layout.values[:, None] * transport_data.values, index=pop_layout.index, columns=transport_data.index) mobility_demand.to_csv(snakemake.output.mobility_demand) diff --git a/workflow/scripts/modify_industry_demand.py b/workflow/scripts/modify_industry_demand.py index 951d6fb1..b0a0c94f 100644 --- a/workflow/scripts/modify_industry_demand.py +++ b/workflow/scripts/modify_industry_demand.py @@ -41,7 +41,7 @@ leitmodell="FORECAST v1.0" year = snakemake.input.industrial_production_per_country_tomorrow.split("_")[-1].split(".")[0] - if (snakemake.config["iiasa_database"]["db_name"] == "ariadne2_intern" + if (snakemake.params.db_name == "ariadne2_intern" and year == "2020"): logger.warning(f"Assuming {leitmodell} uses 2021 as base year instead of 2020.") year = "2021" From dec08aa2cd6d70d0a87c5d286499c47c57735c90 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Fri, 28 Jun 2024 14:39:08 +0200 Subject: [PATCH 06/11] fix snakemake error --- config/config.yaml | 2 +- workflow/Snakefile | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index db2e2294..8d129b24 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -4,7 +4,7 @@ # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#run run: - prefix: 20240627wind_limits + prefix: 20240628public_db name: # - CurrentPolicies - KN2045_Bal_v4 diff --git a/workflow/Snakefile b/workflow/Snakefile index c76f1ab2..8cb411a8 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -247,8 +247,9 @@ use rule build_existing_heating_distribution from pypsaeur with: existing_heating=resources("existing_heating.csv"), rule modify_industry_demand: - input: + params: db_name=config_provider("iiasa_database", "db_name"), + input: ariadne=resources("ariadne_database.csv"), industrial_production_per_country_tomorrow=resources("industrial_production_per_country_tomorrow_{planning_horizons}.csv"), output: From 581ec33e15ce9d0b81141cea1da9b123ca823738 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Fri, 28 Jun 2024 15:15:57 +0200 Subject: [PATCH 07/11] empty plot for empty data frames --- workflow/scripts/plot_ariadne_variables.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/workflow/scripts/plot_ariadne_variables.py b/workflow/scripts/plot_ariadne_variables.py index 7deddb87..00ca55da 100644 --- a/workflow/scripts/plot_ariadne_variables.py +++ b/workflow/scripts/plot_ariadne_variables.py @@ -17,7 +17,7 @@ def ariadne_subplot( regex=drop_regex, ) # Check that all values have the same Unit - assert df.columns.unique(level="Unit").size == 1 + assert df.columns.unique(level="Unit").size <= 1 # Simplify variable names df.columns = pd.Index( @@ -27,8 +27,13 @@ def ariadne_subplot( ), name=df.columns.names[0], ) - - return df.plot.area(ax=ax, title=title, legend=False, stacked=stacked) + if df.empty: + # Create an empty plot if DataFrame is empty + ax.plot([], []) + ax.set_title("Ooops! Empty DataFrame") + return ax + else: + return df.plot.area(ax=ax, title=title, legend=False, stacked=stacked) @@ -193,8 +198,9 @@ def elec_val_plot(df, savepath): opts="", ll="v1.2", sector_opts="None", - planning_horizons="2050", - run="KN2045_Bal_v4" + planning_horizons="2045", + run="8Gt_Bal_v3", + configfiles="config/config.public.yaml" ) df = pd.read_excel( From 2cc872be47598b69f2a41250b07f5cc214d6f46a Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Fri, 28 Jun 2024 16:31:08 +0200 Subject: [PATCH 08/11] Explain new features of pypsa-de --- README.md | 58 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 5e7a6aaa..790ecf6b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# Kopernikus-Projekt Ariadne - Gesamtsystemmodell PyPSA-Eur +# Kopernikus-Projekt Ariadne - Gesamtsystemmodell PyPSA-DE -Dieses Repository enthält das Gesamtsystemmodell für das Kopernikus-Projekt Ariadne, basierend auf der Toolbox PyPSA und dem Datensatz PyPSA-Eur. Das Modell bildet Deutschland mit hoher geographischer Auflösung, mit voller Sektorenkopplung und mit Integration in das europäische Energiesystem ab. +Dieses Repository enthält das Gesamtsystemmodell PyPSA-DE für das Kopernikus-Projekt Ariadne, basierend auf der Toolbox PyPSA und dem Datensatz PyPSA-Eur. Das Modell bildet Deutschland mit hoher geographischer Auflösung, mit voller Sektorenkopplung und mit Integration in das europäische Energiesystem ab. This repository contains the entire scientific project, including data sources and code. The philosophy behind this repository is that no intermediary results are included, but all results are computed from raw data and code. @@ -24,30 +24,43 @@ More information on Git Submodules can be found [here](https://git-scm.com/book/ ## Getting ready -You need [mamba](https://mamba.readthedocs.io/en/latest/) to run the analysis. Using mamba, you can create an environment from within you can run it: +You need conda or [mamba](https://mamba.readthedocs.io/en/latest/) to run the analysis. Using mamba, you can create an environment from within you can run it: mamba env create -f environment.yaml -## Provide login details +## For external users: Use config.public.yaml -The snakemake rule `retrieve_ariadne_database` logs into the IIASA Database via the [`pyam`](https://pyam-iamc.readthedocs.io/en/stable/tutorials/iiasa.html) package. The credentials for logging into this database have to be stored locally on your machine with `ixmp4`. To do this, run +The default workflow configured for this repository assumes access to the internal Ariadne2 database. Users that do not have the required login details can run the analysis based on the data published during the [first phase of the Ariadne project](https://data.ece.iiasa.ac.at/ariadne/). + +This is possible by providing an additional config to the snakemake workflow. For every `snakemake COMMAND` specified in the instructions below, public users may use: + +``` +snakemake --configfile=config/config.public.yaml COMMAND +``` + +If public users wish to edit the default scenario specifications, they should change `scenarios.public.yaml` instead of `scenarios.manual.yaml`. More details on using scenarios are given below. + +## For internal users: Provide login details + +The snakemake rule `retrieve_ariadne_database` logs into the interal Ariadne IIASA Database via the [`pyam`](https://pyam-iamc.readthedocs.io/en/stable/tutorials/iiasa.html) package. The credentials for logging into this database have to be stored locally on your machine with `ixmp4`. To do this activate the project environment and run ``` ixmp4 login ``` -You will be prompted to enter your ``. +You will be prompted to enter your ``. Caveat: These credentials are stored on your machine in plain text. +To switch between public and internal use, the command `ixmp4 logout` may be necessary. + ## Run the analysis -Before running any scenarios, the rule `build_scenarios` must be executed. This will write the file `config/scenarios.automated.yaml` which includes transport shares and ksg goals from the iiasa database as well as the information from the file `config/scenarios.manual.yaml`. +Before running any analysis with scenarios, the rule `build_scenarios` must be executed. This will write the file `config/scenarios.automated.yaml` which includes input data and CO2 targets from the IIASA Ariadne database as well as the information from the manual scenario file. [This file is specified in the default config.yaml via they key `run:scenarios:manual_file` (by default located at `config/scenarios.manual.yaml`)]. snakemake -call build_scenarios -f -Note that the hierarchy of scenario files is the following: `scenarios.automated.yaml` > `config.yaml` > `config.default.yaml` -Changes in the file `scenarios.manual.yaml` are only taken into account if the rule `build_scenarios` is executed. +Note that the hierarchy of scenario files is the following: `scenarios.automated.yaml` > (any `explicitly specified --configfiles`) > `config.yaml `> `config.default.yaml `Changes in the file `scenarios.manual.yaml `are only taken into account if the rule `build_scenarios` is executed. For the first run open config.yaml and set @@ -68,11 +81,30 @@ To generate a PDF of the dependency graph of all steps `build/dag.pdf` run: ## Repo structure * `config`: configurations used in the study -* `cutouts`: very large weather data cutouts supplied by atlite library -* `data`: place for raw data -* `resources`: place for intermediate/processing data for the workflow +* `ariadne-data`: Germany specific data from the Ariadne project +* `workflow`: contains the Snakemake workflow, including the submodule PyPSA-Eur and specific scripts for Germany +* `cutouts`: very large weather data cutouts supplied by atlite library (does not exist initially) +* `data`: place for raw data (does not exist initially) +* `resources`: place for intermediate/processing data for the workflow (does not exist initially) * `results`: will contain all results (does not exist initially) -* `workflow`: contains the Snakemake workflow, including the submodule PyPSA-Eur + +## Differences to PyPSA-EUR + +- Specific cost assumption for Germany: + - Gas, Oil, Coal prices + - electrolysis and heat-pump costs + - Infrastructure costs according to the Netzentwicklungsplan 23 (NEP23) + - option for pessimstic, mean and optimistic cost development +- Transport and Industry demands as well as heating stock imported from the sectoral models in the Ariadne consortium +- More detailed data on CHPs in Germany +- Option for building the German Wasserstoffkernnetz +- The model has been validated against 2020 electricity data for Germany +- National CO2-Targets according to the Klimaschutzgesetz +- Additional constraints that limit maximum capacity of specific technologies +- Import constraints +- Renewable build out according to the Wind-an-Land, Wind-auf-See and Solarstrategie laws +- A comprehensive reporting reporting module that exports Capacity Expansion, Primary/Secondary/Final Energy, CO2 Emissions per Sector, Trade, Investments, ... +- Plotting functionality to compare different scenarios ## License From 72b35a921e2cc71ef5c87827b1d982b4dd7b0b44 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Mon, 1 Jul 2024 10:49:10 +0200 Subject: [PATCH 09/11] some clarifications --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 790ecf6b..4dae07d2 100644 --- a/README.md +++ b/README.md @@ -32,13 +32,13 @@ You need conda or [mamba](https://mamba.readthedocs.io/en/latest/) to run the an The default workflow configured for this repository assumes access to the internal Ariadne2 database. Users that do not have the required login details can run the analysis based on the data published during the [first phase of the Ariadne project](https://data.ece.iiasa.ac.at/ariadne/). -This is possible by providing an additional config to the snakemake workflow. For every `snakemake COMMAND` specified in the instructions below, public users may use: +This is possible by providing an additional config to the snakemake workflow. For every `snakemake COMMAND` specified in the instructions below, public users should use: ``` snakemake --configfile=config/config.public.yaml COMMAND ``` -If public users wish to edit the default scenario specifications, they should change `scenarios.public.yaml` instead of `scenarios.manual.yaml`. More details on using scenarios are given below. +The additional config file specifies the required database, model, and scenario names for Ariadne1. If public users wish to edit the default scenario specifications, they should change `scenarios.public.yaml` instead of `scenarios.manual.yaml`. More details on using scenarios are given below. ## For internal users: Provide login details @@ -52,17 +52,17 @@ You will be prompted to enter your ``. Caveat: These credentials are stored on your machine in plain text. -To switch between public and internal use, the command `ixmp4 logout` may be necessary. +To switch between internal and public use, the command `ixmp4 logout` may be necessary. ## Run the analysis -Before running any analysis with scenarios, the rule `build_scenarios` must be executed. This will write the file `config/scenarios.automated.yaml` which includes input data and CO2 targets from the IIASA Ariadne database as well as the information from the manual scenario file. [This file is specified in the default config.yaml via they key `run:scenarios:manual_file` (by default located at `config/scenarios.manual.yaml`)]. +Before running any analysis with scenarios, the rule `build_scenarios` must be executed. This will create the file `config/scenarios.automated.yaml` which includes input data and CO2 targets from the IIASA Ariadne database as well as the specifications from the manual scenario file. [This file is specified in the default config.yaml via they key `run:scenarios:manual_file` (by default located at `config/scenarios.manual.yaml`)]. snakemake -call build_scenarios -f Note that the hierarchy of scenario files is the following: `scenarios.automated.yaml` > (any `explicitly specified --configfiles`) > `config.yaml `> `config.default.yaml `Changes in the file `scenarios.manual.yaml `are only taken into account if the rule `build_scenarios` is executed. -For the first run open config.yaml and set +For the first run, open config.yaml and set enable: retrieve: true # set to false once initial data is retrieved @@ -80,7 +80,7 @@ To generate a PDF of the dependency graph of all steps `build/dag.pdf` run: ## Repo structure -* `config`: configurations used in the study +* `config`: configuration files * `ariadne-data`: Germany specific data from the Ariadne project * `workflow`: contains the Snakemake workflow, including the submodule PyPSA-Eur and specific scripts for Germany * `cutouts`: very large weather data cutouts supplied by atlite library (does not exist initially) @@ -103,7 +103,7 @@ To generate a PDF of the dependency graph of all steps `build/dag.pdf` run: - Additional constraints that limit maximum capacity of specific technologies - Import constraints - Renewable build out according to the Wind-an-Land, Wind-auf-See and Solarstrategie laws -- A comprehensive reporting reporting module that exports Capacity Expansion, Primary/Secondary/Final Energy, CO2 Emissions per Sector, Trade, Investments, ... +- A comprehensive reporting module that exports Capacity Expansion, Primary/Secondary/Final Energy, CO2 Emissions per Sector, Trade, Investments, ... - Plotting functionality to compare different scenarios ## License From a94ca404a468c09629c7392d330b713ac6e13171 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Thu, 4 Jul 2024 14:41:05 +0200 Subject: [PATCH 10/11] consider land use change emissions --- workflow/scripts/build_scenarios.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/workflow/scripts/build_scenarios.py b/workflow/scripts/build_scenarios.py index 31f0b3c1..911bbfbf 100644 --- a/workflow/scripts/build_scenarios.py +++ b/workflow/scripts/build_scenarios.py @@ -87,10 +87,19 @@ def get_co2_budget(df, source): co2 = ( df.loc["Emissions|CO2","Mt CO2/yr"] + - df.loc["Emissions|CO2|Land-Use Change","Mt CO2-equiv/yr"] ) - + try: + kyoto_land_use_change = ( + df.loc["Emissions|Kyoto Gases|Land-Use Change","Mt CO2-equiv/yr"] + ) + except KeyError: # Key not in Ariadne public database + # Guesstimate of difference from Ariadne 2 data + kyoto_land_use_change = df.loc["Emissions|CO2|Land-Use Change","Mt CO2-equiv/yr"] + 4.5 + ghg = ( df.loc["Emissions|Kyoto Gases","Mt CO2-equiv/yr"] + - kyoto_land_use_change ) nonco2 = ghg - co2 From 0bb044bd05aee619a8450a5f9c62121a6eddeeeb Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Tue, 9 Jul 2024 17:07:40 +0200 Subject: [PATCH 11/11] change key for ariadne 1 DB --- workflow/scripts/build_scenarios.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/workflow/scripts/build_scenarios.py b/workflow/scripts/build_scenarios.py index 911bbfbf..fa101ea5 100644 --- a/workflow/scripts/build_scenarios.py +++ b/workflow/scripts/build_scenarios.py @@ -85,17 +85,21 @@ def get_co2_budget(df, source): # # No Kyoto Gas emissions for Bunkers recorded in Ariadne DB # ) + try: + co2_land_use_change = df.loc["Emissions|CO2|Land-Use Change","Mt CO2-equiv/yr"] + except KeyError: # Key not in Ariadne public database + co2_land_use_change = df.loc["Emissions|CO2|AFOLU","Mt CO2/yr"] + co2 = ( df.loc["Emissions|CO2","Mt CO2/yr"] - - df.loc["Emissions|CO2|Land-Use Change","Mt CO2-equiv/yr"] + - co2_land_use_change ) + try: - kyoto_land_use_change = ( - df.loc["Emissions|Kyoto Gases|Land-Use Change","Mt CO2-equiv/yr"] - ) + kyoto_land_use_change = df.loc["Emissions|Kyoto Gases|Land-Use Change","Mt CO2-equiv/yr"] except KeyError: # Key not in Ariadne public database # Guesstimate of difference from Ariadne 2 data - kyoto_land_use_change = df.loc["Emissions|CO2|Land-Use Change","Mt CO2-equiv/yr"] + 4.5 + kyoto_land_use_change = co2_land_use_change + 4.5 ghg = ( df.loc["Emissions|Kyoto Gases","Mt CO2-equiv/yr"]