From fa19075ca9af480513fa2f686608fa0a297cf6aa Mon Sep 17 00:00:00 2001 From: toniseibold Date: Mon, 2 Dec 2024 11:03:22 +0100 Subject: [PATCH 1/4] first stab at modifying sector ratios --- config/config.yaml | 71 +++++++------- workflow/Snakefile | 37 ++++++++ workflow/scripts/modify_sector_ratios.py | 115 +++++++++++++++++++++++ 3 files changed, 191 insertions(+), 32 deletions(-) create mode 100644 workflow/scripts/modify_sector_ratios.py diff --git a/config/config.yaml b/config/config.yaml index 713449c9..c047e85d 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -4,7 +4,7 @@ # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#run run: - prefix: 20241121-fix-missing-gas-capa + prefix: 20241202-modify-sector-ratios name: # - CurrentPolicies @@ -62,7 +62,7 @@ scenario: ll: - vopt clusters: - - 27 #current options: 27, 49 + - 49 #current options: 27, 49 opts: - '' sector_opts: @@ -206,38 +206,38 @@ clustering: # print(fw.div(fw.sum()).subtract(5e-5).round(4).to_dict().__repr__().replace(",","\n")) focus_weights: # 27 nodes: 8 for Germany, 3 for Italy, 2 each for Denmark, UK and Spain, 1 per each of other 10 "Stromnachbarn" - 'DE': 0.2966 - 'AT': 0.0370 - 'BE': 0.0370 - 'CH': 0.0370 - 'CZ': 0.0370 - 'DK': 0.0741 - 'FR': 0.0370 - 'GB': 0.0741 - 'LU': 0.0370 - 'NL': 0.0370 - 'NO': 0.0370 - 'PL': 0.0370 - 'SE': 0.0370 - 'ES': 0.0741 - 'IT': 0.1111 + # 'DE': 0.2966 + # 'AT': 0.0370 + # 'BE': 0.0370 + # 'CH': 0.0370 + # 'CZ': 0.0370 + # 'DK': 0.0741 + # 'FR': 0.0370 + # 'GB': 0.0741 + # 'LU': 0.0370 + # 'NL': 0.0370 + # 'NO': 0.0370 + # 'PL': 0.0370 + # 'SE': 0.0370 + # 'ES': 0.0741 + # 'IT': 0.1111 # high spatial resolution: change clusters to 49 # 49 nodes: 30 for Germany, 3 for Italy, 2 each for Denmark, UK and Spain, 1 per each of other 10 "Stromnachbarn" - # 'DE': 0.6124 - # 'AT': 0.0204 - # 'BE': 0.0204 - # 'CH': 0.0204 - # 'CZ': 0.0204 - # 'DK': 0.0408 - # 'FR': 0.0204 - # 'GB': 0.0408 - # 'LU': 0.0204 - # 'NL': 0.0204 - # 'NO': 0.0204 - # 'PL': 0.0204 - # 'SE': 0.0204 - # 'ES': 0.0408 - # 'IT': 0.0612 + 'DE': 0.6124 + 'AT': 0.0204 + 'BE': 0.0204 + 'CH': 0.0204 + 'CZ': 0.0204 + 'DK': 0.0408 + 'FR': 0.0204 + 'GB': 0.0408 + 'LU': 0.0204 + 'NL': 0.0204 + 'NO': 0.0204 + 'PL': 0.0204 + 'SE': 0.0204 + 'ES': 0.0408 + 'IT': 0.0612 temporal: resolution_sector: 365H @@ -349,6 +349,13 @@ sector: # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#industry industry: + sector_ratios_fraction_future_DE: + 2020: 0.0 + 2025: 0.1 + 2030: 0.3 + 2035: 0.5 + 2040: 0.8 + 2045: 1.0 ammonia: false St_primary_fraction: 2020: 0.6 diff --git a/workflow/Snakefile b/workflow/Snakefile index 918b2a59..4afe74df 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -413,6 +413,43 @@ use rule build_industrial_production_per_node from pypsaeur with: ), +rule modify_sector_ratios: + params: + future_DE=config_provider("industry", "sector_ratios_fraction_future"), + future_EU=config_provider("industry", "sector_ratios_fraction_future"), + input: + industry_sector_ratios=resources("industry_sector_ratios.csv"), + industrial_energy_demand_per_country_today=resources( + "industrial_energy_demand_per_country_today.csv" + ), + industrial_production_per_country=resources( + "industrial_production_per_country.csv" + ), + sector_ratios=resources("industry_sector_ratios_{planning_horizons}.csv"), + output: + sector_ratios_modified=resources("industry_sector_ratios_{planning_horizons}-modified.csv"), + resources: + mem_mb=2000, + log: + logs("modify_industry_sector_ratios_{planning_horizons}}.log"), + script: + "scripts/modify_sector_ratios.py" + + +ruleorder: modify_sector_ratios > build_industry_sector_ratios_intermediate + +use rule build_industrial_energy_demand_per_node from pypsaeur with: + input: + **{ + k: v + for k, v in rules.build_industrial_energy_demand_per_node.input.items() + if k != "industry_sector_ratios" + }, + industry_sector_ratios=resources( + "industry_sector_ratios_{planning_horizons}-modified.csv" + ), + + rule build_wasserstoff_kernnetz: params: kernnetz=config_provider("wasserstoff_kernnetz"), diff --git a/workflow/scripts/modify_sector_ratios.py b/workflow/scripts/modify_sector_ratios.py new file mode 100644 index 00000000..d77ae15d --- /dev/null +++ b/workflow/scripts/modify_sector_ratios.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2020-2024 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT +""" +Overwrite the sector ratios for Germany to represent climate neutrality in 2045 already compared to 2050 for the rest of Europe. + +Relevant Settings +----------------- + +.. code:: yaml + + industry: + sector_ratios_fraction_future_DE: + +Inputs +------ + +- ``resources/industry_sector_ratios.csv`` +- ``resources/industrial_energy_demand_per_country_today.csv`` +- ``resources/industrial_production_per_country.csv`` + +Outputs +------- + +- ``resources/industry_sector_ratios_{planning_horizons}-modified.csv`` + +""" +import sys +import os +paths = ["workflow/submodules/pypsa-eur/scripts", "../submodules/pypsa-eur/scripts"] +for path in paths: + sys.path.insert(0, os.path.abspath(path)) + +import numpy as np +import pandas as pd +from prepare_sector_network import get + + +def build_industry_sector_ratios_intermediate(): + + # in TWh/a + demand = pd.read_csv( + snakemake.input.industrial_energy_demand_per_country_today, + header=[0, 1], + index_col=0, + ) + + # in Mt/a + production = ( + pd.read_csv(snakemake.input.industrial_production_per_country, index_col=0) + / 1e3 + ).stack() + production.index.names = [None, None] + + # in MWh/t + future_sector_ratios = pd.read_csv( + snakemake.input.industry_sector_ratios, index_col=0 + ) + + today_sector_ratios = demand.div(production, axis=1).replace([np.inf, -np.inf], 0) + + today_sector_ratios.dropna(how="all", axis=1, inplace=True) + + rename = { + "waste": "biomass", + "electricity": "elec", + "solid": "coke", + "gas": "methane", + "other": "biomass", + "liquid": "naphtha", + } + today_sector_ratios = today_sector_ratios.rename(rename).groupby(level=0).sum() + + fraction_DE = get(snakemake.params.future_DE, year) + fraction_EU = get(snakemake.params.future_EU, year) + + intermediate_sector_ratios = {} + # TODO: ideally I would just change the data for Germany without writing the file again + for ct, group in today_sector_ratios.T.groupby(level=0): + if ct == "DE": + fraction_future = fraction_DE + else: + fraction_future = fraction_EU + today_sector_ratios_ct = group.droplevel(0).T.reindex_like(future_sector_ratios) + missing_mask = today_sector_ratios_ct.isna().all() + today_sector_ratios_ct.loc[:, missing_mask] = future_sector_ratios.loc[ + :, missing_mask + ] + today_sector_ratios_ct.loc[:, ~missing_mask] = today_sector_ratios_ct.loc[ + :, ~missing_mask + ].fillna(future_sector_ratios) + intermediate_sector_ratios[ct] = ( + today_sector_ratios_ct * (1 - fraction_future) + + future_sector_ratios * fraction_future + ) + + intermediate_sector_ratios = pd.concat(intermediate_sector_ratios, axis=1) + + intermediate_sector_ratios.to_csv(snakemake.output.sector_ratios_modified) + + +if __name__ == "__main__": + if "snakemake" not in globals(): + + from _helpers import mock_snakemake + + snakemake = mock_snakemake( + "modify_sector_ratios", + planning_horizons="2045", + ) + + year = int(snakemake.wildcards.planning_horizons) + + build_industry_sector_ratios_intermediate() From c11265e7e6c3119b94c0aae44b95bd9fa4b72935 Mon Sep 17 00:00:00 2001 From: toniseibold Date: Mon, 2 Dec 2024 20:49:41 +0100 Subject: [PATCH 2/4] refining sector ratio modification --- config/config.yaml | 4 +-- workflow/Snakefile | 3 +- workflow/scripts/modify_sector_ratios.py | 42 +++++++++++------------- 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index c047e85d..dd375829 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -353,8 +353,8 @@ industry: 2020: 0.0 2025: 0.1 2030: 0.3 - 2035: 0.5 - 2040: 0.8 + 2035: 0.6 + 2040: 0.9 2045: 1.0 ammonia: false St_primary_fraction: diff --git a/workflow/Snakefile b/workflow/Snakefile index 4afe74df..1548138f 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -415,8 +415,7 @@ use rule build_industrial_production_per_node from pypsaeur with: rule modify_sector_ratios: params: - future_DE=config_provider("industry", "sector_ratios_fraction_future"), - future_EU=config_provider("industry", "sector_ratios_fraction_future"), + future_DE=config_provider("industry", "sector_ratios_fraction_future_DE"), input: industry_sector_ratios=resources("industry_sector_ratios.csv"), industrial_energy_demand_per_country_today=resources( diff --git a/workflow/scripts/modify_sector_ratios.py b/workflow/scripts/modify_sector_ratios.py index d77ae15d..a13c457c 100644 --- a/workflow/scripts/modify_sector_ratios.py +++ b/workflow/scripts/modify_sector_ratios.py @@ -72,30 +72,27 @@ def build_industry_sector_ratios_intermediate(): } today_sector_ratios = today_sector_ratios.rename(rename).groupby(level=0).sum() + # custom DE pathway fraction_DE = get(snakemake.params.future_DE, year) - fraction_EU = get(snakemake.params.future_EU, year) - - intermediate_sector_ratios = {} - # TODO: ideally I would just change the data for Germany without writing the file again - for ct, group in today_sector_ratios.T.groupby(level=0): - if ct == "DE": - fraction_future = fraction_DE - else: - fraction_future = fraction_EU - today_sector_ratios_ct = group.droplevel(0).T.reindex_like(future_sector_ratios) - missing_mask = today_sector_ratios_ct.isna().all() - today_sector_ratios_ct.loc[:, missing_mask] = future_sector_ratios.loc[ - :, missing_mask - ] - today_sector_ratios_ct.loc[:, ~missing_mask] = today_sector_ratios_ct.loc[ - :, ~missing_mask - ].fillna(future_sector_ratios) - intermediate_sector_ratios[ct] = ( - today_sector_ratios_ct * (1 - fraction_future) - + future_sector_ratios * fraction_future - ) - intermediate_sector_ratios = pd.concat(intermediate_sector_ratios, axis=1) + intermediate_sector_ratios_DE = {} + + DE_sector_ratios = today_sector_ratios.loc[:, "DE"].reindex_like(future_sector_ratios) + missing_mask = DE_sector_ratios.isna().all() + DE_sector_ratios.loc[:, missing_mask] = future_sector_ratios.loc[:, missing_mask] + DE_sector_ratios.loc[:, ~missing_mask] = DE_sector_ratios.loc[:, ~missing_mask].fillna(future_sector_ratios) + intermediate_sector_ratios_DE["DE"] = ( + DE_sector_ratios * (1 - fraction_DE) + + future_sector_ratios * fraction_DE + ) + # make dictionary to dataframe + intermediate_sector_ratios_DE = pd.concat(intermediate_sector_ratios_DE, axis=1) + + # read in original sector ratios + intermediate_sector_ratios = pd.read_csv(snakemake.input.sector_ratios, header=[0, 1], index_col=0) + + # update DE sector ratios + intermediate_sector_ratios.loc[:, "DE"] = intermediate_sector_ratios_DE["DE"].values intermediate_sector_ratios.to_csv(snakemake.output.sector_ratios_modified) @@ -108,6 +105,7 @@ def build_industry_sector_ratios_intermediate(): snakemake = mock_snakemake( "modify_sector_ratios", planning_horizons="2045", + run="KN2045_Bal_v4", ) year = int(snakemake.wildcards.planning_horizons) From 1ee534e633892dab56cb3db72cdd78f24fba81ec Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 19:54:20 +0000 Subject: [PATCH 3/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- workflow/Snakefile | 7 +++++-- workflow/scripts/build_scenarios.py | 5 +++-- workflow/scripts/modify_sector_ratios.py | 18 ++++++++++++------ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 1548138f..d5ff0e82 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -426,7 +426,9 @@ rule modify_sector_ratios: ), sector_ratios=resources("industry_sector_ratios_{planning_horizons}.csv"), output: - sector_ratios_modified=resources("industry_sector_ratios_{planning_horizons}-modified.csv"), + sector_ratios_modified=resources( + "industry_sector_ratios_{planning_horizons}-modified.csv" + ), resources: mem_mb=2000, log: @@ -435,7 +437,8 @@ rule modify_sector_ratios: "scripts/modify_sector_ratios.py" -ruleorder: modify_sector_ratios > build_industry_sector_ratios_intermediate +ruleorder: modify_sector_ratios > build_industry_sector_ratios_intermediate + use rule build_industrial_energy_demand_per_node from pypsaeur with: input: diff --git a/workflow/scripts/build_scenarios.py b/workflow/scripts/build_scenarios.py index 4fc00ce8..4fba8ec7 100644 --- a/workflow/scripts/build_scenarios.py +++ b/workflow/scripts/build_scenarios.py @@ -178,8 +178,9 @@ def write_to_scenario_yaml(input, output, scenarios, df): df.loc[:, fallback_reference_scenario, :], planning_horizons ) - - if reference_scenario.startswith("KN2045plus"): # Still waiting for REMIND uploads + if reference_scenario.startswith( + "KN2045plus" + ): # Still waiting for REMIND uploads fallback_reference_scenario = reference_scenario co2_budget_source = config[scenario]["co2_budget_DE_source"] diff --git a/workflow/scripts/modify_sector_ratios.py b/workflow/scripts/modify_sector_ratios.py index a13c457c..39383360 100644 --- a/workflow/scripts/modify_sector_ratios.py +++ b/workflow/scripts/modify_sector_ratios.py @@ -26,8 +26,9 @@ - ``resources/industry_sector_ratios_{planning_horizons}-modified.csv`` """ -import sys import os +import sys + paths = ["workflow/submodules/pypsa-eur/scripts", "../submodules/pypsa-eur/scripts"] for path in paths: sys.path.insert(0, os.path.abspath(path)) @@ -77,19 +78,24 @@ def build_industry_sector_ratios_intermediate(): intermediate_sector_ratios_DE = {} - DE_sector_ratios = today_sector_ratios.loc[:, "DE"].reindex_like(future_sector_ratios) + DE_sector_ratios = today_sector_ratios.loc[:, "DE"].reindex_like( + future_sector_ratios + ) missing_mask = DE_sector_ratios.isna().all() DE_sector_ratios.loc[:, missing_mask] = future_sector_ratios.loc[:, missing_mask] - DE_sector_ratios.loc[:, ~missing_mask] = DE_sector_ratios.loc[:, ~missing_mask].fillna(future_sector_ratios) + DE_sector_ratios.loc[:, ~missing_mask] = DE_sector_ratios.loc[ + :, ~missing_mask + ].fillna(future_sector_ratios) intermediate_sector_ratios_DE["DE"] = ( - DE_sector_ratios * (1 - fraction_DE) - + future_sector_ratios * fraction_DE + DE_sector_ratios * (1 - fraction_DE) + future_sector_ratios * fraction_DE ) # make dictionary to dataframe intermediate_sector_ratios_DE = pd.concat(intermediate_sector_ratios_DE, axis=1) # read in original sector ratios - intermediate_sector_ratios = pd.read_csv(snakemake.input.sector_ratios, header=[0, 1], index_col=0) + intermediate_sector_ratios = pd.read_csv( + snakemake.input.sector_ratios, header=[0, 1], index_col=0 + ) # update DE sector ratios intermediate_sector_ratios.loc[:, "DE"] = intermediate_sector_ratios_DE["DE"].values From d086d39104ede3ed8bf59c04fbdde810811170c9 Mon Sep 17 00:00:00 2001 From: toniseibold Date: Mon, 2 Dec 2024 20:55:50 +0100 Subject: [PATCH 4/4] document changes --- Changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Changelog.md b/Changelog.md index 3c0615fa..5a580f84 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,4 +1,5 @@ # Changelog +- Modify industry sector ratios for Germany to reach climate neutrality by 2045 in the industrial sector. - Restricting the maximum capacity of CurrentPolicies and minus scenarios to the 'uba Projektionsbericht' - Restricting Fischer Tropsch capacity addition with config[solving][limit_DE_FT_cap] - Except for Current Policies force a minimum of 5 GW of electrolysis capacity in Germany