diff --git a/docs/hooks/generate_readable_schema.py b/docs/hooks/generate_readable_schema.py index 89ae232e..7bcfa205 100644 --- a/docs/hooks/generate_readable_schema.py +++ b/docs/hooks/generate_readable_schema.py @@ -14,12 +14,15 @@ import jsonschema2md from mkdocs.structure.files import File +from calliope import AttrDict, config from calliope.util import schema TEMPDIR = tempfile.TemporaryDirectory() SCHEMAS = { - "config_schema": schema.CONFIG_SCHEMA, + "config_schema": AttrDict.from_yaml_string( + config.CalliopeConfig().model_yaml_schema() + ), "model_schema": schema.MODEL_SCHEMA, "math_schema": schema.MATH_SCHEMA, "data_table_schema": schema.DATA_TABLE_SCHEMA, diff --git a/requirements/base.txt b/requirements/base.txt index 2bf5f664..6305e13d 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,6 +4,7 @@ geographiclib >= 2, < 3 ipdb >= 0.13, < 0.14 ipykernel < 7 jinja2 >= 3, < 4 +jsonref >= 1.1, < 2 jsonschema >= 4, < 5 natsort >= 8, < 9 netcdf4 >= 1.2, < 1.7 @@ -13,4 +14,4 @@ pyomo >= 6.5, < 6.7.2 pyparsing >= 3.0, < 3.1 ruamel.yaml >= 0.18, < 0.19 typing-extensions >= 4, < 5 -xarray >= 2024.1, < 2024.4 \ No newline at end of file +xarray >= 2024.1, < 2024.4 diff --git a/src/calliope/attrdict.py b/src/calliope/attrdict.py index bd94df7b..f17cf0ef 100644 --- a/src/calliope/attrdict.py +++ b/src/calliope/attrdict.py @@ -9,6 +9,7 @@ import numpy as np import ruamel.yaml as ruamel_yaml +from ruamel.yaml.scalarstring import walk_tree from typing_extensions import Self from calliope.util.tools import relative_path @@ -355,6 +356,9 @@ def to_yaml(self, path=None): result = result.as_dict() + # handle multi-line strings. + walk_tree(result) + if path is not None: with open(path, "w") as f: yaml_.dump(result, f) diff --git a/src/calliope/backend/__init__.py b/src/calliope/backend/__init__.py index d37395d8..84929792 100644 --- a/src/calliope/backend/__init__.py +++ b/src/calliope/backend/__init__.py @@ -15,19 +15,19 @@ from calliope.preprocess import CalliopeMath if TYPE_CHECKING: + from calliope import config from calliope.backend.backend_model import BackendModel def get_model_backend( - name: str, data: xr.Dataset, math: CalliopeMath, **kwargs + build_config: "config.Build", data: xr.Dataset, math: CalliopeMath ) -> "BackendModel": """Assign a backend using the given configuration. Args: - name (str): name of the backend to use. + build_config: Build configuration options. data (Dataset): model data for the backend. math (CalliopeMath): Calliope math. - **kwargs: backend keyword arguments corresponding to model.config.build. Raises: exceptions.BackendError: If invalid backend was requested. @@ -35,10 +35,10 @@ def get_model_backend( Returns: BackendModel: Initialized backend object. """ - match name: + match build_config.backend: case "pyomo": - return PyomoBackendModel(data, math, **kwargs) + return PyomoBackendModel(data, math, build_config) case "gurobi": - return GurobiBackendModel(data, math, **kwargs) + return GurobiBackendModel(data, math, build_config) case _: - raise BackendError(f"Incorrect backend '{name}' requested.") + raise BackendError(f"Incorrect backend '{build_config.backend}' requested.") diff --git a/src/calliope/backend/backend_model.py b/src/calliope/backend/backend_model.py index c52d74ab..21603864 100644 --- a/src/calliope/backend/backend_model.py +++ b/src/calliope/backend/backend_model.py @@ -26,17 +26,13 @@ import numpy as np import xarray as xr -from calliope import exceptions +from calliope import config, exceptions from calliope.attrdict import AttrDict from calliope.backend import helper_functions, parsing from calliope.exceptions import warn as model_warn from calliope.io import load_config from calliope.preprocess.model_math import ORDERED_COMPONENTS_T, CalliopeMath -from calliope.util.schema import ( - MODEL_SCHEMA, - extract_from_schema, - update_then_validate_config, -) +from calliope.util.schema import MODEL_SCHEMA, extract_from_schema if TYPE_CHECKING: from calliope.backend.parsing import T as Tp @@ -69,20 +65,20 @@ class BackendModelGenerator(ABC): _PARAM_UNITS = extract_from_schema(MODEL_SCHEMA, "x-unit") _PARAM_TYPE = extract_from_schema(MODEL_SCHEMA, "x-type") - def __init__(self, inputs: xr.Dataset, math: CalliopeMath, **kwargs): + def __init__( + self, inputs: xr.Dataset, math: CalliopeMath, build_config: config.Build + ): """Abstract base class to build a representation of the optimisation problem. Args: inputs (xr.Dataset): Calliope model data. math (CalliopeMath): Calliope math. - **kwargs (Any): build configuration overrides. + build_config: Build configuration options. """ self._dataset = xr.Dataset() self.inputs = inputs.copy() self.inputs.attrs = deepcopy(inputs.attrs) - self.inputs.attrs["config"]["build"] = update_then_validate_config( - "build", self.inputs.attrs["config"], **kwargs - ) + self.config = build_config self.math: CalliopeMath = deepcopy(math) self._solve_logger = logging.getLogger(__name__ + ".") @@ -200,6 +196,7 @@ def _check_inputs(self): "equation_name": "", "backend_interface": self, "input_data": self.inputs, + "build_config": self.config, "helper_functions": helper_functions._registry["where"], "apply_where": True, "references": set(), @@ -246,7 +243,7 @@ def add_optimisation_components(self) -> None: # The order of adding components matters! # 1. Variables, 2. Global Expressions, 3. Constraints, 4. Objectives self._add_all_inputs_as_parameters() - if self.inputs.attrs["config"]["build"]["pre_validate_math_strings"]: + if self.config.pre_validate_math_strings: self._validate_math_string_parsing() for components in typing.get_args(ORDERED_COMPONENTS_T): component = components.removesuffix("s") @@ -399,7 +396,7 @@ def _add_all_inputs_as_parameters(self) -> None: if param_name in self.parameters.keys(): continue elif ( - self.inputs.attrs["config"]["build"]["mode"] != "operate" + self.config.mode != "operate" and param_name in extract_from_schema(MODEL_SCHEMA, "x-operate-param").keys() ): @@ -606,7 +603,11 @@ class BackendModel(BackendModelGenerator, Generic[T]): """Calliope's backend model functionality.""" def __init__( - self, inputs: xr.Dataset, math: CalliopeMath, instance: T, **kwargs + self, + inputs: xr.Dataset, + math: CalliopeMath, + instance: T, + build_config: config.Build, ) -> None: """Abstract base class to build backend models that interface with solvers. @@ -614,9 +615,9 @@ def __init__( inputs (xr.Dataset): Calliope model data. math (CalliopeMath): Calliope math. instance (T): Interface model instance. - **kwargs: build configuration overrides. + build_config: Build configuration options. """ - super().__init__(inputs, math, **kwargs) + super().__init__(inputs, math, build_config) self._instance = instance self.shadow_prices: ShadowPrices self._has_verbose_strings: bool = False diff --git a/src/calliope/backend/gurobi_backend_model.py b/src/calliope/backend/gurobi_backend_model.py index 2d2e0a48..ab02d9d4 100644 --- a/src/calliope/backend/gurobi_backend_model.py +++ b/src/calliope/backend/gurobi_backend_model.py @@ -14,6 +14,7 @@ import pandas as pd import xarray as xr +from calliope import config from calliope.backend import backend_model, parsing from calliope.exceptions import BackendError, BackendWarning from calliope.exceptions import warn as model_warn @@ -41,19 +42,21 @@ class GurobiBackendModel(backend_model.BackendModel): """gurobipy-specific backend functionality.""" - def __init__(self, inputs: xr.Dataset, math: CalliopeMath, **kwargs) -> None: + def __init__( + self, inputs: xr.Dataset, math: CalliopeMath, build_config: config.Build + ) -> None: """Gurobi solver interface class. Args: inputs (xr.Dataset): Calliope model data. math (CalliopeMath): Calliope math. - **kwargs: passed directly to the solver. + build_config: Build configuration options. """ if importlib.util.find_spec("gurobipy") is None: raise ImportError( "Install the `gurobipy` package to build the optimisation problem with the Gurobi backend." ) - super().__init__(inputs, math, gurobipy.Model(), **kwargs) + super().__init__(inputs, math, gurobipy.Model(), build_config) self._instance: gurobipy.Model self.shadow_prices = GurobiShadowPrices(self) @@ -144,7 +147,7 @@ def _objective_setter( ) -> xr.DataArray: expr = element.evaluate_expression(self, references=references) - if name == self.inputs.attrs["config"].build.objective: + if name == self.config.objective: self._instance.setObjective(expr.item(), sense=sense) self.log("objectives", name, "Objective activated.") diff --git a/src/calliope/backend/parsing.py b/src/calliope/backend/parsing.py index 33c9ea47..5cdd0808 100644 --- a/src/calliope/backend/parsing.py +++ b/src/calliope/backend/parsing.py @@ -311,6 +311,7 @@ def evaluate_where( helper_functions=helper_functions._registry["where"], input_data=backend_interface.inputs, backend_interface=backend_interface, + build_config=backend_interface.config, references=references if references is not None else set(), apply_where=True, ) diff --git a/src/calliope/backend/pyomo_backend_model.py b/src/calliope/backend/pyomo_backend_model.py index 5ba41ba0..46ea3b32 100644 --- a/src/calliope/backend/pyomo_backend_model.py +++ b/src/calliope/backend/pyomo_backend_model.py @@ -26,6 +26,7 @@ from pyomo.opt import SolverFactory # type: ignore from pyomo.util.model_size import build_model_size_report # type: ignore +from calliope import config from calliope.exceptions import BackendError, BackendWarning from calliope.exceptions import warn as model_warn from calliope.preprocess import CalliopeMath @@ -58,15 +59,17 @@ class PyomoBackendModel(backend_model.BackendModel): """Pyomo-specific backend functionality.""" - def __init__(self, inputs: xr.Dataset, math: CalliopeMath, **kwargs) -> None: + def __init__( + self, inputs: xr.Dataset, math: CalliopeMath, build_config: config.Build + ) -> None: """Pyomo solver interface class. Args: inputs (xr.Dataset): Calliope model data. math (CalliopeMath): Calliope math. - **kwargs: passed directly to the solver. + build_config: Build configuration options. """ - super().__init__(inputs, math, pmo.block(), **kwargs) + super().__init__(inputs, math, pmo.block(), build_config) self._instance.parameters = pmo.parameter_dict() self._instance.variables = pmo.variable_dict() @@ -185,7 +188,7 @@ def _objective_setter( ) -> xr.DataArray: expr = element.evaluate_expression(self, references=references) objective = pmo.objective(expr.item(), sense=sense) - if name == self.inputs.attrs["config"].build.objective: + if name == self.config.objective: text = "activated" objective.activate() else: diff --git a/src/calliope/backend/where_parser.py b/src/calliope/backend/where_parser.py index f434a9bf..06f782f6 100644 --- a/src/calliope/backend/where_parser.py +++ b/src/calliope/backend/where_parser.py @@ -17,6 +17,7 @@ from calliope.exceptions import BackendError if TYPE_CHECKING: + from calliope import config from calliope.backend.backend_model import BackendModel @@ -34,6 +35,7 @@ class EvalAttrs(TypedDict): helper_functions: dict[str, Callable] apply_where: NotRequired[bool] references: NotRequired[set] + build_config: config.Build class EvalWhere(expression_parser.EvalToArrayStr): @@ -118,9 +120,7 @@ def as_math_string(self) -> str: # noqa: D102, override return rf"\text{{config.{self.config_option}}}" def as_array(self) -> xr.DataArray: # noqa: D102, override - config_val = ( - self.eval_attrs["input_data"].attrs["config"].build[self.config_option] - ) + config_val = getattr(self.eval_attrs["build_config"], self.config_option) if not isinstance(config_val, int | float | str | bool | np.bool_): raise BackendError( diff --git a/src/calliope/cli.py b/src/calliope/cli.py index a9d811d2..4059de7e 100644 --- a/src/calliope/cli.py +++ b/src/calliope/cli.py @@ -278,9 +278,9 @@ def run( # Else run the model, then save outputs else: click.secho("Starting model run...") - + kwargs = {} if save_logs: - model.config.set_key("solve.save_logs", save_logs) + kwargs["solve.save_logs"] = save_logs if save_csv is None and save_netcdf is None: click.secho( @@ -292,14 +292,13 @@ def run( # If save_netcdf is used, override the 'save_per_spore_path' to point to a # directory of the same name as the planned netcdf - if save_netcdf and model.config.solve.spores_save_per_spore: - model.config.set_key( - "solve.spores_save_per_spore_path", + if save_netcdf and model.config.solve.spores.save_per_spore: + kwargs["solve.spores_save_per_spore_path"] = ( save_netcdf.replace(".nc", "/spore_{}.nc"), ) model.build() - model.solve() + model.solve(**kwargs) termination = model._model_data.attrs.get( "termination_condition", "unknown" ) diff --git a/src/calliope/config.py b/src/calliope/config.py new file mode 100644 index 00000000..e07ee2cb --- /dev/null +++ b/src/calliope/config.py @@ -0,0 +1,369 @@ +# Copyright (C) since 2013 Calliope contributors listed in AUTHORS. +# Licensed under the Apache 2.0 License (see LICENSE file). +"""Implements the Calliope configuration class.""" + +from collections.abc import Hashable +from datetime import datetime +from pathlib import Path +from typing import Annotated, Literal, Self, TypeVar, get_args, overload + +import jsonref +from pydantic import AfterValidator, BaseModel, Field, model_validator +from pydantic_core import PydanticCustomError + +from calliope.attrdict import AttrDict +from calliope.util import tools + +MODES_T = Literal["plan", "operate", "spores"] +CONFIG_T = Literal["init", "build", "solve"] + +# == +# Taken from https://github.com/pydantic/pydantic-core/pull/820#issuecomment-1670475909 +T = TypeVar("T", bound=Hashable) + + +def _validate_unique_list(v: list[T]) -> list[T]: + if len(v) != len(set(v)): + raise PydanticCustomError("unique_list", "List must be unique") + return v + + +UniqueList = Annotated[ + list[T], + AfterValidator(_validate_unique_list), + Field(json_schema_extra={"uniqueItems": True}), +] +# == + + +def hide_from_schema(to_hide: list[str]): + """Hide fields from the generated schema. + + Args: + to_hide (list[str]): List of fields to hide. + """ + + def _hide_from_schema(schema: dict): + for hide in to_hide: + schema.get("properties", {}).pop(hide, None) + return schema + + return _hide_from_schema + + +class ConfigBaseModel(BaseModel): + """A base class for creating pydantic models for Calliope configuration options.""" + + _kwargs: dict = {} + + def update(self, update_dict: dict, deep: bool = False) -> Self: + """Return a new iteration of the model with updated fields. + + Updates are validated and stored in the parent class in the `_kwargs` key. + + Args: + update_dict (dict): Dictionary with which to update the base model. + deep (bool, optional): Set to True to make a deep copy of the model. Defaults to False. + + Returns: + BaseModel: New model instance. + """ + new_dict: dict = {} + # Iterate through dict to be updated and convert any sub-dicts into their respective pydantic model objects + for key, val in update_dict.items(): + key_class = getattr(self, key) + if isinstance(key_class, ConfigBaseModel): + new_dict[key] = key_class.update(val) + key_class._kwargs = val + else: + new_dict[key] = val + updated = super().model_copy(update=new_dict, deep=deep) + updated.model_validate(updated) + self._kwargs = update_dict + return updated + + @overload + def model_yaml_schema(self, filepath: str | Path) -> None: ... + + @overload + def model_yaml_schema(self, filepath: None = None) -> str: ... + + def model_yaml_schema(self, filepath: str | Path | None = None) -> None | str: + """Generate a YAML schema for the class. + + Args: + filepath (str | Path | None, optional): If given, save schema to given path. Defaults to None. + + Returns: + None | str: If `filepath` is given, returns None. Otherwise, returns the YAML string. + """ + # By default, the schema uses $ref/$def cross-referencing for each pydantic model class, + # but this isn't very readable when rendered in our documentation. + # So, we resolve references and then delete all the `$defs` + schema_dict = AttrDict(jsonref.replace_refs(self.model_json_schema())) + schema_dict.del_key("$defs") + return schema_dict.to_yaml(filepath) + + @property + def applied_keyword_overrides(self) -> dict: + """Most recently applied keyword overrides used to update this configuration. + + Returns: + dict: Description of applied overrides. + """ + return self._kwargs + + +class ModeBaseModel(ConfigBaseModel): + """Mode-specific configuration, which will be hidden from the string representation of the model if that mode is not activated.""" + + mode: MODES_T = Field(default="plan") + """Mode in which to run the optimisation.""" + + @model_validator(mode="after") + def update_repr(self) -> Self: + """Hide config from model string representation if mode is not activated.""" + for key, val in self.model_fields.items(): + if key in get_args(MODES_T): + val.repr = self.mode == key + return self + + +class Init(ConfigBaseModel): + """All configuration options used when initialising a Calliope model.""" + + model_config = { + "title": "init", + "extra": "forbid", + "frozen": True, + "json_schema_extra": hide_from_schema(["def_path"]), + "revalidate_instances": "always", + "use_attribute_docstrings": True, + } + + def_path: Path = Field(default=".", repr=False, exclude=True) + """The path to the main model definition YAML file, if one has been used to instantiate the Calliope Model class.""" + + name: str | None = Field(default=None) + """Model name""" + + calliope_version: str | None = Field(default=None) + """Calliope framework version this model is intended for""" + + time_subset: tuple[datetime, datetime] | None = Field(default=None) + """ + Subset of timesteps as an two-element list giving the **inclusive** range. + For example, ["2005-01", "2005-04"] will create a time subset from "2005-01-01 00:00:00" to "2005-04-31 23:59:59". + + Strings must be ISO8601-compatible, i.e. of the form `YYYY-mm-dd HH:MM:SS` (e.g, '2005-01 ', '2005-01-01', '2005-01-01 00:00', ...) + """ + + time_resample: str | None = Field(default=None, pattern="^[0-9]+[a-zA-Z]") + """Setting to adjust time resolution, e.g. '2h' for 2-hourly""" + + time_cluster: Path | None = Field(default=None) + """ + Setting to cluster the timeseries. + Must be a path to a file where each date is linked to a representative date that also exists in the timeseries. + """ + + time_format: str = Field(default="ISO8601") + """ + Timestamp format of all time series data when read from file. + 'ISO8601' means '%Y-%m-%d %H:%M:%S'. + """ + + distance_unit: Literal["km", "m"] = Field(default="km") + """ + Unit of transmission link `distance` (m - metres, km - kilometres). + Automatically derived distances from lat/lon coordinates will be given in this unit. + """ + + @model_validator(mode="before") + @classmethod + def abs_path(cls, data): + """Add model definition path.""" + if data.get("time_cluster", None) is not None: + data["time_cluster"] = tools.relative_path( + data["def_path"], data["time_cluster"] + ) + return data + + +class BuildOperate(ConfigBaseModel): + """Operate mode configuration options used when building a Calliope optimisation problem (`calliope.Model.build`).""" + + model_config = { + "title": "operate", + "extra": "forbid", + "json_schema_extra": hide_from_schema(["start_window_idx"]), + "revalidate_instances": "always", + "use_attribute_docstrings": True, + } + + window: str = Field(default="24h") + """ + Operate mode rolling `window`, given as a pandas frequency string. + See [here](https://pandas.pydata.org/docs/user_guide/timeseries.html#offset-aliases) for a list of frequency aliases. + """ + + horizon: str = Field(default="48h") + """ + Operate mode rolling `horizon`, given as a pandas frequency string. + See [here](https://pandas.pydata.org/docs/user_guide/timeseries.html#offset-aliases) for a list of frequency aliases. + Must be ≥ `window` + """ + + use_cap_results: bool = Field(default=False) + """If the model already contains `plan` mode results, use those optimal capacities as input parameters to the `operate` mode run.""" + + start_window_idx: int = Field(default=0, repr=False, exclude=True) + """Which time window to build. This is used to track the window when re-building the model part way through solving in `operate` mode.""" + + +class Build(ModeBaseModel): + """Base configuration options used when building a Calliope optimisation problem (`calliope.Model.build`).""" + + model_config = { + "title": "build", + "extra": "allow", + "revalidate_instances": "always", + } + add_math: UniqueList[str] = Field(default=[]) + """ + List of references to files which contain additional mathematical formulations to be applied on top of or instead of the base mode math. + If referring to an pre-defined Calliope math file (see documentation for available files), do not append the reference with ".yaml". + If referring to your own math file, ensure the file type is given as a suffix (".yaml" or ".yml"). + Relative paths will be assumed to be relative to the model definition file given when creating a calliope Model (`calliope.Model(model_definition=...)`) + """ + + ignore_mode_math: bool = Field(default=False) + """ + If True, do not initialise the mathematical formulation with the pre-defined math for the given run `mode`. + This option can be used to completely re-define the Calliope mathematical formulation. + """ + + backend: Literal["pyomo", "gurobi"] = Field(default="pyomo") + """Module with which to build the optimisation problem.""" + + ensure_feasibility: bool = Field(default=False) + """ + Whether to include decision variables in the model which will meet unmet demand or consume unused supply in the model so that the optimisation solves successfully. + This should only be used as a debugging option (as any unmet demand/unused supply is a sign of improper model formulation). + """ + + objective: str = Field(default="min_cost_optimisation") + """Name of internal objective function to use, from those defined in the pre-defined math and any applied additional math.""" + + pre_validate_math_strings: bool = Field(default=True) + """ + If true, the Calliope math definition will be scanned for parsing errors _before_ undertaking the much more expensive operation of building the optimisation problem. + You can switch this off (e.g., if you know there are no parsing errors) to reduce overall build time. + """ + + operate: BuildOperate = BuildOperate() + + +class SolveSpores(ConfigBaseModel): + """SPORES configuration options used when solving a Calliope optimisation problem (`calliope.Model.solve`).""" + + number: int = Field(default=3) + """SPORES mode number of iterations after the initial base run.""" + + score_cost_class: str = Field(default="score") + """SPORES mode cost class to vary between iterations after the initial base run.""" + + slack_cost_group: str = Field(default=None) + """SPORES mode cost class to keep below the given `slack` (usually "monetary").""" + + save_per_spore: bool = Field(default=False) + """ + Whether or not to save the result of each SPORES mode run between iterations. + If False, will consolidate all iterations into one dataset after completion of N iterations (defined by `number`) and save that one dataset. + """ + + save_per_spore_path: Path | None = Field(default=None) + """If saving per spore, the path to save to.""" + + skip_cost_op: bool = Field(default=False) + """If the model already contains `plan` mode results, use those as the initial base run results and start with SPORES iterations immediately.""" + + @model_validator(mode="after") + def require_save_per_spore_path(self) -> Self: + """Ensure that path is given if saving per spore.""" + if self.save_per_spore: + if self.save_per_spore_path is None: + raise ValueError( + "Must define `save_per_spore_path` if you want to save each SPORES result separately." + ) + elif not self.save_per_spore_path.is_dir(): + raise ValueError("`save_per_spore_path` must be a directory.") + return self + + +class Solve(ModeBaseModel): + """Base configuration options used when solving a Calliope optimisation problem (`calliope.Model.solve`).""" + + model_config = { + "title": "solve", + "extra": "forbid", + "revalidate_instances": "always", + "json_schema_extra": hide_from_schema(["mode"]), + } + + save_logs: Path | None = Field(default=None) + """If given, should be a path to a directory in which to save optimisation logs.""" + + solver_io: str | None = Field(default=None) + """ + Some solvers have different interfaces that perform differently. + For instance, setting `solver_io="python"` when using the solver `gurobi` tends to reduce the time to send the optimisation problem to the solver. + """ + + solver_options: dict = Field(default={}) + """Any solver options, as key-value pairs, to pass to the chosen solver""" + + solver: str = Field(default="cbc") + """Solver to use. Any solvers that have Pyomo interfaces can be used. Refer to the Pyomo documentation for the latest list.""" + + zero_threshold: float = Field(default=1e-10) + """On postprocessing the optimisation results, values smaller than this threshold will be considered as optimisation artefacts and will be set to zero.""" + + shadow_prices: UniqueList[str] = Field(default=[]) + """Names of model constraints.""" + + spores: SolveSpores = SolveSpores() + + +class CalliopeConfig(ConfigBaseModel): + """Calliope configuration class.""" + + model_config = {"title": "config"} + init: Init = Init() + build: Build = Build() + solve: Solve = Solve() + + @model_validator(mode="before") + @classmethod + def update_solve_mode(cls, data): + """Solve mode should match build mode.""" + data["solve"]["mode"] = data["build"]["mode"] + return data + + def update(self, update_dict: dict, deep: bool = False) -> Self: + """Return a new iteration of the model with updated fields. + + Updates are validated and stored in the parent class in the `_kwargs` key. + + Args: + update_dict (dict): Dictionary with which to update the base model. + deep (bool, optional): Set to True to make a deep copy of the model. Defaults to False. + + Returns: + BaseModel: New model instance. + """ + update_dict_temp = AttrDict(update_dict) + if update_dict_temp.get_key("build.mode", None) is not None: + update_dict_temp.set_key("solve.mode", update_dict_temp["build"]["mode"]) + updated = super().update(update_dict_temp.as_dict(), deep=deep) + return updated diff --git a/src/calliope/config/config_schema.yaml b/src/calliope/config/config_schema.yaml index b9ebe627..41a8c06e 100644 --- a/src/calliope/config/config_schema.yaml +++ b/src/calliope/config/config_schema.yaml @@ -15,172 +15,16 @@ properties: init: type: object description: All configuration options used when initialising a Calliope model - additionalProperties: false - properties: - name: - type: ["null", string] - default: null - description: Model name - calliope_version: - type: ["null", string] - default: null - description: Calliope framework version this model is intended for - time_subset: - oneOf: - - type: "null" - - type: array - minItems: 2 - maxItems: 2 - items: - type: string - description: ISO8601 format datetime strings of the form `YYYY-mm-dd HH:MM:SS` (e.g, '2005-01', '2005-01-01', '2005-01-01 00:00', ...) - default: null - description: >- - Subset of timesteps as an two-element list giving the **inclusive** range. - For example, ['2005-01', '2005-04'] will create a time subset from '2005-01-01 00:00:00' to '2005-04-31 23:59:59'. - time_resample: - type: ["null", string] - default: null - description: setting to adjust time resolution, e.g. "2h" for 2-hourly - pattern: "^[0-9]+[a-zA-Z]" - time_cluster: - type: ["null", string] - default: null - description: setting to cluster the timeseries, must be a path to a file where each date is linked to a representative date that also exists in the timeseries. - time_format: - type: string - default: "ISO8601" - description: Timestamp format of all time series data when read from file. "ISO8601" means "%Y-%m-%d %H:%M:%S". - distance_unit: - type: string - default: km - description: >- - Unit of transmission link `distance` (m - metres, km - kilometres). - Automatically derived distances from lat/lon coordinates will be given in this unit. - enum: [m, km] build: type: object description: > All configuration options used when building a Calliope optimisation problem (`calliope.Model.build`). Additional configuration items will be passed onto math string parsing and can therefore be accessed in the `where` strings by `config.[item-name]`, where "[item-name]" is the name of your own configuration item. - additionalProperties: true - properties: - add_math: - type: array - default: [] - description: List of references to files which contain additional mathematical formulations to be applied on top of or instead of the base mode math. - uniqueItems: true - items: - type: string - description: > - If referring to an pre-defined Calliope math file (see documentation for available files), do not append the reference with ".yaml". - If referring to your own math file, ensure the file type is given as a suffix (".yaml" or ".yml"). - Relative paths will be assumed to be relative to the model definition file given when creating a calliope Model (`calliope.Model(model_definition=...)`). - ignore_mode_math: - type: boolean - default: false - description: >- - If True, do not initialise the mathematical formulation with the pre-defined math for the given run `mode`. - This option can be used to completely re-define the Calliope mathematical formulation. - backend: - type: string - default: pyomo - description: Module with which to build the optimisation problem - ensure_feasibility: - type: boolean - default: false - description: > - whether to include decision variables in the model which will meet unmet demand or consume unused supply in the model so that the optimisation solves successfully. - This should only be used as a debugging option (as any unmet demand/unused supply is a sign of improper model formulation). - mode: - type: string - default: plan - description: Mode in which to run the optimisation. - enum: [plan, spores, operate] - objective: - type: string - default: min_cost_optimisation - description: Name of internal objective function to use, from those defined in the pre-defined math and any applied additional math. - operate_window: - type: string - description: >- - Operate mode rolling `window`, given as a pandas frequency string. - See [here](https://pandas.pydata.org/docs/user_guide/timeseries.html#offset-aliases) for a list of frequency aliases. - operate_horizon: - type: string - description: >- - Operate mode rolling `horizon`, given as a pandas frequency string. - See [here](https://pandas.pydata.org/docs/user_guide/timeseries.html#offset-aliases) for a list of frequency aliases. - Must be ≥ `operate_window` - operate_use_cap_results: - type: boolean - default: false - description: If the model already contains `plan` mode results, use those optimal capacities as input parameters to the `operate` mode run. - pre_validate_math_strings: - type: boolean - default: true - description: >- - If true, the Calliope math definition will be scanned for parsing errors _before_ undertaking the much more expensive operation of building the optimisation problem. - You can switch this off (e.g., if you know there are no parsing errors) to reduce overall build time. solve: type: object description: All configuration options used when solving a Calliope optimisation problem (`calliope.Model.solve`). - additionalProperties: false - properties: - spores_number: - type: integer - default: 3 - description: SPORES mode number of iterations after the initial base run. - spores_score_cost_class: - type: string - default: spores_score - description: SPORES mode cost class to vary between iterations after the initial base run. - spores_slack_cost_group: - type: string - description: SPORES mode cost class to keep below the given `slack` (usually "monetary"). - spores_save_per_spore: - type: boolean - default: false - description: Whether or not to save the result of each SPORES mode run between iterations. If False, will consolidate all iterations into one dataset after completion of N iterations (defined by `spores_number`) and save that one dataset. - spores_save_per_spore_path: - type: string - description: If saving per spore, the path to save to. - spores_skip_cost_op: - type: boolean - default: false - description: If the model already contains `plan` mode results, use those as the initial base run results and start with SPORES iterations immediately. - save_logs: - type: ["null", string] - default: null - description: If given, should be a path to a directory in which to save optimisation logs. - solver_io: - type: ["null", string] - default: null - description: > - Some solvers have different interfaces that perform differently. - For instance, setting `solver_io="python"` when using the solver `gurobi` tends to reduce the time to send the optimisation problem to the solver. - solver_options: - type: ["null", object] - default: null - description: Any solver options, as key-value pairs, to pass to the chosen solver - solver: - type: string - default: cbc - description: Solver to use. Any solvers that have Pyomo interfaces can be used. Refer to the Pyomo documentation for the latest list. - zero_threshold: - type: number - default: 1e-10 - description: On postprocessing the optimisation results, values smaller than this threshold will be considered as optimisation artefacts and will be set to zero. - shadow_prices: - type: array - uniqueItems: true - items: - type: string - description: Names of model constraints. - default: [] - description: List of constraints for which to extract shadow prices. Shadow prices will be added as variables to the model results as `shadow_price_{constraintname}`. parameters: type: [object, "null"] diff --git a/src/calliope/example_models/national_scale/scenarios.yaml b/src/calliope/example_models/national_scale/scenarios.yaml index 58a3dc81..0e34f8f9 100644 --- a/src/calliope/example_models/national_scale/scenarios.yaml +++ b/src/calliope/example_models/national_scale/scenarios.yaml @@ -70,8 +70,9 @@ overrides: init.time_subset: ["2005-01-01", "2005-01-10"] build: mode: operate - operate_window: 12h - operate_horizon: 24h + operate: + window: 12h + horizon: 24h nodes: region1.techs.ccgt.flow_cap: 30000 diff --git a/src/calliope/example_models/urban_scale/scenarios.yaml b/src/calliope/example_models/urban_scale/scenarios.yaml index 12d114cb..d754496d 100644 --- a/src/calliope/example_models/urban_scale/scenarios.yaml +++ b/src/calliope/example_models/urban_scale/scenarios.yaml @@ -51,8 +51,9 @@ overrides: init.time_subset: ["2005-07-01", "2005-07-10"] build: mode: operate - operate_window: 2h - operate_horizon: 48h + operate: + window: 2h + horizon: 48h nodes: X1: diff --git a/src/calliope/model.py b/src/calliope/model.py index ee8c5a77..e6088c21 100644 --- a/src/calliope/model.py +++ b/src/calliope/model.py @@ -12,7 +12,7 @@ import xarray as xr import calliope -from calliope import backend, exceptions, io, preprocess +from calliope import backend, config, exceptions, io, preprocess from calliope.attrdict import AttrDict from calliope.postprocess import postprocess as postprocess_results from calliope.preprocess.data_tables import DataTable @@ -22,10 +22,9 @@ CONFIG_SCHEMA, MODEL_SCHEMA, extract_from_schema, - update_then_validate_config, validate_dict, ) -from calliope.util.tools import climb_template_tree, relative_path +from calliope.util.tools import climb_template_tree if TYPE_CHECKING: from calliope.backend.backend_model import BackendModel @@ -43,7 +42,7 @@ class Model: """A Calliope Model.""" _TS_OFFSET = pd.Timedelta(1, unit="nanoseconds") - ATTRS_SAVED = ("_def_path", "applied_math") + ATTRS_SAVED = ("applied_math", "config") def __init__( self, @@ -74,10 +73,9 @@ def __init__( **kwargs: initialisation overrides. """ self._timings: dict = {} - self.config: AttrDict + self.config: config.CalliopeConfig self.defaults: AttrDict self.applied_math: preprocess.CalliopeMath - self._def_path: str | None = None self.backend: BackendModel self._is_built: bool = False self._is_solved: bool = False @@ -88,20 +86,24 @@ def __init__( LOGGER, self._timings, "model_creation", comment="Model: initialising" ) if isinstance(model_definition, xr.Dataset): + if kwargs: + raise exceptions.ModelError( + "Cannot apply initialisation configuration overrides when loading data from an xarray Dataset." + ) self._init_from_model_data(model_definition) else: if isinstance(model_definition, dict): model_def_dict = AttrDict(model_definition) else: - self._def_path = str(model_definition) + kwargs["def_path"] = str(model_definition) model_def_dict = AttrDict.from_yaml(model_definition) (model_def, applied_overrides) = preprocess.load_scenario_overrides( - model_def_dict, scenario, override_dict, **kwargs + model_def_dict, scenario, override_dict ) self._init_from_model_def_dict( - model_def, applied_overrides, scenario, data_table_dfs + model_def, applied_overrides, scenario, data_table_dfs, **kwargs ) self._model_data.attrs["timestamp_model_creation"] = timestamp_model_creation @@ -144,6 +146,7 @@ def _init_from_model_def_dict( applied_overrides: str, scenario: str | None, data_table_dfs: dict[str, pd.DataFrame] | None = None, + **kwargs, ) -> None: """Initialise the model using pre-processed YAML files and optional dataframes/dicts. @@ -152,6 +155,7 @@ def _init_from_model_def_dict( applied_overrides (str): overrides specified by users scenario (str | None): scenario specified by users data_table_dfs (dict[str, pd.DataFrame] | None, optional): files with additional model information. Defaults to None. + **kwargs: Initialisation configuration overrides. """ # First pass to check top-level keys are all good validate_dict(model_definition, CONFIG_SCHEMA, "Model definition") @@ -162,19 +166,13 @@ def _init_from_model_def_dict( "model_run_creation", comment="Model: preprocessing stage 1 (model_run)", ) - model_config = AttrDict(extract_from_schema(CONFIG_SCHEMA, "default")) - model_config.union(model_definition.pop("config"), allow_override=True) - - init_config = update_then_validate_config("init", model_config) - if init_config["time_cluster"] is not None: - init_config["time_cluster"] = relative_path( - self._def_path, init_config["time_cluster"] - ) + model_config = config.CalliopeConfig(**model_definition.pop("config")) + init_config = model_config.update({"init": kwargs}).init param_metadata = {"default": extract_from_schema(MODEL_SCHEMA, "default")} attributes = { - "calliope_version_defined": init_config["calliope_version"], + "calliope_version_defined": init_config.calliope_version, "calliope_version_initialised": calliope.__version__, "applied_overrides": applied_overrides, "scenario": scenario, @@ -185,11 +183,8 @@ def _init_from_model_def_dict( for table_name, table_dict in model_definition.pop("data_tables", {}).items(): table_dict, _ = climb_template_tree(table_dict, templates, table_name) data_tables.append( - DataTable( - init_config, table_name, table_dict, data_table_dfs, self._def_path - ) + DataTable(table_name, table_dict, data_table_dfs, init_config.def_path) ) - model_data_factory = ModelDataFactory( init_config, model_definition, data_tables, attributes, param_metadata ) @@ -204,9 +199,12 @@ def _init_from_model_def_dict( comment="Model: preprocessing stage 2 (model_data)", ) - self._add_observed_dict("config", model_config) + self._model_data.attrs["name"] = init_config.name + + # Unlike at the build and solve phases, we store the init config overrides in the main model config. + model_config.init = init_config + self.config = model_config - self._model_data.attrs["name"] = init_config["name"] log_time( LOGGER, self._timings, @@ -223,15 +221,15 @@ def _init_from_model_data(self, model_data: xr.Dataset) -> None: model_data (xr.Dataset): Model dataset with input parameters as arrays and configuration stored in the dataset attributes dictionary. """ - if "_def_path" in model_data.attrs: - self._def_path = model_data.attrs.pop("_def_path") if "applied_math" in model_data.attrs: self.applied_math = preprocess.CalliopeMath.from_dict( model_data.attrs.pop("applied_math") ) + if "config" in model_data.attrs: + self.config = config.CalliopeConfig(**model_data.attrs.pop("config")) + self.config.update(model_data.attrs.pop("config_kwarg_overrides")) self._model_data = model_data - self._add_model_data_methods() if self.results: self._is_solved = True @@ -243,47 +241,6 @@ def _init_from_model_data(self, model_data: xr.Dataset) -> None: comment="Model: loaded model_data", ) - def _add_model_data_methods(self): - """Add observed data to `model`. - - 1. Filter model dataset to produce views on the input/results data - 2. Add top-level configuration dictionaries simultaneously to the model data attributes and as attributes of this class. - - """ - self._add_observed_dict("config") - - def _add_observed_dict(self, name: str, dict_to_add: dict | None = None) -> None: - """Add the same dictionary as property of model object and an attribute of the model xarray dataset. - - Args: - name (str): - Name of dictionary which will be set as the model property name and - (if necessary) the dataset attribute name. - dict_to_add (dict | None, optional): - If given, set as both the model property and the dataset attribute, - otherwise set an existing dataset attribute as a model property of the - same name. Defaults to None. - - Raises: - exceptions.ModelError: If `dict_to_add` is not given, it must be an attribute of model data. - TypeError: `dict_to_add` must be a dictionary. - """ - if dict_to_add is None: - try: - dict_to_add = self._model_data.attrs[name] - except KeyError: - raise exceptions.ModelError( - f"Expected the model property `{name}` to be a dictionary attribute of the model dataset. If you are loading the model from a NetCDF file, ensure it is a valid Calliope model." - ) - if not isinstance(dict_to_add, dict): - raise TypeError( - f"Attempted to add dictionary property `{name}` to model, but received argument of type `{type(dict_to_add).__name__}`" - ) - else: - dict_to_add = AttrDict(dict_to_add) - self._model_data.attrs[name] = dict_to_add - setattr(self, name, dict_to_add) - def build( self, force: bool = False, add_math_dict: dict | None = None, **kwargs ) -> None: @@ -310,30 +267,26 @@ def build( comment="Model: backend build starting", ) - backend_config = {**self.config["build"], **kwargs} - mode = backend_config["mode"] + this_build_config = self.config.update({"build": kwargs}).build + mode = this_build_config.mode if mode == "operate": if not self._model_data.attrs["allow_operate_mode"]: raise exceptions.ModelError( "Unable to run this model in operate (i.e. dispatch) mode, probably because " "there exist non-uniform timesteps (e.g. from time clustering)" ) - start_window_idx = backend_config.pop("start_window_idx", 0) - backend_input = self._prepare_operate_mode_inputs( - start_window_idx, **backend_config - ) + backend_input = self._prepare_operate_mode_inputs(this_build_config.operate) else: backend_input = self._model_data - init_math_list = [] if backend_config.get("ignore_mode_math") else [mode] + init_math_list = [] if this_build_config.ignore_mode_math else [mode] end_math_list = [] if add_math_dict is None else [add_math_dict] - full_math_list = init_math_list + backend_config["add_math"] + end_math_list + full_math_list = init_math_list + this_build_config.add_math + end_math_list LOGGER.debug(f"Math preprocessing | Loading math: {full_math_list}") - model_math = preprocess.CalliopeMath(full_math_list, self._def_path) + model_math = preprocess.CalliopeMath(full_math_list, self.config.init.def_path) - backend_name = backend_config.pop("backend") self.backend = backend.get_model_backend( - backend_name, backend_input, model_math, **backend_config + this_build_config, backend_input, model_math ) self.backend.add_optimisation_components() @@ -370,7 +323,7 @@ def solve(self, force: bool = False, warmstart: bool = False, **kwargs) -> None: exceptions.ModelError: Some preprocessing steps will stop a run mode of "operate" from being possible. """ # Check that results exist and are non-empty - if not self._is_built: + if not self.is_built: raise exceptions.ModelError( "You must build the optimisation problem (`.build()`) " "before you can run it." @@ -388,23 +341,27 @@ def solve(self, force: bool = False, warmstart: bool = False, **kwargs) -> None: else: to_drop = [] - run_mode = self.backend.inputs.attrs["config"]["build"]["mode"] + kwargs["mode"] = self.config.build.applied_keyword_overrides.get( + "mode", self.config.build.mode + ) + + this_solve_config = self.config.update({"solve": kwargs}).solve self._model_data.attrs["timestamp_solve_start"] = log_time( LOGGER, self._timings, "solve_start", - comment=f"Optimisation model | starting model in {run_mode} mode.", + comment=f"Optimisation model | starting model in {this_solve_config.mode} mode.", ) - solver_config = update_then_validate_config("solve", self.config, **kwargs) - - shadow_prices = solver_config.get("shadow_prices", []) + shadow_prices = this_solve_config.shadow_prices self.backend.shadow_prices.track_constraints(shadow_prices) - if run_mode == "operate": - results = self._solve_operate(**solver_config) + if this_solve_config.mode == "operate": + results = self._solve_operate(**this_solve_config.model_dump()) else: - results = self.backend._solve(warmstart=warmstart, **solver_config) + results = self.backend._solve( + warmstart=warmstart, **this_solve_config.model_dump() + ) log_time( LOGGER, @@ -417,7 +374,7 @@ def solve(self, force: bool = False, warmstart: bool = False, **kwargs) -> None: # Add additional post-processed result variables to results if results.attrs["termination_condition"] in ["optimal", "feasible"]: results = postprocess_results.postprocess_model_results( - results, self._model_data + results, self._model_data, self.config.solve.zero_threshold ) log_time( @@ -434,7 +391,6 @@ def solve(self, force: bool = False, warmstart: bool = False, **kwargs) -> None: self._model_data = xr.merge( [results, self._model_data], compat="override", combine_attrs="no_conflicts" ) - self._add_model_data_methods() self._model_data.attrs["timestamp_solve_complete"] = log_time( LOGGER, @@ -469,6 +425,7 @@ def to_netcdf(self, path): saved_attrs[attr] = dict(getattr(self, attr)) else: saved_attrs[attr] = getattr(self, attr) + saved_attrs["config_kwarg_overrides"] = self.config.applied_keyword_overrides io.save_netcdf(self._model_data, path, **saved_attrs) @@ -507,28 +464,24 @@ def info(self) -> str: return "\n".join(info_strings) def _prepare_operate_mode_inputs( - self, start_window_idx: int = 0, **config_kwargs + self, operate_config: config.BuildOperate ) -> xr.Dataset: """Slice the input data to just the length of operate mode time horizon. Args: - start_window_idx (int, optional): - Set the operate `window` to start at, based on integer index. - This is used when re-initialising the backend model for shorter time horizons close to the end of the model period. - Defaults to 0. - **config_kwargs: kwargs related to operate mode configuration. + operate_config (config.BuildOperate): operate mode configuration options. Returns: xr.Dataset: Slice of input data. """ - window = config_kwargs["operate_window"] - horizon = config_kwargs["operate_horizon"] self._model_data.coords["windowsteps"] = pd.date_range( self.inputs.timesteps[0].item(), self.inputs.timesteps[-1].item(), - freq=window, + freq=operate_config.window, + ) + horizonsteps = self._model_data.coords["windowsteps"] + pd.Timedelta( + operate_config.horizon ) - horizonsteps = self._model_data.coords["windowsteps"] + pd.Timedelta(horizon) # We require an offset because pandas / xarray slicing is _inclusive_ of both endpoints # where we only want it to be inclusive of the left endpoint. # Except in the last time horizon, where we want it to include the right endpoint. @@ -538,11 +491,11 @@ def _prepare_operate_mode_inputs( self._model_data.coords["horizonsteps"] = clipped_horizonsteps - self._TS_OFFSET sliced_inputs = self._model_data.sel( timesteps=slice( - self._model_data.windowsteps[start_window_idx], - self._model_data.horizonsteps[start_window_idx], + self._model_data.windowsteps[operate_config.start_window_idx], + self._model_data.horizonsteps[operate_config.start_window_idx], ) ) - if config_kwargs.get("operate_use_cap_results", False): + if operate_config.use_cap_results: to_parameterise = extract_from_schema(MODEL_SCHEMA, "x-operate-param") if not self._is_solved: raise exceptions.ModelError( @@ -565,10 +518,7 @@ def _solve_operate(self, **solver_config) -> xr.Dataset: """ if self.backend.inputs.timesteps[0] != self._model_data.timesteps[0]: LOGGER.info("Optimisation model | Resetting model to first time window.") - self.build( - force=True, - **{"mode": "operate", **self.backend.inputs.attrs["config"]["build"]}, - ) + self.build(force=True, **self.config.build.applied_keyword_overrides) LOGGER.info("Optimisation model | Running first time window.") @@ -595,11 +545,9 @@ def _solve_operate(self, **solver_config) -> xr.Dataset: "Optimisation model | Reaching the end of the timeseries. " "Re-building model with shorter time horizon." ) - self.build( - force=True, - start_window_idx=idx + 1, - **self.backend.inputs.attrs["config"]["build"], - ) + build_kwargs = AttrDict(self.config.build.applied_keyword_overrides) + build_kwargs.set_key("operate.start_window_idx", idx + 1) + self.build(force=True, **build_kwargs) else: self.backend._dataset.coords["timesteps"] = new_inputs.timesteps self.backend.inputs.coords["timesteps"] = new_inputs.timesteps diff --git a/src/calliope/postprocess/postprocess.py b/src/calliope/postprocess/postprocess.py index 402b928e..327b1ce2 100644 --- a/src/calliope/postprocess/postprocess.py +++ b/src/calliope/postprocess/postprocess.py @@ -11,7 +11,7 @@ def postprocess_model_results( - results: xr.Dataset, model_data: xr.Dataset + results: xr.Dataset, model_data: xr.Dataset, zero_threshold: float ) -> xr.Dataset: """Post-processing of model results. @@ -22,11 +22,11 @@ def postprocess_model_results( Args: results (xarray.Dataset): Output from the solver backend. model_data (xarray.Dataset): Calliope model data. + zero_threshold (float): Numbers below this value will be assumed to be zero Returns: xarray.Dataset: input-results dataset. """ - zero_threshold = model_data.config.solve.zero_threshold results["capacity_factor"] = capacity_factor(results, model_data) results["systemwide_capacity_factor"] = capacity_factor( results, model_data, systemwide=True diff --git a/src/calliope/preprocess/data_tables.py b/src/calliope/preprocess/data_tables.py index 4a90fbf3..a9e7acf2 100644 --- a/src/calliope/preprocess/data_tables.py +++ b/src/calliope/preprocess/data_tables.py @@ -51,22 +51,20 @@ class DataTable: def __init__( self, - model_config: dict, table_name: str, data_table: DataTableDict, data_table_dfs: dict[str, pd.DataFrame] | None = None, - model_definition_path: Path | None = None, + model_definition_path: Path = Path("."), ): """Load and format a data table from file / in-memory object. Args: - model_config (dict): Model initialisation configuration dictionary. table_name (str): name of the data table. data_table (DataTableDict): Data table definition dictionary. data_table_dfs (dict[str, pd.DataFrame] | None, optional): If given, a dictionary mapping table names in `data_table` to in-memory pandas DataFrames. Defaults to None. - model_definition_path (Path | None, optional): + model_definition_path (Path, optional): If given, the path to the model definition YAML file, relative to which data table filepaths will be set. If None, relative data table filepaths will be considered relative to the current working directory. Defaults to None. @@ -75,7 +73,6 @@ def __init__( self.input = data_table self.dfs = data_table_dfs if data_table_dfs is not None else dict() self.model_definition_path = model_definition_path - self.config = model_config self.columns = self._listify_if_defined("columns") self.index = self._listify_if_defined("rows") diff --git a/src/calliope/preprocess/model_data.py b/src/calliope/preprocess/model_data.py index 7c6d6cc3..89b21386 100644 --- a/src/calliope/preprocess/model_data.py +++ b/src/calliope/preprocess/model_data.py @@ -15,6 +15,7 @@ from calliope import exceptions from calliope.attrdict import AttrDict +from calliope.config import Init from calliope.preprocess import data_tables, time from calliope.util.schema import MODEL_SCHEMA, validate_dict from calliope.util.tools import climb_template_tree, listify @@ -70,7 +71,7 @@ class ModelDataFactory: def __init__( self, - model_config: dict, + init_config: Init, model_definition: ModelDefinition, data_tables: list[data_tables.DataTable], attributes: dict, @@ -81,13 +82,13 @@ def __init__( This includes resampling/clustering timeseries data as necessary. Args: - model_config (dict): Model initialisation configuration (i.e., `config.init`). + init_config (Init): Model initialisation configuration (i.e., `config.init`). model_definition (ModelDefinition): Definition of model nodes and technologies, and their potential `templates`. data_tables (list[data_tables.DataTable]): Pre-loaded data tables that will be used to initialise the dataset before handling definitions given in `model_definition`. attributes (dict): Attributes to attach to the model Dataset. param_attributes (dict[str, dict]): Attributes to attach to the generated model DataArrays. """ - self.config: dict = model_config + self.config: Init = init_config self.model_definition: ModelDefinition = model_definition.copy() self.dataset = xr.Dataset(attrs=AttrDict(attributes)) self.tech_data_from_tables = AttrDict() @@ -244,7 +245,7 @@ def update_time_dimension_and_params(self): raise exceptions.ModelError( "Must define at least one timeseries parameter in a Calliope model." ) - time_subset = self.config.get("time_subset", None) + time_subset = self.config.time_subset if time_subset is not None: self.dataset = time.subset_timeseries(self.dataset, time_subset) self.dataset = time.add_inferred_time_params(self.dataset) @@ -252,11 +253,11 @@ def update_time_dimension_and_params(self): # By default, the model allows operate mode self.dataset.attrs["allow_operate_mode"] = 1 - if self.config["time_resample"] is not None: - self.dataset = time.resample(self.dataset, self.config["time_resample"]) - if self.config["time_cluster"] is not None: + if self.config.time_resample is not None: + self.dataset = time.resample(self.dataset, self.config.time_resample) + if self.config.time_cluster is not None: self.dataset = time.cluster( - self.dataset, self.config["time_cluster"], self.config["time_format"] + self.dataset, self.config.time_cluster, self.config.time_format ) def clean_data_from_undefined_members(self): @@ -324,7 +325,7 @@ def add_link_distances(self): self.dataset.longitude.sel(nodes=node2).item(), )["s12"] distance_array = pd.Series(distances).rename_axis(index="techs").to_xarray() - if self.config["distance_unit"] == "km": + if self.config.distance_unit == "km": distance_array /= 1000 else: LOGGER.debug( @@ -660,7 +661,7 @@ def _add_to_dataset(self, to_add: xr.Dataset, id_: str): """ to_add_numeric_dims = self._update_numeric_dims(to_add, id_) to_add_numeric_ts_dims = time.timeseries_to_datetime( - to_add_numeric_dims, self.config["time_format"], id_ + to_add_numeric_dims, self.config.time_format, id_ ) self.dataset = xr.merge( [to_add_numeric_ts_dims, self.dataset], diff --git a/src/calliope/preprocess/scenarios.py b/src/calliope/preprocess/scenarios.py index 473544fb..88e382a1 100644 --- a/src/calliope/preprocess/scenarios.py +++ b/src/calliope/preprocess/scenarios.py @@ -15,7 +15,6 @@ def load_scenario_overrides( model_definition: dict, scenario: str | None = None, override_dict: dict | None = None, - **kwargs, ) -> tuple[AttrDict, str]: """Apply user-defined overrides to the model definition. @@ -28,8 +27,6 @@ def load_scenario_overrides( override_dict (dict | None, optional): Overrides to apply _after_ `scenario` overrides. Defaults to None. - **kwargs: - initialisation overrides. Returns: tuple[AttrDict, str]: @@ -88,10 +85,6 @@ def load_scenario_overrides( _log_overrides(model_def_dict, model_def_with_overrides) - model_def_with_overrides.union( - AttrDict({"config.init": kwargs}), allow_override=True - ) - return (model_def_with_overrides, ";".join(applied_overrides)) diff --git a/src/calliope/util/schema.py b/src/calliope/util/schema.py index bd98cc77..361cd9a9 100644 --- a/src/calliope/util/schema.py +++ b/src/calliope/util/schema.py @@ -25,20 +25,6 @@ def reset(): importlib.reload(sys.modules[__name__]) -def update_then_validate_config( - config_key: str, config_dict: AttrDict, **update_kwargs -) -> AttrDict: - """Return an updated version of the configuration schema.""" - to_validate = deepcopy(config_dict[config_key]) - to_validate.union(AttrDict(update_kwargs), allow_override=True) - validate_dict( - {"config": {config_key: to_validate}}, - CONFIG_SCHEMA, - f"`{config_key}` configuration", - ) - return to_validate - - def update_model_schema( top_level_property: Literal["nodes", "techs", "parameters"], new_entries: dict, diff --git a/src/calliope/util/tools.py b/src/calliope/util/tools.py index dee2f6ca..3d8d4320 100644 --- a/src/calliope/util/tools.py +++ b/src/calliope/util/tools.py @@ -15,7 +15,7 @@ T = TypeVar("T") -def relative_path(base_path_file, path) -> Path: +def relative_path(base_path_file: str | Path, path: str | Path) -> Path: """Path standardization. If ``path`` is not absolute, it is interpreted as relative to the @@ -23,7 +23,7 @@ def relative_path(base_path_file, path) -> Path: """ # Check if base_path_file is a string because it might be an AttrDict path = Path(path) - if path.is_absolute() or base_path_file is None: + if path.is_absolute(): return path else: base_path_file = Path(base_path_file) diff --git a/tests/common/util.py b/tests/common/util.py index 8ae70da8..94f90dc2 100644 --- a/tests/common/util.py +++ b/tests/common/util.py @@ -95,9 +95,7 @@ def build_lp( math (dict | None, optional): All constraint/global expression/objective math to apply. Defaults to None. backend_name (Literal["pyomo"], optional): Backend to use to create the LP file. Defaults to "pyomo". """ - math = calliope.preprocess.CalliopeMath( - ["plan", *model.config.build.get("add_math", [])] - ) + math = calliope.preprocess.CalliopeMath(["plan", *model.config.build.add_math]) math_to_add = calliope.AttrDict() if isinstance(math_data, dict): diff --git a/tests/conftest.py b/tests/conftest.py index 3d4694c5..0334d0b4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,10 +5,11 @@ import pytest import xarray as xr +from calliope import config from calliope.attrdict import AttrDict from calliope.backend import latex_backend_model, pyomo_backend_model from calliope.preprocess import CalliopeMath -from calliope.util.schema import CONFIG_SCHEMA, MODEL_SCHEMA, extract_from_schema +from calliope.util.schema import MODEL_SCHEMA, extract_from_schema from .common.util import build_test_model as build_model @@ -33,7 +34,7 @@ def foreach(request): @pytest.fixture(scope="session") def config_defaults(): - return AttrDict(extract_from_schema(CONFIG_SCHEMA, "default")) + return AttrDict(config.CalliopeConfig().model_dump()) @pytest.fixture(scope="session") diff --git a/tests/test_core_model.py b/tests/test_core_model.py index e16ebfa4..ddd97800 100644 --- a/tests/test_core_model.py +++ b/tests/test_core_model.py @@ -9,7 +9,6 @@ import calliope.preprocess from .common.util import build_test_model as build_model -from .common.util import check_error_or_warning LOGGER = "calliope.model" @@ -32,40 +31,6 @@ def test_info(self, national_scale_example): def test_info_simple_model(self, simple_supply): simple_supply.info() - def test_update_observed_dict(self, national_scale_example): - national_scale_example.config.build["backend"] = "foo" - assert national_scale_example._model_data.attrs["config"].build.backend == "foo" - - def test_add_observed_dict_from_model_data( - self, national_scale_example, dict_to_add - ): - national_scale_example._model_data.attrs["foo"] = dict_to_add - national_scale_example._add_observed_dict("foo") - assert national_scale_example.foo == dict_to_add - assert national_scale_example._model_data.attrs["foo"] == dict_to_add - - def test_add_observed_dict_from_dict(self, national_scale_example, dict_to_add): - national_scale_example._add_observed_dict("bar", dict_to_add) - assert national_scale_example.bar == dict_to_add - assert national_scale_example._model_data.attrs["bar"] == dict_to_add - - def test_add_observed_dict_not_available(self, national_scale_example): - with pytest.raises(calliope.exceptions.ModelError) as excinfo: - national_scale_example._add_observed_dict("baz") - assert check_error_or_warning( - excinfo, - "Expected the model property `baz` to be a dictionary attribute of the model dataset", - ) - assert not hasattr(national_scale_example, "baz") - - def test_add_observed_dict_not_dict(self, national_scale_example): - with pytest.raises(TypeError) as excinfo: - national_scale_example._add_observed_dict("baz", "bar") - assert check_error_or_warning( - excinfo, - "Attempted to add dictionary property `baz` to model, but received argument of type `str`", - ) - class TestOperateMode: @contextmanager @@ -127,9 +92,7 @@ def rerun_operate_log(self, request, operate_model_and_log): def test_backend_build_mode(self, operate_model_and_log): """Verify that we have run in operate mode""" operate_model, _ = operate_model_and_log - assert ( - operate_model.backend.inputs.attrs["config"]["build"]["mode"] == "operate" - ) + assert operate_model.backend.config.mode == "operate" def test_operate_mode_success(self, operate_model_and_log): """Solving in operate mode should lead to an optimal solution.""" @@ -153,8 +116,8 @@ def test_reset_model_window(self, rerun_operate_log): def test_end_of_horizon(self, operate_model_and_log): """Check that increasingly shorter time horizons are logged as model rebuilds.""" operate_model, log = operate_model_and_log - config = operate_model.backend.inputs.attrs["config"]["build"] - if config["operate_window"] != config["operate_horizon"]: + config = operate_model.backend.config.operate + if config.operate_window != config.operate_horizon: assert "Reaching the end of the timeseries." in log else: assert "Reaching the end of the timeseries." not in log diff --git a/tests/test_preprocess_model_data.py b/tests/test_preprocess_model_data.py index 48bc519c..e3208e1a 100644 --- a/tests/test_preprocess_model_data.py +++ b/tests/test_preprocess_model_data.py @@ -202,10 +202,14 @@ def test_add_link_distances_missing_distance( @pytest.mark.parametrize(("unit", "expected"), [("m", 343834), ("km", 343.834)]) def test_add_link_distances_no_da( - self, my_caplog, model_data_factory_w_params: ModelDataFactory, unit, expected + self, + mocker, + my_caplog, + model_data_factory_w_params: ModelDataFactory, + unit, + expected, ): - _default_distance_unit = model_data_factory_w_params.config["distance_unit"] - model_data_factory_w_params.config["distance_unit"] = unit + mocker.patch.object(ModelDataFactory, "config.distance_unit", return_value=unit) model_data_factory_w_params.clean_data_from_undefined_members() model_data_factory_w_params.dataset["latitude"] = ( pd.Series({"A": 51.507222, "B": 48.8567}) @@ -220,7 +224,6 @@ def test_add_link_distances_no_da( del model_data_factory_w_params.dataset["distance"] model_data_factory_w_params.add_link_distances() - model_data_factory_w_params.config["distance_unit"] = _default_distance_unit assert "Link distance matrix automatically computed" in my_caplog.text assert ( model_data_factory_w_params.dataset["distance"].dropna("techs")