Add config obj replace schemas (#717)

* Added schemas for data tables, parameters, techs, nodes and model definition.

* changed to / from to link_to / link_from

---------

Co-authored-by: Bryn Pickering <17178478+brynpickering@users.noreply.github.com>

Author: irm-codebase

CHANGELOG.md

@@ -35,7 +35,9 @@ This change has occurred to avoid confusion between data "sources" and model ene
 
 ### Internal changes
 
-|changed| Model configuration now uses `pydantic`.
+|changed| updated transmission technologies to/from -> link_to/link_from to avoid conflicts with protected `python` terminology.
+
+|changed| Model configuration, data tables, techs/nodes data, math and general model definition now uses `pydantic`.
 
 |changed| Model definition reading is now defined in a single place (preprocess/model_definition.py).
 

docs/advanced/constraints.md

@@ -133,14 +133,14 @@ To force unidirectionality for a given technology along a given link, you have t
 ```yaml
 techs:
   region1_to_region2:
-    from: region1
-    to: region2
+    link_from: region1
+    link_to: region2
     base_tech: transmission
     one_way: true
 ```
 
 This will only allow transmission from `region1` to `region2`.
-To swap the direction, `to` and `from` must be swapped.
+To swap the direction, `link_to` and `link_from` must be swapped.
 
 ## Per-distance transmission constraints
 

docs/creating/nodes.md

@@ -36,7 +36,7 @@ In the above example, `node_flow_out_max` at `region1` could be used to create [
 ## Understanding node-level parameters
 
 `techs` is the only required parameter in a node.
-This can be an empty dictionary (`techs: {}`), which you may use if your node is just a junction for transmission technologies (which you [**do not define in the `techs` of a node**](techs.md#transmission-technologies) - rather, you define them as separate technologies that connect `from` one node `to` another node).
+This can be an empty dictionary (`techs: {}`), which you may use if your node is just a junction for transmission technologies (which you [**do not define in the `techs` of a node**](techs.md#transmission-technologies) - rather, you define them as separate technologies that connect a `link_from` node with a `link_to` node).
 
 !!! info "See also"
 

docs/creating/techs.md

@@ -85,14 +85,14 @@ Instead, you associate transmission technologies with nodes in `techs`:
 ```yaml
 techs:
   ac_transmission:
-    from: region1  # (1)!
-    to: region2
+    link_from: region1  # (1)!
+    link_to: region2
     flow_cap_max: 100
     ...
 ```
 
-1. The region you specify in `from` or `to` is interchangeable unless you set the parameter `one_way: true`.
-In that case, flow along the transmission line is only allowed from the `from` region to the `to` region.
+1. The region you specify in `link_from` or `link_to` is interchangeable unless you set the parameter `one_way: true`.
+In that case, flow along the transmission line is only allowed from the `link_from` node to the `link_to` node.
 
 ## Understanding tech-level parameters
 
@@ -103,7 +103,7 @@ There are _required_ parameters according to the technology `base_tech`:
 * `supply`: `base_tech` and `carrier_out`.
 * `demand`: `base_tech` and `carrier_in`.
 * `storage`: `base_tech` and `carrier_out` and `carrier_in`.
-* `transmission`: `base_tech` and `carrier_out`, `carrier_in`, `to`, and `from`.
+* `transmission`: `base_tech` and `carrier_out`, `carrier_in`, `link_to`, and `link_from`.
 * `conversion`: `base_tech` and `carrier_out` and `carrier_in`.
 
 For `storage` and `transmission`, it may seem like unnecessary repetition to define both `carrier_out` and `carrier_in` as they are likely the same value.

docs/examples/loading_tabular_data.py

@@ -78,8 +78,8 @@
 #     base_tech: transmission
 #     carrier_in: electricity
 #     carrier_out: electricity
-#     from: A
-#     to: B
+#     link_from: A
+#     link_to: B
 #     flow_cap_max: 8
 #
 # nodes:
@@ -136,8 +136,8 @@
     base_tech: transmission
     carrier_in: electricity
     carrier_out: electricity
-    from: A
-    to: B
+    link_from: A
+    link_to: B
     flow_cap_max: 8
 
 nodes:
@@ -215,8 +215,8 @@
         "demand_tech": {"base_tech": "demand"},
         "transmission_tech": {
             "base_tech": "transmission",
-            "from": "A",
-            "to": "B",
+            "link_from": "A",
+            "link_to": "B",
             "flow_cap_max": 8,
         },
     }
@@ -483,8 +483,8 @@
 #     base_tech: transmission
 #     carrier_in: electricity
 #     carrier_out: electricity
-#     from: A
-#     to: B
+#     link_from: A
+#     link_to: B
 #     flow_cap_max: 8
 #
 # nodes:
@@ -532,8 +532,8 @@
 #     base_tech: transmission
 #     carrier_in: electricity
 #     carrier_out: electricity
-#     from: A
-#     to: B
+#     link_from: A
+#     link_to: B
 #     flow_cap_max: 8
 #
 # nodes:

docs/hooks/dummy_model/model.yaml

@@ -14,8 +14,11 @@ nodes:
 
 techs:
   tech_transmission:
-    from: A
-    to: B
+    base_tech: transmission
+    carrier_in: foo
+    carrier_out: foo
+    link_from: A
+    link_to: B
 
 data_tables:
   techs:

docs/hooks/dummy_model/techs.csv

@@ -54,8 +54,8 @@ transmission_tech,lifetime,25
 transmission_tech,one_way,1.0
 transmission_tech,flow_cap_max,1.0
 transmission_tech,flow_cap_min,1.0
-transmission_tech,from,A
-transmission_tech,to,B
+transmission_tech,link_from,A
+transmission_tech,link_to,B
 demand_tech,sink_unit,per_cap
 conversion_tech,base_tech,conversion
 conversion_tech,cap_method,integer
@@ -64,4 +64,4 @@ supply_tech,cap_method,integer
 supply_tech,source_unit,per_area
 storage_tech,base_tech,storage
 transmission_tech,base_tech,transmission
-demand_tech,base_tech,demand
+demand_tech,base_tech,demand

docs/hooks/generate_readable_schema.py

@@ -14,16 +14,17 @@
 import jsonschema2md
 from mkdocs.structure.files import File
 
-from calliope import config
+from calliope.schemas import config_schema, data_table_schema, math_schema
 from calliope.util import schema
 
 TEMPDIR = tempfile.TemporaryDirectory()
 
+# FIXME: should only use pydantic models instead of YAML
 SCHEMAS = {
-    "config_schema": config.CalliopeConfig().model_no_ref_schema(),
+    "config_schema": config_schema.CalliopeConfig.model_no_ref_schema(),
     "model_schema": schema.MODEL_SCHEMA,
-    "math_schema": schema.MATH_SCHEMA,
-    "data_table_schema": schema.DATA_TABLE_SCHEMA,
+    "math_schema": math_schema.CalliopeMathDef.model_no_ref_schema(),
+    "data_table_schema": data_table_schema.CalliopeDataTable.model_no_ref_schema(),
 }
 
 

docs/installation.md

@@ -7,7 +7,7 @@ Calliope is tested on Linux, macOS, and Windows.
 Running Calliope requires four things:
 
 1. The Python programming language, version {{ min_python_version }} to {{ max_python_version }}.
-2. A number of Python add-on modules including [Pyomo](https://www.pyomo.org/), [Pandas](https://pandas.pydata.org/) and [Xarray](https://xarray.dev/).
+2. A number of Python add-on modules including [Pyomo](https://www.pyomo.org/), [Pandas](https://pandas.pydata.org/) and [Xarray](https://docs.xarray.dev/).
 3. An optimisation solver: Calliope has been tested with CBC, GLPK, Gurobi, and CPLEX. Any other solver that is compatible with Pyomo should also work.
 4. The Calliope software itself.
 

docs/migrating.md

@@ -270,7 +270,7 @@ We have changed the nesting structure for defining technology costs so they are
 ### `links` → transmission links defined in `techs`
 
 The top-level key `links` no longer exists.
-Instead, links are defined as separate transmission technologies in `techs`, including `to`/`from` keys:
+Instead, links are defined as separate transmission technologies in `techs`, including `link_to`/`link_from` keys:
 
 === "v0.6"
 
@@ -298,13 +298,13 @@ Instead, links are defined as separate transmission technologies in `techs`, inc
     ```yaml
     techs:
       x1_to_x2_ac_transmission:
-        from: X1
-        to: X2
+        link_from: X1
+        link_to: X2
         base_tech: transmission
         flow_cap_max: 10
       x1_to_x2_dc_transmission:
-        from: X1
-        to: X2
+        link_from: X1
+        link_to: X2
         base_tech: transmission
         flow_cap_max: 5
     ```

docs/user_defined_math/components.md

@@ -36,7 +36,7 @@ Without a `where` string, all valid members (according to the `definition_matrix
     If a value for a valid variable member is undefined in the referenced parameter, the decision variable will be unbounded for this member.
 1. It can be deactivated so that it does not appear in the built optimisation problem by setting `active: false`.
 1. It can take on a `default` value that will be used in math operations to avoid `NaN` values creeping in.
-The default value should be set such that it has no impact on the optimisation problem if it is included (most of the time, this means setting it to zero).
+The default value should be set such that it has no impact on the optimisation problem if it is included (most of the time, this means `NaN`).
 
 ## Global Expressions
 
@@ -63,7 +63,7 @@ Without a `where` string, all valid members (according to the `definition_matrix
 The equation expressions do _not_ have comparison operators; those are reserved for [constraints](#constraints)
 1. It can be deactivated so that it does not appear in the built optimisation problem by setting `active: false`.
 1. It can take on a `default` value that will be used in math operations to avoid `NaN` values creeping in.
-The default value should be set such that it has no impact on the optimisation problem if it is included (most of the time, this means setting it to zero).
+The default value should be set such that it has no impact on the optimisation problem if it is included (most of the time, this means `NaN`).
 
 ## Constraints
 

pyproject.toml

@@ -54,7 +54,7 @@ max-complexity = 10
 
 # Ignore `E402` (import violations) and `F401` (unused imports) in all `__init__.py` files
 [tool.ruff.lint.per-file-ignores]
-"__init__.py" = ["E402", "F401"]
+"__init__.py" = ["E402", "F401", "D104"]
 "*.ipynb" = ["E402"]
 "tests/*" = ["D"]
 "docs/examples/*" = ["D"]

src/calliope/backend/__init__.py

@@ -15,12 +15,12 @@
 from calliope.preprocess import CalliopeMath
 
 if TYPE_CHECKING:
-    from calliope import config
     from calliope.backend.backend_model import BackendModel
+    from calliope.schemas import config_schema
 
 
 def get_model_backend(
-    build_config: "config.Build", data: xr.Dataset, math: CalliopeMath
+    build_config: "config_schema.Build", data: xr.Dataset, math: CalliopeMath
 ) -> "BackendModel":
     """Assign a backend using the given configuration.
 

src/calliope/backend/backend_model.py

@@ -26,12 +26,13 @@
 import numpy as np
 import xarray as xr
 
-from calliope import config, exceptions
+from calliope import exceptions
 from calliope.attrdict import AttrDict
 from calliope.backend import helper_functions, parsing
 from calliope.exceptions import warn as model_warn
 from calliope.io import load_config, to_yaml
 from calliope.preprocess.model_math import ORDERED_COMPONENTS_T, CalliopeMath
+from calliope.schemas import config_schema
 from calliope.util.schema import MODEL_SCHEMA, extract_from_schema
 
 if TYPE_CHECKING:
@@ -66,7 +67,7 @@ class BackendModelGenerator(ABC):
     _PARAM_TYPE = extract_from_schema(MODEL_SCHEMA, "x-type")
 
     def __init__(
-        self, inputs: xr.Dataset, math: CalliopeMath, build_config: config.Build
+        self, inputs: xr.Dataset, math: CalliopeMath, build_config: config_schema.Build
     ):
         """Abstract base class to build a representation of the optimisation problem.
 
@@ -606,7 +607,7 @@ def __init__(
         self,
         inputs: xr.Dataset,
         math: CalliopeMath,
-        build_config: config.Build,
+        build_config: config_schema.Build,
         instance: T,
     ) -> None:
         """Abstract base class to build backend models that interface with solvers.

src/calliope/backend/gurobi_backend_model.py

@@ -14,11 +14,11 @@
 import pandas as pd
 import xarray as xr
 
-from calliope import config
 from calliope.backend import backend_model, parsing
 from calliope.exceptions import BackendError, BackendWarning
 from calliope.exceptions import warn as model_warn
 from calliope.preprocess import CalliopeMath
+from calliope.schemas import config_schema
 
 if importlib.util.find_spec("gurobipy") is not None:
     import gurobipy
@@ -43,7 +43,7 @@ class GurobiBackendModel(backend_model.BackendModel):
     """gurobipy-specific backend functionality."""
 
     def __init__(
-        self, inputs: xr.Dataset, math: CalliopeMath, build_config: config.Build
+        self, inputs: xr.Dataset, math: CalliopeMath, build_config: config_schema.Build
     ) -> None:
         """Gurobi solver interface class.
 

src/calliope/backend/latex_backend_model.py

@@ -12,10 +12,10 @@
 import pandas as pd
 import xarray as xr
 
-from calliope import config
 from calliope.backend import backend_model, parsing
 from calliope.exceptions import ModelError
 from calliope.preprocess import CalliopeMath
+from calliope.schemas import config_schema
 
 ALLOWED_MATH_FILE_FORMATS = Literal["tex", "rst", "md"]
 LOGGER = logging.getLogger(__name__)
@@ -306,7 +306,7 @@ def __init__(
         self,
         inputs: xr.Dataset,
         math: CalliopeMath,
-        build_config: config.Build,
+        build_config: config_schema.Build,
         include: Literal["all", "valid"] = "all",
     ) -> None:
         """Interface to build a string representation of the mathematical formulation using LaTeX math notation.

src/calliope/backend/pyomo_backend_model.py

@@ -26,10 +26,10 @@
 from pyomo.opt import SolverFactory  # type: ignore
 from pyomo.util.model_size import build_model_size_report  # type: ignore
 
-from calliope import config
 from calliope.exceptions import BackendError, BackendWarning
 from calliope.exceptions import warn as model_warn
 from calliope.preprocess import CalliopeMath
+from calliope.schemas import config_schema
 from calliope.util.logging import LogWriter
 
 from . import backend_model, parsing
@@ -60,7 +60,7 @@ class PyomoBackendModel(backend_model.BackendModel):
     """Pyomo-specific backend functionality."""
 
     def __init__(
-        self, inputs: xr.Dataset, math: CalliopeMath, build_config: config.Build
+        self, inputs: xr.Dataset, math: CalliopeMath, build_config: config_schema.Build
     ) -> None:
         """Pyomo solver interface class.
 

src/calliope/backend/where_parser.py

@@ -13,9 +13,9 @@
 import xarray as xr
 from typing_extensions import NotRequired, TypedDict
 
-from calliope import config
 from calliope.backend import expression_parser
 from calliope.exceptions import BackendError
+from calliope.schemas import config_schema
 from calliope.util import tools
 
 if TYPE_CHECKING:
@@ -35,7 +35,7 @@ class EvalAttrs(TypedDict):
     helper_functions: dict[str, Callable]
     apply_where: NotRequired[bool]
     references: NotRequired[set]
-    build_config: config.Build
+    build_config: config_schema.Build
 
 
 class EvalWhere(expression_parser.EvalToArrayStr):

src/calliope/cli.py

@@ -193,9 +193,13 @@ def _run_setup_model(model_file, scenario, model_format, override_dict):
                 f'extension for "{model_file}". Set format explicitly with '
                 "--model_format."
             )
+    if isinstance(override_dict, str):
+        overrides = io.read_rich_yaml(override_dict)
+    else:
+        overrides = io.AttrDict()
 
     if model_format == "yaml":
-        model = Model(model_file, scenario=scenario, override_dict=override_dict)
+        model = Model(model_file, scenario=scenario, override_dict=overrides)
     elif model_format == "netcdf":
         if scenario is not None or override_dict is not None:
             raise ValueError(