feat!: validate yaml model

Use yaml classes for yaml validation.

BREAKING CHANGE: Validation is more strict than before.

Refs: ECALC-1566

docs/docs/changelog/next.md

@@ -18,4 +18,8 @@ sidebar_position: -1002
 
 ## Breaking changes
 
-
+- This version includes a rewrite of the yaml validation, which should catch more errors than before.
+
+    Main changes:
+    - Economics has been removed. TAX, PRICE and QUOTA keywords will now give an error.
+    - Misplaced keywords will now cause an error instead of being ignored.

pyproject.toml

@@ -78,6 +78,7 @@ coverage = {version="^7.2.5", extras=["toml"]}
 pdoc = "^14.5.1"
 nbconvert = "^7.4.0"
 pytest-xdist = "^3.3.0"
+inline-snapshot = "^0.12.1"
 
 [tool.poetry.extras]
 notebooks = ["jupyter", "matplotlib"]

src/ecalc_cli/main.py

@@ -7,6 +7,7 @@
 from ecalc_cli.commands.run import run
 from ecalc_cli.commands.selftest import selftest
 from ecalc_cli.logger import CLILogConfigurator, LogLevel, logger
+from libecalc.presentation.yaml.model import ModelValidationException
 from libecalc.presentation.yaml.validation_errors import DataValidationError
 
 app = typer.Typer(name="ecalc")
@@ -77,6 +78,8 @@ def main():
     try:
         logger.info("Logging started")
         app()
+    except ModelValidationException as mve:
+        logger.error(str(mve))
     except DataValidationError as de:
         logger.error(de.extended_message)
     except Exception as e:

src/libecalc/fixtures/cases/venting_emitters/venting_emitter_yaml.py

@@ -1,24 +1,21 @@
+from io import StringIO
 from pathlib import Path
 from typing import List
 
-import yaml
-
 from libecalc.common.time_utils import Frequency
 from libecalc.common.units import Unit
 from libecalc.common.utils.rates import RateType
-from libecalc.dto import ResultOptions
 from libecalc.fixtures.case_types import DTOCase
-from libecalc.presentation.yaml.mappers.variables_mapper import map_yaml_to_variables
-from libecalc.presentation.yaml.model import FileResourceService
-from libecalc.presentation.yaml.parse_input import map_yaml_to_dto
-from libecalc.presentation.yaml.yaml_models.pyyaml_yaml_model import PyYamlYamlModel
+from libecalc.presentation.yaml.model import FileResourceService, YamlModel
+from libecalc.presentation.yaml.yaml_entities import ResourceStream
 from libecalc.presentation.yaml.yaml_types.emitters.yaml_venting_emitter import (
     YamlVentingType,
 )
 from libecalc.presentation.yaml.yaml_types.yaml_stream_conditions import (
     YamlEmissionRateUnits,
     YamlOilRateUnits,
 )
+from tests.libecalc.input.mappers.test_model_mapper import OverridableStreamConfigurationService
 
 
 def venting_emitter_yaml_factory(
@@ -83,25 +80,18 @@ def venting_emitter_yaml_factory(
 
         """
 
-    yaml_text = yaml.safe_load(input_text)
-    configuration = PyYamlYamlModel(
-        internal_datamodel=yaml_text,
-        name="venting_emitters",
-        instantiated_through_read=True,
+    configuration_service = OverridableStreamConfigurationService(
+        stream=ResourceStream(
+            name="venting_emitters",
+            stream=StringIO(input_text),
+        )
     )
-    resources = FileResourceService._read_resources(configuration=configuration, working_directory=path)
-    variables = map_yaml_to_variables(
-        configuration,
-        resources=resources,
-        result_options=ResultOptions(
-            start=configuration.start,
-            end=configuration.end,
-            output_frequency=Frequency.YEAR,
-        ),
+    resource_service = FileResourceService(working_directory=path)
+    model = YamlModel(
+        configuration_service=configuration_service, resource_service=resource_service, output_frequency=Frequency.YEAR
     )
 
-    yaml_model = map_yaml_to_dto(configuration=configuration, resources=resources)
-    return DTOCase(ecalc_model=yaml_model, variables=variables)
+    return DTOCase(ecalc_model=model.dto, variables=model.variables)
 
 
 def create_fuel_consumers(include_fuel_consumers: bool) -> str:

src/libecalc/presentation/yaml/model.py

@@ -1,7 +1,8 @@
 import abc
 from datetime import datetime
 from pathlib import Path
-from typing import Callable, Dict, Optional, Protocol
+from textwrap import indent
+from typing import Callable, Dict, List, Optional, Protocol
 
 from libecalc.common.errors.exceptions import EcalcError, InvalidResourceHeaderException
 from libecalc.common.logger import logger
@@ -14,11 +15,18 @@
 )
 from libecalc.presentation.yaml.mappers.variables_mapper import map_yaml_to_variables
 from libecalc.presentation.yaml.parse_input import map_yaml_to_dto
+from libecalc.presentation.yaml.validation_errors import DtoValidationError, ModelValidationError
 from libecalc.presentation.yaml.yaml_entities import (
     Resource,
     ResourceStream,
 )
 from libecalc.presentation.yaml.yaml_models.yaml_model import ReaderType, YamlConfiguration, YamlValidator
+from libecalc.presentation.yaml.yaml_validation_context import (
+    ModelContext,
+    ModelName,
+    YamlModelValidationContext,
+    YamlModelValidationContextNames,
+)
 
 
 class ResourceService(Protocol):
@@ -81,6 +89,29 @@ def get_configuration(self) -> YamlValidator:
             return main_yaml_model
 
 
+class InvalidResourceException(Exception):
+    pass
+
+
+class ModelValidationException(Exception):
+    def __init__(self, errors: List[ModelValidationError]):
+        self._errors = errors
+        super().__init__("Model is not valid")
+
+    def error_count(self) -> int:
+        return len(self._errors)
+
+    def errors(self) -> List[ModelValidationError]:
+        return self._errors
+
+    def __str__(self):
+        msg = "Validation error\n\n"
+        errors = "\n\n".join(map(str, self._errors))
+        errors = indent(errors, "\t")
+        msg += errors
+        return msg
+
+
 class YamlModel:
     """
     Class representing both the yaml and the resources.
@@ -103,10 +134,10 @@ def __init__(
         output_frequency: Frequency,
     ) -> None:
         self._output_frequency = output_frequency
-        configuration = configuration_service.get_configuration()
-        self.resources = resource_service.get_resources(configuration)
-        self.dto = map_yaml_to_dto(configuration=configuration, resources=self.resources)
-        self._configuration = configuration
+        self._configuration = configuration_service.get_configuration()
+        self.resources = resource_service.get_resources(self._configuration)
+        self.is_valid_for_run()
+        self.dto = map_yaml_to_dto(configuration=self._configuration, resources=self.resources)
 
     @property
     def start(self) -> Optional[datetime]:
@@ -133,3 +164,53 @@ def result_options(self) -> ResultOptions:
     @property
     def graph(self) -> ComponentGraph:
         return self.dto.get_graph()
+
+    def _find_resource_from_name(self, filename: str) -> Optional[Resource]:
+        return self.resources.get(filename)
+
+    def _get_token_references(self, yaml_model: YamlValidator) -> List[str]:
+        token_references = []
+        for time_series in yaml_model.time_series:
+            resource = self._find_resource_from_name(time_series.file)
+
+            if resource is None:
+                # Don't add any tokens if the resource is not found
+                continue
+
+            try:
+                headers = resource.headers
+                for header in headers:
+                    token_references.append(f"{time_series.name};{header}")
+            except InvalidResourceException:
+                # Don't add any tokens if resource is invalid (unable to read header)
+                continue
+
+        for reference in yaml_model.variables:
+            token_references.append(f"$var.{reference}")
+
+        return token_references
+
+    @staticmethod
+    def _get_model_types(yaml_model: YamlValidator) -> Dict["ModelName", "ModelContext"]:
+        models = [*yaml_model.models, *yaml_model.facility_inputs]
+        model_types: Dict[ModelName, ModelContext] = {}
+        for model in models:
+            if hasattr(model, "name"):
+                model_types[model.name] = model
+        return model_types
+
+    def _get_validation_context(self, yaml_model: YamlValidator) -> YamlModelValidationContext:
+        return {
+            YamlModelValidationContextNames.resource_file_names: [name for name, resource in self.resources.items()],
+            YamlModelValidationContextNames.expression_tokens: self._get_token_references(yaml_model=yaml_model),
+            YamlModelValidationContextNames.model_types: self._get_model_types(yaml_model=yaml_model),
+        }
+
+    def is_valid_for_run(self) -> bool:
+        try:
+            # Validate model
+            validation_context = self._get_validation_context(yaml_model=self._configuration)
+            self._configuration.validate(validation_context)
+            return True
+        except DtoValidationError as e:
+            raise ModelValidationException(errors=e.errors()) from e