diff --git a/hydrolib/core/dflowfm/ini/models.py b/hydrolib/core/dflowfm/ini/models.py index 484ad755e..1459df78c 100644 --- a/hydrolib/core/dflowfm/ini/models.py +++ b/hydrolib/core/dflowfm/ini/models.py @@ -2,6 +2,7 @@ from abc import ABC from enum import Enum from math import isnan +from re import compile from typing import Any, Callable, List, Literal, Optional, Set, Type, Union from pydantic.v1 import Extra, Field, root_validator @@ -46,6 +47,9 @@ class INIBasedModel(BaseModel, ABC): _header: str _file_path_style_converter = FilePathStyleConverter() + _scientific_notation_regex = compile( + r"([\d.]+)([dD])([+-]?\d{1,3})" + ) # matches a float: 1d9, 1D-3, 1.D+4, etc. class Config: extra = Extra.ignore @@ -123,6 +127,24 @@ def comments_matches_has_comments(cls, v): v = None return v + @validator("*", pre=True, allow_reuse=True) + def replace_fortran_scientific_notation_for_floats(cls, value, field): + if field.type_ != float: + return value + + return cls._replace_fortran_scientific_notation(value) + + @classmethod + def _replace_fortran_scientific_notation(cls, value): + if isinstance(value, str): + return cls._scientific_notation_regex.sub(r"\1e\3", value) + if isinstance(value, list): + for i, v in enumerate(value): + if isinstance(v, str): + value[i] = cls._scientific_notation_regex.sub(r"\1e\3", v) + + return value + @classmethod def validate(cls: Type["INIBasedModel"], value: Any) -> "INIBasedModel": if isinstance(value, Section): diff --git a/hydrolib/core/dflowfm/ini/parser.py b/hydrolib/core/dflowfm/ini/parser.py index 36d5d8811..981637cde 100644 --- a/hydrolib/core/dflowfm/ini/parser.py +++ b/hydrolib/core/dflowfm/ini/parser.py @@ -1,4 +1,3 @@ -import re from enum import IntEnum from pathlib import Path from typing import Callable, Dict, List, Optional, Tuple, Union @@ -372,27 +371,8 @@ def parse(cls, filepath: Path, config: ParserConfig = None) -> Document: config = ParserConfig() parser = cls(config) - progline = re.compile( - r"^([^#]*=\s*)([^#]*)(#.*)?" - ) # matches whole line: "Field = Value Maybe more # optional comment" - progfloat = re.compile( - r"([\d.]+)([dD])([+\-]?\d{1,3})" - ) # matches a float value: 1d9, 1D-3, 1.D+4, etc. - with filepath.open(encoding="utf8") as f: for line in f: - # Replace Fortran scientific notation for doubles - # Match number d/D +/- number (e.g. 1d-05 or 1.23D+01 or 1.d-4) - match = progline.match(line) - if match: # Only process value - line = ( - match.group(1) - + progfloat.sub(r"\1e\3", match.group(2)) - + str(match.group(3) or "") - ) - else: # Process full line - line = progfloat.sub(r"\1e\3", line) - parser.feed_line(line) return parser.finalize() diff --git a/tests/dflowfm/ini/test_ini.py b/tests/dflowfm/ini/test_ini.py index 9784909cc..679cfb7c1 100644 --- a/tests/dflowfm/ini/test_ini.py +++ b/tests/dflowfm/ini/test_ini.py @@ -13,7 +13,6 @@ Property, Section, ) -from hydrolib.core.dflowfm.ini.models import INIBasedModel from hydrolib.core.dflowfm.ini.parser import ( Parser, ParserConfig, @@ -379,12 +378,6 @@ def test_is_datarow(self, line: str, config: ParserConfig, expected_result: bool [ ("someParam = 1.0 # comment text", "someparam", "1.0"), ("someParam = 1.0", "someparam", "1.0"), - ("someParam = 1d0 # comment text", "someparam", "1e0"), - ("someParam = 1d-2", "someparam", "1e-2"), - ("someParam = 1d+2", "someparam", "1e+2"), - ("someParam = 1.d+2", "someparam", "1.e+2"), - ("someParam = -1.d-2", "someparam", "-1.e-2"), - ("someParam1D2D = -1.d-2", "someparam1d2d", "-1.e-2"), ], ) def test_float_values( diff --git a/tests/dflowfm/ini/test_models.py b/tests/dflowfm/ini/test_models.py index 750a6960d..cc7a794c7 100644 --- a/tests/dflowfm/ini/test_models.py +++ b/tests/dflowfm/ini/test_models.py @@ -1,9 +1,10 @@ from math import nan +from typing import List import pytest from pydantic.v1.error_wrappers import ValidationError -from hydrolib.core.dflowfm.ini.models import DataBlockINIBasedModel +from hydrolib.core.dflowfm.ini.models import DataBlockINIBasedModel, INIBasedModel from ...utils import error_occurs_only_once @@ -39,3 +40,111 @@ def test_datablock_with_multiple_nans_should_only_give_error_once(self): expected_message = "NaN is not supported in datablocks." assert error_occurs_only_once(expected_message, str(error.value)) + + +class TestINIBasedModel: + class INIBasedModelTest(INIBasedModel): + id: str + float_value: float + float_values: List[float] + + _random_string: str = "randomString" + _random_float: float = 123.456 + _random_list_of_floats: List[float] = [12.34, 56.78] + + @pytest.mark.parametrize("string_value", ["1d0", "1d-2", "1d+2", "1.d+2", "-1.d-2"]) + def test_scientific_notation_for_string_field_is_parsed_as_string( + self, string_value: str + ): + + test_model = self.INIBasedModelTest( + id=string_value, + float_value=self._random_float, + float_values=self._random_list_of_floats, + ) + + assert test_model.id == string_value + assert test_model.float_value == pytest.approx(self._random_float) + assert test_model.float_values == pytest.approx(self._random_list_of_floats) + + @pytest.mark.parametrize( + "float_as_string, expected_value", + [ + ("1d0", 1e0), + ("1d-2", 1e-2), + ("1d+2", 1e2), + ("1.d+2", 1.0e2), + ("-1.d-2", -1.0e-2), + ], + ) + def test_scientific_notation_for_float_field_is_parsed_as_float( + self, float_as_string: str, expected_value: float + ): + test_model = self.INIBasedModelTest( + id=self._random_string, + float_value=float_as_string, + float_values=self._random_list_of_floats, + ) + + assert test_model.id == self._random_string + assert test_model.float_value == pytest.approx(expected_value) + assert test_model.float_values == pytest.approx(self._random_list_of_floats) + + @pytest.mark.parametrize( + "floats_as_strings, expected_values", + [ + (["1d0", "1d-2"], [1e0, 1e-2]), + (["1d+2", "1.d+2", "-1.d-2"], [1e2, 1.0e2, -1.0e-2]), + ], + ) + def test_scientific_notation_for_list_of_float_field_is_parsed_as_list_of_floats( + self, floats_as_strings: List[str], expected_values: List[float] + ): + test_model = self.INIBasedModelTest( + id=self._random_string, + float_value=self._random_float, + float_values=floats_as_strings, + ) + + assert test_model.id == self._random_string + assert test_model.float_value == pytest.approx(self._random_float) + assert test_model.float_values == pytest.approx(expected_values) + + def test_setting_string_attribute_with_scientific_notation_correctly_parses_value( + self, + ): + test_model = self.INIBasedModelTest( + id=self._random_string, + float_value=self._random_float, + float_values=self._random_list_of_floats, + ) + + test_model.id = "1d1" + + assert test_model.id == "1d1" + + def test_setting_float_attribute_with_scientific_notation_correctly_parses_value( + self, + ): + test_model = self.INIBasedModelTest( + id=self._random_string, + float_value=self._random_float, + float_values=self._random_list_of_floats, + ) + + test_model.float_value = "1d1" + + assert test_model.float_value == pytest.approx(1e1) + + def test_setting_list_of_floats_attribute_with_scientific_notation_correctly_parses_values( + self, + ): + test_model = self.INIBasedModelTest( + id=self._random_string, + float_value=self._random_float, + float_values=self._random_list_of_floats, + ) + + test_model.float_values = ["1d1", "2d-1"] + + assert test_model.float_values == pytest.approx([1e1, 2e-1]) diff --git a/tests/dflowfm/test_storagenode.py b/tests/dflowfm/test_storagenode.py index 378449d47..0918abb0a 100644 --- a/tests/dflowfm/test_storagenode.py +++ b/tests/dflowfm/test_storagenode.py @@ -162,3 +162,39 @@ def _create_required_storage_node_values(usetable: bool) -> dict: ) return values + + +def test_scientific_notation_in_ini_file_are_correctly_parsed_for_strings_and_floats( + tmp_path, +): + + file_data = """ +[General] + fileVersion = 2.00 + fileType = storageNodes + useStreetStorage = 1 + +[StorageNode] + id = 0D05252 + name = 0D05252 + nodeId = Compartment002 + ManholeId = Manhole002 + bedLevel = -2e0 + area = 0.4096000 + streetLevel = 0.000 + storageType = Reservoir + streetStorageArea = 1d2 + CompartmentShape = Unknown + useTable = False + """ + + temporary_file_path_node_file = tmp_path / "nodeFile.ini" + temporary_file_path_node_file.write_text(file_data) + + storage_node_model = StorageNodeModel(temporary_file_path_node_file) + + storage_node = storage_node_model.storagenode[0] + assert storage_node.id == "0D05252" + assert storage_node.name == "0D05252" + assert storage_node.bedlevel == pytest.approx(-2e0) + assert storage_node.streetstoragearea == pytest.approx(1e2)