Ensure only floats check for Fortran scientific notation when parsing (…

…#626)

refs: #559

hydrolib/core/dflowfm/ini/models.py

@@ -2,6 +2,7 @@
 from abc import ABC
 from enum import Enum
 from math import isnan
+from re import compile
 from typing import Any, Callable, List, Literal, Optional, Set, Type, Union
 
 from pydantic.v1 import Extra, Field, root_validator
@@ -46,6 +47,9 @@ class INIBasedModel(BaseModel, ABC):
 
     _header: str
     _file_path_style_converter = FilePathStyleConverter()
+    _scientific_notation_regex = compile(
+        r"([\d.]+)([dD])([+-]?\d{1,3})"
+    )  # matches a float: 1d9, 1D-3, 1.D+4, etc.
 
     class Config:
         extra = Extra.ignore
@@ -123,6 +127,24 @@ def comments_matches_has_comments(cls, v):
             v = None
         return v
 
+    @validator("*", pre=True, allow_reuse=True)
+    def replace_fortran_scientific_notation_for_floats(cls, value, field):
+        if field.type_ != float:
+            return value
+
+        return cls._replace_fortran_scientific_notation(value)
+
+    @classmethod
+    def _replace_fortran_scientific_notation(cls, value):
+        if isinstance(value, str):
+            return cls._scientific_notation_regex.sub(r"\1e\3", value)
+        if isinstance(value, list):
+            for i, v in enumerate(value):
+                if isinstance(v, str):
+                    value[i] = cls._scientific_notation_regex.sub(r"\1e\3", v)
+
+        return value
+
     @classmethod
     def validate(cls: Type["INIBasedModel"], value: Any) -> "INIBasedModel":
         if isinstance(value, Section):

hydrolib/core/dflowfm/ini/parser.py

@@ -1,4 +1,3 @@
-import re
 from enum import IntEnum
 from pathlib import Path
 from typing import Callable, Dict, List, Optional, Tuple, Union
@@ -372,27 +371,8 @@ def parse(cls, filepath: Path, config: ParserConfig = None) -> Document:
             config = ParserConfig()
         parser = cls(config)
 
-        progline = re.compile(
-            r"^([^#]*=\s*)([^#]*)(#.*)?"
-        )  # matches whole line: "Field = Value Maybe more # optional comment"
-        progfloat = re.compile(
-            r"([\d.]+)([dD])([+\-]?\d{1,3})"
-        )  # matches a float value: 1d9, 1D-3, 1.D+4, etc.
-
         with filepath.open(encoding="utf8") as f:
             for line in f:
-                # Replace Fortran scientific notation for doubles
-                # Match number d/D +/- number (e.g. 1d-05 or 1.23D+01 or 1.d-4)
-                match = progline.match(line)
-                if match:  # Only process value
-                    line = (
-                        match.group(1)
-                        + progfloat.sub(r"\1e\3", match.group(2))
-                        + str(match.group(3) or "")
-                    )
-                else:  # Process full line
-                    line = progfloat.sub(r"\1e\3", line)
-
                 parser.feed_line(line)
 
         return parser.finalize()

tests/dflowfm/ini/test_ini.py

@@ -13,7 +13,6 @@
     Property,
     Section,
 )
-from hydrolib.core.dflowfm.ini.models import INIBasedModel
 from hydrolib.core.dflowfm.ini.parser import (
     Parser,
     ParserConfig,
@@ -379,12 +378,6 @@ def test_is_datarow(self, line: str, config: ParserConfig, expected_result: bool
         [
             ("someParam = 1.0 # comment text", "someparam", "1.0"),
             ("someParam = 1.0", "someparam", "1.0"),
-            ("someParam = 1d0 # comment text", "someparam", "1e0"),
-            ("someParam = 1d-2", "someparam", "1e-2"),
-            ("someParam = 1d+2", "someparam", "1e+2"),
-            ("someParam = 1.d+2", "someparam", "1.e+2"),
-            ("someParam = -1.d-2", "someparam", "-1.e-2"),
-            ("someParam1D2D = -1.d-2", "someparam1d2d", "-1.e-2"),
         ],
     )
     def test_float_values(

tests/dflowfm/ini/test_models.py

@@ -1,9 +1,10 @@
 from math import nan
+from typing import List
 
 import pytest
 from pydantic.v1.error_wrappers import ValidationError
 
-from hydrolib.core.dflowfm.ini.models import DataBlockINIBasedModel
+from hydrolib.core.dflowfm.ini.models import DataBlockINIBasedModel, INIBasedModel
 
 from ...utils import error_occurs_only_once
 
@@ -39,3 +40,111 @@ def test_datablock_with_multiple_nans_should_only_give_error_once(self):
 
         expected_message = "NaN is not supported in datablocks."
         assert error_occurs_only_once(expected_message, str(error.value))
+
+
+class TestINIBasedModel:
+    class INIBasedModelTest(INIBasedModel):
+        id: str
+        float_value: float
+        float_values: List[float]
+
+    _random_string: str = "randomString"
+    _random_float: float = 123.456
+    _random_list_of_floats: List[float] = [12.34, 56.78]
+
+    @pytest.mark.parametrize("string_value", ["1d0", "1d-2", "1d+2", "1.d+2", "-1.d-2"])
+    def test_scientific_notation_for_string_field_is_parsed_as_string(
+        self, string_value: str
+    ):
+
+        test_model = self.INIBasedModelTest(
+            id=string_value,
+            float_value=self._random_float,
+            float_values=self._random_list_of_floats,
+        )
+
+        assert test_model.id == string_value
+        assert test_model.float_value == pytest.approx(self._random_float)
+        assert test_model.float_values == pytest.approx(self._random_list_of_floats)
+
+    @pytest.mark.parametrize(
+        "float_as_string, expected_value",
+        [
+            ("1d0", 1e0),
+            ("1d-2", 1e-2),
+            ("1d+2", 1e2),
+            ("1.d+2", 1.0e2),
+            ("-1.d-2", -1.0e-2),
+        ],
+    )
+    def test_scientific_notation_for_float_field_is_parsed_as_float(
+        self, float_as_string: str, expected_value: float
+    ):
+        test_model = self.INIBasedModelTest(
+            id=self._random_string,
+            float_value=float_as_string,
+            float_values=self._random_list_of_floats,
+        )
+
+        assert test_model.id == self._random_string
+        assert test_model.float_value == pytest.approx(expected_value)
+        assert test_model.float_values == pytest.approx(self._random_list_of_floats)
+
+    @pytest.mark.parametrize(
+        "floats_as_strings, expected_values",
+        [
+            (["1d0", "1d-2"], [1e0, 1e-2]),
+            (["1d+2", "1.d+2", "-1.d-2"], [1e2, 1.0e2, -1.0e-2]),
+        ],
+    )
+    def test_scientific_notation_for_list_of_float_field_is_parsed_as_list_of_floats(
+        self, floats_as_strings: List[str], expected_values: List[float]
+    ):
+        test_model = self.INIBasedModelTest(
+            id=self._random_string,
+            float_value=self._random_float,
+            float_values=floats_as_strings,
+        )
+
+        assert test_model.id == self._random_string
+        assert test_model.float_value == pytest.approx(self._random_float)
+        assert test_model.float_values == pytest.approx(expected_values)
+
+    def test_setting_string_attribute_with_scientific_notation_correctly_parses_value(
+        self,
+    ):
+        test_model = self.INIBasedModelTest(
+            id=self._random_string,
+            float_value=self._random_float,
+            float_values=self._random_list_of_floats,
+        )
+
+        test_model.id = "1d1"
+
+        assert test_model.id == "1d1"
+
+    def test_setting_float_attribute_with_scientific_notation_correctly_parses_value(
+        self,
+    ):
+        test_model = self.INIBasedModelTest(
+            id=self._random_string,
+            float_value=self._random_float,
+            float_values=self._random_list_of_floats,
+        )
+
+        test_model.float_value = "1d1"
+
+        assert test_model.float_value == pytest.approx(1e1)
+
+    def test_setting_list_of_floats_attribute_with_scientific_notation_correctly_parses_values(
+        self,
+    ):
+        test_model = self.INIBasedModelTest(
+            id=self._random_string,
+            float_value=self._random_float,
+            float_values=self._random_list_of_floats,
+        )
+
+        test_model.float_values = ["1d1", "2d-1"]
+
+        assert test_model.float_values == pytest.approx([1e1, 2e-1])

tests/dflowfm/test_storagenode.py

@@ -162,3 +162,39 @@ def _create_required_storage_node_values(usetable: bool) -> dict:
         )
 
     return values
+
+
+def test_scientific_notation_in_ini_file_are_correctly_parsed_for_strings_and_floats(
+    tmp_path,
+):
+
+    file_data = """        
+[General]
+    fileVersion           = 2.00
+    fileType              = storageNodes
+    useStreetStorage      = 1
+
+[StorageNode]
+    id                    = 0D05252
+    name                  = 0D05252
+    nodeId                = Compartment002
+    ManholeId             = Manhole002
+    bedLevel              = -2e0
+    area                  = 0.4096000
+    streetLevel           = 0.000
+    storageType           = Reservoir
+    streetStorageArea     = 1d2
+    CompartmentShape      = Unknown
+    useTable              = False            
+    """
+
+    temporary_file_path_node_file = tmp_path / "nodeFile.ini"
+    temporary_file_path_node_file.write_text(file_data)
+
+    storage_node_model = StorageNodeModel(temporary_file_path_node_file)
+
+    storage_node = storage_node_model.storagenode[0]
+    assert storage_node.id == "0D05252"
+    assert storage_node.name == "0D05252"
+    assert storage_node.bedlevel == pytest.approx(-2e0)
+    assert storage_node.streetstoragearea == pytest.approx(1e2)