From 4c64a2d9cf3bf4fcf3b3d0b07ff5f44268a9eb81 Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Wed, 24 Apr 2024 09:01:18 +0200 Subject: [PATCH 01/31] feat: (622) write message on reading file and setting variables based on config.extra --- hydrolib/core/dflowfm/ini/models.py | 17 ++++++ tests/dflowfm/test_mdu.py | 90 +++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/hydrolib/core/dflowfm/ini/models.py b/hydrolib/core/dflowfm/ini/models.py index 484ad755e..d2bb5897b 100644 --- a/hydrolib/core/dflowfm/ini/models.py +++ b/hydrolib/core/dflowfm/ini/models.py @@ -50,6 +50,23 @@ class INIBasedModel(BaseModel, ABC): class Config: extra = Extra.ignore arbitrary_types_allowed = False + + def __init__(self, **data): + for key, _ in data.items(): + self._notify_ingored_field(key) + super().__init__(**data) + + def __setattr__(self, name, value): + self._notify_ingored_field(name) + super().__setattr__(name, value) + + def _notify_ingored_field(self, name): + if name not in self.__fields__ and name not in self._exclude_fields(): + if self.Config.extra == Extra.allow: + print(f"Unknown keyword detected in '{self._header}', '{name}', keyword will be kept in memory but will have no validation.") + else: + print(f"Unknown keyword detected in '{self._header}', '{name}', keyword will be dropped.") + @classmethod def _supports_comments(cls): diff --git a/tests/dflowfm/test_mdu.py b/tests/dflowfm/test_mdu.py index b2ce042b6..3de5eda2c 100644 --- a/tests/dflowfm/test_mdu.py +++ b/tests/dflowfm/test_mdu.py @@ -1,4 +1,8 @@ from pathlib import Path +from unittest import mock +from unittest.mock import patch + +from pydantic.v1.config import Extra from hydrolib.core.basemodel import DiskOnlyFileModel from hydrolib.core.dflowfm.mdu.models import ( @@ -388,3 +392,89 @@ def test_loading_fmmodel_model_with_both_ini_and_xyn_obsfiles(self): assert actual_point.x == expected_point.x assert actual_point.y == expected_point.y assert actual_point.name == expected_point.name + + def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword(self, tmp_path, capsys): + tmp_mdu = """ + [General] + fileVersion = 1.09 + fileType = modelDef + program = D-Flow FM + version = 1.2.100.66357 + autoStart = 0 + pathsRelativeToParent = 0 + unknownkey = something + """ + + tmp_mdu_path = tmp_path / "tmp.mdu" + tmp_mdu_path.write_text(tmp_mdu) + + with patch('hydrolib.core.dflowfm.ini.models.INIBasedModel.Config') as mock_config: + mock_config.extra = Extra.ignore + + FMModel(filepath=tmp_mdu_path) + captured = capsys.readouterr() + + section = "General" + name = "unknownkey" + + assert f"Unknown keyword detected in '{section}', '{name}', keyword will be dropped." in captured.out + + def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword2(self, tmp_path, capsys): + tmp_mdu = """ + [General] + fileVersion = 1.09 + fileType = modelDef + program = D-Flow FM + version = 1.2.100.66357 + autoStart = 0 + pathsRelativeToParent = 0 + unknownkey = something + """ + + tmp_mdu_path = tmp_path / "tmp.mdu" + tmp_mdu_path.write_text(tmp_mdu) + + + with patch('hydrolib.core.dflowfm.ini.models.INIBasedModel.Config') as mock_config: + mock_config.extra = Extra.allow + + FMModel(filepath=tmp_mdu_path) + captured = capsys.readouterr() + + section = "General" + name = "unknownkey" + + assert f"Unknown keyword detected in '{section}', '{name}', keyword will be kept in memory but will have no validation." in captured.out + + def test_mdu_unknown_keywords_loading_does_not_give_message_on_exclude_fields(self, tmp_path, capsys): + tmp_mdu = """ + [General] + fileVersion = 1.09 + fileType = modelDef + program = D-Flow FM + version = 1.2.100.66357 + autoStart = 0 + pathsRelativeToParent = 0 + unknownkey = something + """ + + tmp_mdu_path = tmp_path / "tmp.mdu" + tmp_mdu_path.write_text(tmp_mdu) + + FMModel(filepath=tmp_mdu_path) + captured = capsys.readouterr() + + excluded_fields = ["comments", "datablock", "_header"] + for excluded_field in excluded_fields: + assert excluded_field not in captured.out + + def test_mdu_unknown_keywords_allow_extra_setting_field_gives_message(self, capsys): + model = FMModel() + model.general.Config.extra = Extra.allow + model.general.unknown = "something" + captured = capsys.readouterr() + + section = "General" + name = "unknown" + + assert f"Unknown keyword detected in '{section}', '{name}', keyword will be kept in memory but will have no validation." in captured.out \ No newline at end of file From 4ee025088d04caa89b0926e4ff5bc57f2b3733d6 Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Wed, 24 Apr 2024 09:22:46 +0200 Subject: [PATCH 02/31] feat: (622) resolve failing tests. --- hydrolib/core/dflowfm/ini/models.py | 2 +- tests/dflowfm/test_mdu.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hydrolib/core/dflowfm/ini/models.py b/hydrolib/core/dflowfm/ini/models.py index d2bb5897b..d9b785d13 100644 --- a/hydrolib/core/dflowfm/ini/models.py +++ b/hydrolib/core/dflowfm/ini/models.py @@ -52,9 +52,9 @@ class Config: arbitrary_types_allowed = False def __init__(self, **data): + super().__init__(**data) for key, _ in data.items(): self._notify_ingored_field(key) - super().__init__(**data) def __setattr__(self, name, value): self._notify_ingored_field(name) diff --git a/tests/dflowfm/test_mdu.py b/tests/dflowfm/test_mdu.py index 3de5eda2c..1f9e0bb66 100644 --- a/tests/dflowfm/test_mdu.py +++ b/tests/dflowfm/test_mdu.py @@ -470,10 +470,14 @@ def test_mdu_unknown_keywords_loading_does_not_give_message_on_exclude_fields(se def test_mdu_unknown_keywords_allow_extra_setting_field_gives_message(self, capsys): model = FMModel() + config_extra_setting = model.general.Config.extra # Save the Config.extra to revert back to original setting after the test. + model.general.Config.extra = Extra.allow model.general.unknown = "something" captured = capsys.readouterr() + model.general.Config.extra = config_extra_setting # Revert the Config.extra to the original setting, if this is not done it can affect other tests. + section = "General" name = "unknown" From ca14542b858f4c3500f668ef4d2fae40c6c28821 Mon Sep 17 00:00:00 2001 From: MRVermeulenDeltares Date: Wed, 24 Apr 2024 07:24:07 +0000 Subject: [PATCH 03/31] autoformat: isort & black --- hydrolib/core/dflowfm/ini/models.py | 15 +++--- tests/dflowfm/test_mdu.py | 84 ++++++++++++++++++----------- 2 files changed, 61 insertions(+), 38 deletions(-) diff --git a/hydrolib/core/dflowfm/ini/models.py b/hydrolib/core/dflowfm/ini/models.py index d9b785d13..92ff99a94 100644 --- a/hydrolib/core/dflowfm/ini/models.py +++ b/hydrolib/core/dflowfm/ini/models.py @@ -50,23 +50,26 @@ class INIBasedModel(BaseModel, ABC): class Config: extra = Extra.ignore arbitrary_types_allowed = False - + def __init__(self, **data): super().__init__(**data) for key, _ in data.items(): self._notify_ingored_field(key) - + def __setattr__(self, name, value): self._notify_ingored_field(name) super().__setattr__(name, value) - + def _notify_ingored_field(self, name): if name not in self.__fields__ and name not in self._exclude_fields(): if self.Config.extra == Extra.allow: - print(f"Unknown keyword detected in '{self._header}', '{name}', keyword will be kept in memory but will have no validation.") + print( + f"Unknown keyword detected in '{self._header}', '{name}', keyword will be kept in memory but will have no validation." + ) else: - print(f"Unknown keyword detected in '{self._header}', '{name}', keyword will be dropped.") - + print( + f"Unknown keyword detected in '{self._header}', '{name}', keyword will be dropped." + ) @classmethod def _supports_comments(cls): diff --git a/tests/dflowfm/test_mdu.py b/tests/dflowfm/test_mdu.py index 1f9e0bb66..7124c93dc 100644 --- a/tests/dflowfm/test_mdu.py +++ b/tests/dflowfm/test_mdu.py @@ -392,8 +392,10 @@ def test_loading_fmmodel_model_with_both_ini_and_xyn_obsfiles(self): assert actual_point.x == expected_point.x assert actual_point.y == expected_point.y assert actual_point.name == expected_point.name - - def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword(self, tmp_path, capsys): + + def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword( + self, tmp_path, capsys + ): tmp_mdu = """ [General] fileVersion = 1.09 @@ -404,22 +406,29 @@ def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword(self, tm pathsRelativeToParent = 0 unknownkey = something """ - + tmp_mdu_path = tmp_path / "tmp.mdu" tmp_mdu_path.write_text(tmp_mdu) - - with patch('hydrolib.core.dflowfm.ini.models.INIBasedModel.Config') as mock_config: + + with patch( + "hydrolib.core.dflowfm.ini.models.INIBasedModel.Config" + ) as mock_config: mock_config.extra = Extra.ignore - + FMModel(filepath=tmp_mdu_path) captured = capsys.readouterr() - + section = "General" name = "unknownkey" - - assert f"Unknown keyword detected in '{section}', '{name}', keyword will be dropped." in captured.out - - def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword2(self, tmp_path, capsys): + + assert ( + f"Unknown keyword detected in '{section}', '{name}', keyword will be dropped." + in captured.out + ) + + def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword2( + self, tmp_path, capsys + ): tmp_mdu = """ [General] fileVersion = 1.09 @@ -430,23 +439,29 @@ def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword2(self, t pathsRelativeToParent = 0 unknownkey = something """ - + tmp_mdu_path = tmp_path / "tmp.mdu" tmp_mdu_path.write_text(tmp_mdu) - - - with patch('hydrolib.core.dflowfm.ini.models.INIBasedModel.Config') as mock_config: + + with patch( + "hydrolib.core.dflowfm.ini.models.INIBasedModel.Config" + ) as mock_config: mock_config.extra = Extra.allow - + FMModel(filepath=tmp_mdu_path) captured = capsys.readouterr() - + section = "General" name = "unknownkey" - - assert f"Unknown keyword detected in '{section}', '{name}', keyword will be kept in memory but will have no validation." in captured.out - - def test_mdu_unknown_keywords_loading_does_not_give_message_on_exclude_fields(self, tmp_path, capsys): + + assert ( + f"Unknown keyword detected in '{section}', '{name}', keyword will be kept in memory but will have no validation." + in captured.out + ) + + def test_mdu_unknown_keywords_loading_does_not_give_message_on_exclude_fields( + self, tmp_path, capsys + ): tmp_mdu = """ [General] fileVersion = 1.09 @@ -457,28 +472,33 @@ def test_mdu_unknown_keywords_loading_does_not_give_message_on_exclude_fields(se pathsRelativeToParent = 0 unknownkey = something """ - + tmp_mdu_path = tmp_path / "tmp.mdu" tmp_mdu_path.write_text(tmp_mdu) - + FMModel(filepath=tmp_mdu_path) captured = capsys.readouterr() - + excluded_fields = ["comments", "datablock", "_header"] for excluded_field in excluded_fields: assert excluded_field not in captured.out - + def test_mdu_unknown_keywords_allow_extra_setting_field_gives_message(self, capsys): model = FMModel() - config_extra_setting = model.general.Config.extra # Save the Config.extra to revert back to original setting after the test. - + config_extra_setting = ( + model.general.Config.extra + ) # Save the Config.extra to revert back to original setting after the test. + model.general.Config.extra = Extra.allow model.general.unknown = "something" captured = capsys.readouterr() - - model.general.Config.extra = config_extra_setting # Revert the Config.extra to the original setting, if this is not done it can affect other tests. - + + model.general.Config.extra = config_extra_setting # Revert the Config.extra to the original setting, if this is not done it can affect other tests. + section = "General" name = "unknown" - - assert f"Unknown keyword detected in '{section}', '{name}', keyword will be kept in memory but will have no validation." in captured.out \ No newline at end of file + + assert ( + f"Unknown keyword detected in '{section}', '{name}', keyword will be kept in memory but will have no validation." + in captured.out + ) From 0808f17385de2c291ebe81de298b0530b5d92565 Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Wed, 24 Apr 2024 11:37:59 +0200 Subject: [PATCH 04/31] feat: (622) move related changes to specific class UnknownKeyNotificationManager and update notification layout for file loading --- hydrolib/core/dflowfm/ini/models.py | 88 +++++++++++++++++++++++------ tests/dflowfm/test_mdu.py | 86 +++++++++++----------------- 2 files changed, 104 insertions(+), 70 deletions(-) diff --git a/hydrolib/core/dflowfm/ini/models.py b/hydrolib/core/dflowfm/ini/models.py index 92ff99a94..a3e9a304d 100644 --- a/hydrolib/core/dflowfm/ini/models.py +++ b/hydrolib/core/dflowfm/ini/models.py @@ -28,6 +28,69 @@ logger = logging.getLogger(__name__) +class UnknownKeyNotificationManager(): + """ + Notification manager for unknown keys. + Detects unknown keys and manages the notification to the user. + """ + + def notify_unknown_keywords(self, data : dict[str, Any], section_header : str, fields : dict[str, Any], excluded_fields : Set, config_extra : Extra): + """ + Notify the user of unknown keywords. + + Args: + data (dict[str, Any]) : Input data containing all set properties which are checked on unknown keywords. + section_header (str) : Header of the section in which unknown keys might be detected. + fields (dict[str, Any]) : Known fields of the section. + excluded_fields (Set) : Fields which should be excluded from the check for unknown keywords. + config_extra (Extra) : Setting which determines if unknown keywords are allowed or dropped. + """ + unknown_keywords = self._get_all_unknown_keywords(data, fields, excluded_fields) + + if len(unknown_keywords) == 0: + return + + self._print_list_of_unknown_keywords(section_header, config_extra, unknown_keywords) + + def notify_unknown_keyword(self, name : str, section_header : str, fields : dict[str, Any], excluded_fields : Set, config_extra : Extra): + """ + Notify the user of a unknown keyword. + + Args: + name (str) : Keyword which is checked if it is an unknown keyword. + section_header (str) : Header of the section in which unknown keys might be detected. + fields (dict[str, Any]) : Known fields of the section. + excluded_fields (Set) : Fields which should be excluded from the check for unknown keywords. + config_extra (Extra) : Setting which determines if unknown keywords are allowed or dropped. + """ + if self._is_unknown_keyword(name, fields, excluded_fields): + self._print_single_unknown_keyword(name, section_header, config_extra) + + def _get_all_unknown_keywords(self, data : dict[str, Any], fields : dict[str, Any], excluded_fields : Set) -> list[str]: + list_of_unknown_keywords = [] + for name, _ in data.items(): + if self._is_unknown_keyword(name, fields, excluded_fields): + list_of_unknown_keywords.append(name) + + return list_of_unknown_keywords + + def _is_unknown_keyword(self, name : str, fields : dict[str, Any], excluded_fields : Set): + return name not in fields and name not in excluded_fields + + def _print_list_of_unknown_keywords(self, section_header : str, config_extra : Extra, list_of_unknown_keywords : list[str]): + if config_extra == Extra.allow: + print(f"Unknown keywords are detected in '{section_header}', these keywords will be kept in memory but will have no validation:") + else: + print(f"Unknown keywords are detected in '{section_header}', these keywords will be dropped:") + + for name in list_of_unknown_keywords: + print(name) + + def _print_single_unknown_keyword(self, name : str, section_header : str, config_extra : Extra): + if config_extra == Extra.allow: + print(f"Unknown keyword detected in '{section_header}', '{name}', keyword will be kept in memory but will have no validation.") + else: + print(f"Unknown keyword detected in '{section_header}', '{name}', keyword will be dropped.") class INIBasedModel(BaseModel, ABC): """INIBasedModel defines the base model for blocks/chapters @@ -44,33 +107,22 @@ class INIBasedModel(BaseModel, ABC): descriptions for all data fields. """ - _header: str + _header: str = "" _file_path_style_converter = FilePathStyleConverter() + _unknown_key_notification_manager = UnknownKeyNotificationManager() class Config: extra = Extra.ignore arbitrary_types_allowed = False - + def __init__(self, **data): super().__init__(**data) - for key, _ in data.items(): - self._notify_ingored_field(key) - + self._unknown_key_notification_manager.notify_unknown_keywords(data, self._header, self.__fields__, self._exclude_fields(), self.Config.extra) + def __setattr__(self, name, value): - self._notify_ingored_field(name) + self._unknown_key_notification_manager.notify_unknown_keyword(name, self._header, self.__fields__, self._exclude_fields(), self.Config.extra) super().__setattr__(name, value) - - def _notify_ingored_field(self, name): - if name not in self.__fields__ and name not in self._exclude_fields(): - if self.Config.extra == Extra.allow: - print( - f"Unknown keyword detected in '{self._header}', '{name}', keyword will be kept in memory but will have no validation." - ) - else: - print( - f"Unknown keyword detected in '{self._header}', '{name}', keyword will be dropped." - ) - + @classmethod def _supports_comments(cls): return True diff --git a/tests/dflowfm/test_mdu.py b/tests/dflowfm/test_mdu.py index 7124c93dc..665434b70 100644 --- a/tests/dflowfm/test_mdu.py +++ b/tests/dflowfm/test_mdu.py @@ -392,10 +392,8 @@ def test_loading_fmmodel_model_with_both_ini_and_xyn_obsfiles(self): assert actual_point.x == expected_point.x assert actual_point.y == expected_point.y assert actual_point.name == expected_point.name - - def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword( - self, tmp_path, capsys - ): + + def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword(self, tmp_path, capsys): tmp_mdu = """ [General] fileVersion = 1.09 @@ -406,29 +404,23 @@ def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword( pathsRelativeToParent = 0 unknownkey = something """ - + tmp_mdu_path = tmp_path / "tmp.mdu" tmp_mdu_path.write_text(tmp_mdu) - - with patch( - "hydrolib.core.dflowfm.ini.models.INIBasedModel.Config" - ) as mock_config: + + with patch('hydrolib.core.dflowfm.ini.models.INIBasedModel.Config') as mock_config: mock_config.extra = Extra.ignore - + FMModel(filepath=tmp_mdu_path) captured = capsys.readouterr() - + section = "General" name = "unknownkey" - - assert ( - f"Unknown keyword detected in '{section}', '{name}', keyword will be dropped." - in captured.out - ) - - def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword2( - self, tmp_path, capsys - ): + + assert f"Unknown keywords are detected in '{section}', these keywords will be dropped:" in captured.out + assert name in captured.out + + def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword2(self, tmp_path, capsys): tmp_mdu = """ [General] fileVersion = 1.09 @@ -439,29 +431,24 @@ def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword2( pathsRelativeToParent = 0 unknownkey = something """ - + tmp_mdu_path = tmp_path / "tmp.mdu" tmp_mdu_path.write_text(tmp_mdu) - - with patch( - "hydrolib.core.dflowfm.ini.models.INIBasedModel.Config" - ) as mock_config: + + + with patch('hydrolib.core.dflowfm.ini.models.INIBasedModel.Config') as mock_config: mock_config.extra = Extra.allow - + FMModel(filepath=tmp_mdu_path) captured = capsys.readouterr() - + section = "General" name = "unknownkey" - - assert ( - f"Unknown keyword detected in '{section}', '{name}', keyword will be kept in memory but will have no validation." - in captured.out - ) - - def test_mdu_unknown_keywords_loading_does_not_give_message_on_exclude_fields( - self, tmp_path, capsys - ): + + assert f"Unknown keywords are detected in '{section}', these keywords will be kept in memory but will have no validation:" in captured.out + assert name in captured.out + + def test_mdu_unknown_keywords_loading_does_not_give_message_on_exclude_fields(self, tmp_path, capsys): tmp_mdu = """ [General] fileVersion = 1.09 @@ -472,33 +459,28 @@ def test_mdu_unknown_keywords_loading_does_not_give_message_on_exclude_fields( pathsRelativeToParent = 0 unknownkey = something """ - + tmp_mdu_path = tmp_path / "tmp.mdu" tmp_mdu_path.write_text(tmp_mdu) - + FMModel(filepath=tmp_mdu_path) captured = capsys.readouterr() - + excluded_fields = ["comments", "datablock", "_header"] for excluded_field in excluded_fields: assert excluded_field not in captured.out - + def test_mdu_unknown_keywords_allow_extra_setting_field_gives_message(self, capsys): model = FMModel() - config_extra_setting = ( - model.general.Config.extra - ) # Save the Config.extra to revert back to original setting after the test. - + config_extra_setting = model.general.Config.extra # Save the Config.extra to revert back to original setting after the test. + model.general.Config.extra = Extra.allow model.general.unknown = "something" captured = capsys.readouterr() - - model.general.Config.extra = config_extra_setting # Revert the Config.extra to the original setting, if this is not done it can affect other tests. - + + model.general.Config.extra = config_extra_setting # Revert the Config.extra to the original setting, if this is not done it can affect other tests. + section = "General" name = "unknown" - - assert ( - f"Unknown keyword detected in '{section}', '{name}', keyword will be kept in memory but will have no validation." - in captured.out - ) + + assert f"Unknown keyword detected in '{section}', '{name}', keyword will be kept in memory but will have no validation." in captured.out \ No newline at end of file From 5980d3185e64da7e0ab5702cc7e476cafa2c220a Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Wed, 24 Apr 2024 13:41:22 +0200 Subject: [PATCH 05/31] feat: (622) Move UnknownKeyNotificationManager to util --- hydrolib/core/dflowfm/ini/models.py | 66 +--------------------------- hydrolib/core/dflowfm/ini/util.py | 67 ++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 66 deletions(-) diff --git a/hydrolib/core/dflowfm/ini/models.py b/hydrolib/core/dflowfm/ini/models.py index a3e9a304d..38305cebe 100644 --- a/hydrolib/core/dflowfm/ini/models.py +++ b/hydrolib/core/dflowfm/ini/models.py @@ -24,74 +24,10 @@ INISerializerConfig, write_ini, ) -from .util import make_list_validator +from .util import make_list_validator, UnknownKeyNotificationManager logger = logging.getLogger(__name__) -class UnknownKeyNotificationManager(): - """ - Notification manager for unknown keys. - Detects unknown keys and manages the notification to the user. - """ - - def notify_unknown_keywords(self, data : dict[str, Any], section_header : str, fields : dict[str, Any], excluded_fields : Set, config_extra : Extra): - """ - Notify the user of unknown keywords. - - Args: - data (dict[str, Any]) : Input data containing all set properties which are checked on unknown keywords. - section_header (str) : Header of the section in which unknown keys might be detected. - fields (dict[str, Any]) : Known fields of the section. - excluded_fields (Set) : Fields which should be excluded from the check for unknown keywords. - config_extra (Extra) : Setting which determines if unknown keywords are allowed or dropped. - """ - unknown_keywords = self._get_all_unknown_keywords(data, fields, excluded_fields) - - if len(unknown_keywords) == 0: - return - - self._print_list_of_unknown_keywords(section_header, config_extra, unknown_keywords) - - def notify_unknown_keyword(self, name : str, section_header : str, fields : dict[str, Any], excluded_fields : Set, config_extra : Extra): - """ - Notify the user of a unknown keyword. - - Args: - name (str) : Keyword which is checked if it is an unknown keyword. - section_header (str) : Header of the section in which unknown keys might be detected. - fields (dict[str, Any]) : Known fields of the section. - excluded_fields (Set) : Fields which should be excluded from the check for unknown keywords. - config_extra (Extra) : Setting which determines if unknown keywords are allowed or dropped. - """ - if self._is_unknown_keyword(name, fields, excluded_fields): - self._print_single_unknown_keyword(name, section_header, config_extra) - - def _get_all_unknown_keywords(self, data : dict[str, Any], fields : dict[str, Any], excluded_fields : Set) -> list[str]: - list_of_unknown_keywords = [] - for name, _ in data.items(): - if self._is_unknown_keyword(name, fields, excluded_fields): - list_of_unknown_keywords.append(name) - - return list_of_unknown_keywords - - def _is_unknown_keyword(self, name : str, fields : dict[str, Any], excluded_fields : Set): - return name not in fields and name not in excluded_fields - - def _print_list_of_unknown_keywords(self, section_header : str, config_extra : Extra, list_of_unknown_keywords : list[str]): - if config_extra == Extra.allow: - print(f"Unknown keywords are detected in '{section_header}', these keywords will be kept in memory but will have no validation:") - else: - print(f"Unknown keywords are detected in '{section_header}', these keywords will be dropped:") - - for name in list_of_unknown_keywords: - print(name) - - def _print_single_unknown_keyword(self, name : str, section_header : str, config_extra : Extra): - if config_extra == Extra.allow: - print(f"Unknown keyword detected in '{section_header}', '{name}', keyword will be kept in memory but will have no validation.") - else: - print(f"Unknown keyword detected in '{section_header}', '{name}', keyword will be dropped.") - class INIBasedModel(BaseModel, ABC): """INIBasedModel defines the base model for blocks/chapters inside an INIModel (*.ini file). diff --git a/hydrolib/core/dflowfm/ini/util.py b/hydrolib/core/dflowfm/ini/util.py index d71e1c408..5a7527c24 100644 --- a/hydrolib/core/dflowfm/ini/util.py +++ b/hydrolib/core/dflowfm/ini/util.py @@ -3,8 +3,9 @@ from datetime import datetime from enum import Enum from operator import eq -from typing import Any, Callable, Dict, List, Optional, Type +from typing import Any, Callable, Dict, List, Optional, Set, Type +from pydantic import Extra from pydantic.v1.class_validators import root_validator, validator from pydantic.v1.fields import ModelField from pydantic.v1.main import BaseModel @@ -622,3 +623,67 @@ def rename_keys_for_backwards_compatibility( break return values + +class UnknownKeyNotificationManager(): + """ + Notification manager for unknown keys. + Detects unknown keys and manages the notification to the user. + """ + + def notify_unknown_keywords(self, data : dict[str, Any], section_header : str, fields : dict[str, Any], excluded_fields : Set, config_extra : Extra): + """ + Notify the user of unknown keywords. + + Args: + data (dict[str, Any]) : Input data containing all set properties which are checked on unknown keywords. + section_header (str) : Header of the section in which unknown keys might be detected. + fields (dict[str, Any]) : Known fields of the section. + excluded_fields (Set) : Fields which should be excluded from the check for unknown keywords. + config_extra (Extra) : Setting which determines if unknown keywords are allowed or dropped. + """ + unknown_keywords = self._get_all_unknown_keywords(data, fields, excluded_fields) + + if len(unknown_keywords) == 0: + return + + self._print_list_of_unknown_keywords(section_header, config_extra, unknown_keywords) + + def notify_unknown_keyword(self, name : str, section_header : str, fields : dict[str, Any], excluded_fields : Set, config_extra : Extra): + """ + Notify the user of a unknown keyword. + + Args: + name (str) : Keyword which is checked if it is an unknown keyword. + section_header (str) : Header of the section in which unknown keys might be detected. + fields (dict[str, Any]) : Known fields of the section. + excluded_fields (Set) : Fields which should be excluded from the check for unknown keywords. + config_extra (Extra) : Setting which determines if unknown keywords are allowed or dropped. + """ + if self._is_unknown_keyword(name, fields, excluded_fields): + self._print_single_unknown_keyword(name, section_header, config_extra) + + def _get_all_unknown_keywords(self, data : dict[str, Any], fields : dict[str, Any], excluded_fields : Set) -> list[str]: + list_of_unknown_keywords = [] + for name, _ in data.items(): + if self._is_unknown_keyword(name, fields, excluded_fields): + list_of_unknown_keywords.append(name) + + return list_of_unknown_keywords + + def _is_unknown_keyword(self, name : str, fields : dict[str, Any], excluded_fields : Set): + return name not in fields and name not in excluded_fields + + def _print_list_of_unknown_keywords(self, section_header : str, config_extra : Extra, list_of_unknown_keywords : list[str]): + if config_extra == Extra.allow: + print(f"Unknown keywords are detected in '{section_header}', these keywords will be kept in memory but will have no validation:") + else: + print(f"Unknown keywords are detected in '{section_header}', these keywords will be dropped:") + + for name in list_of_unknown_keywords: + print(name) + + def _print_single_unknown_keyword(self, name : str, section_header : str, config_extra : Extra): + if config_extra == Extra.allow: + print(f"Unknown keyword detected in '{section_header}', '{name}', keyword will be kept in memory but will have no validation.") + else: + print(f"Unknown keyword detected in '{section_header}', '{name}', keyword will be dropped.") \ No newline at end of file From 49ec80f5a10b13a79630d902db52255ba039cfd2 Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Wed, 24 Apr 2024 15:15:59 +0200 Subject: [PATCH 06/31] feat: (622) Add unit tests for UnknownKeyNotificationManager --- tests/dflowfm/ini/test_util.py | 82 ++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/tests/dflowfm/ini/test_util.py b/tests/dflowfm/ini/test_util.py index 3abdb1355..c80bb1031 100644 --- a/tests/dflowfm/ini/test_util.py +++ b/tests/dflowfm/ini/test_util.py @@ -9,6 +9,7 @@ from hydrolib.core.dflowfm.ini.util import ( LocationValidationConfiguration, LocationValidationFieldNames, + UnknownKeyNotificationManager, get_from_subclass_defaults, get_type_based_on_subclass_default_value, rename_keys_for_backwards_compatibility, @@ -356,3 +357,84 @@ class WithoutDefaultProperty(BaseClass): class GrandChildWithDefaultProperty(WithoutDefaultProperty): name: Literal["GrandChildWithDefaultProperty"] = "GrandChildWithDefaultProperty" + + +class TestUnknownKeyNotificationManager(): + + section_header = "section header" + fields= {} + excluded_fields = set() + name = "keyname" + second_name = "second_other" + + @pytest.fixture + def setup(self): + self.section_header = "section header" + self.fields= {} + self.excluded_fields = set() + self.name = "keyname" + self.second_name = "second_other" + + @pytest.mark.parametrize("config_extra, expected_message", + [ + pytest.param(Extra.allow, f"Unknown keyword detected in '{section_header}', '{name}', keyword will be kept in memory but will have no validation."), + pytest.param(Extra.ignore, f"Unknown keyword detected in '{section_header}', '{name}', keyword will be dropped."), + pytest.param(Extra.forbid, f"Unknown keyword detected in '{section_header}', '{name}', keyword will be dropped.") + ]) + def test_unknown_keyword_given_and_unknown_keyword_gives_message_with_keyword_kept(self, config_extra, expected_message, capsys, setup): + uknm = UnknownKeyNotificationManager() + + uknm.notify_unknown_keyword(self.name, self.section_header, self.fields, self.excluded_fields, config_extra) + captured = capsys.readouterr() + + assert expected_message in captured.out + + @pytest.mark.parametrize("config_extra", + [ + pytest.param(Extra.allow), + pytest.param(Extra.ignore), + pytest.param(Extra.forbid) + ]) + def test_unknown_keyword_given_no_unknown_keys_gives_no_message(self, config_extra, capsys, setup): + uknm = UnknownKeyNotificationManager() + self.fields[self.name] = 1 + + uknm.notify_unknown_keyword(self.name, self.section_header, self.fields, self.excluded_fields, config_extra) + captured = capsys.readouterr() + + assert len(captured.out) == 0 + + @pytest.mark.parametrize("config_extra, expected_message", + [ + pytest.param(Extra.allow, f"Unknown keywords are detected in '{section_header}', these keywords will be kept in memory but will have no validation:"), + pytest.param(Extra.ignore, f"Unknown keywords are detected in '{section_header}', these keywords will be dropped:"), + pytest.param(Extra.forbid, f"Unknown keywords are detected in '{section_header}', these keywords will be dropped:") + ]) + def test_unknown_keywords_given_and_unknown_keywords_gives_message_with_keyword_kept(self, config_extra, expected_message, capsys, setup): + uknm = UnknownKeyNotificationManager() + data= {self.name:1, self.second_name:2} + + uknm.notify_unknown_keywords(data, self.section_header, self.fields, self.excluded_fields, config_extra) + captured = capsys.readouterr() + + assert expected_message in captured.out + assert self.name in captured.out + assert self.second_name in captured.out + + @pytest.mark.parametrize("config_extra", + [ + pytest.param(Extra.allow), + pytest.param(Extra.ignore), + pytest.param(Extra.forbid) + ]) + def test_unknown_keywords_given_no_unknown_keys_gives_no_message(self, config_extra, capsys, setup): + uknm = UnknownKeyNotificationManager() + data= {} + data[self.name] = 1 + self.fields[self.name] = 1 + + uknm.notify_unknown_keywords(data, self.section_header, self.fields, self.excluded_fields, config_extra) + captured = capsys.readouterr() + + assert len(captured.out) == 0 + \ No newline at end of file From 9ae839e4714ad7f9656e60efdd000e966230bd70 Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Wed, 24 Apr 2024 15:21:51 +0200 Subject: [PATCH 07/31] feat: (622) Add extra spacing at the end of printing the list per section --- hydrolib/core/dflowfm/ini/util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hydrolib/core/dflowfm/ini/util.py b/hydrolib/core/dflowfm/ini/util.py index 5a7527c24..95e04fcf2 100644 --- a/hydrolib/core/dflowfm/ini/util.py +++ b/hydrolib/core/dflowfm/ini/util.py @@ -681,6 +681,8 @@ def _print_list_of_unknown_keywords(self, section_header : str, config_extra : E for name in list_of_unknown_keywords: print(name) + + print() def _print_single_unknown_keyword(self, name : str, section_header : str, config_extra : Extra): if config_extra == Extra.allow: From 969ee2023ae6fc1505c06d83230ed742f7f4661e Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Wed, 24 Apr 2024 15:36:57 +0200 Subject: [PATCH 08/31] feat: (622) Try to resolve failing tests on python 3.8 --- hydrolib/core/dflowfm/ini/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydrolib/core/dflowfm/ini/models.py b/hydrolib/core/dflowfm/ini/models.py index 38305cebe..4501bbd53 100644 --- a/hydrolib/core/dflowfm/ini/models.py +++ b/hydrolib/core/dflowfm/ini/models.py @@ -43,7 +43,7 @@ class INIBasedModel(BaseModel, ABC): descriptions for all data fields. """ - _header: str = "" + _header: Literal["_header"] = "_header" _file_path_style_converter = FilePathStyleConverter() _unknown_key_notification_manager = UnknownKeyNotificationManager() From 2cae2a9ebbf4a4e2ca8b192ce6b369851da7e7a9 Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Wed, 24 Apr 2024 16:29:54 +0200 Subject: [PATCH 09/31] Revert "feat: (622) Try to resolve failing tests on python 3.8" This reverts commit 969ee2023ae6fc1505c06d83230ed742f7f4661e. --- hydrolib/core/dflowfm/ini/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydrolib/core/dflowfm/ini/models.py b/hydrolib/core/dflowfm/ini/models.py index 4501bbd53..38305cebe 100644 --- a/hydrolib/core/dflowfm/ini/models.py +++ b/hydrolib/core/dflowfm/ini/models.py @@ -43,7 +43,7 @@ class INIBasedModel(BaseModel, ABC): descriptions for all data fields. """ - _header: Literal["_header"] = "_header" + _header: str = "" _file_path_style_converter = FilePathStyleConverter() _unknown_key_notification_manager = UnknownKeyNotificationManager() From b6f2ebc2d62997396a944e028c88835276ed6fda Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Wed, 24 Apr 2024 16:31:04 +0200 Subject: [PATCH 10/31] feat: (622) Resolve problem occuring in python 3.8 --- hydrolib/core/dflowfm/ini/util.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/hydrolib/core/dflowfm/ini/util.py b/hydrolib/core/dflowfm/ini/util.py index 95e04fcf2..88b42398a 100644 --- a/hydrolib/core/dflowfm/ini/util.py +++ b/hydrolib/core/dflowfm/ini/util.py @@ -630,14 +630,14 @@ class UnknownKeyNotificationManager(): Detects unknown keys and manages the notification to the user. """ - def notify_unknown_keywords(self, data : dict[str, Any], section_header : str, fields : dict[str, Any], excluded_fields : Set, config_extra : Extra): + def notify_unknown_keywords(self, data : Dict[str, Any], section_header : str, fields : Dict[str, Any], excluded_fields : Set, config_extra : Extra): """ Notify the user of unknown keywords. Args: - data (dict[str, Any]) : Input data containing all set properties which are checked on unknown keywords. + data (Dict[str, Any]) : Input data containing all set properties which are checked on unknown keywords. section_header (str) : Header of the section in which unknown keys might be detected. - fields (dict[str, Any]) : Known fields of the section. + fields (Dict[str, Any]) : Known fields of the section. excluded_fields (Set) : Fields which should be excluded from the check for unknown keywords. config_extra (Extra) : Setting which determines if unknown keywords are allowed or dropped. """ @@ -648,21 +648,21 @@ def notify_unknown_keywords(self, data : dict[str, Any], section_header : str, self._print_list_of_unknown_keywords(section_header, config_extra, unknown_keywords) - def notify_unknown_keyword(self, name : str, section_header : str, fields : dict[str, Any], excluded_fields : Set, config_extra : Extra): + def notify_unknown_keyword(self, name : str, section_header : str, fields : Dict[str, Any], excluded_fields : Set, config_extra : Extra): """ Notify the user of a unknown keyword. Args: name (str) : Keyword which is checked if it is an unknown keyword. section_header (str) : Header of the section in which unknown keys might be detected. - fields (dict[str, Any]) : Known fields of the section. + fields (Dict[str, Any]) : Known fields of the section. excluded_fields (Set) : Fields which should be excluded from the check for unknown keywords. config_extra (Extra) : Setting which determines if unknown keywords are allowed or dropped. """ if self._is_unknown_keyword(name, fields, excluded_fields): self._print_single_unknown_keyword(name, section_header, config_extra) - def _get_all_unknown_keywords(self, data : dict[str, Any], fields : dict[str, Any], excluded_fields : Set) -> list[str]: + def _get_all_unknown_keywords(self, data : Dict[str, Any], fields : Dict[str, Any], excluded_fields : Set) -> List[str]: list_of_unknown_keywords = [] for name, _ in data.items(): if self._is_unknown_keyword(name, fields, excluded_fields): @@ -670,10 +670,10 @@ def _get_all_unknown_keywords(self, data : dict[str, Any], fields : dict[str, An return list_of_unknown_keywords - def _is_unknown_keyword(self, name : str, fields : dict[str, Any], excluded_fields : Set): + def _is_unknown_keyword(self, name : str, fields : Dict[str, Any], excluded_fields : Set): return name not in fields and name not in excluded_fields - def _print_list_of_unknown_keywords(self, section_header : str, config_extra : Extra, list_of_unknown_keywords : list[str]): + def _print_list_of_unknown_keywords(self, section_header : str, config_extra : Extra, list_of_unknown_keywords : List[str]): if config_extra == Extra.allow: print(f"Unknown keywords are detected in '{section_header}', these keywords will be kept in memory but will have no validation:") else: From 060539c5f1f40c3cdc587f5c8d811676d6a6ecfe Mon Sep 17 00:00:00 2001 From: MRVermeulenDeltares Date: Wed, 24 Apr 2024 14:32:21 +0000 Subject: [PATCH 11/31] autoformat: isort & black --- hydrolib/core/dflowfm/ini/models.py | 25 +++-- hydrolib/core/dflowfm/ini/util.py | 78 +++++++++++---- tests/dflowfm/ini/test_util.py | 149 ++++++++++++++++++---------- tests/dflowfm/test_mdu.py | 84 ++++++++++------ 4 files changed, 227 insertions(+), 109 deletions(-) diff --git a/hydrolib/core/dflowfm/ini/models.py b/hydrolib/core/dflowfm/ini/models.py index 38305cebe..b6eced28b 100644 --- a/hydrolib/core/dflowfm/ini/models.py +++ b/hydrolib/core/dflowfm/ini/models.py @@ -24,10 +24,11 @@ INISerializerConfig, write_ini, ) -from .util import make_list_validator, UnknownKeyNotificationManager +from .util import UnknownKeyNotificationManager, make_list_validator logger = logging.getLogger(__name__) + class INIBasedModel(BaseModel, ABC): """INIBasedModel defines the base model for blocks/chapters inside an INIModel (*.ini file). @@ -50,15 +51,27 @@ class INIBasedModel(BaseModel, ABC): class Config: extra = Extra.ignore arbitrary_types_allowed = False - + def __init__(self, **data): super().__init__(**data) - self._unknown_key_notification_manager.notify_unknown_keywords(data, self._header, self.__fields__, self._exclude_fields(), self.Config.extra) - + self._unknown_key_notification_manager.notify_unknown_keywords( + data, + self._header, + self.__fields__, + self._exclude_fields(), + self.Config.extra, + ) + def __setattr__(self, name, value): - self._unknown_key_notification_manager.notify_unknown_keyword(name, self._header, self.__fields__, self._exclude_fields(), self.Config.extra) + self._unknown_key_notification_manager.notify_unknown_keyword( + name, + self._header, + self.__fields__, + self._exclude_fields(), + self.Config.extra, + ) super().__setattr__(name, value) - + @classmethod def _supports_comments(cls): return True diff --git a/hydrolib/core/dflowfm/ini/util.py b/hydrolib/core/dflowfm/ini/util.py index 88b42398a..ee661389d 100644 --- a/hydrolib/core/dflowfm/ini/util.py +++ b/hydrolib/core/dflowfm/ini/util.py @@ -624,13 +624,21 @@ def rename_keys_for_backwards_compatibility( return values -class UnknownKeyNotificationManager(): + +class UnknownKeyNotificationManager: """ Notification manager for unknown keys. Detects unknown keys and manages the notification to the user. """ - - def notify_unknown_keywords(self, data : Dict[str, Any], section_header : str, fields : Dict[str, Any], excluded_fields : Set, config_extra : Extra): + + def notify_unknown_keywords( + self, + data: Dict[str, Any], + section_header: str, + fields: Dict[str, Any], + excluded_fields: Set, + config_extra: Extra, + ): """ Notify the user of unknown keywords. @@ -642,13 +650,22 @@ def notify_unknown_keywords(self, data : Dict[str, Any], section_header : str, config_extra (Extra) : Setting which determines if unknown keywords are allowed or dropped. """ unknown_keywords = self._get_all_unknown_keywords(data, fields, excluded_fields) - + if len(unknown_keywords) == 0: return - - self._print_list_of_unknown_keywords(section_header, config_extra, unknown_keywords) - - def notify_unknown_keyword(self, name : str, section_header : str, fields : Dict[str, Any], excluded_fields : Set, config_extra : Extra): + + self._print_list_of_unknown_keywords( + section_header, config_extra, unknown_keywords + ) + + def notify_unknown_keyword( + self, + name: str, + section_header: str, + fields: Dict[str, Any], + excluded_fields: Set, + config_extra: Extra, + ): """ Notify the user of a unknown keyword. @@ -661,31 +678,50 @@ def notify_unknown_keyword(self, name : str, section_header : str, fields : Dic """ if self._is_unknown_keyword(name, fields, excluded_fields): self._print_single_unknown_keyword(name, section_header, config_extra) - - def _get_all_unknown_keywords(self, data : Dict[str, Any], fields : Dict[str, Any], excluded_fields : Set) -> List[str]: + + def _get_all_unknown_keywords( + self, data: Dict[str, Any], fields: Dict[str, Any], excluded_fields: Set + ) -> List[str]: list_of_unknown_keywords = [] for name, _ in data.items(): if self._is_unknown_keyword(name, fields, excluded_fields): list_of_unknown_keywords.append(name) - + return list_of_unknown_keywords - - def _is_unknown_keyword(self, name : str, fields : Dict[str, Any], excluded_fields : Set): + + def _is_unknown_keyword( + self, name: str, fields: Dict[str, Any], excluded_fields: Set + ): return name not in fields and name not in excluded_fields - def _print_list_of_unknown_keywords(self, section_header : str, config_extra : Extra, list_of_unknown_keywords : List[str]): + def _print_list_of_unknown_keywords( + self, + section_header: str, + config_extra: Extra, + list_of_unknown_keywords: List[str], + ): if config_extra == Extra.allow: - print(f"Unknown keywords are detected in '{section_header}', these keywords will be kept in memory but will have no validation:") + print( + f"Unknown keywords are detected in '{section_header}', these keywords will be kept in memory but will have no validation:" + ) else: - print(f"Unknown keywords are detected in '{section_header}', these keywords will be dropped:") - + print( + f"Unknown keywords are detected in '{section_header}', these keywords will be dropped:" + ) + for name in list_of_unknown_keywords: print(name) - + print() - def _print_single_unknown_keyword(self, name : str, section_header : str, config_extra : Extra): + def _print_single_unknown_keyword( + self, name: str, section_header: str, config_extra: Extra + ): if config_extra == Extra.allow: - print(f"Unknown keyword detected in '{section_header}', '{name}', keyword will be kept in memory but will have no validation.") + print( + f"Unknown keyword detected in '{section_header}', '{name}', keyword will be kept in memory but will have no validation." + ) else: - print(f"Unknown keyword detected in '{section_header}', '{name}', keyword will be dropped.") \ No newline at end of file + print( + f"Unknown keyword detected in '{section_header}', '{name}', keyword will be dropped." + ) diff --git a/tests/dflowfm/ini/test_util.py b/tests/dflowfm/ini/test_util.py index c80bb1031..79867fa82 100644 --- a/tests/dflowfm/ini/test_util.py +++ b/tests/dflowfm/ini/test_util.py @@ -359,82 +359,131 @@ class GrandChildWithDefaultProperty(WithoutDefaultProperty): name: Literal["GrandChildWithDefaultProperty"] = "GrandChildWithDefaultProperty" -class TestUnknownKeyNotificationManager(): - +class TestUnknownKeyNotificationManager: + section_header = "section header" - fields= {} + fields = {} excluded_fields = set() name = "keyname" second_name = "second_other" - + @pytest.fixture def setup(self): self.section_header = "section header" - self.fields= {} + self.fields = {} self.excluded_fields = set() self.name = "keyname" self.second_name = "second_other" - - @pytest.mark.parametrize("config_extra, expected_message", - [ - pytest.param(Extra.allow, f"Unknown keyword detected in '{section_header}', '{name}', keyword will be kept in memory but will have no validation."), - pytest.param(Extra.ignore, f"Unknown keyword detected in '{section_header}', '{name}', keyword will be dropped."), - pytest.param(Extra.forbid, f"Unknown keyword detected in '{section_header}', '{name}', keyword will be dropped.") - ]) - def test_unknown_keyword_given_and_unknown_keyword_gives_message_with_keyword_kept(self, config_extra, expected_message, capsys, setup): + + @pytest.mark.parametrize( + "config_extra, expected_message", + [ + pytest.param( + Extra.allow, + f"Unknown keyword detected in '{section_header}', '{name}', keyword will be kept in memory but will have no validation.", + ), + pytest.param( + Extra.ignore, + f"Unknown keyword detected in '{section_header}', '{name}', keyword will be dropped.", + ), + pytest.param( + Extra.forbid, + f"Unknown keyword detected in '{section_header}', '{name}', keyword will be dropped.", + ), + ], + ) + def test_unknown_keyword_given_and_unknown_keyword_gives_message_with_keyword_kept( + self, config_extra, expected_message, capsys, setup + ): uknm = UnknownKeyNotificationManager() - uknm.notify_unknown_keyword(self.name, self.section_header, self.fields, self.excluded_fields, config_extra) + uknm.notify_unknown_keyword( + self.name, + self.section_header, + self.fields, + self.excluded_fields, + config_extra, + ) captured = capsys.readouterr() - + assert expected_message in captured.out - - @pytest.mark.parametrize("config_extra", - [ - pytest.param(Extra.allow), - pytest.param(Extra.ignore), - pytest.param(Extra.forbid) - ]) - def test_unknown_keyword_given_no_unknown_keys_gives_no_message(self, config_extra, capsys, setup): + + @pytest.mark.parametrize( + "config_extra", + [ + pytest.param(Extra.allow), + pytest.param(Extra.ignore), + pytest.param(Extra.forbid), + ], + ) + def test_unknown_keyword_given_no_unknown_keys_gives_no_message( + self, config_extra, capsys, setup + ): uknm = UnknownKeyNotificationManager() self.fields[self.name] = 1 - - uknm.notify_unknown_keyword(self.name, self.section_header, self.fields, self.excluded_fields, config_extra) + + uknm.notify_unknown_keyword( + self.name, + self.section_header, + self.fields, + self.excluded_fields, + config_extra, + ) captured = capsys.readouterr() - + assert len(captured.out) == 0 - - @pytest.mark.parametrize("config_extra, expected_message", - [ - pytest.param(Extra.allow, f"Unknown keywords are detected in '{section_header}', these keywords will be kept in memory but will have no validation:"), - pytest.param(Extra.ignore, f"Unknown keywords are detected in '{section_header}', these keywords will be dropped:"), - pytest.param(Extra.forbid, f"Unknown keywords are detected in '{section_header}', these keywords will be dropped:") - ]) - def test_unknown_keywords_given_and_unknown_keywords_gives_message_with_keyword_kept(self, config_extra, expected_message, capsys, setup): + + @pytest.mark.parametrize( + "config_extra, expected_message", + [ + pytest.param( + Extra.allow, + f"Unknown keywords are detected in '{section_header}', these keywords will be kept in memory but will have no validation:", + ), + pytest.param( + Extra.ignore, + f"Unknown keywords are detected in '{section_header}', these keywords will be dropped:", + ), + pytest.param( + Extra.forbid, + f"Unknown keywords are detected in '{section_header}', these keywords will be dropped:", + ), + ], + ) + def test_unknown_keywords_given_and_unknown_keywords_gives_message_with_keyword_kept( + self, config_extra, expected_message, capsys, setup + ): uknm = UnknownKeyNotificationManager() - data= {self.name:1, self.second_name:2} - - uknm.notify_unknown_keywords(data, self.section_header, self.fields, self.excluded_fields, config_extra) + data = {self.name: 1, self.second_name: 2} + + uknm.notify_unknown_keywords( + data, self.section_header, self.fields, self.excluded_fields, config_extra + ) captured = capsys.readouterr() - + assert expected_message in captured.out assert self.name in captured.out assert self.second_name in captured.out - - @pytest.mark.parametrize("config_extra", - [ - pytest.param(Extra.allow), - pytest.param(Extra.ignore), - pytest.param(Extra.forbid) - ]) - def test_unknown_keywords_given_no_unknown_keys_gives_no_message(self, config_extra, capsys, setup): + + @pytest.mark.parametrize( + "config_extra", + [ + pytest.param(Extra.allow), + pytest.param(Extra.ignore), + pytest.param(Extra.forbid), + ], + ) + def test_unknown_keywords_given_no_unknown_keys_gives_no_message( + self, config_extra, capsys, setup + ): uknm = UnknownKeyNotificationManager() - data= {} + data = {} data[self.name] = 1 self.fields[self.name] = 1 - uknm.notify_unknown_keywords(data, self.section_header, self.fields, self.excluded_fields, config_extra) + uknm.notify_unknown_keywords( + data, self.section_header, self.fields, self.excluded_fields, config_extra + ) captured = capsys.readouterr() - + assert len(captured.out) == 0 - \ No newline at end of file diff --git a/tests/dflowfm/test_mdu.py b/tests/dflowfm/test_mdu.py index 665434b70..974b61753 100644 --- a/tests/dflowfm/test_mdu.py +++ b/tests/dflowfm/test_mdu.py @@ -392,8 +392,10 @@ def test_loading_fmmodel_model_with_both_ini_and_xyn_obsfiles(self): assert actual_point.x == expected_point.x assert actual_point.y == expected_point.y assert actual_point.name == expected_point.name - - def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword(self, tmp_path, capsys): + + def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword( + self, tmp_path, capsys + ): tmp_mdu = """ [General] fileVersion = 1.09 @@ -404,23 +406,30 @@ def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword(self, tm pathsRelativeToParent = 0 unknownkey = something """ - + tmp_mdu_path = tmp_path / "tmp.mdu" tmp_mdu_path.write_text(tmp_mdu) - - with patch('hydrolib.core.dflowfm.ini.models.INIBasedModel.Config') as mock_config: + + with patch( + "hydrolib.core.dflowfm.ini.models.INIBasedModel.Config" + ) as mock_config: mock_config.extra = Extra.ignore - + FMModel(filepath=tmp_mdu_path) captured = capsys.readouterr() - + section = "General" name = "unknownkey" - - assert f"Unknown keywords are detected in '{section}', these keywords will be dropped:" in captured.out + + assert ( + f"Unknown keywords are detected in '{section}', these keywords will be dropped:" + in captured.out + ) assert name in captured.out - - def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword2(self, tmp_path, capsys): + + def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword2( + self, tmp_path, capsys + ): tmp_mdu = """ [General] fileVersion = 1.09 @@ -431,24 +440,30 @@ def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword2(self, t pathsRelativeToParent = 0 unknownkey = something """ - + tmp_mdu_path = tmp_path / "tmp.mdu" tmp_mdu_path.write_text(tmp_mdu) - - - with patch('hydrolib.core.dflowfm.ini.models.INIBasedModel.Config') as mock_config: + + with patch( + "hydrolib.core.dflowfm.ini.models.INIBasedModel.Config" + ) as mock_config: mock_config.extra = Extra.allow - + FMModel(filepath=tmp_mdu_path) captured = capsys.readouterr() - + section = "General" name = "unknownkey" - - assert f"Unknown keywords are detected in '{section}', these keywords will be kept in memory but will have no validation:" in captured.out + + assert ( + f"Unknown keywords are detected in '{section}', these keywords will be kept in memory but will have no validation:" + in captured.out + ) assert name in captured.out - - def test_mdu_unknown_keywords_loading_does_not_give_message_on_exclude_fields(self, tmp_path, capsys): + + def test_mdu_unknown_keywords_loading_does_not_give_message_on_exclude_fields( + self, tmp_path, capsys + ): tmp_mdu = """ [General] fileVersion = 1.09 @@ -459,28 +474,33 @@ def test_mdu_unknown_keywords_loading_does_not_give_message_on_exclude_fields(se pathsRelativeToParent = 0 unknownkey = something """ - + tmp_mdu_path = tmp_path / "tmp.mdu" tmp_mdu_path.write_text(tmp_mdu) - + FMModel(filepath=tmp_mdu_path) captured = capsys.readouterr() - + excluded_fields = ["comments", "datablock", "_header"] for excluded_field in excluded_fields: assert excluded_field not in captured.out - + def test_mdu_unknown_keywords_allow_extra_setting_field_gives_message(self, capsys): model = FMModel() - config_extra_setting = model.general.Config.extra # Save the Config.extra to revert back to original setting after the test. - + config_extra_setting = ( + model.general.Config.extra + ) # Save the Config.extra to revert back to original setting after the test. + model.general.Config.extra = Extra.allow model.general.unknown = "something" captured = capsys.readouterr() - - model.general.Config.extra = config_extra_setting # Revert the Config.extra to the original setting, if this is not done it can affect other tests. - + + model.general.Config.extra = config_extra_setting # Revert the Config.extra to the original setting, if this is not done it can affect other tests. + section = "General" name = "unknown" - - assert f"Unknown keyword detected in '{section}', '{name}', keyword will be kept in memory but will have no validation." in captured.out \ No newline at end of file + + assert ( + f"Unknown keyword detected in '{section}', '{name}', keyword will be kept in memory but will have no validation." + in captured.out + ) From 790dfe4921633844bd6972f459c72a4f1ca5d0bd Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Fri, 10 May 2024 11:20:04 +0200 Subject: [PATCH 12/31] feat: (622) Update unknown keyword manager to raise an error instead of a message. --- hydrolib/core/dflowfm/ini/models.py | 17 +--- hydrolib/core/dflowfm/ini/util.py | 69 ++----------- tests/dflowfm/ini/test_util.py | 146 ++++------------------------ 3 files changed, 29 insertions(+), 203 deletions(-) diff --git a/hydrolib/core/dflowfm/ini/models.py b/hydrolib/core/dflowfm/ini/models.py index b6eced28b..84ba53f1e 100644 --- a/hydrolib/core/dflowfm/ini/models.py +++ b/hydrolib/core/dflowfm/ini/models.py @@ -24,7 +24,7 @@ INISerializerConfig, write_ini, ) -from .util import UnknownKeyNotificationManager, make_list_validator +from .util import UnknownKeywordErrorManager, make_list_validator logger = logging.getLogger(__name__) @@ -46,7 +46,7 @@ class INIBasedModel(BaseModel, ABC): _header: str = "" _file_path_style_converter = FilePathStyleConverter() - _unknown_key_notification_manager = UnknownKeyNotificationManager() + _unknown_keyword_error_manager = UnknownKeywordErrorManager() class Config: extra = Extra.ignore @@ -54,24 +54,13 @@ class Config: def __init__(self, **data): super().__init__(**data) - self._unknown_key_notification_manager.notify_unknown_keywords( + self._unknown_keyword_error_manager.raise_error_for_unknown_keywords( data, self._header, self.__fields__, self._exclude_fields(), - self.Config.extra, ) - def __setattr__(self, name, value): - self._unknown_key_notification_manager.notify_unknown_keyword( - name, - self._header, - self.__fields__, - self._exclude_fields(), - self.Config.extra, - ) - super().__setattr__(name, value) - @classmethod def _supports_comments(cls): return True diff --git a/hydrolib/core/dflowfm/ini/util.py b/hydrolib/core/dflowfm/ini/util.py index ee661389d..6b7ff05c9 100644 --- a/hydrolib/core/dflowfm/ini/util.py +++ b/hydrolib/core/dflowfm/ini/util.py @@ -625,19 +625,18 @@ def rename_keys_for_backwards_compatibility( return values -class UnknownKeyNotificationManager: +class UnknownKeywordErrorManager: """ - Notification manager for unknown keys. - Detects unknown keys and manages the notification to the user. + Error manager for unknown keys. + Detects unknown keys and manages the Error to the user. """ - def notify_unknown_keywords( + def raise_error_for_unknown_keywords( self, data: Dict[str, Any], section_header: str, fields: Dict[str, Any], excluded_fields: Set, - config_extra: Extra, ): """ Notify the user of unknown keywords. @@ -647,37 +646,13 @@ def notify_unknown_keywords( section_header (str) : Header of the section in which unknown keys might be detected. fields (Dict[str, Any]) : Known fields of the section. excluded_fields (Set) : Fields which should be excluded from the check for unknown keywords. - config_extra (Extra) : Setting which determines if unknown keywords are allowed or dropped. """ unknown_keywords = self._get_all_unknown_keywords(data, fields, excluded_fields) if len(unknown_keywords) == 0: return - - self._print_list_of_unknown_keywords( - section_header, config_extra, unknown_keywords - ) - - def notify_unknown_keyword( - self, - name: str, - section_header: str, - fields: Dict[str, Any], - excluded_fields: Set, - config_extra: Extra, - ): - """ - Notify the user of a unknown keyword. - - Args: - name (str) : Keyword which is checked if it is an unknown keyword. - section_header (str) : Header of the section in which unknown keys might be detected. - fields (Dict[str, Any]) : Known fields of the section. - excluded_fields (Set) : Fields which should be excluded from the check for unknown keywords. - config_extra (Extra) : Setting which determines if unknown keywords are allowed or dropped. - """ - if self._is_unknown_keyword(name, fields, excluded_fields): - self._print_single_unknown_keyword(name, section_header, config_extra) + + raise ValueError(f"Unknown keywords are detected in section: '{section_header}', '{unknown_keywords}'") def _get_all_unknown_keywords( self, data: Dict[str, Any], fields: Dict[str, Any], excluded_fields: Set @@ -693,35 +668,3 @@ def _is_unknown_keyword( self, name: str, fields: Dict[str, Any], excluded_fields: Set ): return name not in fields and name not in excluded_fields - - def _print_list_of_unknown_keywords( - self, - section_header: str, - config_extra: Extra, - list_of_unknown_keywords: List[str], - ): - if config_extra == Extra.allow: - print( - f"Unknown keywords are detected in '{section_header}', these keywords will be kept in memory but will have no validation:" - ) - else: - print( - f"Unknown keywords are detected in '{section_header}', these keywords will be dropped:" - ) - - for name in list_of_unknown_keywords: - print(name) - - print() - - def _print_single_unknown_keyword( - self, name: str, section_header: str, config_extra: Extra - ): - if config_extra == Extra.allow: - print( - f"Unknown keyword detected in '{section_header}', '{name}', keyword will be kept in memory but will have no validation." - ) - else: - print( - f"Unknown keyword detected in '{section_header}', '{name}', keyword will be dropped." - ) diff --git a/tests/dflowfm/ini/test_util.py b/tests/dflowfm/ini/test_util.py index 79867fa82..f7abeee86 100644 --- a/tests/dflowfm/ini/test_util.py +++ b/tests/dflowfm/ini/test_util.py @@ -9,7 +9,7 @@ from hydrolib.core.dflowfm.ini.util import ( LocationValidationConfiguration, LocationValidationFieldNames, - UnknownKeyNotificationManager, + UnknownKeywordErrorManager, get_from_subclass_defaults, get_type_based_on_subclass_default_value, rename_keys_for_backwards_compatibility, @@ -359,131 +359,25 @@ class GrandChildWithDefaultProperty(WithoutDefaultProperty): name: Literal["GrandChildWithDefaultProperty"] = "GrandChildWithDefaultProperty" -class TestUnknownKeyNotificationManager: +class TestUnknownKeywordErrorManager: - section_header = "section header" - fields = {} - excluded_fields = set() - name = "keyname" - second_name = "second_other" - - @pytest.fixture - def setup(self): - self.section_header = "section header" - self.fields = {} - self.excluded_fields = set() - self.name = "keyname" - self.second_name = "second_other" - - @pytest.mark.parametrize( - "config_extra, expected_message", - [ - pytest.param( - Extra.allow, - f"Unknown keyword detected in '{section_header}', '{name}', keyword will be kept in memory but will have no validation.", - ), - pytest.param( - Extra.ignore, - f"Unknown keyword detected in '{section_header}', '{name}', keyword will be dropped.", - ), - pytest.param( - Extra.forbid, - f"Unknown keyword detected in '{section_header}', '{name}', keyword will be dropped.", - ), - ], - ) - def test_unknown_keyword_given_and_unknown_keyword_gives_message_with_keyword_kept( - self, config_extra, expected_message, capsys, setup - ): - uknm = UnknownKeyNotificationManager() - - uknm.notify_unknown_keyword( - self.name, - self.section_header, - self.fields, - self.excluded_fields, - config_extra, - ) - captured = capsys.readouterr() - - assert expected_message in captured.out - - @pytest.mark.parametrize( - "config_extra", - [ - pytest.param(Extra.allow), - pytest.param(Extra.ignore), - pytest.param(Extra.forbid), - ], - ) - def test_unknown_keyword_given_no_unknown_keys_gives_no_message( - self, config_extra, capsys, setup - ): - uknm = UnknownKeyNotificationManager() - self.fields[self.name] = 1 - - uknm.notify_unknown_keyword( - self.name, - self.section_header, - self.fields, - self.excluded_fields, - config_extra, - ) - captured = capsys.readouterr() - - assert len(captured.out) == 0 - - @pytest.mark.parametrize( - "config_extra, expected_message", - [ - pytest.param( - Extra.allow, - f"Unknown keywords are detected in '{section_header}', these keywords will be kept in memory but will have no validation:", - ), - pytest.param( - Extra.ignore, - f"Unknown keywords are detected in '{section_header}', these keywords will be dropped:", - ), - pytest.param( - Extra.forbid, - f"Unknown keywords are detected in '{section_header}', these keywords will be dropped:", - ), - ], - ) - def test_unknown_keywords_given_and_unknown_keywords_gives_message_with_keyword_kept( - self, config_extra, expected_message, capsys, setup + def test_unknown_keywords_given_when_notify_unknown_keywords_gives_error_with_unknown_keywords( + self ): - uknm = UnknownKeyNotificationManager() - data = {self.name: 1, self.second_name: 2} - - uknm.notify_unknown_keywords( - data, self.section_header, self.fields, self.excluded_fields, config_extra - ) - captured = capsys.readouterr() - - assert expected_message in captured.out - assert self.name in captured.out - assert self.second_name in captured.out - - @pytest.mark.parametrize( - "config_extra", - [ - pytest.param(Extra.allow), - pytest.param(Extra.ignore), - pytest.param(Extra.forbid), - ], - ) - def test_unknown_keywords_given_no_unknown_keys_gives_no_message( - self, config_extra, capsys, setup - ): - uknm = UnknownKeyNotificationManager() - data = {} - data[self.name] = 1 - self.fields[self.name] = 1 - - uknm.notify_unknown_keywords( - data, self.section_header, self.fields, self.excluded_fields, config_extra - ) - captured = capsys.readouterr() + section_header = "section header" + fields = {} + excluded_fields = set() + name = "keyname" + second_name = "second_other" + + ukem = UnknownKeywordErrorManager() + data = {name: 1, second_name: 2} + + expected_message = f"Unknown keywords are detected in section: '{section_header}', '{[name, second_name]}'" + + with pytest.raises(ValueError) as exc_err: + ukem.raise_error_for_unknown_keywords( + data, section_header, fields, excluded_fields + ) - assert len(captured.out) == 0 + assert expected_message in str(exc_err.value) From 987564d5b6548e5dc9a269312a9ba11a8ea88ba3 Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Mon, 27 May 2024 10:16:22 +0200 Subject: [PATCH 13/31] feat: (622) implement review suggestion and remove unused imports. --- hydrolib/core/dflowfm/ini/util.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hydrolib/core/dflowfm/ini/util.py b/hydrolib/core/dflowfm/ini/util.py index 6b7ff05c9..1a0138278 100644 --- a/hydrolib/core/dflowfm/ini/util.py +++ b/hydrolib/core/dflowfm/ini/util.py @@ -5,8 +5,7 @@ from operator import eq from typing import Any, Callable, Dict, List, Optional, Set, Type -from pydantic import Extra -from pydantic.v1.class_validators import root_validator, validator +from pydantic.v1.class_validators import validator from pydantic.v1.fields import ModelField from pydantic.v1.main import BaseModel @@ -658,7 +657,7 @@ def _get_all_unknown_keywords( self, data: Dict[str, Any], fields: Dict[str, Any], excluded_fields: Set ) -> List[str]: list_of_unknown_keywords = [] - for name, _ in data.items(): + for name in data: if self._is_unknown_keyword(name, fields, excluded_fields): list_of_unknown_keywords.append(name) From 741cba7f154d0e5952867c35e7eb7f97cdfbfdec Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Mon, 27 May 2024 10:28:32 +0200 Subject: [PATCH 14/31] feat: (622) Update tests and rename tests --- hydrolib/core/dflowfm/ini/util.py | 2 +- tests/dflowfm/test_mdu.py | 95 ++++++++++--------------------- 2 files changed, 32 insertions(+), 65 deletions(-) diff --git a/hydrolib/core/dflowfm/ini/util.py b/hydrolib/core/dflowfm/ini/util.py index 1a0138278..1591ef364 100644 --- a/hydrolib/core/dflowfm/ini/util.py +++ b/hydrolib/core/dflowfm/ini/util.py @@ -641,7 +641,7 @@ def raise_error_for_unknown_keywords( Notify the user of unknown keywords. Args: - data (Dict[str, Any]) : Input data containing all set properties which are checked on unknown keywords. + data (Dict[str, Any]) : Input data containing all properties which are checked on unknown keywords. section_header (str) : Header of the section in which unknown keys might be detected. fields (Dict[str, Any]) : Known fields of the section. excluded_fields (Set) : Fields which should be excluded from the check for unknown keywords. diff --git a/tests/dflowfm/test_mdu.py b/tests/dflowfm/test_mdu.py index 974b61753..3ee8b3625 100644 --- a/tests/dflowfm/test_mdu.py +++ b/tests/dflowfm/test_mdu.py @@ -1,8 +1,6 @@ from pathlib import Path -from unittest import mock -from unittest.mock import patch -from pydantic.v1.config import Extra +import pytest from hydrolib.core.basemodel import DiskOnlyFileModel from hydrolib.core.dflowfm.mdu.models import ( @@ -392,9 +390,9 @@ def test_loading_fmmodel_model_with_both_ini_and_xyn_obsfiles(self): assert actual_point.x == expected_point.x assert actual_point.y == expected_point.y assert actual_point.name == expected_point.name - - def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword( - self, tmp_path, capsys + + def test_mdu_unknown_keyword_loading_throws_valueerror_for_unknown_keyword( + self, tmp_path ): tmp_mdu = """ [General] @@ -410,25 +408,18 @@ def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword( tmp_mdu_path = tmp_path / "tmp.mdu" tmp_mdu_path.write_text(tmp_mdu) - with patch( - "hydrolib.core.dflowfm.ini.models.INIBasedModel.Config" - ) as mock_config: - mock_config.extra = Extra.ignore + section_header = "General" + name = "unknownkey" + + expected_message = f"Unknown keywords are detected in section: '{section_header}', '{[name]}'" + with pytest.raises(ValueError) as exc_err: FMModel(filepath=tmp_mdu_path) - captured = capsys.readouterr() - - section = "General" - name = "unknownkey" - - assert ( - f"Unknown keywords are detected in '{section}', these keywords will be dropped:" - in captured.out - ) - assert name in captured.out - - def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword2( - self, tmp_path, capsys + + assert expected_message in str(exc_err.value) + + def test_mdu_unknown_keywords_loading_throws_valueerror_for_unknown_keywords( + self, tmp_path ): tmp_mdu = """ [General] @@ -439,30 +430,25 @@ def test_mdu_unknown_keywords_loading_gives_message_for_missing_keyword2( autoStart = 0 pathsRelativeToParent = 0 unknownkey = something + unknownkey2 = something2 """ tmp_mdu_path = tmp_path / "tmp.mdu" tmp_mdu_path.write_text(tmp_mdu) - with patch( - "hydrolib.core.dflowfm.ini.models.INIBasedModel.Config" - ) as mock_config: - mock_config.extra = Extra.allow + section_header = "General" + name = "unknownkey" + name2 = "unknownkey2" + + expected_message = f"Unknown keywords are detected in section: '{section_header}', '{[name, name2]}'" + with pytest.raises(ValueError) as exc_err: FMModel(filepath=tmp_mdu_path) - captured = capsys.readouterr() + + assert expected_message in str(exc_err.value) - section = "General" - name = "unknownkey" - - assert ( - f"Unknown keywords are detected in '{section}', these keywords will be kept in memory but will have no validation:" - in captured.out - ) - assert name in captured.out - - def test_mdu_unknown_keywords_loading_does_not_give_message_on_exclude_fields( - self, tmp_path, capsys + def test_mdu_unknown_keywords_loading_thrown_valueerror_for_unknown_keyword_does_not_include_excluded_fields( + self, tmp_path ): tmp_mdu = """ [General] @@ -478,29 +464,10 @@ def test_mdu_unknown_keywords_loading_does_not_give_message_on_exclude_fields( tmp_mdu_path = tmp_path / "tmp.mdu" tmp_mdu_path.write_text(tmp_mdu) - FMModel(filepath=tmp_mdu_path) - captured = capsys.readouterr() - - excluded_fields = ["comments", "datablock", "_header"] - for excluded_field in excluded_fields: - assert excluded_field not in captured.out + with pytest.raises(ValueError) as exc_err: + model = FMModel(filepath=tmp_mdu_path) - def test_mdu_unknown_keywords_allow_extra_setting_field_gives_message(self, capsys): - model = FMModel() - config_extra_setting = ( - model.general.Config.extra - ) # Save the Config.extra to revert back to original setting after the test. - - model.general.Config.extra = Extra.allow - model.general.unknown = "something" - captured = capsys.readouterr() - - model.general.Config.extra = config_extra_setting # Revert the Config.extra to the original setting, if this is not done it can affect other tests. - - section = "General" - name = "unknown" - - assert ( - f"Unknown keyword detected in '{section}', '{name}', keyword will be kept in memory but will have no validation." - in captured.out - ) + excluded_fields = model._exclude_fields() + assert len(excluded_fields) > 0 + for excluded_field in excluded_fields: + assert excluded_field not in str(exc_err.value) \ No newline at end of file From 19b317d7795b188020cbdbb2738366e0f35ae40d Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Mon, 27 May 2024 12:38:49 +0200 Subject: [PATCH 15/31] feat: (622) add review suggestion check on alias too. --- hydrolib/core/dflowfm/ini/util.py | 14 ++++--- tests/dflowfm/ini/test_util.py | 69 +++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 5 deletions(-) diff --git a/hydrolib/core/dflowfm/ini/util.py b/hydrolib/core/dflowfm/ini/util.py index 1591ef364..084753475 100644 --- a/hydrolib/core/dflowfm/ini/util.py +++ b/hydrolib/core/dflowfm/ini/util.py @@ -634,7 +634,7 @@ def raise_error_for_unknown_keywords( self, data: Dict[str, Any], section_header: str, - fields: Dict[str, Any], + fields: Dict[str, ModelField], excluded_fields: Set, ): """ @@ -643,7 +643,7 @@ def raise_error_for_unknown_keywords( Args: data (Dict[str, Any]) : Input data containing all properties which are checked on unknown keywords. section_header (str) : Header of the section in which unknown keys might be detected. - fields (Dict[str, Any]) : Known fields of the section. + fields (Dict[str, ModelField]) : Known fields of the section. excluded_fields (Set) : Fields which should be excluded from the check for unknown keywords. """ unknown_keywords = self._get_all_unknown_keywords(data, fields, excluded_fields) @@ -654,7 +654,7 @@ def raise_error_for_unknown_keywords( raise ValueError(f"Unknown keywords are detected in section: '{section_header}', '{unknown_keywords}'") def _get_all_unknown_keywords( - self, data: Dict[str, Any], fields: Dict[str, Any], excluded_fields: Set + self, data: Dict[str, Any], fields: Dict[str, ModelField], excluded_fields: Set ) -> List[str]: list_of_unknown_keywords = [] for name in data: @@ -664,6 +664,10 @@ def _get_all_unknown_keywords( return list_of_unknown_keywords def _is_unknown_keyword( - self, name: str, fields: Dict[str, Any], excluded_fields: Set + self, name: str, fields: Dict[str, ModelField], excluded_fields: Set ): - return name not in fields and name not in excluded_fields + for model_field in fields.values(): + if name == model_field.name or name == model_field.alias: + return False + + return name not in excluded_fields diff --git a/tests/dflowfm/ini/test_util.py b/tests/dflowfm/ini/test_util.py index f7abeee86..cad4ebedb 100644 --- a/tests/dflowfm/ini/test_util.py +++ b/tests/dflowfm/ini/test_util.py @@ -1,5 +1,7 @@ from typing import Dict, List, Literal, Optional +from unittest.mock import Mock +from pydantic.v1.fields import ModelField import pytest from pydantic.v1 import Extra from pydantic.v1.class_validators import root_validator @@ -381,3 +383,70 @@ def test_unknown_keywords_given_when_notify_unknown_keywords_gives_error_with_un ) assert expected_message in str(exc_err.value) + + def test_keyword_given_known_as_alias_does_not_throw_exception( + self + ): + section_header = "section header" + excluded_fields = set() + name = "keyname" + + mocked_field = Mock(spec=ModelField) + mocked_field.name = "name" + mocked_field.alias = name + + fields = {"name": mocked_field} + + ukem = UnknownKeywordErrorManager() + data = {name: 1} + + try: + ukem.raise_error_for_unknown_keywords( + data, section_header, fields, excluded_fields + ) + except: + pytest.fail("Exception is thrown, no exception is expected for this test.") + + def test_keyword_given_known_as_name_does_not_throw_exception( + self + ): + section_header = "section header" + excluded_fields = set() + name = "keyname" + + mocked_field = Mock(spec=ModelField) + mocked_field.name = "name" + mocked_field.alias = name + + fields = {name: mocked_field} + + ukem = UnknownKeywordErrorManager() + data = {name: 1} + + try: + ukem.raise_error_for_unknown_keywords( + data, section_header, fields, excluded_fields + ) + except: + pytest.fail("Exception is thrown, no exception is expected for this test.") + + + def test_keyword_given_known_as_excluded_field_does_not_throw_exception( + self + ): + section_header = "section header" + excluded_fields = set() + name = "keyname" + excluded_fields.add(name) + + fields = {} + + ukem = UnknownKeywordErrorManager() + data = {name: 1} + + try: + ukem.raise_error_for_unknown_keywords( + data, section_header, fields, excluded_fields + ) + except: + pytest.fail("Exception is thrown, no exception is expected for this test.") \ No newline at end of file From 712f757c2e3ff1e26b8d1c506fc634fe0297fe18 Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Tue, 28 May 2024 09:49:51 +0200 Subject: [PATCH 16/31] feat: (622) remove unknown keys from mdu files and isshared from crs def files. --- .../special_3d_settings.mdu | 21 ---- .../with_optional_sections.mdu | 96 ------------------- .../dimr_model/dflowfm/FlowFM.mdu | 13 --- .../dimr_model/dflowfm/crsdef.ini | 1 - .../c01_steady-state-flow/Boundary.mdu | 19 ---- .../CrossSectionDefinitions.ini | 1 - tests/data/input/file_load_test/fm.mdu | 13 --- .../emptymodel_with_obscrs.mdu | 17 ---- .../single_ini/emptymodel_with_obscrs.mdu | 17 ---- .../single_pli/emptymodel_with_obscrs.mdu | 17 ---- .../obsfile_cases/both_ini_and_xyn/fm.mdu | 13 --- .../input/obsfile_cases/single_ini/fm.mdu | 13 --- .../input/obsfile_cases/single_xyn/fm.mdu | 13 --- .../resolve_casing_file_load_test/fm.mdu | 13 --- 14 files changed, 267 deletions(-) diff --git a/tests/data/input/dflowfm_individual_files/special_3d_settings.mdu b/tests/data/input/dflowfm_individual_files/special_3d_settings.mdu index 2cb4acc5c..4d11b9a43 100644 --- a/tests/data/input/dflowfm_individual_files/special_3d_settings.mdu +++ b/tests/data/input/dflowfm_individual_files/special_3d_settings.mdu @@ -51,19 +51,13 @@ Dxwuimin2D = 0.1 #------------------------------------------------------------------------------------------------------ CFLMax = 0.7 # Maximum Courant number AdvecType = 33 # Advection type (0: none, 1: Wenneker, 2: Wenneker q(uio-u), 3: Perot q(uio-u), 4: Perot q(ui-u), 5: Perot q(ui-u) without itself) -Newcorio = 1 # 0=No, 1=yes, if jsferic then spatially varying, if icoriolistype==6 then constant (anglat) -Corioadamsbashfordfac = 0.5 Limtyphu = 0 # Limiter type for waterdepth in continuity eqn. (0: none, 1: minmod, 2: van Leer, 3: Kooren, 4: monotone central) Limtypmom = 4 # Limiter type for cell center advection velocity (0: none, 1: minmod, 2: van Leer, 3: Kooren, 4: monotone central) Limtypsa = 4 # Limiter type for salinity transport (0: none, 1: minmod, 2: van Leer, 3: Kooren, 4: monotone central) Vertadvtypsal = 6 # Vertical advection type for salinity (0: none, 1: upwind explicit, 2: central explicit, 3: upwind implicit, 4: central implicit, 5: central implicit but upwind for neg. stratif., 6: higher order explicit, no Forester) Vertadvtyptem = 6 # Vertical advection type for temperature (0: none, 1: upwind explicit, 2: central explicit, 3: upwind implicit, 4: central implicit, 5: central implicit but upwind for neg. stratif., 6: higher order explicit, no Forester) Vertadvtypmom = 6 # Vertical advection type for u1: 0: No, 3: Upwind implicit, 4: Central implicit, 5: QUICKEST implicit., 6: centerbased upwind expl -Vertadvtypmom3onbnd = 0 # vert. adv. u1 bnd UpwimpL: 0=follow javau , 1 = on bnd, 2= on and near bnd -Jarhoxu = 0 # Inlcude density gradient in advection term (0: no, 1: yes, 2: Also in barotrop and baroclin pressure term) -Horadvtypzlayer = 0 # Horizontal advection treatment of z-layers (1: default, 2: sigma-like) Zerozbndinflowadvection = 1 # On waterlevel boundaries set incoming advection velocity to zero (0=no, 1=on inflow, 2=also on outflow) -Zlayercenterbedvel = 1 # reconstruction of center velocity at half closed bedcells (0=no, 1: copy bed link velocities) Icgsolver = 4 # Solver type (1: sobekGS_OMP, 2: sobekGS_OMPthreadsafe, 3: sobekGS, 4: sobekGS + Saadilud, 5: parallel/global Saad, 6: parallel/Petsc, 7: parallel/GS) Maxdegree = 6 # Maximum degree in Gauss elimination FixedWeirScheme = 9 # Fixed weir scheme (0: none, 1: compact stencil, 2: whole tile lifted, full subgrid weir + factor) @@ -72,18 +66,12 @@ Fixedweirtopfrictcoef = -999. Izbndpos = 1 # Position of z boundary (0: D3Dflow, 1: on net boundary, 2: on specified polyline) Tlfsmo = 86400. # Fourier smoothing time (s) on water level boundaries Keepstbndonoutflow = 1 # Keep sal and tem signals on bnd also at outflow, 1=yes, 0=no=default=copy inside value on outflow -Logprofkepsbndin = 0 # inflow: 0=0 keps, 1 = log keps inflow, 2 = log keps in and outflow -Drop3D = 1. # Apply droplosses in 3D if z upwind below bob + 2/3 hu*drop3D -Epshstem = 1.d-3 # Only compute heatflx + evap if depth > epshstem -Zwsbtol = 0. # tolerance for zws(kb-1) at bed Teta0 = 0.55 # Theta of time integration (0.5 < theta < 1) Turbulencemodel = 3 # Turbulence model (0: none, 1: constant, 2: algebraic, 3: k-epsilon, 4: k-tau) Turbulenceadvection = 3 # Turbulence advection (0: none, 3: horizontally explicit and vertically implicit) AntiCreep = 0 # Include anti-creep calculation (0: no, 1: yes) -Eddyviscositybedfacmax = 0. # Limit eddy viscosity at bed ) MinTimestepBreak = 0.1 # smallest allowed timestep (in s), checked on a sliding average of several timesteps. Run will abort when violated. Epshu = 1.d-4 # Threshold water depth for wet and dry cells -jasfer3D = 1 # corrections for spherical coordinates Barocponbnd = 1 #====================================================================================================== @@ -125,12 +113,10 @@ Stanton = 1.3d-3 Dalton = 1.3d-3 # Coefficient for evaporative heat flux, if negative, Ceva = abs(Dalton)*Cdwind Tempmax = 50. # Limit the temperature Tempmin = -5. # Limit the temperature -Soiltempthick = 0. # Use soil temperature buffer if > 0, e.g. 0.2 (m) Heat_eachstep = 1 # 1=heat each timestep, 0=heat each usertimestep RhoairRhowater = 1 # windstress rhoa/rhow: 0=Rhoair/Rhomean, 1=Rhoair/rhow(), 2=rhoa0()/rhow(), 3=rhoa10()/Rhow() SecondaryFlow = 0 # Secondary flow (0: no, 1: yes) IniWithNudge = 2 # Initialize salinity and temperature with nudge variables -Jadelvappos = 0 #====================================================================================================== [wind] @@ -140,7 +126,6 @@ Jadelvappos = 0 ICdtyp = 4 # Wind drag coefficient type (1=Const; 2=Smith&Banke (2 pts); 3=S&B (3 pts); 4=Charnock 1955, 5=Hwang 2005, 6=Wuest 2005, 7=Hersbach 2010 (2 pts) Cdbreakpoints = 2.5d-2 # Wind drag coefficient break points Relativewind = 0.5 # Wind speed relative to top-layer water speed, 1=yes, 0 = no) -Windhuorzwsbased = 0 # Wind hu or zws based , 0 = hu, 1 = zws Windpartialdry = 1 # Reduce windstress on water if link partially dry, only for bedlevtyp=3, 0 = no, 1 = yes = default Rhoair = 1.2265 # Air density (kg/m3) PavBnd = 101330. # Average air pressure on open boundaries (N/m2) (only applied if > 0) @@ -161,7 +146,6 @@ Tzone = 0. DtUser = 600. # Time interval (s) for external forcing update DtNodal = 21600. # Time interval (s) for updating nodal factors in astronomical boundary conditions DtMax = 120. # Maximal computation timestep (s) -Dtfacmax = 1.1 # Max timestep increase factor ( ) DtInit = 60. # Initial computation timestep (s) AutoTimestep = 3 # 0 = no, 1 = 2D (hor. out), 3=3D (hor. out), 5 = 3D (hor. inout + ver. inout), smallest dt @@ -287,8 +271,6 @@ Wrimap_flow_flux_q1_main = 1 # Write flow flu Wrimap_numlimdt = 1 # Write the number times a cell was Courant limiting to map file (1: yes, 0: no). (Consider using Wrimap_flow_analysis instead.) Wrimap_taucurrent = 1 # Write the shear stress to map file (1: yes, 0: no) Wrimap_chezy = 1 # Write the chezy roughness to map file (1: yes, 0: no) -Wrimap_salinity = 1 # Write salinity to map file (1: yes, 0: no) -Wrimap_temperature = 1 # Write temperature to map file (1: yes, 0: no) Wrimap_turbulence = 1 # Write vicww, k and eps to map file (1: yes, 0: no) Wrimap_rain = 1 # Write rainfall rates to map file (1: yes, 0: no) Wrimap_interception = 1 # Write interception to map file (1: yes, 0: no) @@ -321,12 +303,9 @@ wrishp_genstruc = 0 #------------------------------------------------------------------------------------------------------ # Miscellaneous #------------------------------------------------------------------------------------------------------ -Writepart_domain = 1 # Write partition domain info. for postprocessing -WriteDFMinterpretedvalues = 0 # Write DFMinterpretedvalues (1: yes, 0: no) MapOutputTimeVector = # File (*.mpt) containing fixed map output times (s) w.r.t. RefDate FullGridOutput = 0 # Full grid output mode (0: compact, 1: full time-varying grid data) EulerVelocities = 0 # Euler velocities output (0: GLM, 1: Euler velocities) -Wrirst_bnd = 1 # Write waterlevel, bedlevel and coordinates of boundaries to restart files #------------------------------------------------------------------------------------------------------ # Uneditable #------------------------------------------------------------------------------------------------------ diff --git a/tests/data/input/dflowfm_individual_files/with_optional_sections.mdu b/tests/data/input/dflowfm_individual_files/with_optional_sections.mdu index 2803ac6ad..eb91dc289 100644 --- a/tests/data/input/dflowfm_individual_files/with_optional_sections.mdu +++ b/tests/data/input/dflowfm_individual_files/with_optional_sections.mdu @@ -8,12 +8,10 @@ Version = 1.2.143.140737 # Version number fileType = modelDef # File type. Do not edit this. fileVersion = 1.09 # File format version (do not edit this) AutoStart = 0 # Autostart simulation after loading MDU (0: no, 1: autostart, 2: autostartstop) -ModelSpecific = # Optional 'model specific ID', to enable certain custom runtime function calls (instead of via MDU name). PathsRelativeToParent = 0 # Default: 0. Whether or not (1/0) to resolve file names (e.g. inside the *.ext file) relative to their direct parent, instead of to the toplevel MDU working dir. [geometry] NetFile = -BedlevelFile = # Bedlevels points file e.g. *.xyz, only needed for bedlevtype not equal 3 GridEnclosureFile = # Enclosure file to clip outer parts from the grid *.pol DryPointsFile = CrossDefFile = # 1D Cross section definition file (*.ini) @@ -24,7 +22,6 @@ StructureFile = WaterLevIniFile = # Initial water levels sample file *.xyz using flood fill algorithm LandBoundaryFile = # Land boundaries file *.ldb, used for visualization ThinDamFile = -Cutcelllist = # File with names of cutcell polygons, e.g. cutcellpolygons.lst FixedWeirFile = # Polyline file *_fxw.pliz, containing fixed weirs with rows x, y, crest level, left ground level, right ground level PillarFile = # Polyline file *_pillar.pliz, containing four colums with x, y, diameter and Cd coefficient Gulliesfile = # Polyline file *_gul.pliz, containing lowest bed level along talweg x, y, z level @@ -41,8 +38,6 @@ Uniformtyp1Dstreetinlets = -2 # Uniform type s Uniformtyp1Droofgutterpipes = -2 # Uniform type roof gutter pipes 1D2DLinkFile = # File *.ini containing custom parameters for 1D2D links ManholeFile = # File *.ini containing manholes -PipeFile = # File *.pliz containing pipe-based 'culverts' -ShipdefFile = # File *.shd containing ship definitions WaterLevIni = 0. # Initial water levels sample file *.xyz using flood fill algorithm BedlevUni = 5. # Uniform bed level used at missing z values if BedlevType > 2 BedlevType = 3 # Bathymetry specification @@ -54,8 +49,6 @@ BedlevType = 3 # Bathymetry spe # 6: at nodes, face levels max. of cell-center values Blmeanbelow = -999. # If not -999d0, below this level the cell center bed level is the mean of surrouding net nodes Blminabove = -999. # If not -999d0, above this level the cell center bed level is the min. of surrouding net nodes -Groundlayerthickness = -999. # Only in pipes: groundlayer thickness (m) -PartitionFile = # Domain partition polygon file *_part.pol for parallel run AngLat = 0. # Angle of latitude S-N (deg), 0: no Coriolis AngLon = 0. # Angle of longitude E-W (deg), 0: Greenwich, used in solar heat flux computation. Conveyance2D = -1 # -1: R=HU,0: R=H, 1: R=A/P, 2: K=analytic-1D conv, 3: K=analytic-2D conv @@ -66,7 +59,6 @@ Dcenterinside = 1. # Limit cell cen Bamin = 1.d-6 # Minimum grid cell area, in combination with cut cells OpenBoundaryTolerance = 0.1 # Search tolerance factor between boundary polyline and grid cells, in cell size units AllowBndAtBifurcation = 0 # Allow 1d boundary node when connecting branch leads to bifurcation (1: yes, 0: no) -CreateLinks1D2D = 0 # Ruecksichtslos create links between 1D nodes and 2D cells when initializing model (1: yes, 0: no) RenumberFlowNodes = 1 # Renumber the flow nodes (1: yes, 0: no) dxDoubleAt1DEndNodes = 1 # Extend 1D end nodes by 0.5 dx (1: yes, 0: no). ChangeVelocityAtStructures = 0 # Change the flow velocity at structures in the advection calculation @@ -80,9 +72,6 @@ StretchType = -1 # Type of layer Floorlevtoplay = -5. # Floor level of top layer Dztopuniabovez = -100. # Above this level layers will have uniform Dztop, below we use SigmaGrowthFactor Keepzlayeringatbed = 2 # 0:possibly very thin layer at bed, 1:bedlayerthickness == zlayerthickness, 2=equal thickness first two layers -Ihuz = 4 # if keepzlayeratbed>=3 : 1=central from bed til second, 2=all central, 3=from bed till highest equal levels -ihuzcsig = 3 # if keepzlayeratbed>=3 : 1,2,3=av,mx,mn of Leftsig,Rightsig,4=uniform -Zlayeratubybob = 0 # Lowest connected cells governed by bob instead of by bL L/R [volumeTables] useVolumeTables = 0 # Use 1D volume tables (1: yes, 0: no). @@ -91,39 +80,24 @@ useVolumeTables = 0 # Use 1D volume CFLMax = 0.7 # Maximum Courant number EpsMaxlev = 1.d-8 # Stop criterium for non linear iteration EpsMaxlevm = 1.d-8 # Stop criterium for Nested Newton loop in non linear iteration -Lincontin = 0 # Default 0; Set to 1 for linearizing d(Hu)/dx; link to AdvecType AdvecType = 33 # Advection type (0: none, 1: Wenneker, 2: Wenneker q(uio-u), 3: Perot q(uio-u), 4: Perot q(ui-u), 5: Perot q(ui-u) without itself), 33: as 3 using links (faster) AdvecCorrection1D2D = 0 # Advection correction of 1D2D link volume (0: regular advection, 1: link volume au*dx, 2: advection on 1D2D switched off. TimeStepType = 2 # Time step handling (0: only transport, 1: transport + velocity update, 2: full implicit step-reduce, 3: step-Jacobi, 4: explicit) maxNonlinearIterations = 100 # Maximal iterations in non-linear iteration loop before a time step reduction is applied setHorizontalBobsFor1d2d = 0 # bobs are set to 2D bedlevel, to prevent incorrect storage in sewer system (0: no, 1:yes). -Icoriolistype = 5 # 0=No, 5=default, 3,4 no weights, 5-10 Kleptsova hu/hs, 25-30 Ham hs/hu, odd: 2D hs/hu, even: hsk/huk -Newcorio = 1 # 0=prior to 27-11-2019, 1=no normal forcing on open bnds, plus 12 variants ) -Corioadamsbashfordfac = 0.5 # 0=No, 0.5d0=AdamsBashford, only for Newcorio=1) Limtyphu = 0 # Limiter type for waterdepth in continuity eqn. (0: none, 1: minmod, 2: van Leer, 3: Kooren, 4: monotone central) Limtypmom = 4 # Limiter type for cell center advection velocity (0: none, 1: minmod, 2: van Leer, 3: Kooren, 4: monotone central) Limtypsa = 4 # Limiter type for salinity transport (0: none, 1: minmod, 2: van Leer, 3: Kooren, 4: monotone central) -TransportMethod = 1 # Transport method (0: Herman's method, 1: transport module) -TransportTimestepping = 1 # Timestepping method in Transport module, 0 = global, 1 = local (default) TransportAutoTimestepdiff = 0 # Auto Timestepdiff in Transport, 0 : lim diff, no lim Dt_tr, 1 : no lim diff, lim Dt_tr, 2: no lim diff, no lim Dt_tr, 3=implicit (only 2D) Implicitdiffusion2D = 0 # 1 = Yes, 0 = No Vertadvtypsal = 6 # Vertical advection type for salinity (0: none, 1: upwind explicit, 2: central explicit, 3: upwind implicit, 4: central implicit, 5: central implicit but upwind for neg. stratif., 6: higher order explicit, no Forester) Vertadvtyptem = 6 # Vertical advection type for temperature (0: none, 1: upwind explicit, 2: central explicit, 3: upwind implicit, 4: central implicit, 5: central implicit but upwind for neg. stratif., 6: higher order explicit, no Forester) Vertadvtypmom = 6 # Vertical advection type for u1: 0: No, 3: Upwind implicit, 4: Central implicit, 5: QUICKEST implicit., 6: centerbased upwind expl, 7=6 HO -Vertadvtypmom3onbnd = 0 # vert. adv. u1 bnd UpwimpL: 0=follow javau , 1 = on bnd, 2= on and near bnd -Cffacver = 0. # Factor for including (1-CFL) in HO term vertical (0d0: no, 1d0: yes) -Jarhoxu = 0 # Include density gradient in advection term (0: no(strongly advised), 1: yes, 2: Also in barotropic and baroclinic pressure term) -Horadvtypzlayer = 0 # Horizontal advection treatment of z-layers (1: default, 2: sigma-like) Pure1D = 0 # 0 = org 1D advec, 1 = pure1D using vol1_f, 2 = pure1D using vol1 -Zlayercenterbedvel = 1 # reconstruction of center velocity at half closed bedcells (0=no, 1: copy bed link velocities) -Structurelayersactive = 1 # 0=structure flow through all layers, 1=structure flow only through open layers Icgsolver = 6 # Solver type (1: sobekGS_OMP, 2: sobekGS_OMPthreadsafe, 3: sobekGS, 4: sobekGS + Saadilud, 5: parallel/global Saad, 6: parallel/Petsc, 7: parallel/GS) Maxdegree = 6 # Maximum degree in Gauss elimination -Noderivedtypes = 0 # 0=use der. types. , 1 = less, 2 = lesser, 5 = also dealloc der. types -jposhchk = 2 # Check for positive waterdepth (0: no, 1: 0.7*dts, just redo, 2: 1.0*dts, close all links, 3: 0.7*dts, close all links, 4: 1.0*dts, reduce au, 5: 0.7*dts, reduce au) FixedWeirScheme = 0 # Fixed weir scheme (0: none, 1: compact stencil, 2: whole tile lifted, full subgrid weir + factor) FixedWeirContraction = 1. # Fixed weir flow width contraction factor -Fixedweirfrictscheme = 0 # Fixed weir friction scheme (0: friction based on hu, 1: friction based on subgrid weir friction scheme) Fixedweirtopwidth = 3. # Uniform width of the groyne part of fixed weirs Fixedweirtopfrictcoef = -999. # Uniform friction coefficient of the groyne part of fixed weirs Fixedweirtalud = 4. # Uniform talud slope of fixed weirs @@ -131,25 +105,16 @@ FixedweirRelaxationcoef = 0.6 # Fixed weir rel Izbndpos = 1 # Position of z boundary (0: D3Dflow, 1: on net boundary, 2: on specified polyline) Tlfsmo = 86400. # Fourier smoothing time (s) on water level boundaries Keepstbndonoutflow = 1 # Keep sal and tem signals on bnd also at outflow, 1=yes, 0=no=default=copy inside value on outflow -Diffusiononbnd = 1 # On open boundaries, 0 switches off horizontal diffusion Default = 1 Tspinupturblogprof = 0. # During Tspinup force log profiles to avoid almost zero vertical eddy viscosity -Logprofatubndin = 1 # ubnds inflow: 0=uniform U1, 1 = log U1, 2 = user3D -Logprofkepsbndin = 0 # inflow: 0=0 keps, 1 = log keps inflow, 2 = log keps in and outflow Slopedrop2D = 0. # Apply drop losses only if local bed slope > Slopedrop2D, (<=0: no drop losses) Drop1D = 0 # Apply drop losses in 1D (0: no, 1:yes) -Drop3D = 1. # Apply droplosses in 3D if z upwind below bob + 2/3 hu*drop3D Chkadvd = 0.1 # Check advection terms if depth < chkadvdp, => less setbacks -Trsh_u1Lb = 0. # 2D bedfriction in 3D below this threshold (m) -Epshstem = 1.d-3 # Only compute heatflx + evap if depth > epshstem -Zwsbtol = 0. # tolerance for zws(kb-1) at bed Teta0 = 0.55 # Theta of time integration (0.5 < theta < 1) -Jbasqbnddownwindhs = 0 # Water depth scheme at discharge boundaries (0: original hu, 1: downwind hs) cstbnd = 0 # Delft-3D type velocity treatment near boundaries for small coastal models (1: yes, 0: no) Maxitverticalforestersal = 0 # Forester iterations for salinity (0: no vertical filter for salinity, > 0: max nr of iterations) Maxitverticalforestertem = 0 # Forester iterations for temperature (0: no vertical filter for temperature, > 0: max nr of iterations) Turbulencemodel = 3 # Turbulence model (0: none, 1: constant, 2: algebraic, 3: k-epsilon, 4: k-tau) Turbulenceadvection = 3 # Turbulence advection (0: none, 3: horizontally explicit and vertically implicit) -Eddyviscositybedfacmax = 0. # Limit eddy viscosity at bed ) AntiCreep = 0 # Include anti-creep calculation (0: no, 1: yes) Maxwaterleveldiff = 0. # upper bound (in m) on water level changes (<= 0: no bounds). Run will abort when violated. Maxvelocitydiff = 0. # upper bound (in m/s) on velocity changes (<= 0: no bounds). Run will abort when violated. @@ -159,35 +124,14 @@ Velocitywarn = 0. # warning level Velmagnwarn = 0. # warning level (in m/s) on velocity magnitude (<= 0: no check). MinTimestepBreak = 0.1 # smallest allowed timestep (in s), checked on a sliding average of several timesteps. Run will abort when violated. Epshu = 1.d-4 # Threshold water depth for wet and dry cells -SobekDFM_umin = 0. # Minimal velocity treshold for weir losses in Sobek-DFM coupling. -SobekDFM_umin_method = 0 # Method for minimal velocity treshold for weir losses in Sobek-DFM coupling. -SobekDFM_Minimal_1d2d_Embankment = 1.d-2 # Minimal crest height of 1D2D SOBEK-DFM embankments. -sobekDFM_relax = 0.1 # Relaxation factor for SOBEK-DFM coupling algorithm. -jaupwindsrc = 1 # 1st-order upwind advection at sources/sinks (1) or higher-order (0) -jasfer3D = 1 # corrections for spherical coordinates HorizontalMomentumFilter = 0 # apply horizontal filter (1:explicit, 2,3:implicit) or not (0) checkerboardmonitor = 0 # compute and output checkerboard monitor (1) or not (0) -LocSaltLev = 1. # salinity level for case of lock exchange -LocSaltMin = 5. # minimum salinity for case of lock exchange -LocSaltMax = 10. # maximum salinity for case of lock exchange -Numlimdt_baorg = 0 # if previous numlimdt > Numlimdt_baorg keep original cell area ba in cutcell -Baorgfracmin = 0. # Cell area = max(orgcellarea*Baorgfracmin, cutcell area) -SubsUplUpdateS1 = 0 # Update water levels (S1) due to subsidence / uplift -Oceaneddysizefrac = 5.d-2 -Oceaneddysize = 0. -Oceaneddyamp = 5.d-2 -Oceaneddyvel = 0. -Oceaneddyyoff = 0. -Oceaneddyxoff = 0. [physics] UnifFrictCoef = 2.8d-2 # Uniform friction coefficient (0: no friction) UnifFrictType = 1 # Uniform friction type (0: Chezy, 1: Manning, 2: White-Colebrook, 3: idem, WAQUA style) UnifFrictCoef1D = 2.3d-2 # Uniform friction coefficient in 1D links (0: no friction) -UnifFrictCoef1D2D = 2.3d-2 # Uniform friction coefficient in 1D links (0: no friction) UnifFrictCoefLin = 0. # Uniform linear friction coefficient (0: no friction) -UnifFrictCoef1DgrLay = 5.d-2 # Uniform ground layer friction coefficient for ocean models (m/s) (0: no friction) -Umodlin = 1. # Linear friction umod, for ifrctyp=4,5,6 Vicouv = 0.1 # Uniform horizontal eddy viscosity (m2/s) Dicouv = 0.1 # Uniform horizontal eddy diffusivity (m2/s) Vicoww = 1.d-4 # Uniform vertical eddy viscosity (m2/s) @@ -202,8 +146,6 @@ Rhomean = 1023. # Average water Idensform = 2 # Density calulation (0: uniform, 1: Eckart, 2: Unesco, 3: baroclinic case) Ag = 9.813 # Gravitational acceleration TidalForcing = 1 # Tidal forcing, if jsferic=1 (0: no, 1: yes) -SelfAttractionLoading = 0 # Self attraction and loading (0=no, 1=yes, 2=only self attraction) -SelfAttractionLoading_correct_wl_with_ini = 0 # correct water level with initial water level in Self attraction and loading (0=no, 1=yes) ITcap = 1. # Upper limit on internal tides dissipation (W/m^2) Doodsonstart = 55.565 # TRIWAQ: 55.565, D3D: 57.555 Doodsonstop = 375.575 # TRIWAQ: 375.575, D3D: 275.555 @@ -225,18 +167,13 @@ Stanton = 1.3d-3 # Coefficient fo Dalton = 1.3d-3 # Coefficient for evaporative heat flux, if negative, Ceva = abs(Dalton)*Cdwind Tempmax = 50. # Limit the temperature Tempmin = -5. # Limit the temperature -Surftempsmofac = 0. # Hor . Smoothing factor for surface water in heatflx comp. (0.0-1.0), 0=no Heat_eachstep = 1 # 1=heat each timestep, 0=heat each usertimestep RhoairRhowater = 1 # windstress rhoa/rhow: 0=Rhoair/Rhomean, 1=Rhoair/rhow(), 2=rhoa0()/rhow(), 3=rhoa10()/Rhow() SecondaryFlow = 0 # Secondary flow (0: no, 1: yes) BetaSpiral = 0. # Weight factor of the spiral flow intensity on flow dispersion stresses -Equili = 0 # Equilibrium spiral flow intensity (0: no, 1: yes) [sediment] Sedimentmodelnr = 0 # Sediment model nr, (0=no, 1=Krone, 2=SvR2007, 3=E-H, 4=MorphologyModule) -Nr_of_sedfractions = 0 # Nr of sediment fractions, (specify the next parameters for each fraction) -MxgrKrone = 0 # Highest fraction index treated by Krone -Seddenscoupling = 0 # Sed rho coupling (0=no, 1=yes [veg] Vegetationmodelnr = 0 # Vegetation model nr, (0=no, 1=Baptist DFM) @@ -250,12 +187,10 @@ Stemheightstd = 0. # Stem height st ICdtyp = 4 # Wind drag coefficient type (1=Const; 2=Smith&Banke (2 pts); 3=S&B (3 pts); 4=Charnock 1955, 5=Hwang 2005, 6=Wuest 2005, 7=Hersbach 2010 (2 pts) Cdbreakpoints = 2.8618830252431d-2 # Wind drag coefficient break points Relativewind = 1 # Wind speed relative to top-layer water speed, 1=yes, 0 = no) -Windhuorzwsbased = 0 # Wind hu or zws based , 0 = hu, 1 = zws Windpartialdry = 1 # Reduce windstress on water if link partially dry, only for bedlevtyp=3, 0 = no, 1 = yes = default Rhoair = 1.2265 # Air density (kg/m3) PavBnd = 101330. # Average air pressure on open boundaries (N/m2) (only applied if > 0) Pavini = 0. # Average air pressure for initial water level correction (N/m2) (only applied if > 0) -Stericcorrection = 0 # Steric correction on waterlevel bnds, for which sal + temp should be prescribed [grw] groundwater = 0 # 0=No (horizontal) groundwater flow, 1=With groundwater flow @@ -271,13 +206,8 @@ InterceptionModel = 0 # Interception m [waves] Wavemodelnr = 0 # Wave model nr. (0: none, 1: fetch/depth limited hurdlestive, 2: Young-Verhagen, 3: SWAN, 5: uniform, 6: SWAN-NetCDF -Wavenikuradse = 1.d-2 # Wave friction Nikuradse ks coefficient (m), used in Krone-Swart Rouwav = FR84 # Friction model for wave induced shear stress: FR84 (default) or: MS90, HT91, GM79, DS88, BK67, CJ85, OY88, VR04 Gammax = 1. # Maximum wave height/water depth ratio -uorbfac = 1 # Orbital velocities: 0=D3D style; 1=Guza style -jahissigwav = 1 # 1: sign wave height on his output; 0: hrms wave height on his output. Default=1. -jamapsigwav = 0 # 1: sign wave height on map output; 0: hrms wave height on map output. Default=0 (legacy behaviour). -hminlw = 0.2 # Cut-off depth for application of wave forces in momentum balance [time] RefDate = 20111222 # Reference date (yyyymmdd) @@ -285,11 +215,8 @@ Tzone = 0. # Time zone assi DtUser = 600. # Time interval (s) for external forcing update DtNodal = 21600. # Time interval (s) for updating nodal factors in astronomical boundary conditions DtMax = 100. # Maximal computation timestep (s) -Dtfacmax = 1.1 # Max timestep increase factor ( ) DtInit = 60. # Initial computation timestep (s) -Timestepanalysis = 0 # 0=no, 1=see file *.steps AutoTimestep = 3 # 0 = no, 1 = 2D (hor. out), 3=3D (hor. out), 5 = 3D (hor. inout + ver. inout), smallest dt -Autotimestepvisc = 0 # 0 = no, 1 = yes (Time limitation based on explicit diffusive term) AutoTimestepNoStruct = 0 # 0 = no, 1 = yes (Exclude structure links (and neighbours) from time step limitation) AutoTimestepNoQout = 1 # 0 = no, 1 = yes (Exclude negative qin terms from time step limitation) Tunit = M # Time unit for start/stop times (D, H, M or S) @@ -302,7 +229,6 @@ UpdateRoughnessInterval = 86400. # Update interva [restart] RestartFile = # Restart netcdf-file, either *_rst.nc or *_map.nc RestartDateTime = yyyymmddhhmmss # Restart date and time (yyyymmddhhmmss) when restarting from *_map.nc -RstIgnoreBl = 0 # Flag indicating whether bed level from restart should be ignored (0=no (default), 1=yes) [external forcing] ExtForceFile = @@ -318,9 +244,6 @@ TrtDef = # File (*.ttd) i TrtL = # File (*.arl) including distribution of trachytope definitions DtTrt = 600. # Trachytope roughness update time interval (s) TrtMxR = 100 # Maximum recursion level for composite trachytope definitions -TrtCll = # Calibration factor file for roughness from trachytopes (see also [calibration] block) -TrtMnH = 0.1 # Minimum water depth for roughness computations -TrtMth = 1 # Area averaging method, (1=Nikuradse k based, 2=Chezy C based (parallel and serial)) [calibration] UseCalibration = 0 # Activate calibration factor friction multiplier (1 = yes, 0 = no) @@ -332,7 +255,6 @@ OutputDir = # Output directo FlowGeomFile = # Flow geometry NetCDF *_flowgeom.nc ObsFile = CrsFile = -RugFile = # Polyline file *_rug.pli defining runup gauges FouFile = FouUpdateStep = 0 # Fourier update step type: 0=every user time step, 1=every computational timestep, 2=same as history output. HisFile = # HisFile name *_his.nc @@ -341,26 +263,14 @@ HisInterval = 600. 864000. 190252800.# History tim XLSInterval = 0. # Interval (s) XLS history MapInterval = 86400. 64065600. 95515200.# Map times (s), interval, starttime, stoptime (s), if starttime, stoptime are left blank, use whole simulation period RstInterval = 31536000. 864000. 190252800.# Restart times (s), interval, starttime, stoptime (s), if starttime, stoptime are left blank, use whole simulation period -MbaInterval = 0. # Mass balance area output interval (s) -MbaWriteCsv = 0 # Write mass balance area output to a csv-file (1: yes, 0: no) -MbaLumpFromToMba = 0 # Lump MBA from/to other areas mass balance terms (1: yes, 0: no) -MbaLumpBoundaries = 0 # Lump MBA boundary mass balance terms (1: yes, 0: no) -MbaLumpSourceSinks = 0 # Lump MBA source/sink mass balance terms (1: yes, 0: no) -MbaLumpProcesses = 0 # Lump MBA processes mass balance terms (1: yes, 0: no) WaqOutputDir = # Output directory of WAQ communication files (flowgeom, vol, flo, etc.), default: DFM_DELWAQ_. Set to . for current dir. WaqInterval = 0. 864000. 190252800.# DELWAQ output times, given as "interval" "start period" "end period" (s) -WaqHorAggr = # DELWAQ output horizontal aggregation file (*.dwq) -WaqVertAggr = # DELWAQ output vertical aggregation file (*.vag) ClassMapInterval = -999. 864000. 190252800.# Class map times (s), interval, starttime, stoptime (s), if starttime, stoptime are left blank, use whole simulation period ClassMapFile = # ClassMapFile name *_clm.nc WaterlevelClasses = # Class map's list of class values for water levels WaterDepthClasses = # Class map's list of class values for water depths -VelocityMagnitudeClasses = # Class map's list of class values for velocity magnitudes -VelocityDirectionClassesInterval = # Class map's step size of class values for velocity direction StatsInterval = -60. # Screen step output interval in seconds simulation time, if negative in seconds wall clock time -WriteBalancefile = 0 # Write balance file (1: yes, 0: no) TimingsInterval = 86400. # Timings statistics output interval -TimeSplitInterval = 0 s # Time splitting interval, after which a new output file is started. value+unit, e.g. '1 M', valid units: Y,M,D,h,m,s. MapFormat = 4 # Map file format, 1: netCDF, 2: Tecplot, 3: netCFD and Tecplot, 4: NetCDF-UGRID NcFormat = 4 # Format for all NetCDF output files (3: classic, 4: NetCDF4+HDF5) NcNoUnlimited = 0 # Write full-length time-dimension instead of unlimited dimension (1: yes, 0: no). (Might require NcFormat=4.) @@ -424,8 +334,6 @@ Wrimap_flow_flux_q1_main = 0 # Write flow flu Wrimap_numlimdt = 1 # Write the number times a cell was Courant limiting to map file (1: yes, 0: no). (Consider using Wrimap_flow_analysis instead.) Wrimap_taucurrent = 0 # Write the shear stress to map file (1: yes, 0: no) Wrimap_chezy = 0 # Write the chezy roughness to map file (1: yes, 0: no) -Wrimap_salinity = 1 # Write salinity to map file (1: yes, 0: no) -Wrimap_temperature = 1 # Write temperature to map file (1: yes, 0: no) Wrimap_turbulence = 0 # Write vicww, k and eps to map file (1: yes, 0: no) Wrimap_rain = 0 # Write rainfall rates to map file (1: yes, 0: no) Wrimap_interception = 0 # Write interception to map file (1: yes, 0: no) @@ -443,7 +351,6 @@ Wrimap_total_net_inflow_1d2d = 0 # Write current Wrimap_total_net_inflow_lateral = 0 # Write current total lateral net inflow (discharge) and cumulative total net lateral inflow (volume) to map file, only for 1D nodes (1: yes, 0: no) Wrimap_water_level_gradient = 0 # Write water level gradient to map file, only on 1D links (1: yes, 0: no) Wrimap_tidal_potential = 0 # Write tidal potential to map file (1: yes, 0: no) -Writepart_domain = 1 # Write partition domain info. for postprocessing Richardsononoutput = 0 # Write Richardson numbers (1: yes, 0: no) Wrishp_crs = 0 # Write grid-snapped cross sections to shapefile (1: yes, 0: no) Wrishp_obs = 0 # Write grid-snapped observation stations to shapefile (1: yes, 0: no) @@ -456,12 +363,9 @@ Wrishp_src = 0 # Write grid-sna Wrishp_pump = 0 # Write grid-snapped pumps to shapefile (1: yes, 0: no) Wrishp_dryarea = 0 # Write a shape file for dry areas wrishp_genstruc = 0 # Write a shape file for general structures -WriteDFMinterpretedvalues = 0 # Write DFMinterpretedvalues (1: yes, 0: no) -WriteDetailedTimers = 0 # Write detailed timers output file (1: yes, 0: no) MapOutputTimeVector = # File (*.mpt) containing fixed map output times (s) w.r.t. RefDate FullGridOutput = 0 # Full grid output mode for layer positions (0: compact, 1: full time-varying grid layer data) EulerVelocities = 0 # Euler velocities output (0: GLM, 1: Euler velocities) -Wrirst_bnd = 1 # Write waterlevel, bedlevel and coordinates of boundaries to restart files Wrimap_bnd = 0 # Write boundary points to map file (1: yes, 0: no) Wrimap_Qin = 0 # Write sum of all influxes to map file (1: yes, 0: no) Wrimap_every_dt = 0 # Write output to map file every dt, based on start and stop from MapInterval, 0=no (default), 1=yes diff --git a/tests/data/input/e02/c11_korte-woerden-1d/dimr_model/dflowfm/FlowFM.mdu b/tests/data/input/e02/c11_korte-woerden-1d/dimr_model/dflowfm/FlowFM.mdu index 4f53e356b..66831cc74 100644 --- a/tests/data/input/e02/c11_korte-woerden-1d/dimr_model/dflowfm/FlowFM.mdu +++ b/tests/data/input/e02/c11_korte-woerden-1d/dimr_model/dflowfm/FlowFM.mdu @@ -6,7 +6,6 @@ Program = D-Flow FM Version = 1.2.94.66079M fileType = modelDef fileVersion = 1.09 -GuiVersion = 1.5.0.49132 AutoStart = 0 PathsRelativeToParent= 0 @@ -26,7 +25,6 @@ CrossDefFile = crsdef.ini CrossLocFile = crsloc.ini FrictFile = roughness-Main.ini;roughness-Sewer1.ini;roughness-Sewer2.ini StorageNodeFile = nodeFile.ini -BranchFile = branches.gui RoofsFile = ProfdefxyzFile = IniFieldFile = @@ -67,13 +65,11 @@ TimeStepType = 2 Limtyphu = 0 Limtypmom = 4 Limtypsa = 4 -TransportMethod = 1 Vertadvtypsal = 5 Icgsolver = 4 Maxdegree = 6 FixedWeirScheme = 6 FixedWeirContraction= 1 -FixedWeirfrictscheme= 1 Fixedweirtopwidth = 3 Fixedweirtopfrictcoef= -999 Fixedweirtalud = 4 @@ -84,11 +80,9 @@ drop1D = 1 Chkadvd = 0.1 Teta0 = 0.55 Qhrelax = 0.01 -Jbasqbnddownwindhs= 0 cstbnd = 0 Maxitverticalforestersal= 0 Maxitverticalforestertem= 0 -Jaorgsethu = 1 Turbulencemodel = 3 Turbulenceadvection= 3 AntiCreep = 0 @@ -101,7 +95,6 @@ UnifFrictCoef = 0.023 UnifFrictType = 1 UnifFrictCoef1D = 0.023 UnifFrictCoefLin = 0 -Umodlin = 0 Vicouv = 1 Dicouv = 1 Vicoww = 5E-05 @@ -129,7 +122,6 @@ Stanton = -1 Dalton = -1 Backgroundwatertemperature= 6 SecondaryFlow = 0 -EffectSpiral = 0 BetaSpiral = 0 Temperature = 0 @@ -143,7 +135,6 @@ PavIni = 0 [waves] Wavemodelnr = 0 -WaveNikuradse = 0.01 Rouwav = FR84 Gammax = 1 @@ -195,7 +186,6 @@ XLSInterval = MapFile = MapInterval = 60 RstInterval = 86400 -S1incinterval = MapFormat = 4 Wrihis_balance = 1 Wrihis_sourcesink = 1 @@ -254,10 +244,7 @@ WaqInterval = 0 ClassMapFile = WaterlevelClasses = 0 WaterdepthClasses = 0 -VelocityMagnitudeClasses= 0 -VelocityDirectionClassesInterval= 30 ClassMapInterval = 300 StatsInterval = -Writebalancefile = 0 TimingsInterval = Richardsononoutput= 1 diff --git a/tests/data/input/e02/c11_korte-woerden-1d/dimr_model/dflowfm/crsdef.ini b/tests/data/input/e02/c11_korte-woerden-1d/dimr_model/dflowfm/crsdef.ini index e82b4f71f..5ed79a534 100644 --- a/tests/data/input/e02/c11_korte-woerden-1d/dimr_model/dflowfm/crsdef.ini +++ b/tests/data/input/e02/c11_korte-woerden-1d/dimr_model/dflowfm/crsdef.ini @@ -119,4 +119,3 @@ sectionCount = 1 frictionIds = Main frictionPositions = -10.00 10.00 - isShared = 1 diff --git a/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/Boundary.mdu b/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/Boundary.mdu index a19c5641e..57486d280 100644 --- a/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/Boundary.mdu +++ b/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/Boundary.mdu @@ -10,8 +10,6 @@ AutoStart = 0 # Autostart simulation a [geometry] NetFile = Boundary_net.nc # Unstructured grid file *_net.nc -OneDNetworkFile = # 1d networkfile -BedlevelFile = # street_level.xyz , Bedlevels points file e.g. *.xyz, only needed for bedlevtype not equal 3 DryPointsFile = # Dry points file *.xyz (third column dummy z values), or dry areas polygon file *.pol (third column 1/-1: inside/outside) IniFieldFile = initialFields.ini # Initial and parameter field file *.ini LandBoundaryFile = # Land boundaries file *.ldb, used for visualization @@ -25,8 +23,6 @@ ProfdefxyzFile = # Channel profile defini Uniformwidth1D = 1. # Uniform width for channel profiles not specified by profloc Uniformheight1D = 1. # Uniform height for channel profiles not specified by profloc ManholeFile = # File *.ini containing manholes -PipeFile = # File *.pliz containing pipe-based 'culverts' -ShipdefFile = # File *.shd containing ship definitions StructureFile = # File *.ini containing structures CrossLocFile = CrossSectionLocations.ini # Name and location of the file containing the locations of the cross sections CrossDefFile = CrossSectionDefinitions.ini # Name and location of the file containing the definitions of the cross sections @@ -51,27 +47,20 @@ NonLin1D = 0 CFLMax = 0.7 # Maximum Courant number AdvecType = 33 # Advection type (0: none, 1: Wenneker, 2: Wenneker q(uio-u), 3: Perot q(uio-u), 4: Perot q(ui-u), 5: Perot q(ui-u) without itself) TimeStepType = 2 # Time step handling (0: only transport, 1: transport + velocity update, 2: full implicit step-reduce, 3: step-Jacobi, 4: explicit) -Icoriolistype = 5 # 0=No, 1=yes, if jsferic then spatially varying, if icoriolistype==6 then constant (anglat) Limtypmom = 4 # Limiter type for cell center advection velocity (0: none, 1: minmod, 2: van Leer, 3: Kooren, 4: monotone central) Limtypsa = 4 # Limiter type for salinity transport (0: none, 1: minmod, 2: van Leer, 3: Kooren, 4: monotone central) -TransportMethod = 1 # Transport method (0: Herman's method, 1: transport module) -TransportTimestepping = 1 # Timestepping method in Transport module, 0 = global, 1 = local (default) Vertadvtypsal = 6 # Vertical advection type for salinity (0: none, 1: upwind explicit, 2: central explicit, 3: upwind implicit, 4: central implicit, 5: central implicit but upwind for neg. stratif., 6: higher order explicit, no Forester) Vertadvtyptem = 6 # Vertical advection type for temperature (0: none, 1: upwind explicit, 2: central explicit, 3: upwind implicit, 4: central implicit, 5: central implicit but upwind for neg. stratif., 6: higher order explicit, no Forester) Icgsolver = 4 # Solver type (1: sobekGS_OMP, 2: sobekGS_OMPthreadsafe, 3: sobekGS, 4: sobekGS + Saadilud, 5: parallel/global Saad, 6: parallel/Petsc, 7: parallel/GS) -Noderivedtypes = 5 # 0=use der. types. , 1 = less, 2 = lesser, 5 = also dealloc der. types Tlfsmo = 0. # Fourier smoothing time (s) on water level boundaries Slopedrop2D = 0. # Apply drop losses only if local bed slope > Slopedrop2D, (<=0: no drop losses) cstbnd = 0 # Delft-3D type velocity treatment near boundaries for small coastal models (1: yes, 0: no) Epshu = 1.d-4 # Threshold water depth for wet and dry cells -jaupwindsrc = 1 # 1st-order upwind advection at sources/sinks (1) or higher-order (0) -jasfer3D = 0 # corrections for spherical coordinates [physics] UnifFrictCoef = 50 # Uniform friction coefficient (0: no friction) UnifFrictType = 0 # Uniform friction type (0: Chezy, 1: Manning, 2: White-Colebrook, 3: idem, WAQUA style) UnifFrictCoef1D = 50 # Uniform friction coefficient in 1D links (0: no friction) -UnifFrictCoef1D2D = 50 # Uniform friction coefficient in 1D links (0: no friction) UnifFrictCoefLin = 0. # Uniform linear friction coefficient for ocean models (m/s) (0: no friction) Vicouv = 1. # Uniform horizontal eddy viscosity (m2/s) Dicouv = 1. # Uniform horizontal eddy diffusivity (m2/s) @@ -82,7 +71,6 @@ wall_ks = 0. # Wall roughness type (0 Rhomean = 1000. # Average water density (kg/m3) Ag = 9.81 # Gravitational acceleration TidalForcing = 1 # Tidal forcing, if jsferic=1 (0: no, 1: yes) -SelfAttractionLoading = 0 # Self attraction and loading (0=no, 1=yes, 2=only self attraction) VillemonteCD1 = 1. # Calibration coefficient for Villemonte. Default = 1.0. NB. For Bloemberg data set 0.8 is recommended. VillemonteCD2 = 10. # Calibration coefficient for Villemonte. Default = 10.0. NB. For Bloemberg data set 0.8 is recommended. Salinity = 0 # Include salinity, (0=no, 1=yes) @@ -104,10 +92,7 @@ Tzone = 0. # Time zone assigned to DtUser = 3600.000 # Time interval (s) for external forcing update DtNodal = 60. # Time interval (s) for updating nodal factors in astronomical boundary conditions DtMax = 600. # Maximal computation timestep (s) -Dtfacmax = 1.1 # Max timestep increase factor ( ) DtInit = 60. # Initial computation timestep (s) -Timestepanalysis = 0 # 0=no, 1=see file *.steps -Autotimestepdiff = 0 # 0 = no, 1 = yes (Time limitation based on explicit diffusive term) Tunit = S # Time unit for start/stop times (D, H, M or S) TStart = 0. # Start time w.r.t. RefDate (in TUnit) TStop = 259200.0 # Stop time w.r.t. RefDate (in TUnit) @@ -135,7 +120,6 @@ RstInterval = 0. 0. 0. # Restart file output ti WaqInterval = 0. 0. 0. # DELWAQ output times, given as "interval" "start period" "end period" (s) StatsInterval = -60. # Screen step output interval in seconds simulation time, if negative in seconds wall clock time TimingsInterval = 0. # Timings statistics output interval -TimeSplitInterval = 0X # Time splitting interval, after which a new output file is started. value+unit, e.g. '1 M', valid units: Y,M,D,h,m,s. MapFormat = 4 # Map file format, 1: netCDF, 2: Tecplot, 3: netCFD and Tecplot, 4: NetCDF-UGRID Wrihis_balance = 1 # Write mass balance totals to his file (1: yes, 0: no) Wrihis_sourcesink = 1 # Write sources-sinks statistics to his file (1=yes, 0=no) @@ -143,7 +127,6 @@ Wrihis_turbulence = 1 # Write k, eps and vicww Wrihis_wind = 1 # Write wind velocities to his file (1: yes, 0: no) Wrihis_rain = 1 # Write precipitation to his file (1: yes, 0: no) Wrihis_temperature = 1 # Write temperature to his file (1: yes, 0: no) -Wrihis_heatflux = 1 # Write heat flux to his file (1: yes, 0: no) Wrihis_salinity = 1 # Write salinity to his file (1: yes, 0: no) Wrimap_waterlevel_s0 = 1 # Write water levels for previous time step to map file (1: yes, 0: no) Wrimap_waterlevel_s1 = 1 # Write water levels to map file (1: yes, 0: no) @@ -165,8 +148,6 @@ Wrimap_tidal_potential = 1 # Write tidal potential MapOutputTimeVector = # File (*.mpt) containing fixed map output times (s) w.r.t. RefDate FullGridOutput = 0 # Full grid output mode (0: compact, 1: full time-varying grid data) EulerVelocities = 0 # Euler velocities output (0: GLM, 1: Euler velocities) -Wrirst_bnd = 1 # Write waterlevel, bedlevel and coordinates of boundaries to restart files -Writepart_domain = 1 # Write partition domain info. for postprocessing [sediment] MorFile = # Morphology settings file (*.mor) diff --git a/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini b/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini index 99cd7c3a4..e13e6c3b5 100644 --- a/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini +++ b/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini @@ -14,5 +14,4 @@ sectionCount = 1 frictionIds = Main frictionPositions = 0.000 50.000 - isShared = 1 diff --git a/tests/data/input/file_load_test/fm.mdu b/tests/data/input/file_load_test/fm.mdu index 360e81e2b..a2b3b686d 100644 --- a/tests/data/input/file_load_test/fm.mdu +++ b/tests/data/input/file_load_test/fm.mdu @@ -6,7 +6,6 @@ Program = D-Flow FM Version = 1.2.124.69571M fileType = modelDef fileVersion = 1.09 -GuiVersion = 4.5.0.491 AutoStart = 0 PathsRelativeToParent= 0 @@ -69,13 +68,11 @@ TimeStepType = 2 Limtyphu = 0 Limtypmom = 4 Limtypsa = 4 -TransportMethod = 1 Vertadvtypsal = 5 Icgsolver = 4 Maxdegree = 6 FixedWeirScheme = 6 FixedWeirContraction= 1 -FixedWeirfrictscheme= 0 Fixedweirtopwidth = 3 Fixedweirtopfrictcoef= -999 Fixedweirtalud = 4 @@ -86,7 +83,6 @@ Drop1D = 1 Chkadvd = 0.1 Teta0 = 0.55 Qhrelax = 0.01 -Jbasqbnddownwindhs= 0 cstbnd = 0 Maxitverticalforestersal= 0 Maxitverticalforestertem= 0 @@ -101,14 +97,12 @@ EpsMaxlevm = 0.0001 maxNonlinearIterations= 100 setHorizontalBobsFor1d2d= 0 Pure1D = 0 -jposhchk = 2 [physics] UnifFrictCoef = 0.023 UnifFrictType = 1 UnifFrictCoef1D = 0.023 UnifFrictCoefLin = 0 -Umodlin = 0 Vicouv = 1 Dicouv = 1 Vicoww = 5E-05 @@ -136,7 +130,6 @@ Stanton = -1 Dalton = -1 Backgroundwatertemperature= 6 SecondaryFlow = 0 -EffectSpiral = 0 BetaSpiral = 0 Temperature = 0 @@ -150,7 +143,6 @@ PavIni = 0 [waves] Wavemodelnr = 0 -WaveNikuradse = 0.01 Rouwav = FR84 Gammax = 1 @@ -206,7 +198,6 @@ XLSInterval = MapFile = MapInterval = 3600 RstInterval = 0 -S1incinterval = MapFormat = 4 Wrihis_balance = 1 Wrihis_sourcesink = 1 @@ -273,10 +264,7 @@ ClassMapFile = WaterlevelClasses = 0 WaterdepthClasses = 0 ClassMapInterval = 0 -VelocityMagnitudeClasses= 0 -VelocityDirectionClassesInterval= 30 StatsInterval = -Writebalancefile = 0 TimingsInterval = Richardsononoutput= 1 NcFormat = 3 @@ -289,7 +277,6 @@ FouUpdateStep = 0 [volumeTables] useVolumeTables = 1 increment = 0.1 -useVolumeTablesFile= 1 [hydrology] InterceptionModel = 0 diff --git a/tests/data/input/obscrsfile_cases/both_ini_and_pli/emptymodel_with_obscrs.mdu b/tests/data/input/obscrsfile_cases/both_ini_and_pli/emptymodel_with_obscrs.mdu index 93cf91760..acccbafda 100644 --- a/tests/data/input/obscrsfile_cases/both_ini_and_pli/emptymodel_with_obscrs.mdu +++ b/tests/data/input/obscrsfile_cases/both_ini_and_pli/emptymodel_with_obscrs.mdu @@ -10,8 +10,6 @@ AutoStart = 0 # Autostart simulation a [geometry] NetFile = # Unstructured grid file *_net.nc -OneDNetworkFile = # 1d networkfile -BedlevelFile = # street_level.xyz , Bedlevels points file e.g. *.xyz, only needed for bedlevtype not equal 3 DryPointsFile = # Dry points file *.xyz (third column dummy z values), or dry areas polygon file *.pol (third column 1/-1: inside/outside) IniFieldFile = # Initial and parameter field file *.ini LandBoundaryFile = # Land boundaries file *.ldb, used for visualization @@ -25,8 +23,6 @@ ProfdefxyzFile = # Channel profile defini Uniformwidth1D = 1. # Uniform width for channel profiles not specified by profloc Uniformheight1D = 1. # Uniform height for channel profiles not specified by profloc ManholeFile = # File *.ini containing manholes -PipeFile = # File *.pliz containing pipe-based 'culverts' -ShipdefFile = # File *.shd containing ship definitions StructureFile = # File *.ini containing structures CrossLocFile = # Name and location of the file containing the locations of the cross sections CrossDefFile = # Name and location of the file containing the definitions of the cross sections @@ -51,25 +47,20 @@ NonLin1D = 0 CFLMax = 0.7 # Maximum Courant number AdvecType = 33 # Advection type (0: none, 1: Wenneker, 2: Wenneker q(uio-u), 3: Perot q(uio-u), 4: Perot q(ui-u), 5: Perot q(ui-u) without itself) TimeStepType = 2 # Time step handling (0: only transport, 1: transport + velocity update, 2: full implicit step-reduce, 3: step-Jacobi, 4: explicit) -Icoriolistype = 5 # 0=No, 1=yes, if jsferic then spatially varying, if icoriolistype==6 then constant (anglat) Limtypmom = 4 # Limiter type for cell center advection velocity (0: none, 1: minmod, 2: van Leer, 3: Kooren, 4: monotone central) Limtypsa = 4 # Limiter type for salinity transport (0: none, 1: minmod, 2: van Leer, 3: Kooren, 4: monotone central) Vertadvtypsal = 6 # Vertical advection type for salinity (0: none, 1: upwind explicit, 2: central explicit, 3: upwind implicit, 4: central implicit, 5: central implicit but upwind for neg. stratif., 6: higher order explicit, no Forester) Vertadvtyptem = 6 # Vertical advection type for temperature (0: none, 1: upwind explicit, 2: central explicit, 3: upwind implicit, 4: central implicit, 5: central implicit but upwind for neg. stratif., 6: higher order explicit, no Forester) Icgsolver = 4 # Solver type (1: sobekGS_OMP, 2: sobekGS_OMPthreadsafe, 3: sobekGS, 4: sobekGS + Saadilud, 5: parallel/global Saad, 6: parallel/Petsc, 7: parallel/GS) -Noderivedtypes = 5 # 0=use der. types. , 1 = less, 2 = lesser, 5 = also dealloc der. types Tlfsmo = 0. # Fourier smoothing time (s) on water level boundaries Slopedrop2D = 0. # Apply drop losses only if local bed slope > Slopedrop2D, (<=0: no drop losses) cstbnd = 0 # Delft-3D type velocity treatment near boundaries for small coastal models (1: yes, 0: no) Epshu = 1.d-4 # Threshold water depth for wet and dry cells -jaupwindsrc = 1 # 1st-order upwind advection at sources/sinks (1) or higher-order (0) -jasfer3D = 0 # corrections for spherical coordinates [physics] UnifFrictCoef = 50 # Uniform friction coefficient (0: no friction) UnifFrictType = 0 # Uniform friction type (0: Chezy, 1: Manning, 2: White-Colebrook, 3: idem, WAQUA style) UnifFrictCoef1D = 50 # Uniform friction coefficient in 1D links (0: no friction) -UnifFrictCoef1D2D = 50 # Uniform friction coefficient in 1D links (0: no friction) UnifFrictCoefLin = 0. # Uniform linear friction coefficient for ocean models (m/s) (0: no friction) Vicouv = 1. # Uniform horizontal eddy viscosity (m2/s) Dicouv = 1. # Uniform horizontal eddy diffusivity (m2/s) @@ -80,7 +71,6 @@ wall_ks = 0. # Wall roughness type (0 Rhomean = 1000. # Average water density (kg/m3) Ag = 9.81 # Gravitational acceleration TidalForcing = 1 # Tidal forcing, if jsferic=1 (0: no, 1: yes) -SelfAttractionLoading = 0 # Self attraction and loading (0=no, 1=yes, 2=only self attraction) VillemonteCD1 = 1. # Calibration coefficient for Villemonte. Default = 1.0. NB. For Bloemberg data set 0.8 is recommended. VillemonteCD2 = 10. # Calibration coefficient for Villemonte. Default = 10.0. NB. For Bloemberg data set 0.8 is recommended. Salinity = 0 # Include salinity, (0=no, 1=yes) @@ -102,10 +92,7 @@ Tzone = 0. # Time zone assigned to DtUser = 3600.000 # Time interval (s) for external forcing update DtNodal = 60. # Time interval (s) for updating nodal factors in astronomical boundary conditions DtMax = 600. # Maximal computation timestep (s) -Dtfacmax = 1.1 # Max timestep increase factor ( ) DtInit = 60. # Initial computation timestep (s) -Timestepanalysis = 0 # 0=no, 1=see file *.steps -Autotimestepdiff = 0 # 0 = no, 1 = yes (Time limitation based on explicit diffusive term) Tunit = S # Time unit for start/stop times (D, H, M or S) TStart = 0. # Start time w.r.t. RefDate (in TUnit) TStop = 259200.0 # Stop time w.r.t. RefDate (in TUnit) @@ -133,7 +120,6 @@ RstInterval = 0. 0. 0. # Restart file output ti WaqInterval = 0. 0. 0. # DELWAQ output times, given as "interval" "start period" "end period" (s) StatsInterval = -60. # Screen step output interval in seconds simulation time, if negative in seconds wall clock time TimingsInterval = 0. # Timings statistics output interval -TimeSplitInterval = 0X # Time splitting interval, after which a new output file is started. value+unit, e.g. '1 M', valid units: Y,M,D,h,m,s. MapFormat = 4 # Map file format, 1: netCDF, 2: Tecplot, 3: netCFD and Tecplot, 4: NetCDF-UGRID Wrihis_balance = 1 # Write mass balance totals to his file (1: yes, 0: no) Wrihis_sourcesink = 1 # Write sources-sinks statistics to his file (1=yes, 0=no) @@ -141,7 +127,6 @@ Wrihis_turbulence = 1 # Write k, eps and vicww Wrihis_wind = 1 # Write wind velocities to his file (1: yes, 0: no) Wrihis_rain = 1 # Write precipitation to his file (1: yes, 0: no) Wrihis_temperature = 1 # Write temperature to his file (1: yes, 0: no) -Wrihis_heatflux = 1 # Write heat flux to his file (1: yes, 0: no) Wrihis_salinity = 1 # Write salinity to his file (1: yes, 0: no) Wrimap_waterlevel_s0 = 1 # Write water levels for previous time step to map file (1: yes, 0: no) Wrimap_waterlevel_s1 = 1 # Write water levels to map file (1: yes, 0: no) @@ -163,8 +148,6 @@ Wrimap_tidal_potential = 1 # Write tidal potential MapOutputTimeVector = # File (*.mpt) containing fixed map output times (s) w.r.t. RefDate FullGridOutput = 0 # Full grid output mode (0: compact, 1: full time-varying grid data) EulerVelocities = 0 # Euler velocities output (0: GLM, 1: Euler velocities) -Wrirst_bnd = 1 # Write waterlevel, bedlevel and coordinates of boundaries to restart files -Writepart_domain = 1 # Write partition domain info. for postprocessing [sediment] MorFile = # Morphology settings file (*.mor) diff --git a/tests/data/input/obscrsfile_cases/single_ini/emptymodel_with_obscrs.mdu b/tests/data/input/obscrsfile_cases/single_ini/emptymodel_with_obscrs.mdu index 2bc30d392..9a17677c5 100644 --- a/tests/data/input/obscrsfile_cases/single_ini/emptymodel_with_obscrs.mdu +++ b/tests/data/input/obscrsfile_cases/single_ini/emptymodel_with_obscrs.mdu @@ -10,8 +10,6 @@ AutoStart = 0 # Autostart simulation a [geometry] NetFile = # Unstructured grid file *_net.nc -OneDNetworkFile = # 1d networkfile -BedlevelFile = # street_level.xyz , Bedlevels points file e.g. *.xyz, only needed for bedlevtype not equal 3 DryPointsFile = # Dry points file *.xyz (third column dummy z values), or dry areas polygon file *.pol (third column 1/-1: inside/outside) IniFieldFile = # Initial and parameter field file *.ini LandBoundaryFile = # Land boundaries file *.ldb, used for visualization @@ -25,8 +23,6 @@ ProfdefxyzFile = # Channel profile defini Uniformwidth1D = 1. # Uniform width for channel profiles not specified by profloc Uniformheight1D = 1. # Uniform height for channel profiles not specified by profloc ManholeFile = # File *.ini containing manholes -PipeFile = # File *.pliz containing pipe-based 'culverts' -ShipdefFile = # File *.shd containing ship definitions StructureFile = # File *.ini containing structures CrossLocFile = # Name and location of the file containing the locations of the cross sections CrossDefFile = # Name and location of the file containing the definitions of the cross sections @@ -51,25 +47,20 @@ NonLin1D = 0 CFLMax = 0.7 # Maximum Courant number AdvecType = 33 # Advection type (0: none, 1: Wenneker, 2: Wenneker q(uio-u), 3: Perot q(uio-u), 4: Perot q(ui-u), 5: Perot q(ui-u) without itself) TimeStepType = 2 # Time step handling (0: only transport, 1: transport + velocity update, 2: full implicit step-reduce, 3: step-Jacobi, 4: explicit) -Icoriolistype = 5 # 0=No, 1=yes, if jsferic then spatially varying, if icoriolistype==6 then constant (anglat) Limtypmom = 4 # Limiter type for cell center advection velocity (0: none, 1: minmod, 2: van Leer, 3: Kooren, 4: monotone central) Limtypsa = 4 # Limiter type for salinity transport (0: none, 1: minmod, 2: van Leer, 3: Kooren, 4: monotone central) Vertadvtypsal = 6 # Vertical advection type for salinity (0: none, 1: upwind explicit, 2: central explicit, 3: upwind implicit, 4: central implicit, 5: central implicit but upwind for neg. stratif., 6: higher order explicit, no Forester) Vertadvtyptem = 6 # Vertical advection type for temperature (0: none, 1: upwind explicit, 2: central explicit, 3: upwind implicit, 4: central implicit, 5: central implicit but upwind for neg. stratif., 6: higher order explicit, no Forester) Icgsolver = 4 # Solver type (1: sobekGS_OMP, 2: sobekGS_OMPthreadsafe, 3: sobekGS, 4: sobekGS + Saadilud, 5: parallel/global Saad, 6: parallel/Petsc, 7: parallel/GS) -Noderivedtypes = 5 # 0=use der. types. , 1 = less, 2 = lesser, 5 = also dealloc der. types Tlfsmo = 0. # Fourier smoothing time (s) on water level boundaries Slopedrop2D = 0. # Apply drop losses only if local bed slope > Slopedrop2D, (<=0: no drop losses) cstbnd = 0 # Delft-3D type velocity treatment near boundaries for small coastal models (1: yes, 0: no) Epshu = 1.d-4 # Threshold water depth for wet and dry cells -jaupwindsrc = 1 # 1st-order upwind advection at sources/sinks (1) or higher-order (0) -jasfer3D = 0 # corrections for spherical coordinates [physics] UnifFrictCoef = 50 # Uniform friction coefficient (0: no friction) UnifFrictType = 0 # Uniform friction type (0: Chezy, 1: Manning, 2: White-Colebrook, 3: idem, WAQUA style) UnifFrictCoef1D = 50 # Uniform friction coefficient in 1D links (0: no friction) -UnifFrictCoef1D2D = 50 # Uniform friction coefficient in 1D links (0: no friction) UnifFrictCoefLin = 0. # Uniform linear friction coefficient for ocean models (m/s) (0: no friction) Vicouv = 1. # Uniform horizontal eddy viscosity (m2/s) Dicouv = 1. # Uniform horizontal eddy diffusivity (m2/s) @@ -80,7 +71,6 @@ wall_ks = 0. # Wall roughness type (0 Rhomean = 1000. # Average water density (kg/m3) Ag = 9.81 # Gravitational acceleration TidalForcing = 1 # Tidal forcing, if jsferic=1 (0: no, 1: yes) -SelfAttractionLoading = 0 # Self attraction and loading (0=no, 1=yes, 2=only self attraction) VillemonteCD1 = 1. # Calibration coefficient for Villemonte. Default = 1.0. NB. For Bloemberg data set 0.8 is recommended. VillemonteCD2 = 10. # Calibration coefficient for Villemonte. Default = 10.0. NB. For Bloemberg data set 0.8 is recommended. Salinity = 0 # Include salinity, (0=no, 1=yes) @@ -102,10 +92,7 @@ Tzone = 0. # Time zone assigned to DtUser = 3600.000 # Time interval (s) for external forcing update DtNodal = 60. # Time interval (s) for updating nodal factors in astronomical boundary conditions DtMax = 600. # Maximal computation timestep (s) -Dtfacmax = 1.1 # Max timestep increase factor ( ) DtInit = 60. # Initial computation timestep (s) -Timestepanalysis = 0 # 0=no, 1=see file *.steps -Autotimestepdiff = 0 # 0 = no, 1 = yes (Time limitation based on explicit diffusive term) Tunit = S # Time unit for start/stop times (D, H, M or S) TStart = 0. # Start time w.r.t. RefDate (in TUnit) TStop = 259200.0 # Stop time w.r.t. RefDate (in TUnit) @@ -133,7 +120,6 @@ RstInterval = 0. 0. 0. # Restart file output ti WaqInterval = 0. 0. 0. # DELWAQ output times, given as "interval" "start period" "end period" (s) StatsInterval = -60. # Screen step output interval in seconds simulation time, if negative in seconds wall clock time TimingsInterval = 0. # Timings statistics output interval -TimeSplitInterval = 0X # Time splitting interval, after which a new output file is started. value+unit, e.g. '1 M', valid units: Y,M,D,h,m,s. MapFormat = 4 # Map file format, 1: netCDF, 2: Tecplot, 3: netCFD and Tecplot, 4: NetCDF-UGRID Wrihis_balance = 1 # Write mass balance totals to his file (1: yes, 0: no) Wrihis_sourcesink = 1 # Write sources-sinks statistics to his file (1=yes, 0=no) @@ -141,7 +127,6 @@ Wrihis_turbulence = 1 # Write k, eps and vicww Wrihis_wind = 1 # Write wind velocities to his file (1: yes, 0: no) Wrihis_rain = 1 # Write precipitation to his file (1: yes, 0: no) Wrihis_temperature = 1 # Write temperature to his file (1: yes, 0: no) -Wrihis_heatflux = 1 # Write heat flux to his file (1: yes, 0: no) Wrihis_salinity = 1 # Write salinity to his file (1: yes, 0: no) Wrimap_waterlevel_s0 = 1 # Write water levels for previous time step to map file (1: yes, 0: no) Wrimap_waterlevel_s1 = 1 # Write water levels to map file (1: yes, 0: no) @@ -163,8 +148,6 @@ Wrimap_tidal_potential = 1 # Write tidal potential MapOutputTimeVector = # File (*.mpt) containing fixed map output times (s) w.r.t. RefDate FullGridOutput = 0 # Full grid output mode (0: compact, 1: full time-varying grid data) EulerVelocities = 0 # Euler velocities output (0: GLM, 1: Euler velocities) -Wrirst_bnd = 1 # Write waterlevel, bedlevel and coordinates of boundaries to restart files -Writepart_domain = 1 # Write partition domain info. for postprocessing [sediment] MorFile = # Morphology settings file (*.mor) diff --git a/tests/data/input/obscrsfile_cases/single_pli/emptymodel_with_obscrs.mdu b/tests/data/input/obscrsfile_cases/single_pli/emptymodel_with_obscrs.mdu index f5bcb80a5..703c8cffd 100644 --- a/tests/data/input/obscrsfile_cases/single_pli/emptymodel_with_obscrs.mdu +++ b/tests/data/input/obscrsfile_cases/single_pli/emptymodel_with_obscrs.mdu @@ -10,8 +10,6 @@ AutoStart = 0 # Autostart simulation a [geometry] NetFile = # Unstructured grid file *_net.nc -OneDNetworkFile = # 1d networkfile -BedlevelFile = # street_level.xyz , Bedlevels points file e.g. *.xyz, only needed for bedlevtype not equal 3 DryPointsFile = # Dry points file *.xyz (third column dummy z values), or dry areas polygon file *.pol (third column 1/-1: inside/outside) IniFieldFile = # Initial and parameter field file *.ini LandBoundaryFile = # Land boundaries file *.ldb, used for visualization @@ -25,8 +23,6 @@ ProfdefxyzFile = # Channel profile defini Uniformwidth1D = 1. # Uniform width for channel profiles not specified by profloc Uniformheight1D = 1. # Uniform height for channel profiles not specified by profloc ManholeFile = # File *.ini containing manholes -PipeFile = # File *.pliz containing pipe-based 'culverts' -ShipdefFile = # File *.shd containing ship definitions StructureFile = # File *.ini containing structures CrossLocFile = # Name and location of the file containing the locations of the cross sections CrossDefFile = # Name and location of the file containing the definitions of the cross sections @@ -51,25 +47,20 @@ NonLin1D = 0 CFLMax = 0.7 # Maximum Courant number AdvecType = 33 # Advection type (0: none, 1: Wenneker, 2: Wenneker q(uio-u), 3: Perot q(uio-u), 4: Perot q(ui-u), 5: Perot q(ui-u) without itself) TimeStepType = 2 # Time step handling (0: only transport, 1: transport + velocity update, 2: full implicit step-reduce, 3: step-Jacobi, 4: explicit) -Icoriolistype = 5 # 0=No, 1=yes, if jsferic then spatially varying, if icoriolistype==6 then constant (anglat) Limtypmom = 4 # Limiter type for cell center advection velocity (0: none, 1: minmod, 2: van Leer, 3: Kooren, 4: monotone central) Limtypsa = 4 # Limiter type for salinity transport (0: none, 1: minmod, 2: van Leer, 3: Kooren, 4: monotone central) Vertadvtypsal = 6 # Vertical advection type for salinity (0: none, 1: upwind explicit, 2: central explicit, 3: upwind implicit, 4: central implicit, 5: central implicit but upwind for neg. stratif., 6: higher order explicit, no Forester) Vertadvtyptem = 6 # Vertical advection type for temperature (0: none, 1: upwind explicit, 2: central explicit, 3: upwind implicit, 4: central implicit, 5: central implicit but upwind for neg. stratif., 6: higher order explicit, no Forester) Icgsolver = 4 # Solver type (1: sobekGS_OMP, 2: sobekGS_OMPthreadsafe, 3: sobekGS, 4: sobekGS + Saadilud, 5: parallel/global Saad, 6: parallel/Petsc, 7: parallel/GS) -Noderivedtypes = 5 # 0=use der. types. , 1 = less, 2 = lesser, 5 = also dealloc der. types Tlfsmo = 0. # Fourier smoothing time (s) on water level boundaries Slopedrop2D = 0. # Apply drop losses only if local bed slope > Slopedrop2D, (<=0: no drop losses) cstbnd = 0 # Delft-3D type velocity treatment near boundaries for small coastal models (1: yes, 0: no) Epshu = 1.d-4 # Threshold water depth for wet and dry cells -jaupwindsrc = 1 # 1st-order upwind advection at sources/sinks (1) or higher-order (0) -jasfer3D = 0 # corrections for spherical coordinates [physics] UnifFrictCoef = 50 # Uniform friction coefficient (0: no friction) UnifFrictType = 0 # Uniform friction type (0: Chezy, 1: Manning, 2: White-Colebrook, 3: idem, WAQUA style) UnifFrictCoef1D = 50 # Uniform friction coefficient in 1D links (0: no friction) -UnifFrictCoef1D2D = 50 # Uniform friction coefficient in 1D links (0: no friction) UnifFrictCoefLin = 0. # Uniform linear friction coefficient for ocean models (m/s) (0: no friction) Vicouv = 1. # Uniform horizontal eddy viscosity (m2/s) Dicouv = 1. # Uniform horizontal eddy diffusivity (m2/s) @@ -80,7 +71,6 @@ wall_ks = 0. # Wall roughness type (0 Rhomean = 1000. # Average water density (kg/m3) Ag = 9.81 # Gravitational acceleration TidalForcing = 1 # Tidal forcing, if jsferic=1 (0: no, 1: yes) -SelfAttractionLoading = 0 # Self attraction and loading (0=no, 1=yes, 2=only self attraction) VillemonteCD1 = 1. # Calibration coefficient for Villemonte. Default = 1.0. NB. For Bloemberg data set 0.8 is recommended. VillemonteCD2 = 10. # Calibration coefficient for Villemonte. Default = 10.0. NB. For Bloemberg data set 0.8 is recommended. Salinity = 0 # Include salinity, (0=no, 1=yes) @@ -102,10 +92,7 @@ Tzone = 0. # Time zone assigned to DtUser = 3600.000 # Time interval (s) for external forcing update DtNodal = 60. # Time interval (s) for updating nodal factors in astronomical boundary conditions DtMax = 600. # Maximal computation timestep (s) -Dtfacmax = 1.1 # Max timestep increase factor ( ) DtInit = 60. # Initial computation timestep (s) -Timestepanalysis = 0 # 0=no, 1=see file *.steps -Autotimestepdiff = 0 # 0 = no, 1 = yes (Time limitation based on explicit diffusive term) Tunit = S # Time unit for start/stop times (D, H, M or S) TStart = 0. # Start time w.r.t. RefDate (in TUnit) TStop = 259200.0 # Stop time w.r.t. RefDate (in TUnit) @@ -133,7 +120,6 @@ RstInterval = 0. 0. 0. # Restart file output ti WaqInterval = 0. 0. 0. # DELWAQ output times, given as "interval" "start period" "end period" (s) StatsInterval = -60. # Screen step output interval in seconds simulation time, if negative in seconds wall clock time TimingsInterval = 0. # Timings statistics output interval -TimeSplitInterval = 0X # Time splitting interval, after which a new output file is started. value+unit, e.g. '1 M', valid units: Y,M,D,h,m,s. MapFormat = 4 # Map file format, 1: netCDF, 2: Tecplot, 3: netCFD and Tecplot, 4: NetCDF-UGRID Wrihis_balance = 1 # Write mass balance totals to his file (1: yes, 0: no) Wrihis_sourcesink = 1 # Write sources-sinks statistics to his file (1=yes, 0=no) @@ -141,7 +127,6 @@ Wrihis_turbulence = 1 # Write k, eps and vicww Wrihis_wind = 1 # Write wind velocities to his file (1: yes, 0: no) Wrihis_rain = 1 # Write precipitation to his file (1: yes, 0: no) Wrihis_temperature = 1 # Write temperature to his file (1: yes, 0: no) -Wrihis_heatflux = 1 # Write heat flux to his file (1: yes, 0: no) Wrihis_salinity = 1 # Write salinity to his file (1: yes, 0: no) Wrimap_waterlevel_s0 = 1 # Write water levels for previous time step to map file (1: yes, 0: no) Wrimap_waterlevel_s1 = 1 # Write water levels to map file (1: yes, 0: no) @@ -163,8 +148,6 @@ Wrimap_tidal_potential = 1 # Write tidal potential MapOutputTimeVector = # File (*.mpt) containing fixed map output times (s) w.r.t. RefDate FullGridOutput = 0 # Full grid output mode (0: compact, 1: full time-varying grid data) EulerVelocities = 0 # Euler velocities output (0: GLM, 1: Euler velocities) -Wrirst_bnd = 1 # Write waterlevel, bedlevel and coordinates of boundaries to restart files -Writepart_domain = 1 # Write partition domain info. for postprocessing [sediment] MorFile = # Morphology settings file (*.mor) diff --git a/tests/data/input/obsfile_cases/both_ini_and_xyn/fm.mdu b/tests/data/input/obsfile_cases/both_ini_and_xyn/fm.mdu index f1f6f760a..ea30adf98 100644 --- a/tests/data/input/obsfile_cases/both_ini_and_xyn/fm.mdu +++ b/tests/data/input/obsfile_cases/both_ini_and_xyn/fm.mdu @@ -6,7 +6,6 @@ Program = D-Flow FM Version = 1.2.124.69571M fileType = modelDef fileVersion = 1.09 -GuiVersion = 4.5.0.491 AutoStart = 0 PathsRelativeToParent= 0 @@ -69,13 +68,11 @@ TimeStepType = 2 Limtyphu = 0 Limtypmom = 4 Limtypsa = 4 -TransportMethod = 1 Vertadvtypsal = 5 Icgsolver = 4 Maxdegree = 6 FixedWeirScheme = 6 FixedWeirContraction= 1 -FixedWeirfrictscheme= 0 Fixedweirtopwidth = 3 Fixedweirtopfrictcoef= -999 Fixedweirtalud = 4 @@ -86,7 +83,6 @@ Drop1D = 1 Chkadvd = 0.1 Teta0 = 0.55 Qhrelax = 0.01 -Jbasqbnddownwindhs= 0 cstbnd = 0 Maxitverticalforestersal= 0 Maxitverticalforestertem= 0 @@ -101,14 +97,12 @@ EpsMaxlevm = 0.0001 maxNonlinearIterations= 100 setHorizontalBobsFor1d2d= 0 Pure1D = 0 -jposhchk = 2 [physics] UnifFrictCoef = 0.023 UnifFrictType = 1 UnifFrictCoef1D = 0.023 UnifFrictCoefLin = 0 -Umodlin = 0 Vicouv = 1 Dicouv = 1 Vicoww = 5E-05 @@ -136,7 +130,6 @@ Stanton = -1 Dalton = -1 Backgroundwatertemperature= 6 SecondaryFlow = 0 -EffectSpiral = 0 BetaSpiral = 0 Temperature = 0 @@ -150,7 +143,6 @@ PavIni = 0 [waves] Wavemodelnr = 0 -WaveNikuradse = 0.01 Rouwav = FR84 Gammax = 1 @@ -206,7 +198,6 @@ XLSInterval = MapFile = MapInterval = 3600 RstInterval = 0 -S1incinterval = MapFormat = 4 Wrihis_balance = 1 Wrihis_sourcesink = 1 @@ -273,10 +264,7 @@ ClassMapFile = WaterlevelClasses = 0 WaterdepthClasses = 0 ClassMapInterval = 0 -VelocityMagnitudeClasses= 0 -VelocityDirectionClassesInterval= 30 StatsInterval = -Writebalancefile = 0 TimingsInterval = Richardsononoutput= 1 NcFormat = 3 @@ -289,7 +277,6 @@ FouUpdateStep = 0 [volumeTables] useVolumeTables = 1 increment = 0.1 -useVolumeTablesFile= 1 [hydrology] InterceptionModel = 0 diff --git a/tests/data/input/obsfile_cases/single_ini/fm.mdu b/tests/data/input/obsfile_cases/single_ini/fm.mdu index e313abc11..2aa64c392 100644 --- a/tests/data/input/obsfile_cases/single_ini/fm.mdu +++ b/tests/data/input/obsfile_cases/single_ini/fm.mdu @@ -6,7 +6,6 @@ Program = D-Flow FM Version = 1.2.124.69571M fileType = modelDef fileVersion = 1.09 -GuiVersion = 4.5.0.491 AutoStart = 0 PathsRelativeToParent= 0 @@ -69,13 +68,11 @@ TimeStepType = 2 Limtyphu = 0 Limtypmom = 4 Limtypsa = 4 -TransportMethod = 1 Vertadvtypsal = 5 Icgsolver = 4 Maxdegree = 6 FixedWeirScheme = 6 FixedWeirContraction= 1 -FixedWeirfrictscheme= 0 Fixedweirtopwidth = 3 Fixedweirtopfrictcoef= -999 Fixedweirtalud = 4 @@ -86,7 +83,6 @@ Drop1D = 1 Chkadvd = 0.1 Teta0 = 0.55 Qhrelax = 0.01 -Jbasqbnddownwindhs= 0 cstbnd = 0 Maxitverticalforestersal= 0 Maxitverticalforestertem= 0 @@ -101,14 +97,12 @@ EpsMaxlevm = 0.0001 maxNonlinearIterations= 100 setHorizontalBobsFor1d2d= 0 Pure1D = 0 -jposhchk = 2 [physics] UnifFrictCoef = 0.023 UnifFrictType = 1 UnifFrictCoef1D = 0.023 UnifFrictCoefLin = 0 -Umodlin = 0 Vicouv = 1 Dicouv = 1 Vicoww = 5E-05 @@ -136,7 +130,6 @@ Stanton = -1 Dalton = -1 Backgroundwatertemperature= 6 SecondaryFlow = 0 -EffectSpiral = 0 BetaSpiral = 0 Temperature = 0 @@ -150,7 +143,6 @@ PavIni = 0 [waves] Wavemodelnr = 0 -WaveNikuradse = 0.01 Rouwav = FR84 Gammax = 1 @@ -206,7 +198,6 @@ XLSInterval = MapFile = MapInterval = 3600 RstInterval = 0 -S1incinterval = MapFormat = 4 Wrihis_balance = 1 Wrihis_sourcesink = 1 @@ -273,10 +264,7 @@ ClassMapFile = WaterlevelClasses = 0 WaterdepthClasses = 0 ClassMapInterval = 0 -VelocityMagnitudeClasses= 0 -VelocityDirectionClassesInterval= 30 StatsInterval = -Writebalancefile = 0 TimingsInterval = Richardsononoutput= 1 NcFormat = 3 @@ -289,7 +277,6 @@ FouUpdateStep = 0 [volumeTables] useVolumeTables = 1 increment = 0.1 -useVolumeTablesFile= 1 [hydrology] InterceptionModel = 0 diff --git a/tests/data/input/obsfile_cases/single_xyn/fm.mdu b/tests/data/input/obsfile_cases/single_xyn/fm.mdu index 776c9d216..75c0c613c 100644 --- a/tests/data/input/obsfile_cases/single_xyn/fm.mdu +++ b/tests/data/input/obsfile_cases/single_xyn/fm.mdu @@ -6,7 +6,6 @@ Program = D-Flow FM Version = 1.2.124.69571M fileType = modelDef fileVersion = 1.09 -GuiVersion = 4.5.0.491 AutoStart = 0 PathsRelativeToParent= 0 @@ -69,13 +68,11 @@ TimeStepType = 2 Limtyphu = 0 Limtypmom = 4 Limtypsa = 4 -TransportMethod = 1 Vertadvtypsal = 5 Icgsolver = 4 Maxdegree = 6 FixedWeirScheme = 6 FixedWeirContraction= 1 -FixedWeirfrictscheme= 0 Fixedweirtopwidth = 3 Fixedweirtopfrictcoef= -999 Fixedweirtalud = 4 @@ -86,7 +83,6 @@ Drop1D = 1 Chkadvd = 0.1 Teta0 = 0.55 Qhrelax = 0.01 -Jbasqbnddownwindhs= 0 cstbnd = 0 Maxitverticalforestersal= 0 Maxitverticalforestertem= 0 @@ -101,14 +97,12 @@ EpsMaxlevm = 0.0001 maxNonlinearIterations= 100 setHorizontalBobsFor1d2d= 0 Pure1D = 0 -jposhchk = 2 [physics] UnifFrictCoef = 0.023 UnifFrictType = 1 UnifFrictCoef1D = 0.023 UnifFrictCoefLin = 0 -Umodlin = 0 Vicouv = 1 Dicouv = 1 Vicoww = 5E-05 @@ -136,7 +130,6 @@ Stanton = -1 Dalton = -1 Backgroundwatertemperature= 6 SecondaryFlow = 0 -EffectSpiral = 0 BetaSpiral = 0 Temperature = 0 @@ -150,7 +143,6 @@ PavIni = 0 [waves] Wavemodelnr = 0 -WaveNikuradse = 0.01 Rouwav = FR84 Gammax = 1 @@ -206,7 +198,6 @@ XLSInterval = MapFile = MapInterval = 3600 RstInterval = 0 -S1incinterval = MapFormat = 4 Wrihis_balance = 1 Wrihis_sourcesink = 1 @@ -273,10 +264,7 @@ ClassMapFile = WaterlevelClasses = 0 WaterdepthClasses = 0 ClassMapInterval = 0 -VelocityMagnitudeClasses= 0 -VelocityDirectionClassesInterval= 30 StatsInterval = -Writebalancefile = 0 TimingsInterval = Richardsononoutput= 1 NcFormat = 3 @@ -289,7 +277,6 @@ FouUpdateStep = 0 [volumeTables] useVolumeTables = 1 increment = 0.1 -useVolumeTablesFile= 1 [hydrology] InterceptionModel = 0 diff --git a/tests/data/input/resolve_casing_file_load_test/fm.mdu b/tests/data/input/resolve_casing_file_load_test/fm.mdu index 2e3a03563..84e5f2b05 100644 --- a/tests/data/input/resolve_casing_file_load_test/fm.mdu +++ b/tests/data/input/resolve_casing_file_load_test/fm.mdu @@ -6,7 +6,6 @@ Program = D-Flow FM Version = 1.2.124.69571M fileType = modelDef fileVersion = 1.09 -GuiVersion = 4.5.0.491 AutoStart = 0 PathsRelativeToParent= 1 @@ -69,13 +68,11 @@ TimeStepType = 2 Limtyphu = 0 Limtypmom = 4 Limtypsa = 4 -TransportMethod = 1 Vertadvtypsal = 5 Icgsolver = 4 Maxdegree = 6 FixedWeirScheme = 6 FixedWeirContraction= 1 -FixedWeirfrictscheme= 0 Fixedweirtopwidth = 3 Fixedweirtopfrictcoef= -999 Fixedweirtalud = 4 @@ -86,7 +83,6 @@ Drop1D = 1 Chkadvd = 0.1 Teta0 = 0.55 Qhrelax = 0.01 -Jbasqbnddownwindhs= 0 cstbnd = 0 Maxitverticalforestersal= 0 Maxitverticalforestertem= 0 @@ -101,14 +97,12 @@ EpsMaxlevm = 0.0001 maxNonlinearIterations= 100 setHorizontalBobsFor1d2d= 0 Pure1D = 0 -jposhchk = 2 [physics] UnifFrictCoef = 0.023 UnifFrictType = 1 UnifFrictCoef1D = 0.023 UnifFrictCoefLin = 0 -Umodlin = 0 Vicouv = 1 Dicouv = 1 Vicoww = 5E-05 @@ -136,7 +130,6 @@ Stanton = -1 Dalton = -1 Backgroundwatertemperature= 6 SecondaryFlow = 0 -EffectSpiral = 0 BetaSpiral = 0 Temperature = 0 @@ -150,7 +143,6 @@ PavIni = 0 [waves] Wavemodelnr = 0 -WaveNikuradse = 0.01 Rouwav = FR84 Gammax = 1 @@ -206,7 +198,6 @@ XLSInterval = MapFile = MapInterval = 3600 RstInterval = 0 -S1incinterval = MapFormat = 4 Wrihis_balance = 1 Wrihis_sourcesink = 1 @@ -273,10 +264,7 @@ ClassMapFile = WaterlevelClasses = 0 WaterdepthClasses = 0 ClassMapInterval = 0 -VelocityMagnitudeClasses= 0 -VelocityDirectionClassesInterval= 30 StatsInterval = -Writebalancefile = 0 TimingsInterval = Richardsononoutput= 1 NcFormat = 3 @@ -289,7 +277,6 @@ FouUpdateStep = 0 [volumeTables] useVolumeTables = 1 increment = 0.1 -useVolumeTablesFile= 1 [hydrology] InterceptionModel = 0 From 3c3894f45d01a97d8e8169ddfb7b4bae10d0c80c Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Tue, 28 May 2024 09:50:34 +0200 Subject: [PATCH 17/31] feat: (622) resolve failing tests. --- hydrolib/core/dflowfm/bc/models.py | 5 ++ hydrolib/core/dflowfm/crosssection/models.py | 5 ++ hydrolib/core/dflowfm/ext/models.py | 5 ++ hydrolib/core/dflowfm/ini/models.py | 8 +- hydrolib/core/dflowfm/ini/util.py | 26 ++++++ tests/dflowfm/ini/test_util.py | 21 +++++ tests/dflowfm/test_storagenode.py | 1 - tests/dflowfm/test_structure.py | 89 ++++---------------- 8 files changed, 84 insertions(+), 76 deletions(-) diff --git a/hydrolib/core/dflowfm/bc/models.py b/hydrolib/core/dflowfm/bc/models.py index 40fc942b0..5bb62bd43 100644 --- a/hydrolib/core/dflowfm/bc/models.py +++ b/hydrolib/core/dflowfm/bc/models.py @@ -28,6 +28,7 @@ from hydrolib.core.dflowfm.ini.parser import Parser, ParserConfig from hydrolib.core.dflowfm.ini.serializer import DataBlockINIBasedSerializerConfig from hydrolib.core.dflowfm.ini.util import ( + ForcingUnknownKeywordErrorManager, get_enum_validator, get_from_subclass_defaults, get_split_string_on_delimiter_validator, @@ -176,6 +177,10 @@ class ForcingBase(DataBlockINIBasedModel): quantityunitpair: List[ScalarOrVectorQUP] """List[ScalarOrVectorQUP]: List of header lines for one or more quantities and their unit. Describes the columns in the actual datablock.""" + @classmethod + def _get_unknown_keyword_error_manager(cls): + return ForcingUnknownKeywordErrorManager() + def _exclude_fields(self) -> Set: return {"quantityunitpair"}.union(super()._exclude_fields()) diff --git a/hydrolib/core/dflowfm/crosssection/models.py b/hydrolib/core/dflowfm/crosssection/models.py index 554429478..61426793c 100644 --- a/hydrolib/core/dflowfm/crosssection/models.py +++ b/hydrolib/core/dflowfm/crosssection/models.py @@ -7,6 +7,7 @@ from hydrolib.core.dflowfm.friction.models import FrictionType from hydrolib.core.dflowfm.ini.models import INIBasedModel, INIGeneral, INIModel from hydrolib.core.dflowfm.ini.util import ( + CrsUnknownKeywordErrorManager, LocationValidationConfiguration, LocationValidationFieldNames, get_enum_validator, @@ -73,6 +74,10 @@ class Comments(INIBasedModel.Comments): def _get_identifier(self, data: dict) -> Optional[str]: return data.get("id") + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return CrsUnknownKeywordErrorManager() @classmethod def _duplicate_keys_as_list(cls): diff --git a/hydrolib/core/dflowfm/ext/models.py b/hydrolib/core/dflowfm/ext/models.py index 624c6bce8..e2a2df245 100644 --- a/hydrolib/core/dflowfm/ext/models.py +++ b/hydrolib/core/dflowfm/ext/models.py @@ -18,6 +18,7 @@ from hydrolib.core.dflowfm.ini.serializer import INISerializerConfig from hydrolib.core.dflowfm.ini.util import ( LocationValidationConfiguration, + MeteoUnknownKeywordErrorManager, get_enum_validator, get_split_string_on_delimiter_validator, make_list_validator, @@ -264,6 +265,10 @@ class Comments(INIBasedModel.Comments): interpolationmethod: Optional[MeteoInterpolationMethod] = Field( alias="interpolationMethod" ) + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MeteoUnknownKeywordErrorManager() def is_intermediate_link(self) -> bool: return True diff --git a/hydrolib/core/dflowfm/ini/models.py b/hydrolib/core/dflowfm/ini/models.py index 6ee29715c..211b1b4be 100644 --- a/hydrolib/core/dflowfm/ini/models.py +++ b/hydrolib/core/dflowfm/ini/models.py @@ -47,7 +47,6 @@ class INIBasedModel(BaseModel, ABC): _header: str = "" _file_path_style_converter = FilePathStyleConverter() - _unknown_keyword_error_manager = UnknownKeywordErrorManager() _scientific_notation_regex = compile( r"([\d.]+)([dD])([+-]?\d{1,3})" ) # matches a float: 1d9, 1D-3, 1.D+4, etc. @@ -58,12 +57,17 @@ class Config: def __init__(self, **data): super().__init__(**data) - self._unknown_keyword_error_manager.raise_error_for_unknown_keywords( + unknown_keyword_error_manager = self._get_unknown_keyword_error_manager() + unknown_keyword_error_manager.raise_error_for_unknown_keywords( data, self._header, self.__fields__, self._exclude_fields(), ) + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return UnknownKeywordErrorManager() @classmethod def _supports_comments(cls): diff --git a/hydrolib/core/dflowfm/ini/util.py b/hydrolib/core/dflowfm/ini/util.py index 084753475..cdd49928b 100644 --- a/hydrolib/core/dflowfm/ini/util.py +++ b/hydrolib/core/dflowfm/ini/util.py @@ -671,3 +671,29 @@ def _is_unknown_keyword( return False return name not in excluded_fields + +class ExtendedUnknownKeywordErrorManager(UnknownKeywordErrorManager): + """ + Extended Error manager for unknown keys, this class can filter out specific keys which should not trigger an unknown key error. + """ + + _field_specific : set = {} + + def _is_unknown_keyword( + self, name: str, fields: Dict[str, ModelField], excluded_fields: Set + ): + is_unknown_keyword = super()._is_unknown_keyword(name, fields, excluded_fields) + + if name in self._field_specific: + return False + + return is_unknown_keyword + +class ForcingUnknownKeywordErrorManager(ExtendedUnknownKeywordErrorManager): + _field_specific : set = {'timeinterpolation', 'quantity', 'unit', 'manholename', 'vertpositionindex', 'time_interpolation', 'vertical_position_specification', 'vertical_interpolation', 'vertical_position_type', 'vertical_position_type', 'vertical_position', 'vector'} + +class CrsUnknownKeywordErrorManager(ExtendedUnknownKeywordErrorManager): + _field_specific : set = {'template', 'height', 'width', 'r', 'r1', 'r2', 'r3', 'a', 'a1'} + +class MeteoUnknownKeywordErrorManager(ExtendedUnknownKeywordErrorManager): + _field_specific : set = {'locationtype', 'locationfile'} diff --git a/tests/dflowfm/ini/test_util.py b/tests/dflowfm/ini/test_util.py index cad4ebedb..635285aaa 100644 --- a/tests/dflowfm/ini/test_util.py +++ b/tests/dflowfm/ini/test_util.py @@ -9,6 +9,7 @@ from hydrolib.core.basemodel import BaseModel from hydrolib.core.dflowfm.ini.util import ( + ExtendedUnknownKeywordErrorManager, LocationValidationConfiguration, LocationValidationFieldNames, UnknownKeywordErrorManager, @@ -444,6 +445,26 @@ def test_keyword_given_known_as_excluded_field_does_not_throw_exception( ukem = UnknownKeywordErrorManager() data = {name: 1} + try: + ukem.raise_error_for_unknown_keywords( + data, section_header, fields, excluded_fields + ) + except: + pytest.fail("Exception is thrown, no exception is expected for this test.") + + def test_keyword_given_known_as_extended_field_does_not_throw_exception( + self + ): + section_header = "section header" + excluded_fields = set() + name = "keyname" + + fields = {} + + ukem = ExtendedUnknownKeywordErrorManager() + ukem._field_specific = {name} + data = {name: 1} + try: ukem.raise_error_for_unknown_keywords( data, section_header, fields, excluded_fields diff --git a/tests/dflowfm/test_storagenode.py b/tests/dflowfm/test_storagenode.py index 0918abb0a..984b67aaf 100644 --- a/tests/dflowfm/test_storagenode.py +++ b/tests/dflowfm/test_storagenode.py @@ -184,7 +184,6 @@ def test_scientific_notation_in_ini_file_are_correctly_parsed_for_strings_and_fl streetLevel = 0.000 storageType = Reservoir streetStorageArea = 1d2 - CompartmentShape = Unknown useTable = False """ diff --git a/tests/dflowfm/test_structure.py b/tests/dflowfm/test_structure.py index 082ef7e49..31a1e0331 100644 --- a/tests/dflowfm/test_structure.py +++ b/tests/dflowfm/test_structure.py @@ -286,7 +286,6 @@ def test_culvert_parses_flowdirection_case_insensitive(input, expected): length="1", inletlosscoeff="1", outletlosscoeff="1", - inletlosvalveonoffscoeff="1", valveonoff="1", valveopeningheight="1", numlosscoeff="1", @@ -319,7 +318,6 @@ def test_culvert_parses_subtype_case_insensitive(input, expected): length="1", inletlosscoeff="1", outletlosscoeff="1", - inletlosvalveonoffscoeff="1", valveonoff="1", valveopeningheight="1", numlosscoeff="1", @@ -440,7 +438,7 @@ def test_bridge_construction_with_parser(self): assert bridge.friction == 70 assert bridge.length == 9.75 - def test_bridge_with_unknown_parameter_is_ignored(self): + def test_bridge_with_unknown_parameter_throws_valueerror(self): parser = Parser(ParserConfig()) input_str = inspect.cleandoc( @@ -472,24 +470,12 @@ def test_bridge_with_unknown_parameter_is_ignored(self): document = parser.finalize() - wrapper = WrapperTest[Bridge].parse_obj({"val": document.sections[0]}) - bridge = wrapper.val + expected_message = "Unknown keywords are detected in section: 'Structure', '['unknown']'" - assert bridge.dict().get("unknown") is None # type: ignore + with pytest.raises(ValueError) as exc_err: + WrapperTest[Bridge].parse_obj({"val": document.sections[0]}) - assert bridge.id == "RS1-KBR31" - assert bridge.name == "RS1-KBR31name" - assert bridge.branchid == "riv_RS1_264" - assert bridge.chainage == 104.184 - assert bridge.type == "bridge" - assert bridge.allowedflowdir == FlowDirection.both - assert bridge.csdefid == "W_980.1S_0" - assert bridge.shift == 0.0 - assert bridge.inletlosscoeff == 1 - assert bridge.outletlosscoeff == 1 - assert bridge.frictiontype == FrictionType.strickler - assert bridge.friction == 70 - assert bridge.length == 9.75 + assert expected_message in str(exc_err.value) def test_bridge_with_missing_required_parameters(self): parser = Parser(ParserConfig()) @@ -1617,7 +1603,7 @@ def test_weir_comments_construction_with_parser(self): assert weir.comments.crestwidth is None assert weir.comments.usevelocityheight == "My own special comment 2" - def test_weir_with_unknown_parameter_is_ignored(self): + def test_weir_with_unknown_parameter_throws_valueerror(self): parser = Parser(ParserConfig()) input_str = inspect.cleandoc( @@ -1645,21 +1631,13 @@ def test_weir_with_unknown_parameter_is_ignored(self): document = parser.finalize() - wrapper = WrapperTest[Weir].parse_obj({"val": document.sections[0]}) - weir = wrapper.val + expected_message = "Unknown keywords are detected in section: 'Structure', '['unknown']'" - assert weir.dict().get("unknown") is None # type: ignore - - assert weir.id == "weir_id" - assert weir.name == "weir" - assert weir.branchid == "branch" - assert weir.chainage == 3.0 - assert weir.type == "weir" - assert weir.allowedflowdir == FlowDirection.positive - assert weir.crestlevel == 10.5 - assert weir.crestwidth is None - assert weir.usevelocityheight == False + with pytest.raises(ValueError) as exc_err: + WrapperTest[Weir].parse_obj({"val": document.sections[0]}) + assert expected_message in str(exc_err.value) + @pytest.mark.parametrize( "input,expected", _get_allowedflowdir_cases(), @@ -2184,7 +2162,7 @@ def test_weir_comments_construction_with_parser(self): assert struct.comments.gateopeningwidth is None assert struct.comments.usevelocityheight == "My own special comment 2" - def test_general_structure_with_unknown_parameter_is_ignored(self): + def test_general_structure_with_unknown_parameter_throws_valueerror(self): parser = Parser(ParserConfig()) input_str = inspect.cleandoc( @@ -2236,47 +2214,12 @@ def test_general_structure_with_unknown_parameter_is_ignored(self): document = parser.finalize() - wrapper = WrapperTest[GeneralStructure].parse_obj({"val": document.sections[0]}) - struct = wrapper.val + expected_message = "Unknown keywords are detected in section: 'Structure', '['unknown']'" - assert struct.dict().get("unknown") is None # type: ignore + with pytest.raises(ValueError) as exc_err: + WrapperTest[GeneralStructure].parse_obj({"val": document.sections[0]}) - assert struct.id == "id" - assert struct.name == "extravagante_waarde" - assert struct.branchid == "stump" - assert struct.chainage == 13.53 - assert struct.type == "generalStructure" - assert struct.allowedflowdir == FlowDirection.positive - assert struct.upstream1width == 111.0 - assert struct.upstream1level == 112.0 - assert struct.upstream2width == 113.0 - assert struct.upstream2level == 114.0 - assert struct.crestwidth == 115.0 - assert struct.crestlevel == 116.0 - assert struct.crestlength == 117.0 - assert struct.downstream1width == 118.0 - assert struct.downstream1level == 119.0 - assert struct.downstream2width == 119.1 - assert struct.downstream2level == 119.2 - assert struct.gateloweredgelevel == 119.3 - assert struct.posfreegateflowcoeff == 119.4 - assert struct.posdrowngateflowcoeff == 119.5 - assert struct.posfreeweirflowcoeff == 119.6 - assert struct.posdrownweirflowcoeff == 119.7 - assert struct.poscontrcoeffreegate == 119.8 - assert struct.negfreegateflowcoeff == 119.9 - assert struct.negdrowngateflowcoeff == 118.1 - assert struct.negfreeweirflowcoeff == 118.2 - assert struct.negdrownweirflowcoeff == 118.3 - assert struct.negcontrcoeffreegate == 118.4 - assert struct.extraresistance == 118.5 - assert struct.gateheight == 118.6 - assert struct.gateopeningwidth == 118.7 - assert ( - struct.gateopeninghorizontaldirection - == GateOpeningHorizontalDirection.from_right - ) - assert struct.usevelocityheight == False + assert expected_message in str(exc_err.value) def _create_required_general_structure_values(self) -> dict: general_structure_values = dict() From c4e3483dcfce5d4d09aa434f639273d4a1f16cfa Mon Sep 17 00:00:00 2001 From: MRVermeulenDeltares Date: Tue, 28 May 2024 07:51:58 +0000 Subject: [PATCH 18/31] autoformat: isort & black --- hydrolib/core/dflowfm/bc/models.py | 2 +- hydrolib/core/dflowfm/crosssection/models.py | 4 +- hydrolib/core/dflowfm/ext/models.py | 4 +- hydrolib/core/dflowfm/ini/models.py | 4 +- hydrolib/core/dflowfm/ini/util.py | 57 ++++++++++++++----- tests/dflowfm/ini/test_util.py | 58 ++++++++------------ tests/dflowfm/test_mdu.py | 18 +++--- tests/dflowfm/test_structure.py | 14 +++-- 8 files changed, 94 insertions(+), 67 deletions(-) diff --git a/hydrolib/core/dflowfm/bc/models.py b/hydrolib/core/dflowfm/bc/models.py index 5bb62bd43..64eced7df 100644 --- a/hydrolib/core/dflowfm/bc/models.py +++ b/hydrolib/core/dflowfm/bc/models.py @@ -177,7 +177,7 @@ class ForcingBase(DataBlockINIBasedModel): quantityunitpair: List[ScalarOrVectorQUP] """List[ScalarOrVectorQUP]: List of header lines for one or more quantities and their unit. Describes the columns in the actual datablock.""" - @classmethod + @classmethod def _get_unknown_keyword_error_manager(cls): return ForcingUnknownKeywordErrorManager() diff --git a/hydrolib/core/dflowfm/crosssection/models.py b/hydrolib/core/dflowfm/crosssection/models.py index 61426793c..afd8bb781 100644 --- a/hydrolib/core/dflowfm/crosssection/models.py +++ b/hydrolib/core/dflowfm/crosssection/models.py @@ -74,8 +74,8 @@ class Comments(INIBasedModel.Comments): def _get_identifier(self, data: dict) -> Optional[str]: return data.get("id") - - @classmethod + + @classmethod def _get_unknown_keyword_error_manager(cls): return CrsUnknownKeywordErrorManager() diff --git a/hydrolib/core/dflowfm/ext/models.py b/hydrolib/core/dflowfm/ext/models.py index e2a2df245..160c960b0 100644 --- a/hydrolib/core/dflowfm/ext/models.py +++ b/hydrolib/core/dflowfm/ext/models.py @@ -265,8 +265,8 @@ class Comments(INIBasedModel.Comments): interpolationmethod: Optional[MeteoInterpolationMethod] = Field( alias="interpolationMethod" ) - - @classmethod + + @classmethod def _get_unknown_keyword_error_manager(cls): return MeteoUnknownKeywordErrorManager() diff --git a/hydrolib/core/dflowfm/ini/models.py b/hydrolib/core/dflowfm/ini/models.py index 211b1b4be..bb13f35c3 100644 --- a/hydrolib/core/dflowfm/ini/models.py +++ b/hydrolib/core/dflowfm/ini/models.py @@ -64,8 +64,8 @@ def __init__(self, **data): self.__fields__, self._exclude_fields(), ) - - @classmethod + + @classmethod def _get_unknown_keyword_error_manager(cls): return UnknownKeywordErrorManager() diff --git a/hydrolib/core/dflowfm/ini/util.py b/hydrolib/core/dflowfm/ini/util.py index cdd49928b..dfb3cd42c 100644 --- a/hydrolib/core/dflowfm/ini/util.py +++ b/hydrolib/core/dflowfm/ini/util.py @@ -650,8 +650,10 @@ def raise_error_for_unknown_keywords( if len(unknown_keywords) == 0: return - - raise ValueError(f"Unknown keywords are detected in section: '{section_header}', '{unknown_keywords}'") + + raise ValueError( + f"Unknown keywords are detected in section: '{section_header}', '{unknown_keywords}'" + ) def _get_all_unknown_keywords( self, data: Dict[str, Any], fields: Dict[str, ModelField], excluded_fields: Set @@ -669,31 +671,58 @@ def _is_unknown_keyword( for model_field in fields.values(): if name == model_field.name or name == model_field.alias: return False - + return name not in excluded_fields - + + class ExtendedUnknownKeywordErrorManager(UnknownKeywordErrorManager): """ Extended Error manager for unknown keys, this class can filter out specific keys which should not trigger an unknown key error. """ - - _field_specific : set = {} - + + _field_specific: set = {} + def _is_unknown_keyword( self, name: str, fields: Dict[str, ModelField], excluded_fields: Set ): is_unknown_keyword = super()._is_unknown_keyword(name, fields, excluded_fields) - + if name in self._field_specific: return False - + return is_unknown_keyword - + + class ForcingUnknownKeywordErrorManager(ExtendedUnknownKeywordErrorManager): - _field_specific : set = {'timeinterpolation', 'quantity', 'unit', 'manholename', 'vertpositionindex', 'time_interpolation', 'vertical_position_specification', 'vertical_interpolation', 'vertical_position_type', 'vertical_position_type', 'vertical_position', 'vector'} + _field_specific: set = { + "timeinterpolation", + "quantity", + "unit", + "manholename", + "vertpositionindex", + "time_interpolation", + "vertical_position_specification", + "vertical_interpolation", + "vertical_position_type", + "vertical_position_type", + "vertical_position", + "vector", + } + class CrsUnknownKeywordErrorManager(ExtendedUnknownKeywordErrorManager): - _field_specific : set = {'template', 'height', 'width', 'r', 'r1', 'r2', 'r3', 'a', 'a1'} - + _field_specific: set = { + "template", + "height", + "width", + "r", + "r1", + "r2", + "r3", + "a", + "a1", + } + + class MeteoUnknownKeywordErrorManager(ExtendedUnknownKeywordErrorManager): - _field_specific : set = {'locationtype', 'locationfile'} + _field_specific: set = {"locationtype", "locationfile"} diff --git a/tests/dflowfm/ini/test_util.py b/tests/dflowfm/ini/test_util.py index 635285aaa..1f8425a6b 100644 --- a/tests/dflowfm/ini/test_util.py +++ b/tests/dflowfm/ini/test_util.py @@ -1,11 +1,11 @@ from typing import Dict, List, Literal, Optional from unittest.mock import Mock -from pydantic.v1.fields import ModelField import pytest from pydantic.v1 import Extra from pydantic.v1.class_validators import root_validator from pydantic.v1.error_wrappers import ValidationError +from pydantic.v1.fields import ModelField from hydrolib.core.basemodel import BaseModel from hydrolib.core.dflowfm.ini.util import ( @@ -363,19 +363,18 @@ class GrandChildWithDefaultProperty(WithoutDefaultProperty): class TestUnknownKeywordErrorManager: - def test_unknown_keywords_given_when_notify_unknown_keywords_gives_error_with_unknown_keywords( - self + self, ): section_header = "section header" fields = {} excluded_fields = set() name = "keyname" second_name = "second_other" - + ukem = UnknownKeywordErrorManager() data = {name: 1, second_name: 2} - + expected_message = f"Unknown keywords are detected in section: '{section_header}', '{[name, second_name]}'" with pytest.raises(ValueError) as exc_err: @@ -385,32 +384,28 @@ def test_unknown_keywords_given_when_notify_unknown_keywords_gives_error_with_un assert expected_message in str(exc_err.value) - def test_keyword_given_known_as_alias_does_not_throw_exception( - self - ): + def test_keyword_given_known_as_alias_does_not_throw_exception(self): section_header = "section header" excluded_fields = set() name = "keyname" - + mocked_field = Mock(spec=ModelField) mocked_field.name = "name" mocked_field.alias = name fields = {"name": mocked_field} - + ukem = UnknownKeywordErrorManager() data = {name: 1} - + try: ukem.raise_error_for_unknown_keywords( data, section_header, fields, excluded_fields ) except: pytest.fail("Exception is thrown, no exception is expected for this test.") - - def test_keyword_given_known_as_name_does_not_throw_exception( - self - ): + + def test_keyword_given_known_as_name_does_not_throw_exception(self): section_header = "section header" excluded_fields = set() name = "keyname" @@ -418,56 +413,51 @@ def test_keyword_given_known_as_name_does_not_throw_exception( mocked_field = Mock(spec=ModelField) mocked_field.name = "name" mocked_field.alias = name - + fields = {name: mocked_field} - + ukem = UnknownKeywordErrorManager() data = {name: 1} - + try: ukem.raise_error_for_unknown_keywords( data, section_header, fields, excluded_fields ) except: pytest.fail("Exception is thrown, no exception is expected for this test.") - - - def test_keyword_given_known_as_excluded_field_does_not_throw_exception( - self - ): + + def test_keyword_given_known_as_excluded_field_does_not_throw_exception(self): section_header = "section header" excluded_fields = set() name = "keyname" excluded_fields.add(name) - + fields = {} - + ukem = UnknownKeywordErrorManager() data = {name: 1} - + try: ukem.raise_error_for_unknown_keywords( data, section_header, fields, excluded_fields ) except: pytest.fail("Exception is thrown, no exception is expected for this test.") - - def test_keyword_given_known_as_extended_field_does_not_throw_exception( - self - ): + + def test_keyword_given_known_as_extended_field_does_not_throw_exception(self): section_header = "section header" excluded_fields = set() name = "keyname" - + fields = {} - + ukem = ExtendedUnknownKeywordErrorManager() ukem._field_specific = {name} data = {name: 1} - + try: ukem.raise_error_for_unknown_keywords( data, section_header, fields, excluded_fields ) except: - pytest.fail("Exception is thrown, no exception is expected for this test.") \ No newline at end of file + pytest.fail("Exception is thrown, no exception is expected for this test.") diff --git a/tests/dflowfm/test_mdu.py b/tests/dflowfm/test_mdu.py index 3ee8b3625..9748a3b5e 100644 --- a/tests/dflowfm/test_mdu.py +++ b/tests/dflowfm/test_mdu.py @@ -390,7 +390,7 @@ def test_loading_fmmodel_model_with_both_ini_and_xyn_obsfiles(self): assert actual_point.x == expected_point.x assert actual_point.y == expected_point.y assert actual_point.name == expected_point.name - + def test_mdu_unknown_keyword_loading_throws_valueerror_for_unknown_keyword( self, tmp_path ): @@ -410,14 +410,16 @@ def test_mdu_unknown_keyword_loading_throws_valueerror_for_unknown_keyword( section_header = "General" name = "unknownkey" - - expected_message = f"Unknown keywords are detected in section: '{section_header}', '{[name]}'" + + expected_message = ( + f"Unknown keywords are detected in section: '{section_header}', '{[name]}'" + ) with pytest.raises(ValueError) as exc_err: FMModel(filepath=tmp_mdu_path) - + assert expected_message in str(exc_err.value) - + def test_mdu_unknown_keywords_loading_throws_valueerror_for_unknown_keywords( self, tmp_path ): @@ -439,12 +441,12 @@ def test_mdu_unknown_keywords_loading_throws_valueerror_for_unknown_keywords( section_header = "General" name = "unknownkey" name2 = "unknownkey2" - + expected_message = f"Unknown keywords are detected in section: '{section_header}', '{[name, name2]}'" with pytest.raises(ValueError) as exc_err: FMModel(filepath=tmp_mdu_path) - + assert expected_message in str(exc_err.value) def test_mdu_unknown_keywords_loading_thrown_valueerror_for_unknown_keyword_does_not_include_excluded_fields( @@ -470,4 +472,4 @@ def test_mdu_unknown_keywords_loading_thrown_valueerror_for_unknown_keyword_does excluded_fields = model._exclude_fields() assert len(excluded_fields) > 0 for excluded_field in excluded_fields: - assert excluded_field not in str(exc_err.value) \ No newline at end of file + assert excluded_field not in str(exc_err.value) diff --git a/tests/dflowfm/test_structure.py b/tests/dflowfm/test_structure.py index 31a1e0331..9c1b535ac 100644 --- a/tests/dflowfm/test_structure.py +++ b/tests/dflowfm/test_structure.py @@ -470,7 +470,9 @@ def test_bridge_with_unknown_parameter_throws_valueerror(self): document = parser.finalize() - expected_message = "Unknown keywords are detected in section: 'Structure', '['unknown']'" + expected_message = ( + "Unknown keywords are detected in section: 'Structure', '['unknown']'" + ) with pytest.raises(ValueError) as exc_err: WrapperTest[Bridge].parse_obj({"val": document.sections[0]}) @@ -1631,13 +1633,15 @@ def test_weir_with_unknown_parameter_throws_valueerror(self): document = parser.finalize() - expected_message = "Unknown keywords are detected in section: 'Structure', '['unknown']'" + expected_message = ( + "Unknown keywords are detected in section: 'Structure', '['unknown']'" + ) with pytest.raises(ValueError) as exc_err: WrapperTest[Weir].parse_obj({"val": document.sections[0]}) assert expected_message in str(exc_err.value) - + @pytest.mark.parametrize( "input,expected", _get_allowedflowdir_cases(), @@ -2214,7 +2218,9 @@ def test_general_structure_with_unknown_parameter_throws_valueerror(self): document = parser.finalize() - expected_message = "Unknown keywords are detected in section: 'Structure', '['unknown']'" + expected_message = ( + "Unknown keywords are detected in section: 'Structure', '['unknown']'" + ) with pytest.raises(ValueError) as exc_err: WrapperTest[GeneralStructure].parse_obj({"val": document.sections[0]}) From 88c872f93b830107ca122a3c776a98019e232fe2 Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Tue, 28 May 2024 10:30:51 +0200 Subject: [PATCH 19/31] feat: (622) resolve code smells. --- hydrolib/core/dflowfm/ini/util.py | 3 +-- tests/dflowfm/ini/test_util.py | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/hydrolib/core/dflowfm/ini/util.py b/hydrolib/core/dflowfm/ini/util.py index dfb3cd42c..7bd79dc09 100644 --- a/hydrolib/core/dflowfm/ini/util.py +++ b/hydrolib/core/dflowfm/ini/util.py @@ -680,7 +680,7 @@ class ExtendedUnknownKeywordErrorManager(UnknownKeywordErrorManager): Extended Error manager for unknown keys, this class can filter out specific keys which should not trigger an unknown key error. """ - _field_specific: set = {} + _field_specific: set = set() def _is_unknown_keyword( self, name: str, fields: Dict[str, ModelField], excluded_fields: Set @@ -704,7 +704,6 @@ class ForcingUnknownKeywordErrorManager(ExtendedUnknownKeywordErrorManager): "vertical_position_specification", "vertical_interpolation", "vertical_position_type", - "vertical_position_type", "vertical_position", "vector", } diff --git a/tests/dflowfm/ini/test_util.py b/tests/dflowfm/ini/test_util.py index 1f8425a6b..23261e09b 100644 --- a/tests/dflowfm/ini/test_util.py +++ b/tests/dflowfm/ini/test_util.py @@ -402,7 +402,7 @@ def test_keyword_given_known_as_alias_does_not_throw_exception(self): ukem.raise_error_for_unknown_keywords( data, section_header, fields, excluded_fields ) - except: + except ValueError: pytest.fail("Exception is thrown, no exception is expected for this test.") def test_keyword_given_known_as_name_does_not_throw_exception(self): @@ -423,7 +423,7 @@ def test_keyword_given_known_as_name_does_not_throw_exception(self): ukem.raise_error_for_unknown_keywords( data, section_header, fields, excluded_fields ) - except: + except ValueError: pytest.fail("Exception is thrown, no exception is expected for this test.") def test_keyword_given_known_as_excluded_field_does_not_throw_exception(self): @@ -441,7 +441,7 @@ def test_keyword_given_known_as_excluded_field_does_not_throw_exception(self): ukem.raise_error_for_unknown_keywords( data, section_header, fields, excluded_fields ) - except: + except ValueError: pytest.fail("Exception is thrown, no exception is expected for this test.") def test_keyword_given_known_as_extended_field_does_not_throw_exception(self): @@ -459,5 +459,5 @@ def test_keyword_given_known_as_extended_field_does_not_throw_exception(self): ukem.raise_error_for_unknown_keywords( data, section_header, fields, excluded_fields ) - except: + except ValueError: pytest.fail("Exception is thrown, no exception is expected for this test.") From c1eb7086253b7fb6e6794c432f53559d88c19b1f Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Tue, 28 May 2024 14:12:10 +0200 Subject: [PATCH 20/31] feat: (622) make a MduUnknownKeywordErrorManager for only checking on the mdu. --- hydrolib/core/dflowfm/bc/models.py | 5 -- hydrolib/core/dflowfm/crosssection/models.py | 5 -- hydrolib/core/dflowfm/ext/models.py | 5 -- hydrolib/core/dflowfm/ini/models.py | 6 +- hydrolib/core/dflowfm/ini/util.py | 90 ++++++++------------ hydrolib/core/dflowfm/mdu/models.py | 76 +++++++++++++++++ tests/dflowfm/ini/test_util.py | 29 ++----- 7 files changed, 118 insertions(+), 98 deletions(-) diff --git a/hydrolib/core/dflowfm/bc/models.py b/hydrolib/core/dflowfm/bc/models.py index 64eced7df..40fc942b0 100644 --- a/hydrolib/core/dflowfm/bc/models.py +++ b/hydrolib/core/dflowfm/bc/models.py @@ -28,7 +28,6 @@ from hydrolib.core.dflowfm.ini.parser import Parser, ParserConfig from hydrolib.core.dflowfm.ini.serializer import DataBlockINIBasedSerializerConfig from hydrolib.core.dflowfm.ini.util import ( - ForcingUnknownKeywordErrorManager, get_enum_validator, get_from_subclass_defaults, get_split_string_on_delimiter_validator, @@ -177,10 +176,6 @@ class ForcingBase(DataBlockINIBasedModel): quantityunitpair: List[ScalarOrVectorQUP] """List[ScalarOrVectorQUP]: List of header lines for one or more quantities and their unit. Describes the columns in the actual datablock.""" - @classmethod - def _get_unknown_keyword_error_manager(cls): - return ForcingUnknownKeywordErrorManager() - def _exclude_fields(self) -> Set: return {"quantityunitpair"}.union(super()._exclude_fields()) diff --git a/hydrolib/core/dflowfm/crosssection/models.py b/hydrolib/core/dflowfm/crosssection/models.py index afd8bb781..554429478 100644 --- a/hydrolib/core/dflowfm/crosssection/models.py +++ b/hydrolib/core/dflowfm/crosssection/models.py @@ -7,7 +7,6 @@ from hydrolib.core.dflowfm.friction.models import FrictionType from hydrolib.core.dflowfm.ini.models import INIBasedModel, INIGeneral, INIModel from hydrolib.core.dflowfm.ini.util import ( - CrsUnknownKeywordErrorManager, LocationValidationConfiguration, LocationValidationFieldNames, get_enum_validator, @@ -75,10 +74,6 @@ class Comments(INIBasedModel.Comments): def _get_identifier(self, data: dict) -> Optional[str]: return data.get("id") - @classmethod - def _get_unknown_keyword_error_manager(cls): - return CrsUnknownKeywordErrorManager() - @classmethod def _duplicate_keys_as_list(cls): return True diff --git a/hydrolib/core/dflowfm/ext/models.py b/hydrolib/core/dflowfm/ext/models.py index 160c960b0..624c6bce8 100644 --- a/hydrolib/core/dflowfm/ext/models.py +++ b/hydrolib/core/dflowfm/ext/models.py @@ -18,7 +18,6 @@ from hydrolib.core.dflowfm.ini.serializer import INISerializerConfig from hydrolib.core.dflowfm.ini.util import ( LocationValidationConfiguration, - MeteoUnknownKeywordErrorManager, get_enum_validator, get_split_string_on_delimiter_validator, make_list_validator, @@ -266,10 +265,6 @@ class Comments(INIBasedModel.Comments): alias="interpolationMethod" ) - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MeteoUnknownKeywordErrorManager() - def is_intermediate_link(self) -> bool: return True diff --git a/hydrolib/core/dflowfm/ini/models.py b/hydrolib/core/dflowfm/ini/models.py index bb13f35c3..c92f4d5db 100644 --- a/hydrolib/core/dflowfm/ini/models.py +++ b/hydrolib/core/dflowfm/ini/models.py @@ -25,7 +25,7 @@ INISerializerConfig, write_ini, ) -from .util import UnknownKeywordErrorManager, make_list_validator +from .util import DefaultUnknownKeywordErrorManager, UnknownKeywordErrorManager, make_list_validator logger = logging.getLogger(__name__) @@ -66,8 +66,8 @@ def __init__(self, **data): ) @classmethod - def _get_unknown_keyword_error_manager(cls): - return UnknownKeywordErrorManager() + def _get_unknown_keyword_error_manager(cls) -> UnknownKeywordErrorManager: + return DefaultUnknownKeywordErrorManager() @classmethod def _supports_comments(cls): diff --git a/hydrolib/core/dflowfm/ini/util.py b/hydrolib/core/dflowfm/ini/util.py index 7bd79dc09..fc83bca0d 100644 --- a/hydrolib/core/dflowfm/ini/util.py +++ b/hydrolib/core/dflowfm/ini/util.py @@ -1,5 +1,6 @@ """util.py provides additional utility methods related to handling ini files. """ +from abc import ABC, abstractmethod from datetime import datetime from enum import Enum from operator import eq @@ -623,13 +624,12 @@ def rename_keys_for_backwards_compatibility( return values - -class UnknownKeywordErrorManager: +class UnknownKeywordErrorManager(ABC): """ - Error manager for unknown keys. - Detects unknown keys and manages the Error to the user. + Base Error manager for unknown keys. """ + @abstractmethod def raise_error_for_unknown_keywords( self, data: Dict[str, Any], @@ -646,6 +646,36 @@ def raise_error_for_unknown_keywords( fields (Dict[str, ModelField]) : Known fields of the section. excluded_fields (Set) : Fields which should be excluded from the check for unknown keywords. """ + + +class DefaultUnknownKeywordErrorManager(UnknownKeywordErrorManager): + """ + Default Error manager for unknown keys. + Does no checks for unknown keywords. + """ + + def raise_error_for_unknown_keywords( + self, + data: Dict[str, Any], + section_header: str, + fields: Dict[str, ModelField], + excluded_fields: Set, + ): + return + +class MduUnknownKeywordErrorManager(UnknownKeywordErrorManager): + """ + Error manager for unknown keys. + Detects unknown keys and manages the Error to the user. + """ + + def raise_error_for_unknown_keywords( + self, + data: Dict[str, Any], + section_header: str, + fields: Dict[str, ModelField], + excluded_fields: Set, + ): unknown_keywords = self._get_all_unknown_keywords(data, fields, excluded_fields) if len(unknown_keywords) == 0: @@ -673,55 +703,3 @@ def _is_unknown_keyword( return False return name not in excluded_fields - - -class ExtendedUnknownKeywordErrorManager(UnknownKeywordErrorManager): - """ - Extended Error manager for unknown keys, this class can filter out specific keys which should not trigger an unknown key error. - """ - - _field_specific: set = set() - - def _is_unknown_keyword( - self, name: str, fields: Dict[str, ModelField], excluded_fields: Set - ): - is_unknown_keyword = super()._is_unknown_keyword(name, fields, excluded_fields) - - if name in self._field_specific: - return False - - return is_unknown_keyword - - -class ForcingUnknownKeywordErrorManager(ExtendedUnknownKeywordErrorManager): - _field_specific: set = { - "timeinterpolation", - "quantity", - "unit", - "manholename", - "vertpositionindex", - "time_interpolation", - "vertical_position_specification", - "vertical_interpolation", - "vertical_position_type", - "vertical_position", - "vector", - } - - -class CrsUnknownKeywordErrorManager(ExtendedUnknownKeywordErrorManager): - _field_specific: set = { - "template", - "height", - "width", - "r", - "r1", - "r2", - "r3", - "a", - "a1", - } - - -class MeteoUnknownKeywordErrorManager(ExtendedUnknownKeywordErrorManager): - _field_specific: set = {"locationtype", "locationfile"} diff --git a/hydrolib/core/dflowfm/mdu/models.py b/hydrolib/core/dflowfm/mdu/models.py index e6d620ed9..a25fa91e7 100644 --- a/hydrolib/core/dflowfm/mdu/models.py +++ b/hydrolib/core/dflowfm/mdu/models.py @@ -17,6 +17,7 @@ from hydrolib.core.dflowfm.ini.models import INIBasedModel, INIGeneral, INIModel from hydrolib.core.dflowfm.ini.serializer import INISerializerConfig from hydrolib.core.dflowfm.ini.util import ( + MduUnknownKeywordErrorManager, get_split_string_on_delimiter_validator, validate_datetime_string, ) @@ -71,6 +72,10 @@ class Comments(INIBasedModel.Comments): fileversion: str = Field("1.09", alias="fileVersion") autostart: Optional[AutoStartOption] = Field(AutoStartOption.no, alias="autoStart") pathsrelativetoparent: bool = Field(False, alias="pathsRelativeToParent") + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MduUnknownKeywordErrorManager() class Numerics(INIBasedModel): @@ -368,6 +373,10 @@ class Comments(INIBasedModel.Comments): velocitywarn: float = Field(0.0, alias="velocityWarn") adveccorrection1d2d: int = Field(0, alias="advecCorrection1D2D") fixedweirtalud: float = Field(4.0, alias="fixedWeirTalud") + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MduUnknownKeywordErrorManager() class VolumeTables(INIBasedModel): @@ -399,6 +408,10 @@ class Comments(INIBasedModel.Comments): usevolumetables: bool = Field(False, alias="useVolumeTables") increment: float = Field(0.2, alias="increment") usevolumetablefile: bool = Field(False, alias="useVolumeTableFile") + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MduUnknownKeywordErrorManager() class Physics(INIBasedModel): @@ -613,6 +626,10 @@ class Comments(INIBasedModel.Comments): iniwithnudge: int = Field(0, alias="iniWithNudge") secondaryflow: bool = Field(False, alias="secondaryFlow") betaspiral: float = Field(0.0, alias="betaSpiral") + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MduUnknownKeywordErrorManager() class Sediment(INIBasedModel): @@ -643,6 +660,9 @@ class Comments(INIBasedModel.Comments): default_factory=lambda: DiskOnlyFileModel(None), alias="SedFile" ) + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MduUnknownKeywordErrorManager() class Wind(INIBasedModel): """ @@ -714,6 +734,10 @@ def list_delimiter(cls) -> str: "cdbreakpoints", "windspeedbreakpoints", ) + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MduUnknownKeywordErrorManager() class Waves(INIBasedModel): @@ -745,6 +769,10 @@ class Comments(INIBasedModel.Comments): wavemodelnr: int = Field(3, alias="waveModelNr") rouwav: str = Field("FR84", alias="rouWav") gammax: float = Field(0.5, alias="gammaX") + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MduUnknownKeywordErrorManager() class Time(INIBasedModel): @@ -827,6 +855,10 @@ class Comments(INIBasedModel.Comments): @validator("startdatetime", "stopdatetime") def _validate_datetime(cls, value, field): return validate_datetime_string(value, field) + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MduUnknownKeywordErrorManager() class Restart(INIBasedModel): @@ -864,6 +896,10 @@ class Comments(INIBasedModel.Comments): @validator("restartdatetime") def _validate_datetime(cls, value, field): return validate_datetime_string(value, field) + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MduUnknownKeywordErrorManager() class ExternalForcing(INIBasedModel): @@ -915,6 +951,10 @@ class Comments(INIBasedModel.Comments): def is_intermediate_link(self) -> bool: return True + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MduUnknownKeywordErrorManager() class Hydrology(INIBasedModel): @@ -937,6 +977,10 @@ class Comments(INIBasedModel.Comments): _header: Literal["Hydrology"] = "Hydrology" interceptionmodel: bool = Field(False, alias="interceptionModel") + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MduUnknownKeywordErrorManager() class Trachytopes(INIBasedModel): @@ -977,6 +1021,10 @@ class Comments(INIBasedModel.Comments): trtl: Optional[Path] = Field(None, alias="trtL") dttrt: float = Field(60.0, alias="dtTrt") trtmxr: Optional[int] = Field(None, alias="trtMxR") + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MduUnknownKeywordErrorManager() ObsFile = Union[XYNModel, ObservationPointModel] @@ -1679,6 +1727,10 @@ class Comments(INIBasedModel.Comments): def is_intermediate_link(self) -> bool: return True + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MduUnknownKeywordErrorManager() class Geometry(INIBasedModel): @@ -2050,6 +2102,10 @@ class Comments(INIBasedModel.Comments): def is_intermediate_link(self) -> bool: return True + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MduUnknownKeywordErrorManager() class Calibration(INIBasedModel): @@ -2090,6 +2146,10 @@ class Comments(INIBasedModel.Comments): areafile: DiskOnlyFileModel = Field( default_factory=lambda: DiskOnlyFileModel(None), alias="AreaFile" ) + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MduUnknownKeywordErrorManager() class InfiltrationMethod(IntEnum): @@ -2162,6 +2222,10 @@ class Comments(INIBasedModel.Comments): bgrwuni: Optional[float] = Field(None, alias="bgrwuni") h_unsatini: Optional[float] = Field(0.2, alias="h_unsatini") sgrwini: Optional[float] = Field(None, alias="sgrwini") + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MduUnknownKeywordErrorManager() class ProcessFluxIntegration(IntEnum): @@ -2249,6 +2313,10 @@ class Comments(INIBasedModel.Comments): wriwaqbot3doutput: Optional[bool] = Field(False, alias="Wriwaqbot3Doutput") volumedrythreshold: Optional[float] = Field(1e-3, alias="VolumeDryThreshold") depthdrythreshold: Optional[float] = Field(1e-3, alias="DepthDryThreshold") + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MduUnknownKeywordErrorManager() class ParticlesThreeDType(IntEnum): @@ -2307,6 +2375,10 @@ class Comments(INIBasedModel.Comments): threedtype: Optional[ParticlesThreeDType] = Field( ParticlesThreeDType.DepthAveraged, alias="3Dtype" ) + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MduUnknownKeywordErrorManager() class VegetationModelNr(IntEnum): @@ -2360,6 +2432,10 @@ class Comments(INIBasedModel.Comments): rhoveg: Optional[float] = Field(0.0, alias="Rhoveg") stemheightstd: Optional[float] = Field(0.0, alias="Stemheightstd") densvegminbap: Optional[float] = Field(0.0, alias="Densvegminbap") + + @classmethod + def _get_unknown_keyword_error_manager(cls): + return MduUnknownKeywordErrorManager() class FMModel(INIModel): diff --git a/tests/dflowfm/ini/test_util.py b/tests/dflowfm/ini/test_util.py index 23261e09b..b1afe2b33 100644 --- a/tests/dflowfm/ini/test_util.py +++ b/tests/dflowfm/ini/test_util.py @@ -9,10 +9,9 @@ from hydrolib.core.basemodel import BaseModel from hydrolib.core.dflowfm.ini.util import ( - ExtendedUnknownKeywordErrorManager, LocationValidationConfiguration, LocationValidationFieldNames, - UnknownKeywordErrorManager, + MduUnknownKeywordErrorManager, get_from_subclass_defaults, get_type_based_on_subclass_default_value, rename_keys_for_backwards_compatibility, @@ -372,7 +371,7 @@ def test_unknown_keywords_given_when_notify_unknown_keywords_gives_error_with_un name = "keyname" second_name = "second_other" - ukem = UnknownKeywordErrorManager() + ukem = MduUnknownKeywordErrorManager() data = {name: 1, second_name: 2} expected_message = f"Unknown keywords are detected in section: '{section_header}', '{[name, second_name]}'" @@ -395,7 +394,7 @@ def test_keyword_given_known_as_alias_does_not_throw_exception(self): fields = {"name": mocked_field} - ukem = UnknownKeywordErrorManager() + ukem = MduUnknownKeywordErrorManager() data = {name: 1} try: @@ -416,7 +415,7 @@ def test_keyword_given_known_as_name_does_not_throw_exception(self): fields = {name: mocked_field} - ukem = UnknownKeywordErrorManager() + ukem = MduUnknownKeywordErrorManager() data = {name: 1} try: @@ -434,25 +433,7 @@ def test_keyword_given_known_as_excluded_field_does_not_throw_exception(self): fields = {} - ukem = UnknownKeywordErrorManager() - data = {name: 1} - - try: - ukem.raise_error_for_unknown_keywords( - data, section_header, fields, excluded_fields - ) - except ValueError: - pytest.fail("Exception is thrown, no exception is expected for this test.") - - def test_keyword_given_known_as_extended_field_does_not_throw_exception(self): - section_header = "section header" - excluded_fields = set() - name = "keyname" - - fields = {} - - ukem = ExtendedUnknownKeywordErrorManager() - ukem._field_specific = {name} + ukem = MduUnknownKeywordErrorManager() data = {name: 1} try: From 4f846522d993cf8760bf4652a3e5e7d3e3da68a3 Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Tue, 28 May 2024 14:16:14 +0200 Subject: [PATCH 21/31] feat: (622) revert structure test cases. --- tests/dflowfm/test_structure.py | 96 +++++++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 21 deletions(-) diff --git a/tests/dflowfm/test_structure.py b/tests/dflowfm/test_structure.py index 9c1b535ac..220620c4b 100644 --- a/tests/dflowfm/test_structure.py +++ b/tests/dflowfm/test_structure.py @@ -438,7 +438,7 @@ def test_bridge_construction_with_parser(self): assert bridge.friction == 70 assert bridge.length == 9.75 - def test_bridge_with_unknown_parameter_throws_valueerror(self): + def test_bridge_with_unknown_parameter_is_ignored(self): parser = Parser(ParserConfig()) input_str = inspect.cleandoc( @@ -470,14 +470,24 @@ def test_bridge_with_unknown_parameter_throws_valueerror(self): document = parser.finalize() - expected_message = ( - "Unknown keywords are detected in section: 'Structure', '['unknown']'" - ) + wrapper = WrapperTest[Bridge].parse_obj({"val": document.sections[0]}) + bridge = wrapper.val - with pytest.raises(ValueError) as exc_err: - WrapperTest[Bridge].parse_obj({"val": document.sections[0]}) + assert bridge.dict().get("unknown") is None # type: ignore - assert expected_message in str(exc_err.value) + assert bridge.id == "RS1-KBR31" + assert bridge.name == "RS1-KBR31name" + assert bridge.branchid == "riv_RS1_264" + assert bridge.chainage == 104.184 + assert bridge.type == "bridge" + assert bridge.allowedflowdir == FlowDirection.both + assert bridge.csdefid == "W_980.1S_0" + assert bridge.shift == 0.0 + assert bridge.inletlosscoeff == 1 + assert bridge.outletlosscoeff == 1 + assert bridge.frictiontype == FrictionType.strickler + assert bridge.friction == 70 + assert bridge.length == 9.75 def test_bridge_with_missing_required_parameters(self): parser = Parser(ParserConfig()) @@ -1605,7 +1615,7 @@ def test_weir_comments_construction_with_parser(self): assert weir.comments.crestwidth is None assert weir.comments.usevelocityheight == "My own special comment 2" - def test_weir_with_unknown_parameter_throws_valueerror(self): + def test_weir_with_unknown_parameter_is_ignored(self): parser = Parser(ParserConfig()) input_str = inspect.cleandoc( @@ -1633,14 +1643,20 @@ def test_weir_with_unknown_parameter_throws_valueerror(self): document = parser.finalize() - expected_message = ( - "Unknown keywords are detected in section: 'Structure', '['unknown']'" - ) + wrapper = WrapperTest[Weir].parse_obj({"val": document.sections[0]}) + weir = wrapper.val - with pytest.raises(ValueError) as exc_err: - WrapperTest[Weir].parse_obj({"val": document.sections[0]}) + assert weir.dict().get("unknown") is None # type: ignore - assert expected_message in str(exc_err.value) + assert weir.id == "weir_id" + assert weir.name == "weir" + assert weir.branchid == "branch" + assert weir.chainage == 3.0 + assert weir.type == "weir" + assert weir.allowedflowdir == FlowDirection.positive + assert weir.crestlevel == 10.5 + assert weir.crestwidth is None + assert weir.usevelocityheight == False @pytest.mark.parametrize( "input,expected", @@ -2166,7 +2182,7 @@ def test_weir_comments_construction_with_parser(self): assert struct.comments.gateopeningwidth is None assert struct.comments.usevelocityheight == "My own special comment 2" - def test_general_structure_with_unknown_parameter_throws_valueerror(self): + def test_general_structure_with_unknown_parameter_is_ignored(self): parser = Parser(ParserConfig()) input_str = inspect.cleandoc( @@ -2218,14 +2234,52 @@ def test_general_structure_with_unknown_parameter_throws_valueerror(self): document = parser.finalize() - expected_message = ( - "Unknown keywords are detected in section: 'Structure', '['unknown']'" - ) + wrapper = WrapperTest[GeneralStructure].parse_obj({"val": document.sections[0]}) + struct = wrapper.val - with pytest.raises(ValueError) as exc_err: - WrapperTest[GeneralStructure].parse_obj({"val": document.sections[0]}) + assert struct.dict().get("unknown") is None # type: ignore - assert expected_message in str(exc_err.value) + assert struct.id == "id" + assert struct.name == "extravagante_waarde" + assert struct.branchid == "stump" + assert struct.chainage == 13.53 + assert struct.type == "generalStructure" + assert struct.allowedflowdir == FlowDirection.positive + assert struct.upstream1width == 111.0 + assert struct.upstream1level == 112.0 + assert struct.upstream2width == 113.0 + assert struct.upstream2level == 114.0 + assert struct.crestwidth == 115.0 + assert struct.crestlevel == 116.0 + assert struct.crestlength == 117.0 + assert struct.downstream1width == 118.0 + assert struct.downstream1level == 119.0 + assert struct.downstream2width == 119.1 + assert struct.downstream2level == 119.2 + assert struct.gateloweredgelevel == 119.3 + assert struct.posfreegateflowcoeff == 119.4 + assert struct.posdrowngateflowcoeff == 119.5 + assert struct.posfreeweirflowcoeff == 119.6 + assert struct.posdrownweirflowcoeff == 119.7 + assert struct.poscontrcoeffreegate == 119.8 + assert struct.negfreegateflowcoeff == 119.9 + assert struct.negdrowngateflowcoeff == 118.1 + assert struct.negfreeweirflowcoeff == 118.2 + assert struct.negdrownweirflowcoeff == 118.3 + assert struct.negcontrcoeffreegate == 118.4 + assert struct.extraresistance == 118.5 + assert struct.gateheight == 118.6 + assert struct.gateopeningwidth == 118.7 + assert ( + struct.gateopeninghorizontaldirection + == GateOpeningHorizontalDirection.from_right + ) + assert struct.usevelocityheight == False + + def _create_required_general_structure_values(self) -> dict: + general_structure_values = dict() + general_structure_values.update(create_structure_values("generalStructure")) + return general_structure_values def _create_required_general_structure_values(self) -> dict: general_structure_values = dict() From d247030fad4fcc785477e168074a5b8a9d779533 Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Tue, 28 May 2024 14:20:20 +0200 Subject: [PATCH 22/31] feat: (622) revert crs def ini file changes. --- .../e02/c11_korte-woerden-1d/dimr_model/dflowfm/crsdef.ini | 1 + .../c01_steady-state-flow/CrossSectionDefinitions.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/data/input/e02/c11_korte-woerden-1d/dimr_model/dflowfm/crsdef.ini b/tests/data/input/e02/c11_korte-woerden-1d/dimr_model/dflowfm/crsdef.ini index 5ed79a534..2d3ab349e 100644 --- a/tests/data/input/e02/c11_korte-woerden-1d/dimr_model/dflowfm/crsdef.ini +++ b/tests/data/input/e02/c11_korte-woerden-1d/dimr_model/dflowfm/crsdef.ini @@ -119,3 +119,4 @@ sectionCount = 1 frictionIds = Main frictionPositions = -10.00 10.00 + isShared = 1 \ No newline at end of file diff --git a/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini b/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini index e13e6c3b5..47930ff9f 100644 --- a/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini +++ b/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini @@ -14,4 +14,4 @@ sectionCount = 1 frictionIds = Main frictionPositions = 0.000 50.000 - + isShared = 1 From 5a368a47399e119cfa5cebe2c4962fd7220ef059 Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen <102953403+MRVermeulenDeltares@users.noreply.github.com> Date: Tue, 28 May 2024 14:21:48 +0200 Subject: [PATCH 23/31] Update CrossSectionDefinitions.ini --- .../c01_steady-state-flow/CrossSectionDefinitions.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini b/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini index 47930ff9f..195bbb9d4 100644 --- a/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini +++ b/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini @@ -14,4 +14,4 @@ sectionCount = 1 frictionIds = Main frictionPositions = 0.000 50.000 - isShared = 1 + isShared = 1 From 4e04665eef652f84dddb149583fdb3b14d4e5f06 Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen <102953403+MRVermeulenDeltares@users.noreply.github.com> Date: Tue, 28 May 2024 14:22:27 +0200 Subject: [PATCH 24/31] Update CrossSectionDefinitions.ini --- .../c01_steady-state-flow/CrossSectionDefinitions.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini b/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini index 195bbb9d4..0a4ee8a20 100644 --- a/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini +++ b/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini @@ -14,4 +14,4 @@ sectionCount = 1 frictionIds = Main frictionPositions = 0.000 50.000 - isShared = 1 + isShared = 1 From 719c4125b178182b8379237b8e6a94134411ceb4 Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen <102953403+MRVermeulenDeltares@users.noreply.github.com> Date: Tue, 28 May 2024 14:22:56 +0200 Subject: [PATCH 25/31] Update CrossSectionDefinitions.ini --- .../c01_steady-state-flow/CrossSectionDefinitions.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini b/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini index 0a4ee8a20..39c1edb55 100644 --- a/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini +++ b/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini @@ -15,3 +15,4 @@ frictionIds = Main frictionPositions = 0.000 50.000 isShared = 1 + From 5ce7b27f75d3155d81beaf7c9eebf8155250fc39 Mon Sep 17 00:00:00 2001 From: MRVermeulenDeltares Date: Tue, 28 May 2024 12:24:08 +0000 Subject: [PATCH 26/31] autoformat: isort & black --- hydrolib/core/dflowfm/ini/models.py | 6 ++++- hydrolib/core/dflowfm/ini/util.py | 2 ++ hydrolib/core/dflowfm/mdu/models.py | 37 +++++++++++++++-------------- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/hydrolib/core/dflowfm/ini/models.py b/hydrolib/core/dflowfm/ini/models.py index c92f4d5db..d5a2ca5b1 100644 --- a/hydrolib/core/dflowfm/ini/models.py +++ b/hydrolib/core/dflowfm/ini/models.py @@ -25,7 +25,11 @@ INISerializerConfig, write_ini, ) -from .util import DefaultUnknownKeywordErrorManager, UnknownKeywordErrorManager, make_list_validator +from .util import ( + DefaultUnknownKeywordErrorManager, + UnknownKeywordErrorManager, + make_list_validator, +) logger = logging.getLogger(__name__) diff --git a/hydrolib/core/dflowfm/ini/util.py b/hydrolib/core/dflowfm/ini/util.py index fc83bca0d..eeb9283db 100644 --- a/hydrolib/core/dflowfm/ini/util.py +++ b/hydrolib/core/dflowfm/ini/util.py @@ -624,6 +624,7 @@ def rename_keys_for_backwards_compatibility( return values + class UnknownKeywordErrorManager(ABC): """ Base Error manager for unknown keys. @@ -663,6 +664,7 @@ def raise_error_for_unknown_keywords( ): return + class MduUnknownKeywordErrorManager(UnknownKeywordErrorManager): """ Error manager for unknown keys. diff --git a/hydrolib/core/dflowfm/mdu/models.py b/hydrolib/core/dflowfm/mdu/models.py index a25fa91e7..fa849d030 100644 --- a/hydrolib/core/dflowfm/mdu/models.py +++ b/hydrolib/core/dflowfm/mdu/models.py @@ -72,7 +72,7 @@ class Comments(INIBasedModel.Comments): fileversion: str = Field("1.09", alias="fileVersion") autostart: Optional[AutoStartOption] = Field(AutoStartOption.no, alias="autoStart") pathsrelativetoparent: bool = Field(False, alias="pathsRelativeToParent") - + @classmethod def _get_unknown_keyword_error_manager(cls): return MduUnknownKeywordErrorManager() @@ -373,7 +373,7 @@ class Comments(INIBasedModel.Comments): velocitywarn: float = Field(0.0, alias="velocityWarn") adveccorrection1d2d: int = Field(0, alias="advecCorrection1D2D") fixedweirtalud: float = Field(4.0, alias="fixedWeirTalud") - + @classmethod def _get_unknown_keyword_error_manager(cls): return MduUnknownKeywordErrorManager() @@ -408,7 +408,7 @@ class Comments(INIBasedModel.Comments): usevolumetables: bool = Field(False, alias="useVolumeTables") increment: float = Field(0.2, alias="increment") usevolumetablefile: bool = Field(False, alias="useVolumeTableFile") - + @classmethod def _get_unknown_keyword_error_manager(cls): return MduUnknownKeywordErrorManager() @@ -626,7 +626,7 @@ class Comments(INIBasedModel.Comments): iniwithnudge: int = Field(0, alias="iniWithNudge") secondaryflow: bool = Field(False, alias="secondaryFlow") betaspiral: float = Field(0.0, alias="betaSpiral") - + @classmethod def _get_unknown_keyword_error_manager(cls): return MduUnknownKeywordErrorManager() @@ -664,6 +664,7 @@ class Comments(INIBasedModel.Comments): def _get_unknown_keyword_error_manager(cls): return MduUnknownKeywordErrorManager() + class Wind(INIBasedModel): """ The `[Wind]` section in an MDU file. @@ -734,7 +735,7 @@ def list_delimiter(cls) -> str: "cdbreakpoints", "windspeedbreakpoints", ) - + @classmethod def _get_unknown_keyword_error_manager(cls): return MduUnknownKeywordErrorManager() @@ -769,7 +770,7 @@ class Comments(INIBasedModel.Comments): wavemodelnr: int = Field(3, alias="waveModelNr") rouwav: str = Field("FR84", alias="rouWav") gammax: float = Field(0.5, alias="gammaX") - + @classmethod def _get_unknown_keyword_error_manager(cls): return MduUnknownKeywordErrorManager() @@ -855,7 +856,7 @@ class Comments(INIBasedModel.Comments): @validator("startdatetime", "stopdatetime") def _validate_datetime(cls, value, field): return validate_datetime_string(value, field) - + @classmethod def _get_unknown_keyword_error_manager(cls): return MduUnknownKeywordErrorManager() @@ -896,7 +897,7 @@ class Comments(INIBasedModel.Comments): @validator("restartdatetime") def _validate_datetime(cls, value, field): return validate_datetime_string(value, field) - + @classmethod def _get_unknown_keyword_error_manager(cls): return MduUnknownKeywordErrorManager() @@ -951,7 +952,7 @@ class Comments(INIBasedModel.Comments): def is_intermediate_link(self) -> bool: return True - + @classmethod def _get_unknown_keyword_error_manager(cls): return MduUnknownKeywordErrorManager() @@ -977,7 +978,7 @@ class Comments(INIBasedModel.Comments): _header: Literal["Hydrology"] = "Hydrology" interceptionmodel: bool = Field(False, alias="interceptionModel") - + @classmethod def _get_unknown_keyword_error_manager(cls): return MduUnknownKeywordErrorManager() @@ -1021,7 +1022,7 @@ class Comments(INIBasedModel.Comments): trtl: Optional[Path] = Field(None, alias="trtL") dttrt: float = Field(60.0, alias="dtTrt") trtmxr: Optional[int] = Field(None, alias="trtMxR") - + @classmethod def _get_unknown_keyword_error_manager(cls): return MduUnknownKeywordErrorManager() @@ -1727,7 +1728,7 @@ class Comments(INIBasedModel.Comments): def is_intermediate_link(self) -> bool: return True - + @classmethod def _get_unknown_keyword_error_manager(cls): return MduUnknownKeywordErrorManager() @@ -2102,7 +2103,7 @@ class Comments(INIBasedModel.Comments): def is_intermediate_link(self) -> bool: return True - + @classmethod def _get_unknown_keyword_error_manager(cls): return MduUnknownKeywordErrorManager() @@ -2146,7 +2147,7 @@ class Comments(INIBasedModel.Comments): areafile: DiskOnlyFileModel = Field( default_factory=lambda: DiskOnlyFileModel(None), alias="AreaFile" ) - + @classmethod def _get_unknown_keyword_error_manager(cls): return MduUnknownKeywordErrorManager() @@ -2222,7 +2223,7 @@ class Comments(INIBasedModel.Comments): bgrwuni: Optional[float] = Field(None, alias="bgrwuni") h_unsatini: Optional[float] = Field(0.2, alias="h_unsatini") sgrwini: Optional[float] = Field(None, alias="sgrwini") - + @classmethod def _get_unknown_keyword_error_manager(cls): return MduUnknownKeywordErrorManager() @@ -2313,7 +2314,7 @@ class Comments(INIBasedModel.Comments): wriwaqbot3doutput: Optional[bool] = Field(False, alias="Wriwaqbot3Doutput") volumedrythreshold: Optional[float] = Field(1e-3, alias="VolumeDryThreshold") depthdrythreshold: Optional[float] = Field(1e-3, alias="DepthDryThreshold") - + @classmethod def _get_unknown_keyword_error_manager(cls): return MduUnknownKeywordErrorManager() @@ -2375,7 +2376,7 @@ class Comments(INIBasedModel.Comments): threedtype: Optional[ParticlesThreeDType] = Field( ParticlesThreeDType.DepthAveraged, alias="3Dtype" ) - + @classmethod def _get_unknown_keyword_error_manager(cls): return MduUnknownKeywordErrorManager() @@ -2432,7 +2433,7 @@ class Comments(INIBasedModel.Comments): rhoveg: Optional[float] = Field(0.0, alias="Rhoveg") stemheightstd: Optional[float] = Field(0.0, alias="Stemheightstd") densvegminbap: Optional[float] = Field(0.0, alias="Densvegminbap") - + @classmethod def _get_unknown_keyword_error_manager(cls): return MduUnknownKeywordErrorManager() From 9fe75b68dba232712c87e150a85bb4c14be5ba01 Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Tue, 28 May 2024 14:25:28 +0200 Subject: [PATCH 27/31] feat: (622) revert crs def ini file changes. --- .../e02/c11_korte-woerden-1d/dimr_model/dflowfm/crsdef.ini | 2 +- .../c01_steady-state-flow/CrossSectionDefinitions.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data/input/e02/c11_korte-woerden-1d/dimr_model/dflowfm/crsdef.ini b/tests/data/input/e02/c11_korte-woerden-1d/dimr_model/dflowfm/crsdef.ini index 2d3ab349e..e82b4f71f 100644 --- a/tests/data/input/e02/c11_korte-woerden-1d/dimr_model/dflowfm/crsdef.ini +++ b/tests/data/input/e02/c11_korte-woerden-1d/dimr_model/dflowfm/crsdef.ini @@ -119,4 +119,4 @@ sectionCount = 1 frictionIds = Main frictionPositions = -10.00 10.00 - isShared = 1 \ No newline at end of file + isShared = 1 diff --git a/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini b/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini index 39c1edb55..99cd7c3a4 100644 --- a/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini +++ b/tests/data/input/e02/f101_1D-boundaries/c01_steady-state-flow/CrossSectionDefinitions.ini @@ -14,5 +14,5 @@ sectionCount = 1 frictionIds = Main frictionPositions = 0.000 50.000 - isShared = 1 + isShared = 1 From 7ed3f01352ffba70f27096621957220b84330bcc Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Tue, 28 May 2024 14:48:35 +0200 Subject: [PATCH 28/31] feat: (622) revert structure test cases. --- tests/dflowfm/test_storagenode.py | 1 + tests/dflowfm/test_structure.py | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/dflowfm/test_storagenode.py b/tests/dflowfm/test_storagenode.py index 984b67aaf..0918abb0a 100644 --- a/tests/dflowfm/test_storagenode.py +++ b/tests/dflowfm/test_storagenode.py @@ -184,6 +184,7 @@ def test_scientific_notation_in_ini_file_are_correctly_parsed_for_strings_and_fl streetLevel = 0.000 storageType = Reservoir streetStorageArea = 1d2 + CompartmentShape = Unknown useTable = False """ diff --git a/tests/dflowfm/test_structure.py b/tests/dflowfm/test_structure.py index 220620c4b..082ef7e49 100644 --- a/tests/dflowfm/test_structure.py +++ b/tests/dflowfm/test_structure.py @@ -286,6 +286,7 @@ def test_culvert_parses_flowdirection_case_insensitive(input, expected): length="1", inletlosscoeff="1", outletlosscoeff="1", + inletlosvalveonoffscoeff="1", valveonoff="1", valveopeningheight="1", numlosscoeff="1", @@ -318,6 +319,7 @@ def test_culvert_parses_subtype_case_insensitive(input, expected): length="1", inletlosscoeff="1", outletlosscoeff="1", + inletlosvalveonoffscoeff="1", valveonoff="1", valveopeningheight="1", numlosscoeff="1", @@ -2281,11 +2283,6 @@ def _create_required_general_structure_values(self) -> dict: general_structure_values.update(create_structure_values("generalStructure")) return general_structure_values - def _create_required_general_structure_values(self) -> dict: - general_structure_values = dict() - general_structure_values.update(create_structure_values("generalStructure")) - return general_structure_values - def _create_general_structure_values(self) -> dict: general_structure_values = dict( allowedflowdir=FlowDirection.positive, From 1a77689acfa39c7f562a2e7bb0d34e21e7388a16 Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Thu, 30 May 2024 13:48:23 +0200 Subject: [PATCH 29/31] feat: (622) Pair program to implement root_validator --- hydrolib/core/dflowfm/crosssection/models.py | 8 +++ hydrolib/core/dflowfm/ext/models.py | 8 +++ hydrolib/core/dflowfm/ini/models.py | 34 +++++---- hydrolib/core/dflowfm/ini/util.py | 43 +---------- hydrolib/core/dflowfm/mdu/models.py | 76 +++++--------------- hydrolib/core/dflowfm/storagenode/models.py | 8 +++ hydrolib/core/dflowfm/structure/models.py | 8 +++ tests/dflowfm/ini/test_util.py | 10 +-- 8 files changed, 76 insertions(+), 119 deletions(-) diff --git a/hydrolib/core/dflowfm/crosssection/models.py b/hydrolib/core/dflowfm/crosssection/models.py index 554429478..9fd8098d7 100644 --- a/hydrolib/core/dflowfm/crosssection/models.py +++ b/hydrolib/core/dflowfm/crosssection/models.py @@ -9,6 +9,7 @@ from hydrolib.core.dflowfm.ini.util import ( LocationValidationConfiguration, LocationValidationFieldNames, + UnknownKeywordErrorManager, get_enum_validator, get_from_subclass_defaults, get_split_string_on_delimiter_validator, @@ -70,6 +71,13 @@ class Comments(INIBasedModel.Comments): id: str = Field(alias="id") type: str = Field(alias="type") thalweg: Optional[float] + + @classmethod + def _get_unknown_keyword_error_manager(cls) -> Optional[UnknownKeywordErrorManager]: + """ + The CrossSectionDefinition does not currently support raising an error on unknown keywords. + """ + return None def _get_identifier(self, data: dict) -> Optional[str]: return data.get("id") diff --git a/hydrolib/core/dflowfm/ext/models.py b/hydrolib/core/dflowfm/ext/models.py index 624c6bce8..ceeb65a5d 100644 --- a/hydrolib/core/dflowfm/ext/models.py +++ b/hydrolib/core/dflowfm/ext/models.py @@ -18,6 +18,7 @@ from hydrolib.core.dflowfm.ini.serializer import INISerializerConfig from hydrolib.core.dflowfm.ini.util import ( LocationValidationConfiguration, + UnknownKeywordErrorManager, get_enum_validator, get_split_string_on_delimiter_validator, make_list_validator, @@ -248,6 +249,13 @@ class Comments(INIBasedModel.Comments): ) comments: Comments = Comments() + + @classmethod + def _get_unknown_keyword_error_manager(cls) -> Optional[UnknownKeywordErrorManager]: + """ + The Meteo does not currently support raising an error on unknown keywords. + """ + return None _disk_only_file_model_should_not_be_none = ( validator_set_default_disk_only_file_model_when_none() diff --git a/hydrolib/core/dflowfm/ini/models.py b/hydrolib/core/dflowfm/ini/models.py index d5a2ca5b1..37ec7bebd 100644 --- a/hydrolib/core/dflowfm/ini/models.py +++ b/hydrolib/core/dflowfm/ini/models.py @@ -26,7 +26,6 @@ write_ini, ) from .util import ( - DefaultUnknownKeywordErrorManager, UnknownKeywordErrorManager, make_list_validator, ) @@ -59,19 +58,9 @@ class Config: extra = Extra.ignore arbitrary_types_allowed = False - def __init__(self, **data): - super().__init__(**data) - unknown_keyword_error_manager = self._get_unknown_keyword_error_manager() - unknown_keyword_error_manager.raise_error_for_unknown_keywords( - data, - self._header, - self.__fields__, - self._exclude_fields(), - ) - @classmethod - def _get_unknown_keyword_error_manager(cls) -> UnknownKeywordErrorManager: - return DefaultUnknownKeywordErrorManager() + def _get_unknown_keyword_error_manager(cls) -> Optional[UnknownKeywordErrorManager]: + return UnknownKeywordErrorManager() @classmethod def _supports_comments(cls): @@ -121,6 +110,18 @@ class Config: comments: Optional[Comments] = Comments() + @root_validator(pre=True) + def _validate_unknown_keywords(cls, values): + unknown_keyword_error_manager = cls._get_unknown_keyword_error_manager() + if unknown_keyword_error_manager: + unknown_keyword_error_manager.raise_error_for_unknown_keywords( + values, + cls._header, + cls.__fields__, + cls._exclude_fields(), + ) + return values + @root_validator(pre=True) def _skip_nones_and_set_header(cls, values): """Drop None fields for known fields.""" @@ -237,6 +238,13 @@ class DataBlockINIBasedModel(INIBasedModel): datablock: Datablock = [] _make_lists = make_list_validator("datablock") + + @classmethod + def _get_unknown_keyword_error_manager(cls) -> Optional[UnknownKeywordErrorManager]: + """ + The DataBlockINIBasedModel does not need to raise an error on unknown keywords. + """ + return None def _to_section( self, diff --git a/hydrolib/core/dflowfm/ini/util.py b/hydrolib/core/dflowfm/ini/util.py index eeb9283db..9fcfca63d 100644 --- a/hydrolib/core/dflowfm/ini/util.py +++ b/hydrolib/core/dflowfm/ini/util.py @@ -1,6 +1,5 @@ """util.py provides additional utility methods related to handling ini files. """ -from abc import ABC, abstractmethod from datetime import datetime from enum import Enum from operator import eq @@ -625,47 +624,7 @@ def rename_keys_for_backwards_compatibility( return values -class UnknownKeywordErrorManager(ABC): - """ - Base Error manager for unknown keys. - """ - - @abstractmethod - def raise_error_for_unknown_keywords( - self, - data: Dict[str, Any], - section_header: str, - fields: Dict[str, ModelField], - excluded_fields: Set, - ): - """ - Notify the user of unknown keywords. - - Args: - data (Dict[str, Any]) : Input data containing all properties which are checked on unknown keywords. - section_header (str) : Header of the section in which unknown keys might be detected. - fields (Dict[str, ModelField]) : Known fields of the section. - excluded_fields (Set) : Fields which should be excluded from the check for unknown keywords. - """ - - -class DefaultUnknownKeywordErrorManager(UnknownKeywordErrorManager): - """ - Default Error manager for unknown keys. - Does no checks for unknown keywords. - """ - - def raise_error_for_unknown_keywords( - self, - data: Dict[str, Any], - section_header: str, - fields: Dict[str, ModelField], - excluded_fields: Set, - ): - return - - -class MduUnknownKeywordErrorManager(UnknownKeywordErrorManager): +class UnknownKeywordErrorManager: """ Error manager for unknown keys. Detects unknown keys and manages the Error to the user. diff --git a/hydrolib/core/dflowfm/mdu/models.py b/hydrolib/core/dflowfm/mdu/models.py index fa849d030..7325d70fa 100644 --- a/hydrolib/core/dflowfm/mdu/models.py +++ b/hydrolib/core/dflowfm/mdu/models.py @@ -17,7 +17,6 @@ from hydrolib.core.dflowfm.ini.models import INIBasedModel, INIGeneral, INIModel from hydrolib.core.dflowfm.ini.serializer import INISerializerConfig from hydrolib.core.dflowfm.ini.util import ( - MduUnknownKeywordErrorManager, get_split_string_on_delimiter_validator, validate_datetime_string, ) @@ -73,10 +72,6 @@ class Comments(INIBasedModel.Comments): autostart: Optional[AutoStartOption] = Field(AutoStartOption.no, alias="autoStart") pathsrelativetoparent: bool = Field(False, alias="pathsRelativeToParent") - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MduUnknownKeywordErrorManager() - class Numerics(INIBasedModel): """ @@ -374,9 +369,7 @@ class Comments(INIBasedModel.Comments): adveccorrection1d2d: int = Field(0, alias="advecCorrection1D2D") fixedweirtalud: float = Field(4.0, alias="fixedWeirTalud") - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MduUnknownKeywordErrorManager() + class VolumeTables(INIBasedModel): @@ -409,9 +402,7 @@ class Comments(INIBasedModel.Comments): increment: float = Field(0.2, alias="increment") usevolumetablefile: bool = Field(False, alias="useVolumeTableFile") - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MduUnknownKeywordErrorManager() + class Physics(INIBasedModel): @@ -627,9 +618,7 @@ class Comments(INIBasedModel.Comments): secondaryflow: bool = Field(False, alias="secondaryFlow") betaspiral: float = Field(0.0, alias="betaSpiral") - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MduUnknownKeywordErrorManager() + class Sediment(INIBasedModel): @@ -660,9 +649,7 @@ class Comments(INIBasedModel.Comments): default_factory=lambda: DiskOnlyFileModel(None), alias="SedFile" ) - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MduUnknownKeywordErrorManager() + class Wind(INIBasedModel): @@ -736,9 +723,7 @@ def list_delimiter(cls) -> str: "windspeedbreakpoints", ) - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MduUnknownKeywordErrorManager() + class Waves(INIBasedModel): @@ -771,9 +756,7 @@ class Comments(INIBasedModel.Comments): rouwav: str = Field("FR84", alias="rouWav") gammax: float = Field(0.5, alias="gammaX") - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MduUnknownKeywordErrorManager() + class Time(INIBasedModel): @@ -857,9 +840,7 @@ class Comments(INIBasedModel.Comments): def _validate_datetime(cls, value, field): return validate_datetime_string(value, field) - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MduUnknownKeywordErrorManager() + class Restart(INIBasedModel): @@ -898,9 +879,7 @@ class Comments(INIBasedModel.Comments): def _validate_datetime(cls, value, field): return validate_datetime_string(value, field) - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MduUnknownKeywordErrorManager() + class ExternalForcing(INIBasedModel): @@ -953,9 +932,7 @@ class Comments(INIBasedModel.Comments): def is_intermediate_link(self) -> bool: return True - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MduUnknownKeywordErrorManager() + class Hydrology(INIBasedModel): @@ -979,9 +956,7 @@ class Comments(INIBasedModel.Comments): _header: Literal["Hydrology"] = "Hydrology" interceptionmodel: bool = Field(False, alias="interceptionModel") - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MduUnknownKeywordErrorManager() + class Trachytopes(INIBasedModel): @@ -1023,9 +998,7 @@ class Comments(INIBasedModel.Comments): dttrt: float = Field(60.0, alias="dtTrt") trtmxr: Optional[int] = Field(None, alias="trtMxR") - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MduUnknownKeywordErrorManager() + ObsFile = Union[XYNModel, ObservationPointModel] @@ -1729,9 +1702,7 @@ class Comments(INIBasedModel.Comments): def is_intermediate_link(self) -> bool: return True - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MduUnknownKeywordErrorManager() + class Geometry(INIBasedModel): @@ -2104,9 +2075,7 @@ class Comments(INIBasedModel.Comments): def is_intermediate_link(self) -> bool: return True - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MduUnknownKeywordErrorManager() + class Calibration(INIBasedModel): @@ -2148,9 +2117,7 @@ class Comments(INIBasedModel.Comments): default_factory=lambda: DiskOnlyFileModel(None), alias="AreaFile" ) - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MduUnknownKeywordErrorManager() + class InfiltrationMethod(IntEnum): @@ -2224,9 +2191,7 @@ class Comments(INIBasedModel.Comments): h_unsatini: Optional[float] = Field(0.2, alias="h_unsatini") sgrwini: Optional[float] = Field(None, alias="sgrwini") - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MduUnknownKeywordErrorManager() + class ProcessFluxIntegration(IntEnum): @@ -2315,9 +2280,7 @@ class Comments(INIBasedModel.Comments): volumedrythreshold: Optional[float] = Field(1e-3, alias="VolumeDryThreshold") depthdrythreshold: Optional[float] = Field(1e-3, alias="DepthDryThreshold") - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MduUnknownKeywordErrorManager() + class ParticlesThreeDType(IntEnum): @@ -2377,9 +2340,7 @@ class Comments(INIBasedModel.Comments): ParticlesThreeDType.DepthAveraged, alias="3Dtype" ) - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MduUnknownKeywordErrorManager() + class VegetationModelNr(IntEnum): @@ -2434,9 +2395,6 @@ class Comments(INIBasedModel.Comments): stemheightstd: Optional[float] = Field(0.0, alias="Stemheightstd") densvegminbap: Optional[float] = Field(0.0, alias="Densvegminbap") - @classmethod - def _get_unknown_keyword_error_manager(cls): - return MduUnknownKeywordErrorManager() class FMModel(INIModel): diff --git a/hydrolib/core/dflowfm/storagenode/models.py b/hydrolib/core/dflowfm/storagenode/models.py index 45d4debf8..842fcd260 100644 --- a/hydrolib/core/dflowfm/storagenode/models.py +++ b/hydrolib/core/dflowfm/storagenode/models.py @@ -6,6 +6,7 @@ from hydrolib.core.dflowfm.ini.models import INIBasedModel, INIGeneral, INIModel from hydrolib.core.dflowfm.ini.util import ( + UnknownKeywordErrorManager, get_enum_validator, get_split_string_on_delimiter_validator, make_list_validator, @@ -161,6 +162,13 @@ class Comments(INIBasedModel.Comments): interpolate: Optional[Interpolation] = Field( Interpolation.linear.value, alias="interpolate" ) + + @classmethod + def _get_unknown_keyword_error_manager(cls) -> Optional[UnknownKeywordErrorManager]: + """ + The StorageNode does not currently support raising an error on unknown keywords. + """ + return None _interpolation_validator = get_enum_validator("interpolate", enum=Interpolation) _nodetype_validator = get_enum_validator("nodetype", enum=NodeType) diff --git a/hydrolib/core/dflowfm/structure/models.py b/hydrolib/core/dflowfm/structure/models.py index 04ee9496e..8290eee35 100644 --- a/hydrolib/core/dflowfm/structure/models.py +++ b/hydrolib/core/dflowfm/structure/models.py @@ -18,6 +18,7 @@ from hydrolib.core.dflowfm.friction.models import FrictionType from hydrolib.core.dflowfm.ini.models import INIBasedModel, INIGeneral, INIModel from hydrolib.core.dflowfm.ini.util import ( + UnknownKeywordErrorManager, get_enum_validator, get_from_subclass_defaults, get_split_string_on_delimiter_validator, @@ -82,6 +83,13 @@ class Comments(INIBasedModel.Comments): _loc_coord_fields = {"numcoordinates", "xcoordinates", "ycoordinates"} _loc_branch_fields = {"branchid", "chainage"} _loc_all_fields = _loc_coord_fields | _loc_branch_fields + + @classmethod + def _get_unknown_keyword_error_manager(cls) -> Optional[UnknownKeywordErrorManager]: + """ + The Structure does not currently support raising an error on unknown keywords. + """ + return None _split_to_list = get_split_string_on_delimiter_validator( "xcoordinates", "ycoordinates" diff --git a/tests/dflowfm/ini/test_util.py b/tests/dflowfm/ini/test_util.py index b1afe2b33..19f4abe77 100644 --- a/tests/dflowfm/ini/test_util.py +++ b/tests/dflowfm/ini/test_util.py @@ -11,7 +11,7 @@ from hydrolib.core.dflowfm.ini.util import ( LocationValidationConfiguration, LocationValidationFieldNames, - MduUnknownKeywordErrorManager, + UnknownKeywordErrorManager, get_from_subclass_defaults, get_type_based_on_subclass_default_value, rename_keys_for_backwards_compatibility, @@ -371,7 +371,7 @@ def test_unknown_keywords_given_when_notify_unknown_keywords_gives_error_with_un name = "keyname" second_name = "second_other" - ukem = MduUnknownKeywordErrorManager() + ukem = UnknownKeywordErrorManager() data = {name: 1, second_name: 2} expected_message = f"Unknown keywords are detected in section: '{section_header}', '{[name, second_name]}'" @@ -394,7 +394,7 @@ def test_keyword_given_known_as_alias_does_not_throw_exception(self): fields = {"name": mocked_field} - ukem = MduUnknownKeywordErrorManager() + ukem = UnknownKeywordErrorManager() data = {name: 1} try: @@ -415,7 +415,7 @@ def test_keyword_given_known_as_name_does_not_throw_exception(self): fields = {name: mocked_field} - ukem = MduUnknownKeywordErrorManager() + ukem = UnknownKeywordErrorManager() data = {name: 1} try: @@ -433,7 +433,7 @@ def test_keyword_given_known_as_excluded_field_does_not_throw_exception(self): fields = {} - ukem = MduUnknownKeywordErrorManager() + ukem = UnknownKeywordErrorManager() data = {name: 1} try: From 57d10e3fc7ee35aef510289f4c021e98eeb94d54 Mon Sep 17 00:00:00 2001 From: MRVermeulenDeltares Date: Thu, 30 May 2024 11:49:44 +0000 Subject: [PATCH 30/31] autoformat: isort & black --- hydrolib/core/dflowfm/crosssection/models.py | 2 +- hydrolib/core/dflowfm/ext/models.py | 2 +- hydrolib/core/dflowfm/ini/models.py | 7 ++-- hydrolib/core/dflowfm/mdu/models.py | 35 -------------------- hydrolib/core/dflowfm/storagenode/models.py | 2 +- hydrolib/core/dflowfm/structure/models.py | 2 +- 6 files changed, 6 insertions(+), 44 deletions(-) diff --git a/hydrolib/core/dflowfm/crosssection/models.py b/hydrolib/core/dflowfm/crosssection/models.py index 9fd8098d7..731ec7047 100644 --- a/hydrolib/core/dflowfm/crosssection/models.py +++ b/hydrolib/core/dflowfm/crosssection/models.py @@ -71,7 +71,7 @@ class Comments(INIBasedModel.Comments): id: str = Field(alias="id") type: str = Field(alias="type") thalweg: Optional[float] - + @classmethod def _get_unknown_keyword_error_manager(cls) -> Optional[UnknownKeywordErrorManager]: """ diff --git a/hydrolib/core/dflowfm/ext/models.py b/hydrolib/core/dflowfm/ext/models.py index ceeb65a5d..4e11e1d17 100644 --- a/hydrolib/core/dflowfm/ext/models.py +++ b/hydrolib/core/dflowfm/ext/models.py @@ -249,7 +249,7 @@ class Comments(INIBasedModel.Comments): ) comments: Comments = Comments() - + @classmethod def _get_unknown_keyword_error_manager(cls) -> Optional[UnknownKeywordErrorManager]: """ diff --git a/hydrolib/core/dflowfm/ini/models.py b/hydrolib/core/dflowfm/ini/models.py index 37ec7bebd..e5a30a22d 100644 --- a/hydrolib/core/dflowfm/ini/models.py +++ b/hydrolib/core/dflowfm/ini/models.py @@ -25,10 +25,7 @@ INISerializerConfig, write_ini, ) -from .util import ( - UnknownKeywordErrorManager, - make_list_validator, -) +from .util import UnknownKeywordErrorManager, make_list_validator logger = logging.getLogger(__name__) @@ -238,7 +235,7 @@ class DataBlockINIBasedModel(INIBasedModel): datablock: Datablock = [] _make_lists = make_list_validator("datablock") - + @classmethod def _get_unknown_keyword_error_manager(cls) -> Optional[UnknownKeywordErrorManager]: """ diff --git a/hydrolib/core/dflowfm/mdu/models.py b/hydrolib/core/dflowfm/mdu/models.py index 7325d70fa..e6d620ed9 100644 --- a/hydrolib/core/dflowfm/mdu/models.py +++ b/hydrolib/core/dflowfm/mdu/models.py @@ -370,8 +370,6 @@ class Comments(INIBasedModel.Comments): fixedweirtalud: float = Field(4.0, alias="fixedWeirTalud") - - class VolumeTables(INIBasedModel): """ The `[VolumeTables]` section in an MDU file. @@ -403,8 +401,6 @@ class Comments(INIBasedModel.Comments): usevolumetablefile: bool = Field(False, alias="useVolumeTableFile") - - class Physics(INIBasedModel): """ The `[Physics]` section in an MDU file. @@ -619,8 +615,6 @@ class Comments(INIBasedModel.Comments): betaspiral: float = Field(0.0, alias="betaSpiral") - - class Sediment(INIBasedModel): class Comments(INIBasedModel.Comments): sedimentmodelnr: Optional[str] = Field( @@ -650,8 +644,6 @@ class Comments(INIBasedModel.Comments): ) - - class Wind(INIBasedModel): """ The `[Wind]` section in an MDU file. @@ -724,8 +716,6 @@ def list_delimiter(cls) -> str: ) - - class Waves(INIBasedModel): """ The `[Waves]` section in an MDU file. @@ -757,8 +747,6 @@ class Comments(INIBasedModel.Comments): gammax: float = Field(0.5, alias="gammaX") - - class Time(INIBasedModel): """ The `[Time]` section in an MDU file. @@ -841,8 +829,6 @@ def _validate_datetime(cls, value, field): return validate_datetime_string(value, field) - - class Restart(INIBasedModel): """ The `[Restart]` section in an MDU file. @@ -880,8 +866,6 @@ def _validate_datetime(cls, value, field): return validate_datetime_string(value, field) - - class ExternalForcing(INIBasedModel): """ The `[External Forcing]` section in an MDU file. @@ -933,8 +917,6 @@ def is_intermediate_link(self) -> bool: return True - - class Hydrology(INIBasedModel): """ The `[Hydrology]` section in an MDU file. @@ -957,8 +939,6 @@ class Comments(INIBasedModel.Comments): interceptionmodel: bool = Field(False, alias="interceptionModel") - - class Trachytopes(INIBasedModel): """ The `[Trachytopes]` section in an MDU file. @@ -999,8 +979,6 @@ class Comments(INIBasedModel.Comments): trtmxr: Optional[int] = Field(None, alias="trtMxR") - - ObsFile = Union[XYNModel, ObservationPointModel] ObsCrsFile = Union[PolyFile, ObservationCrossSectionModel] @@ -1703,8 +1681,6 @@ def is_intermediate_link(self) -> bool: return True - - class Geometry(INIBasedModel): """ The `[Geometry]` section in an MDU file. @@ -2076,8 +2052,6 @@ def is_intermediate_link(self) -> bool: return True - - class Calibration(INIBasedModel): """ The `[Calibration]` section in an MDU file. @@ -2118,8 +2092,6 @@ class Comments(INIBasedModel.Comments): ) - - class InfiltrationMethod(IntEnum): """ Enum class containing the valid values for the Infiltrationmodel @@ -2192,8 +2164,6 @@ class Comments(INIBasedModel.Comments): sgrwini: Optional[float] = Field(None, alias="sgrwini") - - class ProcessFluxIntegration(IntEnum): """ Enum class containing the valid values for the ProcessFluxIntegration @@ -2281,8 +2251,6 @@ class Comments(INIBasedModel.Comments): depthdrythreshold: Optional[float] = Field(1e-3, alias="DepthDryThreshold") - - class ParticlesThreeDType(IntEnum): """ Enum class containing the valid values for the 3Dtype @@ -2341,8 +2309,6 @@ class Comments(INIBasedModel.Comments): ) - - class VegetationModelNr(IntEnum): """ Enum class containing the valid values for the VegetationModelNr @@ -2396,7 +2362,6 @@ class Comments(INIBasedModel.Comments): densvegminbap: Optional[float] = Field(0.0, alias="Densvegminbap") - class FMModel(INIModel): """ The overall FM model that contains the contents of the toplevel MDU file. diff --git a/hydrolib/core/dflowfm/storagenode/models.py b/hydrolib/core/dflowfm/storagenode/models.py index 842fcd260..39131b1b2 100644 --- a/hydrolib/core/dflowfm/storagenode/models.py +++ b/hydrolib/core/dflowfm/storagenode/models.py @@ -162,7 +162,7 @@ class Comments(INIBasedModel.Comments): interpolate: Optional[Interpolation] = Field( Interpolation.linear.value, alias="interpolate" ) - + @classmethod def _get_unknown_keyword_error_manager(cls) -> Optional[UnknownKeywordErrorManager]: """ diff --git a/hydrolib/core/dflowfm/structure/models.py b/hydrolib/core/dflowfm/structure/models.py index 8290eee35..ed51ef7f8 100644 --- a/hydrolib/core/dflowfm/structure/models.py +++ b/hydrolib/core/dflowfm/structure/models.py @@ -83,7 +83,7 @@ class Comments(INIBasedModel.Comments): _loc_coord_fields = {"numcoordinates", "xcoordinates", "ycoordinates"} _loc_branch_fields = {"branchid", "chainage"} _loc_all_fields = _loc_coord_fields | _loc_branch_fields - + @classmethod def _get_unknown_keyword_error_manager(cls) -> Optional[UnknownKeywordErrorManager]: """ From f007066466196d385ae0f3a969e8aa0cf86b819e Mon Sep 17 00:00:00 2001 From: Marlon Vermeulen Date: Thu, 30 May 2024 15:35:55 +0200 Subject: [PATCH 31/31] feat: (622) add review suggestions typehinting and documentation. --- hydrolib/core/dflowfm/ini/util.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/hydrolib/core/dflowfm/ini/util.py b/hydrolib/core/dflowfm/ini/util.py index 9fcfca63d..a3049ecee 100644 --- a/hydrolib/core/dflowfm/ini/util.py +++ b/hydrolib/core/dflowfm/ini/util.py @@ -635,8 +635,17 @@ def raise_error_for_unknown_keywords( data: Dict[str, Any], section_header: str, fields: Dict[str, ModelField], - excluded_fields: Set, - ): + excluded_fields: Set[str], + ) -> None: + """ + Notify the user of unknown keywords. + + Args: + data (Dict[str, Any]) : Input data containing all properties which are checked on unknown keywords. + section_header (str) : Header of the section in which unknown keys might be detected. + fields (Dict[str, ModelField]) : Known fields of the section. + excluded_fields (Set[str]) : Fields which should be excluded from the check for unknown keywords. + """ unknown_keywords = self._get_all_unknown_keywords(data, fields, excluded_fields) if len(unknown_keywords) == 0: