From bf75f024e48b515d4cd7c93cd3f997de4fc5b1ef Mon Sep 17 00:00:00 2001 From: Arthur van Dam Date: Fri, 2 Jun 2023 07:52:37 +0200 Subject: [PATCH] fix: Rainfall Runoff .bui file with multiple stations gives parse error Refs: #548 --- hydrolib/core/rr/meteo/parser.py | 12 +- hydrolib/core/rr/meteo/serializer.py | 2 +- .../input/rr_individual_files/DEFAULT_rr.BUI | 188 ++++++++++++++++++ .../input/rr_sample_trimmed/rr/default.bui | 26 ++- tests/rr/meteo/test_bui.py | 45 ++++- 5 files changed, 247 insertions(+), 26 deletions(-) create mode 100644 tests/data/input/rr_individual_files/DEFAULT_rr.BUI diff --git a/hydrolib/core/rr/meteo/parser.py b/hydrolib/core/rr/meteo/parser.py index 681b00d95..9badfe475 100644 --- a/hydrolib/core/rr/meteo/parser.py +++ b/hydrolib/core/rr/meteo/parser.py @@ -159,8 +159,8 @@ def parse(filepath: Path) -> Dict: Dict: Parsed values. """ - def get_station_ids(line: str) -> List[str]: - return [s_id for s_id in line.split(",")] + def get_station_ids(lines: List[str]) -> List[str]: + return [s_id.strip("'\"") for s_id in lines] def parse_events_and_timestep(line: str) -> Tuple[int, int]: n_events_timestep = line.split() @@ -171,16 +171,18 @@ def parse_events_and_timestep(line: str) -> Tuple[int, int]: for line in filepath.read_text(encoding="utf8").splitlines() if not line.startswith("*") ] + number_of_stations = int(bui_lines[1]) + last_station_line = 1 + number_of_stations - n_events, timestep = parse_events_and_timestep(bui_lines[3]) + n_events, timestep = parse_events_and_timestep(bui_lines[last_station_line + 1]) return dict( default_dataset=bui_lines[0], number_of_stations=bui_lines[1], - name_of_stations=get_station_ids(bui_lines[2]), + name_of_stations=get_station_ids(bui_lines[2 : last_station_line + 1]), number_of_events=n_events, seconds_per_timestep=timestep, precipitation_events=BuiEventListParser.parse( - "\n".join(bui_lines[4:]), n_events, timestep + "\n".join(bui_lines[last_station_line + 2 :]), n_events, timestep ), ) diff --git a/hydrolib/core/rr/meteo/serializer.py b/hydrolib/core/rr/meteo/serializer.py index 5044a033b..c9ed416fd 100644 --- a/hydrolib/core/rr/meteo/serializer.py +++ b/hydrolib/core/rr/meteo/serializer.py @@ -216,7 +216,7 @@ def serialize_stations_ids(data_to_serialize: List[str]) -> str: Returns: str: Serialized string. """ - return str.join(" ", data_to_serialize) + return "\n".join(f"'{station_id}'" for station_id in data_to_serialize) def write_bui_file( diff --git a/tests/data/input/rr_individual_files/DEFAULT_rr.BUI b/tests/data/input/rr_individual_files/DEFAULT_rr.BUI new file mode 100644 index 000000000..4e35c1ea8 --- /dev/null +++ b/tests/data/input/rr_individual_files/DEFAULT_rr.BUI @@ -0,0 +1,188 @@ +*Name of this file: c:\ResultE8\DEFAULT.BUI +*Date and time of construction: 00/00/2000 00:00:00. +1 +*Aantal stations +7 +*Namen van stations +'De Bilt' +'Cabauw' +'Den Helder' +'Texelhors' +'Berkhout' +'IJmuiden' +'Wijk aan Zee' +*Aantal gebeurtenissen (omdat het 1 bui betreft is dit altijd 1) +*en het aantal seconden per waarnemingstijdstap +1 3600 +*Elke commentaarregel wordt begonnen met een * (asterisk). +*Eerste record bevat startdatum en -tijd, lengte van de gebeurtenis in dd hh mm ss +*Het format is: yyyymmdd:hhmmss:ddhhmmss*Daarna voor elk station de neerslag in mm per tijdstap. +2021 4 20 7 0 0 7 0 0 0 +0.000 0.020 0.011 0.000 0.100 0.000 0.007 +0.000 0.020 0.011 0.000 0.100 0.000 0.007 +0.000 0.022 0.012 0.000 0.108 0.000 0.008 +0.000 0.025 0.013 0.000 0.123 0.000 0.009 +0.000 0.028 0.015 0.000 0.138 0.000 0.010 +0.000 0.031 0.017 0.000 0.153 0.000 0.011 +0.000 0.037 0.020 0.000 0.185 0.000 0.013 +0.000 0.047 0.026 0.000 0.235 0.000 0.016 +0.000 0.064 0.035 0.000 0.318 0.000 0.022 +0.000 0.087 0.048 0.000 0.433 0.000 0.030 +0.000 0.122 0.067 0.000 0.610 0.000 0.043 +0.000 0.500 0.275 0.000 2.500 0.000 0.175 +0.000 0.500 0.275 0.000 2.500 0.000 0.175 +0.000 0.122 0.067 0.000 0.610 0.000 0.043 +0.000 0.086 0.048 0.000 0.432 0.000 0.030 +0.000 0.064 0.035 0.000 0.318 0.000 0.022 +0.000 0.047 0.026 0.000 0.235 0.000 0.016 +0.000 0.037 0.020 0.000 0.185 0.000 0.013 +0.000 0.031 0.017 0.000 0.153 0.000 0.011 +0.000 0.027 0.015 0.000 0.137 0.000 0.010 +0.000 0.025 0.014 0.000 0.124 0.000 0.009 +0.000 0.023 0.012 0.000 0.113 0.000 0.008 +0.000 0.020 0.011 0.000 0.102 0.000 0.007 +0.000 0.018 0.010 0.000 0.091 0.000 0.006 +0.236 0.284 0.207 0.000 0.245 0.205 0.311 +0.236 0.284 0.207 0.000 0.245 0.205 0.311 +0.254 0.305 0.223 0.000 0.263 0.220 0.334 +0.289 0.348 0.254 0.000 0.300 0.251 0.381 +0.325 0.391 0.285 0.000 0.337 0.282 0.428 +0.360 0.433 0.316 0.000 0.374 0.313 0.474 +0.437 0.525 0.383 0.000 0.453 0.379 0.575 +0.555 0.667 0.486 0.000 0.576 0.482 0.731 +0.749 0.902 0.657 0.000 0.778 0.651 0.987 +1.021 1.228 0.895 0.000 1.060 0.887 1.345 +1.440 1.732 1.263 0.000 1.495 1.251 1.897 +5.900 7.100 5.175 0.000 6.125 5.125 7.775 +5.900 7.100 5.175 0.000 6.125 5.125 7.775 +1.440 1.732 1.263 0.000 1.495 1.251 1.897 +1.021 1.228 0.895 0.000 1.060 0.887 1.345 +0.749 0.902 0.657 0.000 0.778 0.651 0.987 +0.555 0.667 0.486 0.000 0.576 0.482 0.731 +0.437 0.525 0.383 0.000 0.453 0.379 0.575 +0.360 0.433 0.316 0.000 0.374 0.313 0.474 +0.324 0.390 0.285 0.000 0.337 0.282 0.428 +0.294 0.353 0.258 0.000 0.305 0.255 0.387 +0.267 0.321 0.234 0.000 0.277 0.232 0.352 +0.240 0.289 0.211 0.000 0.250 0.209 0.317 +0.214 0.257 0.188 0.000 0.222 0.186 0.282 +0.038 0.054 0.037 0.000 0.108 0.051 0.046 +0.038 0.054 0.037 0.000 0.108 0.051 0.046 +0.041 0.058 0.040 0.000 0.116 0.055 0.049 +0.047 0.066 0.045 0.000 0.132 0.062 0.056 +0.052 0.074 0.051 0.000 0.149 0.070 0.063 +0.058 0.082 0.056 0.000 0.165 0.078 0.070 +0.070 0.100 0.068 0.000 0.200 0.094 0.085 +0.089 0.127 0.087 0.000 0.254 0.120 0.108 +0.121 0.171 0.117 0.000 0.343 0.162 0.146 +0.164 0.234 0.160 0.000 0.467 0.221 0.199 +0.232 0.329 0.226 0.000 0.659 0.311 0.281 +0.950 1.350 0.925 0.000 2.700 1.275 1.150 +0.950 1.350 0.925 0.000 2.700 1.275 1.150 +0.232 0.329 0.226 0.000 0.659 0.311 0.281 +0.164 0.234 0.160 0.000 0.467 0.221 0.199 +0.121 0.171 0.117 0.000 0.343 0.162 0.146 +0.089 0.127 0.087 0.000 0.254 0.120 0.108 +0.070 0.100 0.068 0.000 0.200 0.094 0.085 +0.058 0.082 0.056 0.000 0.165 0.078 0.070 +0.052 0.074 0.051 0.000 0.148 0.070 0.063 +0.047 0.067 0.046 0.000 0.134 0.063 0.057 +0.043 0.061 0.042 0.000 0.122 0.058 0.052 +0.039 0.055 0.038 0.000 0.110 0.052 0.047 +0.034 0.049 0.034 0.000 0.098 0.046 0.042 +0.426 0.500 0.488 0.400 0.224 0.182 0.383 +0.426 0.500 0.488 0.400 0.224 0.182 0.383 +0.458 0.538 0.525 0.430 0.241 0.196 0.412 +0.522 0.613 0.598 0.490 0.274 0.223 0.469 +0.586 0.688 0.671 0.550 0.308 0.250 0.527 +0.650 0.763 0.744 0.610 0.342 0.278 0.584 +0.788 0.925 0.903 0.740 0.414 0.337 0.709 +1.001 1.175 1.147 0.940 0.526 0.428 0.900 +1.353 1.588 1.549 1.270 0.711 0.578 1.216 +1.842 2.163 2.111 1.730 0.969 0.787 1.656 +2.599 3.050 2.977 2.440 1.366 1.110 2.336 +10.650 12.500 12.200 10.000 5.600 4.550 9.575 +10.650 12.500 12.200 10.000 5.600 4.550 9.575 +2.599 3.050 2.977 2.440 1.366 1.110 2.336 +1.842 2.163 2.111 1.730 0.969 0.787 1.656 +1.353 1.588 1.549 1.270 0.711 0.578 1.216 +1.001 1.175 1.147 0.940 0.526 0.428 0.900 +0.788 0.925 0.903 0.740 0.414 0.337 0.709 +0.650 0.762 0.744 0.610 0.342 0.278 0.584 +0.586 0.687 0.671 0.550 0.308 0.250 0.527 +0.530 0.622 0.607 0.498 0.279 0.226 0.476 +0.482 0.566 0.552 0.452 0.253 0.206 0.433 +0.434 0.510 0.497 0.408 0.228 0.185 0.390 +0.386 0.453 0.442 0.362 0.203 0.165 0.347 +0.322 0.166 0.152 0.000 0.367 0.203 0.061 +0.322 0.166 0.152 0.000 0.367 0.203 0.061 +0.346 0.178 0.163 0.000 0.395 0.218 0.066 +0.394 0.203 0.186 0.000 0.450 0.249 0.075 +0.443 0.228 0.209 0.000 0.505 0.279 0.084 +0.491 0.253 0.232 0.000 0.560 0.310 0.093 +0.596 0.307 0.281 0.000 0.679 0.376 0.113 +0.757 0.390 0.357 0.000 0.862 0.477 0.143 +1.022 0.527 0.483 0.000 1.165 0.645 0.194 +1.393 0.718 0.657 0.000 1.587 0.878 0.264 +1.964 1.013 0.927 0.000 2.239 1.238 0.372 +8.050 4.150 3.800 0.000 9.175 5.075 1.525 +8.050 4.150 3.800 0.000 9.175 5.075 1.525 +1.964 1.013 0.927 0.000 2.239 1.238 0.372 +1.393 0.718 0.657 0.000 1.587 0.878 0.264 +1.022 0.527 0.483 0.000 1.165 0.645 0.194 +0.757 0.390 0.357 0.000 0.862 0.477 0.143 +0.596 0.307 0.281 0.000 0.679 0.376 0.113 +0.491 0.253 0.232 0.000 0.560 0.310 0.093 +0.443 0.228 0.209 0.000 0.505 0.279 0.084 +0.401 0.207 0.189 0.000 0.457 0.253 0.076 +0.364 0.188 0.172 0.000 0.415 0.230 0.069 +0.328 0.169 0.155 0.000 0.374 0.207 0.062 +0.292 0.150 0.138 0.000 0.333 0.184 0.055 +0.040 0.076 0.040 0.304 0.195 0.109 0.930 +0.040 0.076 0.040 0.304 0.195 0.109 0.930 +0.043 0.082 0.043 0.327 0.210 0.117 1.000 +0.049 0.093 0.049 0.372 0.239 0.134 1.139 +0.055 0.105 0.055 0.418 0.268 0.150 1.279 +0.061 0.116 0.061 0.464 0.297 0.166 1.418 +0.074 0.141 0.074 0.562 0.361 0.202 1.721 +0.094 0.179 0.094 0.714 0.458 0.256 2.186 +0.127 0.241 0.127 0.965 0.619 0.346 2.953 +0.173 0.329 0.173 1.315 0.843 0.471 4.022 +0.244 0.464 0.244 1.854 1.190 0.665 5.673 +1.000 1.900 1.000 7.600 4.875 2.725 23.250 +1.000 1.900 1.000 7.600 4.875 2.725 23.250 +0.244 0.464 0.244 1.854 1.190 0.665 5.673 +0.173 0.329 0.173 1.315 0.843 0.471 4.022 +0.127 0.241 0.127 0.965 0.619 0.346 2.953 +0.094 0.179 0.094 0.714 0.458 0.256 2.186 +0.074 0.141 0.074 0.562 0.361 0.202 1.721 +0.061 0.116 0.061 0.464 0.297 0.166 1.418 +0.055 0.104 0.055 0.418 0.268 0.150 1.279 +0.050 0.095 0.050 0.378 0.243 0.136 1.157 +0.045 0.086 0.045 0.344 0.221 0.123 1.052 +0.041 0.077 0.041 0.310 0.199 0.111 0.948 +0.036 0.069 0.036 0.275 0.177 0.099 0.843 +0.604 0.432 0.410 0.229 0.718 0.551 1.132 +0.604 0.432 0.410 0.229 0.718 0.551 1.132 +0.649 0.464 0.441 0.246 0.772 0.592 1.217 +0.740 0.529 0.502 0.281 0.880 0.675 1.387 +0.831 0.594 0.564 0.315 0.987 0.758 1.557 +0.921 0.659 0.625 0.349 1.095 0.840 1.726 +1.117 0.799 0.759 0.424 1.328 1.019 2.094 +1.419 1.015 0.964 0.538 1.687 1.295 2.660 +1.918 1.372 1.302 0.727 2.280 1.749 3.594 +2.612 1.868 1.773 0.990 3.105 2.383 4.896 +3.684 2.635 2.501 1.397 4.380 3.361 6.905 +15.100 10.800 10.250 5.725 17.950 13.775 28.300 +15.100 10.800 10.250 5.725 17.950 13.775 28.300 +3.684 2.635 2.501 1.397 4.380 3.361 6.905 +2.612 1.868 1.773 0.990 3.105 2.383 4.896 +1.918 1.372 1.302 0.727 2.280 1.749 3.594 +1.419 1.015 0.963 0.538 1.687 1.295 2.660 +1.117 0.799 0.759 0.424 1.328 1.019 2.094 +0.921 0.659 0.625 0.349 1.095 0.840 1.726 +0.830 0.594 0.564 0.315 0.987 0.758 1.556 +0.751 0.537 0.510 0.285 0.893 0.685 1.408 +0.683 0.489 0.464 0.259 0.812 0.623 1.280 +0.615 0.440 0.418 0.233 0.732 0.561 1.154 +0.547 0.391 0.371 0.207 0.651 0.499 1.026 diff --git a/tests/data/input/rr_sample_trimmed/rr/default.bui b/tests/data/input/rr_sample_trimmed/rr/default.bui index e0ff7b80c..1e3d08bec 100644 --- a/tests/data/input/rr_sample_trimmed/rr/default.bui +++ b/tests/data/input/rr_sample_trimmed/rr/default.bui @@ -1,19 +1,17 @@ -*Name of this file: \SOB_LITE\FIXED\DEFAULT.BUI -*Date and time of construction: 17-04-1997 17:13:50 -*Enige algemene wenken: -*Gebruik de default dataset voor overige invoer (altijd 1) +*Name of this file: C:\checkouts\HYDROLIB-core_git\tests\data\input\rr_sample_trimmed\rr\default.bui +*Date and time of construction: 31-05-23 14:33:12 +*Comments are following an * (asterisk) and written above variables 1 -*Aantal stations +*Number of stations 1 -*Namen van stations -’Station1’ -*Aantal gebeurtenissen (omdat het 1 bui betreft is dit altijd 1) -*en het aantal seconden per waarnemingstijdstap (10800 = 3x3600) +*Station Name +'Station1' +*Number_of_events seconds_per_timestamp 1 10800 -*Elke commentaarregel wordt begonnen met een * (asteriks). -*Eerste record bevat startdatum en -tijd, lengte van de gebeurtenis in dd hh mm ss -*Het format is: yyyymmdd:hhmmss:ddhhmmss -*Daarna voor elk station de neerslag in mm per tijdstap. +* Event 1 duration days:1 hours:3 minutes:0 seconds:0 +* Start date and time of the event: yyyy mm dd hh mm ss +* Duration of the event : dd hh mm ss +* Rainfall value per time step [mm/time step] 1996 1 1 0 0 0 1 3 0 0 0.2 0.2 @@ -23,4 +21,4 @@ 0.2 0.2 0.2 -0.2 +0.2 \ No newline at end of file diff --git a/tests/rr/meteo/test_bui.py b/tests/rr/meteo/test_bui.py index 812e031af..b1d1b91a4 100644 --- a/tests/rr/meteo/test_bui.py +++ b/tests/rr/meteo/test_bui.py @@ -40,7 +40,7 @@ def default_bui_file() -> Path: @staticmethod def default_bui_station() -> str: """Just a wrapper to return the default value for the default station name""" - return "’Station1’" + return "Station1" @staticmethod def bui_model() -> BuiModel: @@ -89,7 +89,7 @@ def test_given_nwrw_file_loads_model(self): model = BuiModel(filepath=test_file) event_list = model.precipitation_events assert len(event_list) == 404 - station_events = model.get_station_events("'De Bilt'") + station_events = model.get_station_events("De Bilt") first_event = station_events[datetime(1955, 1, 15, 16, 45)] assert first_event[0] == 0.30 assert first_event[-1] == 0 @@ -103,7 +103,7 @@ def test_given_t_sewer_file_loads_model(self): model = BuiModel(filepath=test_file) event_list = model.precipitation_events assert len(event_list) == 10 - station_events = model.get_station_events("'De Bilt'") + station_events = model.get_station_events("De Bilt") first_event = station_events[datetime(2000, 1, 10)] assert first_event[0] == 0.30 assert first_event[-1] == 0 @@ -117,6 +117,39 @@ class TestBuiModel: all the methods in the BuiModel class. """ + def test_given_bui_file_loads_model(self): + test_file = test_input_dir / "rr_individual_files" / "DEFAULT_rr.BUI" + expected_number_of_events = 1 + expected_number_of_stations = 7 + expected_timeseries_length = 168 + + assert test_file.is_file() + model = BuiModel(filepath=test_file) + event_list = model.precipitation_events + + assert len(event_list) == expected_number_of_events + assert model.number_of_events == expected_number_of_events + + assert len(model.name_of_stations) == expected_number_of_stations + assert model.number_of_stations == expected_number_of_stations + + assert event_list[0].start_time == datetime(2021, 4, 20, 7, 0) + assert model.precipitation_events[0].timeseries_length == timedelta(days=7) + assert ( + len(event_list[0].precipitation_per_timestep) + == expected_timeseries_length + ) + + assert model.name_of_stations == [ + "De Bilt", + "Cabauw", + "Den Helder", + "Texelhors", + "Berkhout", + "IJmuiden", + "Wijk aan Zee", + ] + def test_given_filepath_all_properties_loaded(self): test_file = BuiTestData.default_bui_file() model = BuiModel(filepath=test_file) @@ -135,7 +168,7 @@ def test_save_default_verify_expected_text(self): *Number of stations 1 *Station Name - ’Station1’ + 'Station1' *Number_of_events seconds_per_timestamp 1 10800 * Event 1 duration days:1 hours:3 minutes:0 seconds:0 @@ -445,7 +478,7 @@ def test_given_dict_serialize_into_text(self): *Number of stations 1 *Station Name - ’Station1’ + 'Station1' *Number_of_events seconds_per_timestamp 1 10800 * Event 1 duration days:1 hours:3 minutes:0 seconds:0 @@ -466,7 +499,7 @@ def test_given_dict_serialize_into_text(self): def test_given_station_ids_serialize_into_text(self): stations_ids = ["Hello", "World", BuiTestData.default_bui_station()] serialized_text = BuiSerializer.serialize_stations_ids(stations_ids) - assert serialized_text == "Hello World ’Station1’" + assert serialized_text == "'Hello'\n'World'\n'Station1'" def test_given_precipitation_serialize_into_text(self): precipitation_per_timestep = [[1.23], [2.34], [3.45], [4.56]]