From 21d8b47aa5c6cbe81b3d93d80aa176cb4e212e52 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 23 Sep 2024 15:51:50 +0100 Subject: [PATCH 1/8] plug in peasant solution --- esmvalcore/cmor/_fixes/icon/icon.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/esmvalcore/cmor/_fixes/icon/icon.py b/esmvalcore/cmor/_fixes/icon/icon.py index 707a47f20c..70b45f5d08 100644 --- a/esmvalcore/cmor/_fixes/icon/icon.py +++ b/esmvalcore/cmor/_fixes/icon/icon.py @@ -503,6 +503,15 @@ def _fix_invalid_time_units(time_coord): category=FutureWarning, ) new_datetimes = np.array(rounded_datetimes.dt.to_pydatetime()) + + # Pandas round doesn't work (at all!) anymore + # so we have been pinning pandas + # see eg https://github.com/ESMValGroup/ESMValCore/pull/2394 + # until they fix https://github.com/pandas-dev/pandas/issues/57002 + # we plug in a peasant's workaround + def round_mins(dp): + return (dp.replace(second=0) + timedelta(minutes=dp.second // 30)) + new_datetimes = [round_mins(dp) for dp in new_datetimes] new_dt_points = date2num(np.array(new_datetimes), new_t_units) # Modify time coordinate in place From 1cbef92766d2c91e67a12b47771a5ea1b49fb284 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 23 Sep 2024 15:53:11 +0100 Subject: [PATCH 2/8] unpin pandas --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 00696ef452..9cfe3bfe56 100644 --- a/environment.yml +++ b/environment.yml @@ -29,7 +29,7 @@ dependencies: - netcdf4 - numpy !=1.24.3,<2.0.0 # avoid pulling 2.0.0rcX - packaging - - pandas !=2.2.0,!=2.2.1,!=2.2.2 # github.com/ESMValGroup/ESMValCore/pull/2305 and #2349 + - pandas - pillow - pip !=21.3 - prov From 6b26f5305d36a48b90a23e1cdebd47d41f0f3066 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 23 Sep 2024 15:53:16 +0100 Subject: [PATCH 3/8] unpin pandas --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e1645d3085..4f8081b382 100755 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ 'netCDF4', 'numpy!=1.24.3,<2.0.0', # avoid pulling 2.0.0rc1 'packaging', - 'pandas!=2.2.0,!=2.2.1,!=2.2.2', # GH #2305 #2349 etc + 'pandas', 'pillow', 'prov', 'psutil', From bb79e8ae385b56b9934652a7adc53689c3165e77 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 23 Sep 2024 15:53:26 +0100 Subject: [PATCH 4/8] run GA --- .github/workflows/run-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index ac294e9f44..8fd85ed7ed 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -21,6 +21,7 @@ on: push: branches: - main + - round_pandas # run the test only if the PR is to main # turn it on if required #pull_request: From 909963c0be6edd297f210a7782ef28342bc6c883 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 23 Sep 2024 16:30:07 +0100 Subject: [PATCH 5/8] make sure microsecond is also zero --- esmvalcore/cmor/_fixes/icon/icon.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/esmvalcore/cmor/_fixes/icon/icon.py b/esmvalcore/cmor/_fixes/icon/icon.py index 70b45f5d08..cc6bb7cfd9 100644 --- a/esmvalcore/cmor/_fixes/icon/icon.py +++ b/esmvalcore/cmor/_fixes/icon/icon.py @@ -510,7 +510,9 @@ def _fix_invalid_time_units(time_coord): # until they fix https://github.com/pandas-dev/pandas/issues/57002 # we plug in a peasant's workaround def round_mins(dp): - return (dp.replace(second=0) + timedelta(minutes=dp.second // 30)) + dp = dp.replace(second=0) + timedelta(minutes=dp.second // 30) + dp = dp.replace(microsecond=0) + return dp new_datetimes = [round_mins(dp) for dp in new_datetimes] new_dt_points = date2num(np.array(new_datetimes), new_t_units) From 8007e97912df00ea4c0a8afba4636f6b7e9e455d Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 23 Sep 2024 16:42:03 +0100 Subject: [PATCH 6/8] unrun GA --- .github/workflows/run-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 8fd85ed7ed..ac294e9f44 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -21,7 +21,6 @@ on: push: branches: - main - - round_pandas # run the test only if the PR is to main # turn it on if required #pull_request: From 841ca4d5bb18d7d01c6ea6a1bdad53ae1cf8da5e Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Thu, 26 Sep 2024 10:25:04 +0200 Subject: [PATCH 7/8] Restored original icon.py --- esmvalcore/cmor/_fixes/icon/icon.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/esmvalcore/cmor/_fixes/icon/icon.py b/esmvalcore/cmor/_fixes/icon/icon.py index cc6bb7cfd9..707a47f20c 100644 --- a/esmvalcore/cmor/_fixes/icon/icon.py +++ b/esmvalcore/cmor/_fixes/icon/icon.py @@ -503,17 +503,6 @@ def _fix_invalid_time_units(time_coord): category=FutureWarning, ) new_datetimes = np.array(rounded_datetimes.dt.to_pydatetime()) - - # Pandas round doesn't work (at all!) anymore - # so we have been pinning pandas - # see eg https://github.com/ESMValGroup/ESMValCore/pull/2394 - # until they fix https://github.com/pandas-dev/pandas/issues/57002 - # we plug in a peasant's workaround - def round_mins(dp): - dp = dp.replace(second=0) + timedelta(minutes=dp.second // 30) - dp = dp.replace(microsecond=0) - return dp - new_datetimes = [round_mins(dp) for dp in new_datetimes] new_dt_points = date2num(np.array(new_datetimes), new_t_units) # Modify time coordinate in place From 157b861e320bbc04eec3535d59a74922da224fc4 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Thu, 26 Sep 2024 10:28:34 +0200 Subject: [PATCH 8/8] Fix pandas rounding bug --- esmvalcore/cmor/_fixes/icon/icon.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/esmvalcore/cmor/_fixes/icon/icon.py b/esmvalcore/cmor/_fixes/icon/icon.py index 707a47f20c..51cb5a0274 100644 --- a/esmvalcore/cmor/_fixes/icon/icon.py +++ b/esmvalcore/cmor/_fixes/icon/icon.py @@ -492,8 +492,11 @@ def _fix_invalid_time_units(time_coord): # Finally, add date and day fraction to get final datetime and convert # it to correct units. Note: we also round to next second, otherwise # this results in times that are off by 1s (e.g., 13:59:59 instead of - # 14:00:00). - rounded_datetimes = (year_month_day + day_float).round('s') + # 14:00:00). We round elements individually since rounding the + # pd.Series object directly is broken + # (https://github.com/pandas-dev/pandas/issues/57002). + datetimes = year_month_day + day_float + rounded_datetimes = pd.Series(dt.round('s') for dt in datetimes) with warnings.catch_warnings(): # We already fixed the deprecated code as recommended in the # warning, but it still shows up -> ignore it