From cd46572034018e9fc786fdae4d4d1d590e171306 Mon Sep 17 00:00:00 2001 From: Chong Gao Date: Tue, 8 Oct 2024 21:36:35 +0800 Subject: [PATCH] Update test case related to LEACY datetime format to unblock nightly CI (#11544) Signed-off-by: Chong Gao Co-authored-by: Chong Gao --- docs/compatibility.md | 5 +++-- integration_tests/src/main/python/date_time_test.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/compatibility.md b/docs/compatibility.md index e8812eb7d87..1cc0e80902a 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -662,9 +662,10 @@ LEGACY timeParserPolicy support has the following limitations when running on th - The proleptic Gregorian calendar is used instead of the hybrid Julian+Gregorian calendar that Spark uses in legacy mode - When format is `yyyyMMdd`, GPU only supports 8 digit strings. Spark supports like 7 digit - `2024101` string while GPU does not support. + `2024101` string while GPU does not support. Only tested `UTC` and `Asia/Shanghai` timezones. - When format is `yyyymmdd`, GPU only supports 8 digit strings. Spark supports like 7 digit - `2024101` string while GPU does not support. + `2024101` string while GPU does not support. Only tested `UTC` and `Asia/Shanghai` timezones. + ## Formatting dates and timestamps as strings diff --git a/integration_tests/src/main/python/date_time_test.py b/integration_tests/src/main/python/date_time_test.py index cd0745cd07c..a38cac3c0a7 100644 --- a/integration_tests/src/main/python/date_time_test.py +++ b/integration_tests/src/main/python/date_time_test.py @@ -463,9 +463,9 @@ def test_to_timestamp(parser_policy): @pytest.mark.skipif(not is_supported_time_zone(), reason="not all time zones are supported now, refer to https://github.com/NVIDIA/spark-rapids/issues/6839, please update after all time zones are supported") @pytest.mark.parametrize("format", ['yyyyMMdd', 'yyyymmdd'], ids=idfn) # Test years after 1900, refer to issues: https://github.com/NVIDIA/spark-rapids/issues/11543, https://github.com/NVIDIA/spark-rapids/issues/11539 -@pytest.mark.parametrize("data_gen_regexp", ['(19[0-9]{2})([0-5][0-9])([0-3][0-9])', '(19[0-9]{2})([0-9]{4})'], ids=idfn) -def test_formats_for_legacy_mode(format, data_gen_regexp): - gen = StringGen(data_gen_regexp) +@pytest.mark.skipif(get_test_tz() != "Asia/Shanghai" and get_test_tz() != "UTC", reason="https://github.com/NVIDIA/spark-rapids/issues/11562") +def test_formats_for_legacy_mode(format): + gen = StringGen('(19[0-9]{2}|[2-9][0-9]{3})([0-9]{4})') assert_gpu_and_cpu_are_equal_sql( lambda spark : unary_op_df(spark, gen), "tab",