sync signature

apache · Nov 20, 2023 · 8085383 · 8085383
1 parent b1cfc38
commit 8085383
Show file tree

Hide file tree

Showing 7 changed files with 64 additions and 34 deletions.
diff --git a/datafusion/common/src/scalar.rs b/datafusion/common/src/scalar.rs
@@ -950,6 +950,15 @@ impl ScalarValue {
             ScalarValue::TimestampSecond(Some(v), tz) => {
                 Ok(ScalarValue::TimestampSecond(Some(-v), tz.clone()))
             }
+            ScalarValue::TimestampNanosecond(Some(v), tz) => {
+                Ok(ScalarValue::TimestampNanosecond(Some(-v), tz.clone()))
+            }
+            ScalarValue::TimestampMicrosecond(Some(v), tz) => {
+                Ok(ScalarValue::TimestampMicrosecond(Some(-v), tz.clone()))
+            }
+            ScalarValue::TimestampMillisecond(Some(v), tz) => {
+                Ok(ScalarValue::TimestampMillisecond(Some(-v), tz.clone()))
+            }
             value => _internal_err!(
                 "Can not run arithmetic negative on scalar value {value:?}"
             ),

diff --git a/datafusion/core/tests/sql/timestamp.rs b/datafusion/core/tests/sql/timestamp.rs
@@ -742,7 +742,7 @@ async fn test_arrow_typeof() -> Result<()> {
         "+-----------------------------------------------------------------------+",
         "| arrow_typeof(date_trunc(Utf8(\"microsecond\"),to_timestamp(Int64(61)))) |",
         "+-----------------------------------------------------------------------+",
-        "| Timestamp(Second, None)                                               |",
+        "| Timestamp(Nanosecond, None)                                           |",
         "+-----------------------------------------------------------------------+",
     ];
     assert_batches_eq!(expected, &actual);

diff --git a/datafusion/expr/src/built_in_function.rs b/datafusion/expr/src/built_in_function.rs
@@ -761,13 +761,10 @@ impl BuiltinScalarFunction {
                     return plan_err!("The to_hex function can only accept integers.");
                 }
             }),
-            BuiltinScalarFunction::ToTimestamp => Ok(match &input_expr_types[0] {
-                Int64 => Timestamp(Second, None),
-                _ => Timestamp(Nanosecond, None),
-            }),
+            BuiltinScalarFunction::ToTimestamp
+            | BuiltinScalarFunction::ToTimestampNanos => Ok(Timestamp(Nanosecond, None)),
             BuiltinScalarFunction::ToTimestampMillis => Ok(Timestamp(Millisecond, None)),
             BuiltinScalarFunction::ToTimestampMicros => Ok(Timestamp(Microsecond, None)),
-            BuiltinScalarFunction::ToTimestampNanos => Ok(Timestamp(Nanosecond, None)),
             BuiltinScalarFunction::ToTimestampSeconds => Ok(Timestamp(Second, None)),
             BuiltinScalarFunction::FromUnixtime => Ok(Timestamp(Second, None)),
             BuiltinScalarFunction::Now => {

diff --git a/datafusion/physical-expr/src/datetime_expressions.rs b/datafusion/physical-expr/src/datetime_expressions.rs
@@ -966,9 +966,11 @@ pub fn to_timestamp_invoke(args: &[ColumnarValue]) -> Result<ColumnarValue> {
     }
 
     match args[0].data_type() {
-        DataType::Int64 => {
-            cast_column(&args[0], &DataType::Timestamp(TimeUnit::Second, None), None)
-        }
+        DataType::Int64 => cast_column(
+            &cast_column(&args[0], &DataType::Timestamp(TimeUnit::Second, None), None)?,
+            &DataType::Timestamp(TimeUnit::Nanosecond, None),
+            None,
+        ),
         DataType::Timestamp(_, None) => cast_column(
             &args[0],
             &DataType::Timestamp(TimeUnit::Nanosecond, None),

diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs
@@ -226,18 +226,21 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
             SQLExpr::Cast {
                 expr, data_type, ..
             } => {
-                let mut dt = self.convert_data_type(&data_type)?;
+                let dt = self.convert_data_type(&data_type)?;
                 let expr =
                     self.sql_expr_to_logical_expr(*expr, schema, planner_context)?;
 
-                // int/floats should come as seconds rather as nanoseconds
-                dt = match &dt {
+                // int/floats input should be treated as seconds rather as nanoseconds
+                let expr = match &dt {
                     DataType::Timestamp(TimeUnit::Nanosecond, tz)
                         if expr.get_type(schema)? == DataType::Int64 =>
                     {
-                        DataType::Timestamp(TimeUnit::Second, tz.clone())
+                        Expr::Cast(Cast::new(
+                            Box::new(expr),
+                            DataType::Timestamp(TimeUnit::Second, tz.clone()),
+                        ))
                     }
-                    _ => dt,
+                    _ => expr,
                 };
 
                 Ok(Expr::Cast(Cast::new(Box::new(expr), dt)))

diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rs
@@ -597,20 +597,14 @@ fn select_neg_filter() {
 fn select_compound_filter() {
     let sql = "SELECT id, first_name, last_name \
                    FROM person WHERE state = 'CO' AND age >= 21 AND age <= 65";
-    let expected = "Projection: person.id, person.first_name, person.last_name\
-            \n  Filter: person.state = Utf8(\"CO\") AND person.age >= Int64(21) AND person.age <= Int64(65)\
-            \n    TableScan: person";
+    let expected = "Projection: person.id, person.first_name, person.last_name\n  Filter: person.state = Utf8(\"CO\") AND person.age >= Int64(21) AND person.age <= Int64(65)\n    TableScan: person";
     quick_test(sql, expected);
 }
 
 #[test]
 fn test_timestamp_filter() {
     let sql = "SELECT state FROM person WHERE birth_date < CAST (158412331400600000 as timestamp)";
-
-    let expected = "Projection: person.state\
-            \n  Filter: person.birth_date < CAST(Int64(158412331400600000) AS Timestamp(Second, None))\
-            \n    TableScan: person";
-
+    let expected = "Projection: person.state\n  Filter: person.birth_date < CAST(CAST(Int64(158412331400600000) AS Timestamp(Second, None)) AS Timestamp(Nanosecond, None))\n    TableScan: person";
     quick_test(sql, expected);
 }
 

diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.slt
@@ -1788,20 +1788,45 @@ SELECT TIMESTAMPTZ '2020-01-01 00:00:00Z' = TIMESTAMP '2020-01-01'
 ----
 true
 
-# verify to_timestamp edge cases to be in sync with postgresql
-query PPPPPPPP
-SELECT to_timestamp(null), to_timestamp(-62125747200), to_timestamp(0), to_timestamp(1926632005177), to_timestamp(1926632005), to_timestamp(1), to_timestamp(-1), to_timestamp(0-1)
+# verify timestamp cast with integer input
+query PPPPPP
+SELECT to_timestamp(null), to_timestamp(0), to_timestamp(1926632005), to_timestamp(1), to_timestamp(-1), to_timestamp(0-1)
 ----
-NULL 0001-04-25T00:00:00 1970-01-01T00:00:00 +63022-07-16T12:59:37 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59
+NULL 1970-01-01T00:00:00 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59
 
-# verify timestamp cast from i64 is in sync with to_timestamp(i64)
-query PPPPPPPP
-SELECT null::timestamp, -62125747200::timestamp, 0::timestamp, 1926632005177::timestamp, 1926632005::timestamp, 1::timestamp, -1::timestamp, (0-1)::timestamp
+# verify timestamp cast with integer input timestamp literal syntax
+query PPPPPP
+SELECT null::timestamp, 0::timestamp, 1926632005::timestamp, 1::timestamp, -1::timestamp, (0-1)::timestamp
 ----
-NULL 0001-04-25T00:00:00 1970-01-01T00:00:00 +63022-07-16T12:59:37 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59
+NULL 1970-01-01T00:00:00 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59
 
-# verify timestamp cast from i64 is in sync with to_timestamp(i64) using CAST syntax
-query PPPPPPPP
-SELECT cast(null as timestamp), cast(-62125747200 as timestamp), cast(0 as timestamp), cast(1926632005177 as timestamp), cast(1926632005 as timestamp), cast(1 as timestamp), cast(-1 as timestamp), cast(0-1 as timestamp)
+# verify timestamp cast with integer input timestamp literal syntax using CAST syntax
+query PPPPPP
+SELECT cast(null as timestamp), cast(0 as timestamp), cast(1926632005 as timestamp), cast(1 as timestamp), cast(-1 as timestamp), cast(0-1 as timestamp)
 ----
-NULL 0001-04-25T00:00:00 1970-01-01T00:00:00 +63022-07-16T12:59:37 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59
+NULL 1970-01-01T00:00:00 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:59:59 1969-12-31T23:59:59
+
+# verify timestamp output types 
+query TTT
+SELECT arrow_typeof(to_timestamp(1)), arrow_typeof(to_timestamp(null)), arrow_typeof(to_timestamp('2023-01-10 12:34:56.000'))
+----
+Timestamp(Nanosecond, None) Timestamp(Nanosecond, None) Timestamp(Nanosecond, None)
+
+# verify timestamp output types using timestamp literal syntax
+query TTT
+SELECT arrow_typeof(1::timestamp), arrow_typeof(null::timestamp), arrow_typeof('2023-01-10 12:34:56.000'::timestamp)
+----
+Timestamp(Nanosecond, None) Timestamp(Nanosecond, None) Timestamp(Nanosecond, None)
+
+# verify timestamp output types using CAST syntax
+query TTT
+SELECT arrow_typeof(cast(1 as timestamp)), arrow_typeof(cast(null as timestamp)), arrow_typeof(cast('2023-01-10 12:34:56.000' as timestamp))
+----
+Timestamp(Nanosecond, None) Timestamp(Nanosecond, None) Timestamp(Nanosecond, None)
+
+
+# verify extreme values (expects default precision to be microsecond instead of nanoseconds. Work pending)
+#query PPPPPPPP
+#SELECT to_timestamp(-62125747200), to_timestamp(1926632005177), -62125747200::timestamp, 1926632005177::timestamp, cast(-62125747200 as timestamp), cast(1926632005177 as timestamp)
+#----
+#0001-04-25T00:00:00 +63022-07-16T12:59:37 0001-04-25T00:00:00 +63022-07-16T12:59:37 0001-04-25T00:00:00 +63022-07-16T12:59:37