From 664c4b06766f81988c08259610ff8da829fd166e Mon Sep 17 00:00:00 2001 From: tangruilin Date: Wed, 31 Jan 2024 09:44:52 +0800 Subject: [PATCH] [task #5568] add_to_unixtime_function Signed-off-by: tangruilin --- datafusion/expr/src/built_in_function.rs | 8 +++++ .../physical-expr/src/datetime_expressions.rs | 31 +++++++++++++++++++ datafusion/physical-expr/src/functions.rs | 3 ++ datafusion/proto/proto/datafusion.proto | 2 ++ datafusion/proto/src/generated/pbjson.rs | 3 ++ datafusion/proto/src/generated/prost.rs | 3 ++ .../proto/src/logical_plan/from_proto.rs | 11 +++++++ datafusion/proto/src/logical_plan/to_proto.rs | 1 + .../sqllogictest/test_files/timestamps.slt | 19 ++++++++++++ 9 files changed, 81 insertions(+) diff --git a/datafusion/expr/src/built_in_function.rs b/datafusion/expr/src/built_in_function.rs index d10045ff9f60..2dc6b4a31330 100644 --- a/datafusion/expr/src/built_in_function.rs +++ b/datafusion/expr/src/built_in_function.rs @@ -277,6 +277,8 @@ pub enum BuiltinScalarFunction { FromUnixtime, /// to_date ToDate, + /// to_unixtime + ToUnixtime, ///now Now, ///current_date @@ -475,6 +477,7 @@ impl BuiltinScalarFunction { BuiltinScalarFunction::Struct => Volatility::Immutable, BuiltinScalarFunction::FromUnixtime => Volatility::Immutable, BuiltinScalarFunction::ToDate => Volatility::Immutable, + BuiltinScalarFunction::ToUnixtime => Volatility::Immutable, BuiltinScalarFunction::ArrowTypeof => Volatility::Immutable, BuiltinScalarFunction::OverLay => Volatility::Immutable, BuiltinScalarFunction::Levenshtein => Volatility::Immutable, @@ -785,6 +788,7 @@ impl BuiltinScalarFunction { BuiltinScalarFunction::ToTimestampSeconds => Ok(Timestamp(Second, None)), BuiltinScalarFunction::FromUnixtime => Ok(Timestamp(Second, None)), BuiltinScalarFunction::ToDate => Ok(Date32), + BuiltinScalarFunction::ToUnixtime => Ok(Int64), BuiltinScalarFunction::Now => { Ok(Timestamp(Nanosecond, Some("+00:00".into()))) } @@ -1063,6 +1067,9 @@ impl BuiltinScalarFunction { Signature::uniform(1, vec![Int64], self.volatility()) } BuiltinScalarFunction::ToDate => Signature::variadic_any(self.volatility()), + BuiltinScalarFunction::ToUnixtime => { + Signature::variadic_any(self.volatility()) + } BuiltinScalarFunction::Digest => Signature::one_of( vec![ Exact(vec![Utf8, Utf8]), @@ -1496,6 +1503,7 @@ impl BuiltinScalarFunction { BuiltinScalarFunction::ToTimestampNanos => &["to_timestamp_nanos"], BuiltinScalarFunction::FromUnixtime => &["from_unixtime"], BuiltinScalarFunction::ToDate => &["to_date"], + BuiltinScalarFunction::ToUnixtime => &["to_unixtime"], // hashing functions BuiltinScalarFunction::Digest => &["digest"], diff --git a/datafusion/physical-expr/src/datetime_expressions.rs b/datafusion/physical-expr/src/datetime_expressions.rs index fd57485555c6..57b1d23a19f6 100644 --- a/datafusion/physical-expr/src/datetime_expressions.rs +++ b/datafusion/physical-expr/src/datetime_expressions.rs @@ -1878,6 +1878,37 @@ pub fn from_unixtime_invoke(args: &[ColumnarValue]) -> Result { } } +/// to_unixtime() SQL function implementation +pub fn to_unixtime_invoke(args: &[ColumnarValue]) -> Result { + if args.is_empty() { + return exec_err!( + "to_unixtime function requires 1 or more arguments, got {}", + args.len() + ); + } + + // validate that any args after the first one are Utf8 + if args.len() > 1 { + if let Some(value) = validate_to_timestamp_data_types(args, "to_unixtime") { + return value; + } + } + + match args[0].data_type() { + DataType::Null | DataType::Int32 | DataType::Int64 => Ok(args[0].clone()), + DataType::Timestamp(_, None) => cast_column(&args[0], &DataType::Int64, None), + DataType::Utf8 => { + cast_column(&to_timestamp_seconds(args)?, &DataType::Int64, None) + } + other => { + exec_err!( + "Unsupported data type {:?} for function to_unixtime function", + other + ) + } + } +} + #[cfg(test)] mod tests { use std::sync::Arc; diff --git a/datafusion/physical-expr/src/functions.rs b/datafusion/physical-expr/src/functions.rs index 584901d37d28..2cf2674b7678 100644 --- a/datafusion/physical-expr/src/functions.rs +++ b/datafusion/physical-expr/src/functions.rs @@ -523,6 +523,9 @@ pub fn create_physical_fun( BuiltinScalarFunction::FromUnixtime => { Arc::new(datetime_expressions::from_unixtime_invoke) } + BuiltinScalarFunction::ToUnixtime => { + Arc::new(datetime_expressions::to_unixtime_invoke) + } BuiltinScalarFunction::ToDate => Arc::new(datetime_expressions::to_date_invoke), BuiltinScalarFunction::InitCap => Arc::new(|args| match args[0].data_type() { DataType::Utf8 => { diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto index 9015d9ee9b00..9d6aaf75eddc 100644 --- a/datafusion/proto/proto/datafusion.proto +++ b/datafusion/proto/proto/datafusion.proto @@ -1,5 +1,6 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -684,6 +685,7 @@ enum ScalarFunction { RegexpLike = 135; ToChar = 136; ToDate = 137; + ToUnixtime = 138; } message ScalarFunctionNode { diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs index f42e362a4f99..087b34db2fc0 100644 --- a/datafusion/proto/src/generated/pbjson.rs +++ b/datafusion/proto/src/generated/pbjson.rs @@ -22450,6 +22450,7 @@ impl serde::Serialize for ScalarFunction { Self::RegexpLike => "RegexpLike", Self::ToChar => "ToChar", Self::ToDate => "ToDate", + Self::ToUnixtime => "ToUnixtime", }; serializer.serialize_str(variant) } @@ -22590,6 +22591,7 @@ impl<'de> serde::Deserialize<'de> for ScalarFunction { "RegexpLike", "ToChar", "ToDate", + "ToUnixtime", ]; struct GeneratedVisitor; @@ -22759,6 +22761,7 @@ impl<'de> serde::Deserialize<'de> for ScalarFunction { "RegexpLike" => Ok(ScalarFunction::RegexpLike), "ToChar" => Ok(ScalarFunction::ToChar), "ToDate" => Ok(ScalarFunction::ToDate), + "ToUnixtime" => Ok(ScalarFunction::ToUnixtime), _ => Err(serde::de::Error::unknown_variant(value, FIELDS)), } } diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs index 97a620bac268..93fe2168d9a0 100644 --- a/datafusion/proto/src/generated/prost.rs +++ b/datafusion/proto/src/generated/prost.rs @@ -2771,6 +2771,7 @@ pub enum ScalarFunction { RegexpLike = 135, ToChar = 136, ToDate = 137, + ToUnixtime = 138, } impl ScalarFunction { /// String value of the enum field names used in the ProtoBuf definition. @@ -2908,6 +2909,7 @@ impl ScalarFunction { ScalarFunction::RegexpLike => "RegexpLike", ScalarFunction::ToChar => "ToChar", ScalarFunction::ToDate => "ToDate", + ScalarFunction::ToUnixtime => "ToUnixtime", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -3042,6 +3044,7 @@ impl ScalarFunction { "RegexpLike" => Some(Self::RegexpLike), "ToChar" => Some(Self::ToChar), "ToDate" => Some(Self::ToDate), + "ToUnixtime" => Some(Self::ToUnixtime), _ => None, } } diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rs index b2f7355765d2..4bd051c92eae 100644 --- a/datafusion/proto/src/logical_plan/from_proto.rs +++ b/datafusion/proto/src/logical_plan/from_proto.rs @@ -571,6 +571,7 @@ impl From<&protobuf::ScalarFunction> for BuiltinScalarFunction { ScalarFunction::SubstrIndex => Self::SubstrIndex, ScalarFunction::FindInSet => Self::FindInSet, ScalarFunction::ToDate => Self::ToDate, + ScalarFunction::ToUnixtime => Self::ToUnixtime, } } } @@ -1821,6 +1822,16 @@ pub fn parse_expr( args, ))) } + ScalarFunction::ToUnixtime => { + let args: Vec<_> = args + .iter() + .map(|expr| parse_expr(expr, registry)) + .collect::>()?; + Ok(Expr::ScalarFunction(expr::ScalarFunction::new( + BuiltinScalarFunction::ToUnixtime, + args, + ))) + } } } ExprType::ScalarUdfExpr(protobuf::ScalarUdfExprNode { fun_name, args }) => { diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rs index 2aa149f1606c..f92ac8ad9a32 100644 --- a/datafusion/proto/src/logical_plan/to_proto.rs +++ b/datafusion/proto/src/logical_plan/to_proto.rs @@ -1550,6 +1550,7 @@ impl TryFrom<&BuiltinScalarFunction> for protobuf::ScalarFunction { BuiltinScalarFunction::SubstrIndex => Self::SubstrIndex, BuiltinScalarFunction::FindInSet => Self::FindInSet, BuiltinScalarFunction::ToDate => Self::ToDate, + BuiltinScalarFunction::ToUnixtime => Self::ToUnixtime, }; Ok(scalar_function) diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.slt index d7085631777c..ab9214854be7 100644 --- a/datafusion/sqllogictest/test_files/timestamps.slt +++ b/datafusion/sqllogictest/test_files/timestamps.slt @@ -2678,3 +2678,22 @@ SELECT to_char(null, '%d-%m-%Y'); statement ok drop table formats; + +########## +## to_unixtime tests +########## + +query I +select to_unixtime('2020-09-08T12:00:00+00:00'); +---- +1599566400 + +query I +select to_unixtime('01-14-2023 01:01:30+05:30', '%q', '%d-%m-%Y %H/%M/%S', '%+', '%m-%d-%Y %H:%M:%S%#z'); +---- +1673638290 + +query I +select to_unixtime('03:59:00.123456789 05-17-2023', '%c', '%+', '%H:%M:%S%.f %m-%d-%Y'); +---- +1684295940