Add to_seconds support for tiflash (#4866)

close #4679

dbms/src/Common/MyTime.cpp

@@ -992,6 +992,15 @@ int calcDayNum(int year, int month, int day)
     return delsum + year / 4 - temp;
 }
 
+UInt64 calcSeconds(int year, int month, int day, int hour, int minute, int second)
+{
+    if (year == 0 && month == 0)
+        return 0;
+    Int32 current_days = calcDayNum(year, month, day);
+    return current_days * MyTimeBase::SECOND_IN_ONE_DAY + hour * MyTimeBase::SECOND_IN_ONE_HOUR
+        + minute * MyTimeBase::SECOND_IN_ONE_MINUTE + second;
+}
+
 size_t maxFormattedDateTimeStringLength(const String & format)
 {
     size_t result = 0;
@@ -1142,7 +1151,7 @@ UInt64 addSeconds(UInt64 t, Int64 delta)
         return t;
     }
     MyDateTime my_time(t);
-    Int64 current_second = my_time.hour * 3600 + my_time.minute * 60 + my_time.second;
+    Int64 current_second = my_time.hour * MyTimeBase::SECOND_IN_ONE_HOUR + my_time.minute * MyTimeBase::SECOND_IN_ONE_MINUTE + my_time.second;
     current_second += delta;
     if (current_second >= 0)
     {
@@ -1161,9 +1170,9 @@ UInt64 addSeconds(UInt64 t, Int64 delta)
         current_second += days * MyTimeBase::SECOND_IN_ONE_DAY;
         addDays(my_time, -days);
     }
-    my_time.hour = current_second / 3600;
-    my_time.minute = (current_second % 3600) / 60;
-    my_time.second = current_second % 60;
+    my_time.hour = current_second / MyTimeBase::SECOND_IN_ONE_HOUR;
+    my_time.minute = (current_second % MyTimeBase::SECOND_IN_ONE_HOUR) / MyTimeBase::SECOND_IN_ONE_MINUTE;
+    my_time.second = current_second % MyTimeBase::SECOND_IN_ONE_MINUTE;
     return my_time.toPackedUInt();
 }
 

dbms/src/Common/MyTime.h

@@ -24,6 +24,9 @@ namespace DB
 struct MyTimeBase
 {
     static constexpr Int64 SECOND_IN_ONE_DAY = 86400;
+    static constexpr Int64 SECOND_IN_ONE_HOUR = 3600;
+    static constexpr Int64 SECOND_IN_ONE_MINUTE = 60;
+
 
     // copied from https://github.com/pingcap/tidb/blob/master/types/time.go
     // Core time bit fields.
@@ -193,6 +196,9 @@ std::pair<time_t, UInt32> roundTimeByFsp(time_t second, UInt64 nano_second, UInt
 
 int calcDayNum(int year, int month, int day);
 
+// returns seconds since '0000-00-00'
+UInt64 calcSeconds(int year, int month, int day, int hour, int minute, int second);
+
 size_t maxFormattedDateTimeStringLength(const String & format);
 
 inline time_t getEpochSecond(const MyDateTime & my_time, const DateLUTImpl & time_zone)

dbms/src/Flash/Coprocessor/DAGUtils.cpp

@@ -573,7 +573,7 @@ const std::unordered_map<tipb::ScalarFuncSig, String> scalar_func_map({
     //{tipb::ScalarFuncSig::TimeToSec, "cast"},
     //{tipb::ScalarFuncSig::TimestampAdd, "cast"},
     //{tipb::ScalarFuncSig::ToDays, "cast"},
-    //{tipb::ScalarFuncSig::ToSeconds, "cast"},
+    {tipb::ScalarFuncSig::ToSeconds, "tidbToSeconds"},
     //{tipb::ScalarFuncSig::UTCTimeWithArg, "cast"},
     //{tipb::ScalarFuncSig::UTCTimestampWithoutArg, "cast"},
     //{tipb::ScalarFuncSig::Timestamp1Arg, "cast"},

dbms/src/Functions/FunctionsDateTime.cpp

@@ -137,6 +137,7 @@ void registerFunctionsDateTime(FunctionFactory & factory)
     factory.registerFunction<FunctionToTiDBDayOfWeek>();
     factory.registerFunction<FunctionToTiDBDayOfYear>();
     factory.registerFunction<FunctionToTiDBWeekOfYear>();
+    factory.registerFunction<FunctionToTiDBToSeconds>();
 
     factory.registerFunction<FunctionToTimeZone>();
     factory.registerFunction<FunctionToLastDay>();

dbms/src/Functions/FunctionsDateTime.h

@@ -3277,6 +3277,42 @@ struct TiDBWeekOfYearTransformerImpl
     }
 };
 
+template <typename ToFieldType>
+struct TiDBToSecondsTransformerImpl
+{
+    static constexpr auto name = "tidbToSeconds";
+
+    static void execute(const Context & context,
+                        const ColumnVector<DataTypeMyTimeBase::FieldType>::Container & vec_from,
+                        typename ColumnVector<ToFieldType>::Container & vec_to,
+                        typename ColumnVector<UInt8>::Container & vec_null_map)
+    {
+        bool is_null = false;
+        for (size_t i = 0; i < vec_from.size(); ++i)
+        {
+            MyTimeBase val(vec_from[i]);
+            vec_to[i] = execute(context, val, is_null);
+            vec_null_map[i] = is_null;
+            is_null = false;
+        }
+    }
+
+    static ToFieldType execute(const Context & context, const MyTimeBase & val, bool & is_null)
+    {
+        // TiDB returns normal value if one of month/day is zero for to_seconds function, while MySQL return null if either of them is zero.
+        // TiFlash aligns with MySQL to align the behavior with other functions like last_day.
+        if (val.month == 0 || val.day == 0)
+        {
+            context.getDAGContext()->handleInvalidTime(
+                fmt::format("Invalid time value: month({}) or day({}) is zero", val.month, val.day),
+                Errors::Types::WrongValue);
+            is_null = true;
+            return 0;
+        }
+        return static_cast<ToFieldType>(calcSeconds(val.year, val.month, val.day, val.hour, val.minute, val.second));
+    }
+};
+
 // Similar to FunctionDateOrDateTimeToSomething, but also handle nullable result and mysql sql mode.
 template <typename ToDataType, template <typename> class Transformer, bool return_nullable>
 class FunctionMyDateOrMyDateTimeToSomething : public IFunction
@@ -3376,6 +3412,7 @@ using FunctionToLastDay = FunctionMyDateOrMyDateTimeToSomething<DataTypeMyDate,
 using FunctionToTiDBDayOfWeek = FunctionMyDateOrMyDateTimeToSomething<DataTypeUInt16, TiDBDayOfWeekTransformerImpl, return_nullable>;
 using FunctionToTiDBDayOfYear = FunctionMyDateOrMyDateTimeToSomething<DataTypeUInt16, TiDBDayOfYearTransformerImpl, return_nullable>;
 using FunctionToTiDBWeekOfYear = FunctionMyDateOrMyDateTimeToSomething<DataTypeUInt16, TiDBWeekOfYearTransformerImpl, return_nullable>;
+using FunctionToTiDBToSeconds = FunctionMyDateOrMyDateTimeToSomething<DataTypeUInt64, TiDBToSecondsTransformerImpl, return_nullable>;
 
 using FunctionToRelativeYearNum = FunctionDateOrDateTimeToSomething<DataTypeUInt16, ToRelativeYearNumImpl>;
 using FunctionToRelativeQuarterNum = FunctionDateOrDateTimeToSomething<DataTypeUInt32, ToRelativeQuarterNumImpl>;

dbms/src/Functions/tests/gtest_toseconds.cpp

@@ -0,0 +1,99 @@
+// Copyright 2022 PingCAP, Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <Common/MyTime.h>
+#include <Functions/FunctionFactory.h>
+#include <TestUtils/FunctionTestUtils.h>
+#include <TestUtils/TiFlashTestBasic.h>
+
+#include <string>
+#include <vector>
+
+namespace DB::tests
+{
+class TestToSeconds : public DB::tests::FunctionTest
+{
+};
+
+TEST_F(TestToSeconds, TestAll)
+try
+{
+    DAGContext * dag_context = context.getDAGContext();
+    UInt64 ori_flags = dag_context->getFlags();
+    dag_context->addFlag(TiDBSQLFlags::TRUNCATE_AS_WARNING);
+    /// ColumnVector(nullable)
+    const String func_name = "tidbToSeconds";
+    static auto const nullable_datetime_type_ptr = makeNullable(std::make_shared<DataTypeMyDateTime>(6));
+    static auto const datetime_type_ptr = std::make_shared<DataTypeMyDateTime>(6);
+    static auto const date_type_ptr = std::make_shared<DataTypeMyDate>();
+    auto data_col_ptr = createColumn<Nullable<DataTypeMyDateTime::FieldType>>(
+                            {
+                                {}, // Null
+                                MyDateTime(0, 0, 0, 0, 0, 0, 0).toPackedUInt(),
+                                MyDateTime(0, 1, 1, 0, 0, 0, 0).toPackedUInt(),
+                                MyDateTime(1969, 1, 2, 1, 1, 1, 1).toPackedUInt(),
+                                MyDateTime(2000, 12, 31, 10, 10, 10, 700).toPackedUInt(),
+                                MyDateTime(2022, 3, 13, 6, 7, 8, 9).toPackedUInt(),
+                            })
+                            .column;
+    auto input_col = ColumnWithTypeAndName(data_col_ptr, nullable_datetime_type_ptr, "input");
+    auto output_col = createColumn<Nullable<UInt64>>({{}, {}, 86400, 62135773261ULL, 63145476610ULL, 63814370828ULL});
+    ASSERT_COLUMN_EQ(output_col, executeFunction(func_name, input_col));
+
+    /// ColumnVector(non-null)
+    data_col_ptr = createColumn<DataTypeMyDateTime::FieldType>(
+                       {
+                           MyDateTime(0, 0, 0, 0, 0, 0, 0).toPackedUInt(),
+                           MyDateTime(1969, 1, 2, 1, 1, 1, 1).toPackedUInt(),
+                           MyDateTime(2000, 12, 31, 10, 10, 10, 700).toPackedUInt(),
+                           MyDateTime(2022, 3, 13, 6, 7, 8, 9).toPackedUInt(),
+                       })
+                       .column;
+    input_col = ColumnWithTypeAndName(data_col_ptr, datetime_type_ptr, "input");
+    output_col = createColumn<Nullable<UInt64>>({{}, 62135773261ULL, 63145476610ULL, 63814370828ULL});
+    ASSERT_COLUMN_EQ(output_col, executeFunction(func_name, input_col));
+
+    /// ColumnConst(non-null)
+    input_col = ColumnWithTypeAndName(createConstColumn<DataTypeMyDateTime::FieldType>(1, MyDateTime(2022, 3, 13, 6, 7, 8, 9).toPackedUInt()).column, datetime_type_ptr, "input");
+    output_col = createConstColumn<Nullable<UInt64>>(1, {63814370828ULL});
+    ASSERT_COLUMN_EQ(output_col, executeFunction(func_name, input_col));
+
+    /// ColumnConst(nullable)
+    input_col = ColumnWithTypeAndName(createConstColumn<Nullable<DataTypeMyDateTime::FieldType>>(1, MyDateTime(2022, 3, 13, 6, 7, 8, 9).toPackedUInt()).column, nullable_datetime_type_ptr, "input");
+    output_col = createConstColumn<Nullable<UInt64>>(1, {63814370828ULL});
+    ASSERT_COLUMN_EQ(output_col, executeFunction(func_name, input_col));
+
+    /// ColumnConst(nullable(null))
+    input_col = ColumnWithTypeAndName(createConstColumn<Nullable<DataTypeMyDateTime::FieldType>>(1, {}).column, nullable_datetime_type_ptr, "input");
+    output_col = createConstColumn<Nullable<UInt64>>(1, {});
+    ASSERT_COLUMN_EQ(output_col, executeFunction(func_name, input_col));
+
+    /// MyDate ColumnVector(non-null)
+    data_col_ptr = createColumn<DataTypeMyDate::FieldType>(
+                       {
+                           MyDate(0000, 0, 1).toPackedUInt(),
+                           MyDate(0000, 1, 1).toPackedUInt(),
+                           MyDate(1969, 1, 1).toPackedUInt(),
+                           MyDate(2000, 12, 1).toPackedUInt(),
+                           MyDate(2022, 3, 14).toPackedUInt(),
+                       })
+                       .column;
+    input_col = ColumnWithTypeAndName(data_col_ptr, date_type_ptr, "input");
+    output_col = createColumn<Nullable<UInt64>>({{}, 86400, 62135683200ULL, 63142848000ULL, 63814435200ULL});
+    ASSERT_COLUMN_EQ(output_col, executeFunction(func_name, input_col));
+    dag_context->setFlags(ori_flags);
+}
+CATCH
+
+} // namespace DB::tests

tests/fullstack-test/expr/to_seconds.test

@@ -0,0 +1,61 @@
+# Copyright 2022 PingCAP, Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+mysql> drop table if exists test.t1;
+mysql> create table test.t1(c1 varchar(100), c2 datetime, c3 date);
+mysql> insert into test.t1 values('', '1999-10-10 10:10:10.123', '1999-01-10'), ('200', '1999-02-10 10:10:10.123', '1999-11-10'), ('1999-01-10', '1999-10-10 10:10:10.123', '1999-01-10');
+# leap year
+mysql> insert into test.t1 values('2000-2-10', '2000-2-10 10:10:10', '2000-2-10');
+# non leap year
+mysql> insert into test.t1 values('2001-2-10', '2001-2-10 10:10:10', '2001-2-10');
+# zero day
+mysql> insert into test.t1 values('2000-2-0', '2000-2-10 10:10:10', '2000-2-10');
+mysql> alter table test.t1 set tiflash replica 1;
+func> wait_table test t1
+#mysql> set @@tidb_isolation_read_engines='tiflash'; set @@tidb_enforce_mpp = 1; select c1, to_seconds(c1) from test.t1 order by 1;
+#+------------+----------------+
+#| c1         | to_seconds(c1) |
+#+------------+----------------+
+#|            |           NULL |
+#| 1999-01-10 |    63083145600 |
+#| 200        |           NULL |
+#| 2000-2-0   |           NULL |
+#| 2000-2-10  |    63117360000 |
+#| 2001-2-10  |    63148982400 |
+#+------------+----------------+
+mysql> set @@tidb_isolation_read_engines='tiflash'; set @@tidb_enforce_mpp = 1; select c2, to_seconds(c2) from test.t1 order by 1;
++---------------------+----------------+
+| c2                  | to_seconds(c2) |
++---------------------+----------------+
+| 1999-02-10 10:10:10 |    63085860610 |
+| 1999-10-10 10:10:10 |    63106769410 |
+| 1999-10-10 10:10:10 |    63106769410 |
+| 2000-02-10 10:10:10 |    63117396610 |
+| 2000-02-10 10:10:10 |    63117396610 |
+| 2001-02-10 10:10:10 |    63149019010 |
++---------------------+----------------+
+mysql> set @@tidb_isolation_read_engines='tiflash'; set @@tidb_enforce_mpp = 1; select c3, to_seconds(c3) from test.t1 order by 1;
++------------+----------------+
+| c3         | to_seconds(c3) |
++------------+----------------+
+| 1999-01-10 |    63083145600 |
+| 1999-01-10 |    63083145600 |
+| 1999-11-10 |    63109411200 |
+| 2000-02-10 |    63117360000 |
+| 2000-02-10 |    63117360000 |
+| 2001-02-10 |    63148982400 |
++------------+----------------+
+
+mysql> drop table if exists test.t1;
+