Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix the wrong result bug when casting string as datetime with time zone or illegal chars (#9255) #9355

Closed
wants to merge 2 commits into from
Closed

Fix the wrong result bug when casting string as datetime with time zone or illegal chars (#9255) #9355

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e25801e
This is an automated cherry-pick of #9255
solotzg Aug 9, 2024
cc1b3e6
Merge branch 'release-6.1' into cherry-pick-9255-to-release-6.1
solotzg Sep 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
232 changes: 232 additions & 0 deletions dbms/src/Common/MyTime.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,29 @@ std::tuple<int, String, String, String, String> getTimeZone(const String & liter
// TODO: make unified helper
bool isPunctuation(char c)
{
<<<<<<< HEAD
return (c >= 0x21 && c <= 0x2F) || (c >= 0x3A && c <= 0x40) || (c >= 0x5B && c <= 0x60) || (c >= 0x7B && c <= 0x7E);
=======
auto [tz_idx, tz_sign, tz_hour, tz_sep, tz_minute] = getTimeZone(format);
int end = format.length() - 1;
if (tz_idx != -1)
{
end = tz_idx - 1;
}
int idx = -1;
for (int i = end; i >= 0; i--)
{
if (format[i] != '+' && format[i] != '-' && isPunctuation(format[i]))
{
if (format[i] == '.')
{
idx = i;
}
break;
}
}
return idx;
>>>>>>> 7b17f5b39e (Fix the wrong result bug when casting string as datetime with time zone or illegal chars (#9255))
}

std::tuple<std::vector<String>, String, bool, String, String, String, String> splitDatetime(String format)
Expand Down Expand Up @@ -561,7 +583,217 @@ bool checkTimeValid(Int32 year, Int32 month, Int32 day, Int32 hour, Int32 minute
return day <= getLastDay(year, month);
}

<<<<<<< HEAD
std::pair<Field, bool> parseMyDateTimeAndJudgeIsDate(const String & str, int8_t fsp, bool needCheckTimeValid)
=======
bool noNeedCheckTime(Int32, Int32, Int32, Int32, Int32, Int32)
{
return true;
}

UInt64 addSeconds(UInt64 t, Int64 delta)
{
// todo support zero date
if (t == 0)
{
return t;
}
MyDateTime my_time(t);
Int64 current_second = my_time.hour * MyTimeBase::SECOND_IN_ONE_HOUR
+ my_time.minute * MyTimeBase::SECOND_IN_ONE_MINUTE + my_time.second;
current_second += delta;
if (current_second >= 0)
{
Int64 days = current_second / MyTimeBase::SECOND_IN_ONE_DAY;
current_second = current_second % MyTimeBase::SECOND_IN_ONE_DAY;
if (days != 0)
addDays(my_time, days);
}
else
{
Int64 days = (-current_second) / MyTimeBase::SECOND_IN_ONE_DAY;
if ((-current_second) % MyTimeBase::SECOND_IN_ONE_DAY != 0)
{
days++;
}
current_second += days * MyTimeBase::SECOND_IN_ONE_DAY;
addDays(my_time, -days);
}
my_time.hour = current_second / MyTimeBase::SECOND_IN_ONE_HOUR;
my_time.minute = (current_second % MyTimeBase::SECOND_IN_ONE_HOUR) / MyTimeBase::SECOND_IN_ONE_MINUTE;
my_time.second = current_second % MyTimeBase::SECOND_IN_ONE_MINUTE;
return my_time.toPackedUInt();
}

// Return true if the time is invalid.
inline bool getDatetime(const Int64 & num, MyDateTime & result)
{
UInt64 ymd = num / 1000000;
UInt64 hms = num - ymd * 1000000;

UInt64 year = ymd / 10000;
ymd %= 10000;
UInt64 month = ymd / 100;
UInt64 day = ymd % 100;

UInt64 hour = hms / 10000;
hms %= 10000;
UInt64 minute = hms / 100;
UInt64 second = hms % 100;

if (toCoreTimeChecked(year, month, day, hour, minute, second, 0, result))
{
return true;
}
return !result.isValid(true, false);
}

// Convert a integer number to DateTime and return true if the result is NULL.
// If number is invalid(according to SQL_MODE), return NULL and handle the error with DAGContext.
// This function may throw exception.
inline bool numberToDateTime(Int64 number, MyDateTime & result, bool allowZeroDate)
{
MyDateTime datetime(0);
if (number == 0)
{
if (allowZeroDate)
{
result = datetime;
return false;
}
return true;
}

// datetime type
if (number >= 10000101000000)
{
return getDatetime(number, result);
}

// check MMDD
if (number < 101)
{
return true;
}

// check YYMMDD: 2000-2069
if (number <= 69 * 10000 + 1231)
{
number = (number + 20000000) * 1000000;
return getDatetime(number, result);
}

if (number < 70 * 10000 + 101)
{
return true;
}

// check YYMMDD
if (number <= 991231)
{
number = (number + 19000000) * 1000000;
return getDatetime(number, result);
}

// check hour/min/second
if (number <= 99991231)
{
number *= 1000000;
return getDatetime(number, result);
}

// check MMDDHHMMSS
if (number < 101000000)
{
return true;
}

// check YYMMDDhhmmss: 2000-2069
if (number <= 69 * 10000000000 + 1231235959)
{
number += 20000000000000;
return getDatetime(number, result);
}

// check YYYYMMDDhhmmss
if (number < 70 * 10000000000 + 101000000)
{
return true;
}

// check YYMMDDHHMMSS
if (number <= 991231235959)
{
number += 19000000000000;
return getDatetime(number, result);
}

return getDatetime(number, result);
}

// returns frac, overflow, matched. eg., "999" fsp=2 will overflow.
std::tuple<UInt32, bool, bool> parseFrac(const std::string_view str, int8_t fsp)
{
if (str.empty())
{
return {0, false, true};
}
if (fsp == -1)
{
fsp = 6;
}
if (fsp < 0 || fsp > 6)
{
return {0, false, false};
}
try
{
int end_pos = static_cast<size_t>(fsp) >= str.size() ? str.size() : (fsp + 1);
UInt32 tmp = 0;
int size = 0;
for (int i = 0; i < end_pos; ++i)
{
if (auto c = str[i]; c >= '0' && c <= '9')
{
tmp = tmp * 10 + c - '0';
size++;
}
else
{
break;
}
}

if (fsp >= size)
{
return {tmp * std::pow(10, 6 - size), false, true};
}

tmp = (tmp + 5) / 10;
if (tmp >= std::pow(10, fsp))
{
// overflow
return {0, true, true};
}
// Get the final frac, with 6 digit number
// 1236 round 3 -> 124 -> 124000
// 0312 round 2 -> 3 -> 30000
// 999 round 2 -> 100 -> overflow
return {tmp * std::pow(10, 6 - fsp), false, true};
}
catch (std::exception & e)
{
return {0, false, false};
}
}

// isFloat is true means that the input string is float format like "1212.111"
std::pair<Field, bool> parseMyDateTimeAndJudgeIsDate(
const String & str,
int8_t fsp,
CheckTimeFunc checkTimeFunc,
bool isFloat)
>>>>>>> 7b17f5b39e (Fix the wrong result bug when casting string as datetime with time zone or illegal chars (#9255))
{
Int32 year = 0, month = 0, day = 0, hour = 0, minute = 0, second = 0, delta_hour = 0, delta_minute = 0;

Expand Down
61 changes: 61 additions & 0 deletions dbms/src/Common/tests/gtest_mytime.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,12 +115,73 @@ try
{"2020-10-10 10.10", "2020-10-10 10:10:00.000000"},
{"2018.01.01", "2018-01-01 00:00:00.000000"},
{"2020--12-10 11:11:11..123456", "2020-12-10 11:11:11.123456"},
{"2020-01-01 12:00:00.1234xxxx -0600 PST", "2020-01-01 12:00:00.123400"},
{"2020-01-01 12:00:00.123456-05:00", "2020-01-01 17:00:00.123456"}, // tidb/issues/49555
};
DataTypeMyDateTime type_with_fraction(6);
for (auto & [str, expected] : cases_with_fsp)
{
checkParseMyDateTime(str, expected, type_with_fraction);
}
<<<<<<< HEAD
=======
cases_with_fsp = {
{"2012-12-31 11:30:45", "2012-12-31 11:30:45"},
{"0000-00-00 00:00:00", "0000-00-00 00:00:00"},
{"0001-01-01 00:00:00", "0001-01-01 00:00:00"},
{"00-12-31 11:30:45", "2000-12-31 11:30:45"},
{"12-12-31 11:30:45", "2012-12-31 11:30:45"},
{"2012-12-31", "2012-12-31 00:00:00"},
{"20121231", "2012-12-31 00:00:00"},
{"121231", "2012-12-31 00:00:00"},
{"2012^12^31 11+30+45", "2012-12-31 11:30:45"},
{"2012^12^31T11+30+45", "2012-12-31 11:30:45"},
{"2012-2-1 11:30:45", "2012-02-01 11:30:45"},
{"12-2-1 11:30:45", "2012-02-01 11:30:45"},
{"20121231113045", "2012-12-31 11:30:45"},
{"121231113045", "2012-12-31 11:30:45"},
{"2012-02-29", "2012-02-29 00:00:00"},
{"00-00-00", "0000-00-00 00:00:00"},
// {"00-00-00 00:00:00.123", "2000-00-00 00:00:00.123"},
{"11111111111", "2011-11-11 11:11:01"},
{"1701020301.", "2017-01-02 03:01:00"},
// {"1701020304.1", "2017-01-02 03:04:01.0"},
// {"1701020302.11", "2017-01-02 03:02:11.00"},
{"170102036", "2017-01-02 03:06:00"},
{"170102039.", "2017-01-02 03:09:00"},
// {"170102037.11", "2017-01-02 03:07:11.00"},
{"2018-01-01 18", "2018-01-01 18:00:00"},
{"18-01-01 18", "2018-01-01 18:00:00"},
// {"2018.01.01", "2018-01-01 00:00:00.00"},
// {"2020.10.10 10.10.10", "2020-10-10 10:10:10.00"},
// {"2020-10-10 10-10.10", "2020-10-10 10:10:10.00"},
// {"2020-10-10 10.10", "2020-10-10 10:10:00.00"},
// {"2018.01.01", "2018-01-01 00:00:00.00"},
{"2018.01.01 00:00:00", "2018-01-01 00:00:00"},
{"2018/01/01-00:00:00", "2018-01-01 00:00:00"},
{"4710072", "2047-10-07 02:00:00"},
{"2016-06-01 00:00:00 00:00:00", "2016-06-01 00:00:00"},
{"2020-06-01 00:00:00ads!,?*da;dsx", "2020-06-01 00:00:00"},

{"2020-05-28 23:59:59 00:00:00", "2020-05-28 23:59:59"},
{"2020-05-28 23:59:59-00:00:00", "2020-05-28 23:59:59"},
{"2020-05-28 23:59:59T T00:00:00", "2020-05-28 23:59:59"},
{"2020-10-22 10:31-10:12", "2020-10-22 10:31:10"},
{"2018.01.01 01:00:00", "2018-01-01 01:00:00"},

// {"2020-01-01 12:00:00.123456+05:00", "2020-01-01 07:00:00.123456"}
};
DataTypeMyDateTime type_with_zero_fraction(0);
for (auto & [str, expected] : cases_with_fsp)
{
checkParseMyDateTime(str, expected, type_with_zero_fraction);
}
DataTypeMyDateTime tp(2);
checkParseMyDateTime("2010-12-31 23:59:59.99999", "2011-01-01 00:00:00.00", tp);
checkParseMyDateTime("2010-12-31 23:59:59.99xxxxx -0600 PST", "2010-12-31 23:59:59.99", tp);
checkParseMyDateTime("2020-01-01 12:00:00.123456 +0600 PST", "2020-01-01 12:00:00.12", tp);
checkParseMyDateTime("2020-01-01 12:00:00.123456 -0600 PST", "2020-01-01 12:00:00.12", tp);
>>>>>>> 7b17f5b39e (Fix the wrong result bug when casting string as datetime with time zone or illegal chars (#9255))
}
catch (Exception & e)
{
Expand Down