diff --git a/velox/type/Tokenizer.cpp b/velox/type/Tokenizer.cpp index 56015f5ad793..c48598d9a7e1 100644 --- a/velox/type/Tokenizer.cpp +++ b/velox/type/Tokenizer.cpp @@ -54,11 +54,13 @@ std::unique_ptr Tokenizer::computeNext() { return nullptr; } + // No need to tokenizing the path by dot because Spark treats dot as regular character. + /* if (tryMatch(DOT)) { std::unique_ptr token = matchPathSegment(); firstSegment = false; return token; - } + }*/ if (tryMatch(OPEN_BRACKET)) { std::unique_ptr token = tryMatch(QUOTE) @@ -144,8 +146,9 @@ std::unique_ptr Tokenizer::matchUnquotedSubscript() { } bool Tokenizer::isUnquotedPathCharacter(char c) { + // Add dot here because Spark treats dot as regular character. return c == ':' || c == '$' || c == '-' || c == '/' || c == '@' || c == '|' || - c == '#' || isUnquotedSubscriptCharacter(c); + c == '#' || c == '.' || isUnquotedSubscriptCharacter(c); } bool Tokenizer::isUnquotedSubscriptCharacter(char c) {