Disable tokenizing the path by dot (oap-project#109)

zhejiangxiaomai · Mar 29, 2023 · 7b45b43 · 7b45b43
1 parent 73a86ee
commit 7b45b43
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/velox/type/Tokenizer.cpp b/velox/type/Tokenizer.cpp
@@ -54,11 +54,13 @@ std::unique_ptr<Subfield::PathElement> Tokenizer::computeNext() {
     return nullptr;
   }
 
+  // No need to tokenizing the path by dot because Spark treats dot as regular character.
+  /*
   if (tryMatch(DOT)) {
     std::unique_ptr<Subfield::PathElement> token = matchPathSegment();
     firstSegment = false;
     return token;
-  }
+  }*/
 
   if (tryMatch(OPEN_BRACKET)) {
     std::unique_ptr<Subfield::PathElement> token = tryMatch(QUOTE)
@@ -144,8 +146,9 @@ std::unique_ptr<Subfield::PathElement> Tokenizer::matchUnquotedSubscript() {
 }
 
 bool Tokenizer::isUnquotedPathCharacter(char c) {
+  // Add dot here because Spark treats dot as regular character.
   return c == ':' || c == '$' || c == '-' || c == '/' || c == '@' || c == '|' ||
-      c == '#' || isUnquotedSubscriptCharacter(c);
+      c == '#' || c == '.' || isUnquotedSubscriptCharacter(c);
 }
 
 bool Tokenizer::isUnquotedSubscriptCharacter(char c) {