feat(YQL): add tokenize method for YQL (#205)

* feat(YQL): add tokenize method for yql

* feat(YQL): code review

* ci: add eslint rule

.eslintrc

@@ -12,6 +12,7 @@
         "consistent-return": "off",
         "no-negated-condition": "off",
         "@typescript-eslint/explicit-function-return-type": "error",
+        "@typescript-eslint/array-type": ["error", { "default": "array" }],
         "object-shorthand": "error",
         "no-implicit-globals": "off",
         "camelcase": "off",

src/autocomplete/databases/yql/index.ts

@@ -2,7 +2,9 @@ import {CursorPosition} from '../../shared/autocomplete-types';
 import {yqlAutocompleteData, yqlAutocompleteDataYQ} from './yql-autocomplete';
 import {parseQuery, parseQueryWithoutCursor} from '../../shared/autocomplete';
 import {separateQueryAndCursor} from '../../shared/parse-query-with-cursor';
-import {YqlAutocompleteResult} from './types';
+import {YqlAutocompleteResult, YqlTokenizeResult} from './types';
+import {tokenize} from '../../shared/tokenize';
+import {YQLLexer} from './generated/YQLLexer';
 
 export type {YqlAutocompleteResult};
 
@@ -63,3 +65,7 @@ export function parseYqlQueryWithCursor(queryWithCursor: string): YqlAutocomplet
 export function parseYqQueryWithCursor(queryWithCursor: string): YqlAutocompleteResult {
     return parseYqQuery(...separateQueryAndCursor(queryWithCursor));
 }
+
+export function tokenizeYqlQuery(query: string): YqlTokenizeResult {
+    return tokenize(YQLLexer, YQLLexer.symbolicNames, query);
+}

src/autocomplete/databases/yql/types.ts

@@ -1,4 +1,5 @@
 import {AutocompleteResultBase, TableIndexSuggestion} from '../../shared/autocomplete-types';
+import {TokenizeResult} from '../../shared/tokenize';
 
 export type EntitySuggestion =
     | 'suggestObject'
@@ -58,3 +59,5 @@ export interface YqlAutocompleteResult extends AutocompleteResultBase {
     suggestTableHints?: string;
     suggestEntitySettings?: YQLEntity;
 }
+
+export interface YqlTokenizeResult extends TokenizeResult {}

src/autocomplete/shared/query.ts

@@ -58,3 +58,8 @@ export function createParser<L extends LexerType, P extends ParserType>(
 
     return parser;
 }
+
+export function createLexer<L extends LexerType>(Lexer: LexerConstructor<L>, query: string): L {
+    const inputStream = CharStream.fromString(query);
+    return new Lexer(inputStream);
+}

src/autocomplete/shared/tokenize.ts

@@ -0,0 +1,52 @@
+import {LexerConstructor, ParserSyntaxError} from './autocomplete-types';
+import {Lexer as LexerType} from 'antlr4ng';
+import {createLexer} from './query';
+import {SqlErrorListener} from './sql-error-listener';
+import {tokenDictionary} from '../databases/yql/helpers';
+
+export type Token = {
+    ruleName: string;
+    startIndex: number;
+};
+
+export type TokenizeResult = {
+    tokens: Token[];
+    errors: ParserSyntaxError[];
+};
+
+const EOF = -1;
+
+export function tokenize<L extends LexerType>(
+    Lexer: LexerConstructor<L>,
+    symbolicNames: (string | null)[],
+    query: string,
+): TokenizeResult {
+    const lexer = createLexer(Lexer, query);
+    const errorListener = new SqlErrorListener(tokenDictionary.SPACE);
+
+    lexer.removeErrorListeners();
+    lexer.addErrorListener(errorListener);
+
+    const tokens: Token[] = [];
+
+    let done = false;
+    do {
+        const token = lexer.nextToken();
+        if (token === null || token.type === EOF) {
+            done = true;
+        } else {
+            const tokenName = symbolicNames[token.type];
+            if (tokenName) {
+                tokens.push({
+                    ruleName: tokenName,
+                    startIndex: token.column,
+                });
+            }
+        }
+    } while (!done);
+
+    return {
+        tokens,
+        errors: errorListener.errors,
+    };
+}