feat(flinksql): collect comment, type attribute for entity

src/grammar/flink/FlinkSqlParser.g4

@@ -156,8 +156,9 @@ createTable
 simpleCreateTable
     : KW_CREATE KW_TEMPORARY? KW_TABLE ifNotExists? tablePathCreate LR_BRACKET columnOptionDefinition (
         COMMA columnOptionDefinition
-    )* (COMMA watermarkDefinition)? (COMMA tableConstraint)? (COMMA selfDefinitionClause)? RR_BRACKET commentSpec? partitionDefinition? withOption
-        likeDefinition?
+    )* (COMMA watermarkDefinition)? (COMMA tableConstraint)? (COMMA selfDefinitionClause)? RR_BRACKET (
+        KW_COMMENT comment=STRING_LITERAL
+    )? partitionDefinition? withOption likeDefinition?
     ;
 
 /*
@@ -175,7 +176,7 @@ columnOptionDefinition
     ;
 
 physicalColumnDefinition
-    : columnNameCreate columnType columnConstraint? commentSpec?
+    : colName=columnNameCreate columnType columnConstraint? (KW_COMMENT comment=STRING_LITERAL)?
     ;
 
 columnNameCreate
@@ -193,8 +194,8 @@ columnNameList
     ;
 
 columnType
-    : typeName=(KW_DATE | KW_BOOLEAN | KW_NULL)
-    | typeName=(
+    : colType=(KW_DATE | KW_BOOLEAN | KW_NULL)
+    | colType=(
         KW_CHAR
         | KW_VARCHAR
         | KW_STRING
@@ -210,12 +211,12 @@ columnType
         | KW_TIMESTAMP_LTZ
         | KW_DATETIME
     ) lengthOneDimension?
-    | typeName=KW_TIMESTAMP lengthOneDimension? ((KW_WITHOUT | KW_WITH) KW_LOCAL? KW_TIME KW_ZONE)?
-    | typeName=(KW_DECIMAL | KW_DEC | KW_NUMERIC | KW_FLOAT | KW_DOUBLE) lengthTwoOptionalDimension?
-    | type=(KW_ARRAY | KW_MULTISET) lengthOneTypeDimension?
-    | type=KW_MAP mapTypeDimension?
-    | type=KW_ROW rowTypeDimension?
-    | type=KW_RAW lengthTwoStringDimension?
+    | colType=KW_TIMESTAMP lengthOneDimension? ((KW_WITHOUT | KW_WITH) KW_LOCAL? KW_TIME KW_ZONE)?
+    | colType=(KW_DECIMAL | KW_DEC | KW_NUMERIC | KW_FLOAT | KW_DOUBLE) lengthTwoOptionalDimension?
+    | colType=(KW_ARRAY | KW_MULTISET) lengthOneTypeDimension?
+    | colType=KW_MAP mapTypeDimension?
+    | colType=KW_ROW rowTypeDimension?
+    | colType=KW_RAW lengthTwoStringDimension?
     ;
 
 lengthOneDimension
@@ -247,20 +248,16 @@ columnConstraint
     | KW_NOT? KW_NULL
     ;
 
-commentSpec
-    : KW_COMMENT STRING_LITERAL
-    ;
-
 metadataColumnDefinition
-    : columnNameCreate columnType KW_METADATA (KW_FROM metadataKey)? KW_VIRTUAL?
+    : colName=columnNameCreate columnType KW_METADATA (KW_FROM metadataKey)? KW_VIRTUAL?
     ;
 
 metadataKey
     : STRING_LITERAL
     ;
 
 computedColumnDefinition
-    : columnNameCreate KW_AS computedColumnExpression commentSpec?
+    : colName=columnNameCreate KW_AS computedColumnExpression (KW_COMMENT comment=STRING_LITERAL)?
     ;
 
 // 计算表达式
@@ -316,11 +313,13 @@ createCatalog
     ;
 
 createDatabase
-    : KW_CREATE KW_DATABASE ifNotExists? databasePathCreate commentSpec? withOption
+    : KW_CREATE KW_DATABASE ifNotExists? databasePathCreate (KW_COMMENT comment=STRING_LITERAL)? withOption
     ;
 
 createView
-    : KW_CREATE KW_TEMPORARY? KW_VIEW ifNotExists? viewPathCreate columnNameList? commentSpec? KW_AS queryStatement
+    : KW_CREATE KW_TEMPORARY? KW_VIEW ifNotExists? viewPathCreate columnNameList? (
+        KW_COMMENT comment=STRING_LITERAL
+    )? KW_AS queryStatement
     ;
 
 createFunction

src/lib/flink/FlinkSqlParserListener.ts

@@ -44,7 +44,6 @@ import { LengthOneTypeDimensionContext } from "./FlinkSqlParser.js";
 import { MapTypeDimensionContext } from "./FlinkSqlParser.js";
 import { RowTypeDimensionContext } from "./FlinkSqlParser.js";
 import { ColumnConstraintContext } from "./FlinkSqlParser.js";
-import { CommentSpecContext } from "./FlinkSqlParser.js";
 import { MetadataColumnDefinitionContext } from "./FlinkSqlParser.js";
 import { MetadataKeyContext } from "./FlinkSqlParser.js";
 import { ComputedColumnDefinitionContext } from "./FlinkSqlParser.js";
@@ -588,16 +587,6 @@ export class FlinkSqlParserListener implements ParseTreeListener {
      * @param ctx the parse tree
      */
     exitColumnConstraint?: (ctx: ColumnConstraintContext) => void;
-    /**
-     * Enter a parse tree produced by `FlinkSqlParser.commentSpec`.
-     * @param ctx the parse tree
-     */
-    enterCommentSpec?: (ctx: CommentSpecContext) => void;
-    /**
-     * Exit a parse tree produced by `FlinkSqlParser.commentSpec`.
-     * @param ctx the parse tree
-     */
-    exitCommentSpec?: (ctx: CommentSpecContext) => void;
     /**
      * Enter a parse tree produced by `FlinkSqlParser.metadataColumnDefinition`.
      * @param ctx the parse tree

src/lib/flink/FlinkSqlParserVisitor.ts

@@ -44,7 +44,6 @@ import { LengthOneTypeDimensionContext } from "./FlinkSqlParser.js";
 import { MapTypeDimensionContext } from "./FlinkSqlParser.js";
 import { RowTypeDimensionContext } from "./FlinkSqlParser.js";
 import { ColumnConstraintContext } from "./FlinkSqlParser.js";
-import { CommentSpecContext } from "./FlinkSqlParser.js";
 import { MetadataColumnDefinitionContext } from "./FlinkSqlParser.js";
 import { MetadataKeyContext } from "./FlinkSqlParser.js";
 import { ComputedColumnDefinitionContext } from "./FlinkSqlParser.js";
@@ -447,12 +446,6 @@ export class FlinkSqlParserVisitor<Result> extends AbstractParseTreeVisitor<Resu
      * @return the visitor result
      */
     visitColumnConstraint?: (ctx: ColumnConstraintContext) => Result;
-    /**
-     * Visit a parse tree produced by `FlinkSqlParser.commentSpec`.
-     * @param ctx the parse tree
-     * @return the visitor result
-     */
-    visitCommentSpec?: (ctx: CommentSpecContext) => Result;
     /**
      * Visit a parse tree produced by `FlinkSqlParser.metadataColumnDefinition`.
      * @param ctx the parse tree

src/parser/common/entityCollector.ts

@@ -1,8 +1,8 @@
-import { ParserRuleContext } from 'antlr4ng';
-import { EntityContextType } from './types';
-import { WordPosition, TextPosition } from './textAndWord';
-import { ctxToText, ctxToWord } from './textAndWord';
+import { ParserRuleContext, Token } from 'antlr4ng';
+
 import { SimpleStack } from './simpleStack';
+import { ctxToText, TextPosition, tokenToWord, WordPosition, WordRange } from './textAndWord';
+import { EntityContextType } from './types';
 
 /**
  * TODO: more stmt type should be supported.
@@ -58,28 +58,66 @@ const baseAlias: BaseAliasContext = {
     origin: null,
     alias: null,
 };
+/**
+ * @key comment 实体的comment统一同comment命名
+ * @key colType 实体中字段多一个type属性，如果直接用type命名，很容易在编译中重名，前后都会被加上下划线，所以用colType命名
+ * */
+export enum attrName {
+    comment = '_comment',
+    colType = '_colType',
+}
+
+export const attrNameInRule = {
+    [attrName.comment]: 'comment',
+    [attrName.colType]: 'colType',
+} as const;
 
+type ParserRuleContextWithAttr = ParserRuleContext & {
+    [k in attrName]?: Token;
+};
 export interface EntityContext extends BaseAliasContext {
     readonly entityContextType: EntityContextType;
     readonly text: string;
     readonly position: WordPosition;
     readonly belongStmt: StmtContext;
     relatedEntities: EntityContext[] | null;
-    columns: EntityContext[] | null;
+    columns: ColumnEntityContext[] | null;
+    comment?: WordRange;
+}
+
+export interface FuncEntityContext extends EntityContext {
+    argMode?: WordRange;
+    argName?: WordRange;
+    argType?: WordRange;
+}
+export interface ColumnEntityContext extends EntityContext {
+    colType?: WordRange;
+    comment?: WordRange;
+}
+
+interface attrInfo {
+    needCollectAttr: boolean;
+    attrList: attrName[];
+    endContext: string;
 }
 
 export function toEntityContext(
     ctx: ParserRuleContext,
     type: EntityContextType,
     input: string,
     belongStmt: StmtContext,
+    attrInfo?: attrInfo,
     alias?: BaseAliasContext
-): EntityContext | null {
-    const word = ctxToWord(ctx, input);
+): EntityContext | FuncEntityContext | ColumnEntityContext | null {
+    const word = ctxToText(ctx, input);
     if (!word) return null;
-    const { text, ...position } = word;
+    const { text, startLine, endLine, ...rest } = word;
+    const position = {
+        ...rest,
+        line: startLine,
+    };
     const finalAlias = Object.assign({}, baseAlias, alias ?? {});
-    return {
+    const extraInfo: ColumnEntityContext = {
         entityContextType: type,
         text,
         position,
@@ -88,8 +126,61 @@ export function toEntityContext(
         columns: null,
         ...finalAlias,
     };
+    if (attrInfo?.needCollectAttr) {
+        for (let k = 0; k < attrInfo?.attrList?.length; k++) {
+            const attributeName: attrName = attrInfo?.attrList[k];
+            const attrToken = findAttribute(ctx, attributeName, attrInfo?.endContext);
+            if (attrToken) {
+                const attrVal: WordRange = tokenToWord(attrToken, input);
+                extraInfo[attrNameInRule[attributeName]] = attrVal;
+            }
+        }
+    }
+    return extraInfo;
+}
+
+export function findAttribute(
+    ctx: ParserRuleContextWithAttr | null,
+    keyName: attrName,
+    endContextName: string
+): Token | null {
+    const parent: ParserRuleContextWithAttr | null = ctx?.parent || null;
+    let attrVal: Token | null = null;
+    if (parent?.[keyName]) {
+        attrVal = parent?.[keyName] || null;
+        return attrVal;
+    } else {
+        if (parent?.constructor?.name !== endContextName) {
+            attrVal = findAttribute(parent, keyName, endContextName);
+        }
+        if (!attrVal) {
+            if (parent?.children) {
+                attrVal = findAttributeChildren(parent, keyName);
+            }
+        }
+    }
+    return attrVal;
 }
 
+function findAttributeChildren(
+    ctx: ParserRuleContextWithAttr | null,
+    keyName: attrName
+): Token | null {
+    const visitChildren = ctx?.children || [];
+    let attrVal: Token | null = null;
+    if (visitChildren.length) {
+        for (let i = 0; i < visitChildren.length; i++) {
+            const child = <ParserRuleContextWithAttr | null>visitChildren[i] || null;
+            if (child?.[keyName]) {
+                attrVal = child?.[keyName] || null;
+                return attrVal;
+            } else {
+                attrVal = findAttributeChildren(child, keyName);
+            }
+        }
+    }
+    return attrVal;
+}
 /**
  * @todo: Handle alias, includes column alias, table alias, query as alias and so on.
  * @todo: [may be need] Combine the entities in each clause.
@@ -179,13 +270,15 @@ export abstract class EntityCollector {
     protected pushEntity(
         ctx: ParserRuleContext,
         type: EntityContextType,
+        attrInfo?: attrInfo,
         alias?: BaseAliasContext
     ) {
         const entityContext = toEntityContext(
             ctx,
             type,
             this._input,
             this._stmtStack.peek(),
+            attrInfo,
             alias
         );
         if (entityContext) {

src/parser/common/textAndWord.ts

@@ -40,7 +40,7 @@ export interface TextSlice extends TextPosition {
 /**
  * Convert Token to Word
  */
-export function tokenToWord(token: Token, input: string): WordPosition & { text: string } {
+export function tokenToWord(token: Token, input: string): WordRange {
     const startIndex = token.start;
     const endIndex = token.stop;
     const text = token.text ?? '';
@@ -54,29 +54,6 @@ export function tokenToWord(token: Token, input: string): WordPosition & { text:
     };
 }
 
-/**
- * Convert ParserRuleContext to Word
- */
-export function ctxToWord(
-    ctx: ParserRuleContext,
-    input: string
-): (WordPosition & { text: string }) | null {
-    if (!ctx.start || !ctx.stop) {
-        return null;
-    }
-    const startIndex = ctx.start.start;
-    const endIndex = ctx.stop.stop;
-    const text = input.slice(startIndex, endIndex + 1);
-    return {
-        text,
-        line: ctx.start.line,
-        startIndex,
-        endIndex,
-        startColumn: ctx.start.column + 1,
-        endColumn: ctx.stop.column + 1 + (ctx.stop.text?.length ?? 0),
-    };
-}
-
 /**
  * Convert ParserRuleContext to Text
  */