feat: collect entity's attribute(#333)

* feat(trinosql): collect trino sql's attribute(comment,alias,colType) * feat(hivesql): collect hive sql's attribute(comment,alias,colType) * feat(impalasql): collect attribute(comment, colType, alias) * feat(sparksql): collect entity's attribute (comment,alias, colType) * feat: update endContextList of collect attribute * feat(postgresql): collect hive sql's attribute(alias,colType) * feat: update interface of attrInfo and alter entitycollect ts file * feat(mysql): collect entity's attribute(comment,colType,alias) * ci: fix check-types problem --------- Co-authored-by: zhaoge <>
DTStack · Aug 27, 2024 · a3b6b7e · a3b6b7e
1 parent b823744
commit a3b6b7e
Show file tree

Hide file tree

Showing 39 changed files with 6,506 additions and 5,363 deletions.
diff --git a/src/grammar/hive/HiveSqlParser.g4 b/src/grammar/hive/HiveSqlParser.g4
@@ -887,7 +887,9 @@ tableConstraint
     ;
 
 columnNameTypeConstraint
-    : colName=columnNameCreate columnType columnConstraint? (KW_COMMENT comment=StringLiteral)?
+    : colName=columnNameCreate colType=columnType columnConstraint? (
+        KW_COMMENT comment=StringLiteral
+    )?
     ;
 
 columnConstraint
@@ -1363,10 +1365,12 @@ joinToken
     ;
 
 lateralView
-    : KW_LATERAL KW_VIEW KW_OUTER function_ tableAlias (KW_AS id_ (COMMA id_)*)?
+    : KW_LATERAL KW_VIEW KW_OUTER function_ alias=tableAlias (KW_AS id_ (COMMA id_)*)?
     | COMMA? KW_LATERAL (
-        KW_VIEW function_ tableAlias (KW_AS id_ (COMMA id_)*)?
-        | KW_TABLE LPAREN valuesClause RPAREN KW_AS? tableAlias (LPAREN id_ (COMMA id_)* RPAREN)?
+        KW_VIEW function_ alias=tableAlias (KW_AS id_ (COMMA id_)*)?
+        | KW_TABLE LPAREN valuesClause RPAREN KW_AS? alias=tableAlias (
+            LPAREN id_ (COMMA id_)* RPAREN
+        )?
     )
     ;
 

diff --git a/src/grammar/impala/ImpalaSqlParser.g4 b/src/grammar/impala/ImpalaSqlParser.g4
@@ -96,20 +96,20 @@ createKuduTableAsSelect
     : KW_CREATE KW_EXTERNAL? KW_TABLE ifNotExists? tableNameCreate (
         LPAREN kuduTableElement (COMMA kuduTableElement)* (COMMA KW_PRIMARY KW_KEY columnAliases)? RPAREN
     )? (KW_PRIMARY KW_KEY columnAliases?)? (KW_PARTITION KW_BY kuduPartitionClause)? (
-        KW_COMMENT stringLiteral
+        commentClause
     )? KW_STORED KW_AS KW_KUDU (KW_TBLPROPERTIES tblProp=properties)? (KW_AS queryStatement)?
     ;
 
 createView
-    : KW_CREATE KW_VIEW ifNotExists? viewNameCreate viewColumns? (KW_COMMENT stringLiteral)? (
+    : KW_CREATE KW_VIEW ifNotExists? viewNameCreate viewColumns? commentClause? (
         KW_TBLPROPERTIES tblProp=properties
     )? KW_AS queryStatement
     ;
 
 createSchema
-    : KW_CREATE (KW_SCHEMA | KW_DATABASE) ifNotExists? databaseNameCreate (
-        KW_COMMENT comment=stringLiteral
-    )? (KW_LOCATION location=stringLiteral)?
+    : KW_CREATE (KW_SCHEMA | KW_DATABASE) ifNotExists? databaseNameCreate (commentClause)? (
+        KW_LOCATION location=stringLiteral
+    )?
     ;
 
 createRole
@@ -119,14 +119,14 @@ createRole
 createAggregateFunction
     : KW_CREATE KW_AGGREGATE? KW_FUNCTION ifNotExists? functionNameCreate (
         LPAREN (type (COMMA type)*)? RPAREN
-    )? KW_RETURNS type (KW_INTERMEDIATE type)? KW_LOCATION STRING (KW_INIT_FN EQ STRING)? KW_UPDATE_FN EQ STRING KW_MERGE_FN EQ STRING (
+    )? KW_RETURNS returnType=type (KW_INTERMEDIATE type)? KW_LOCATION STRING (KW_INIT_FN EQ STRING)? KW_UPDATE_FN EQ STRING KW_MERGE_FN EQ STRING (
         KW_PREPARE_FN EQ STRING
     )? (KW_CLOSEFN EQ STRING)? (KW_SERIALIZE_FN EQ STRING)? (KW_FINALIZE_FN EQ STRING)?
     ;
 
 createFunction
     : KW_CREATE KW_FUNCTION ifNotExists? functionNameCreate (LPAREN (type (COMMA type)*)? RPAREN)? (
-        KW_RETURNS type
+        KW_RETURNS returnType=type
     )? KW_LOCATION STRING KW_SYMBOL EQ symbol=stringLiteral
     ;
 
@@ -569,11 +569,9 @@ tableOrViewPath
     ;
 
 createCommonItem
-    : (KW_SORT KW_BY columnAliases)? (KW_COMMENT comment=stringLiteral)? (
-        KW_ROW KW_FORMAT rowFormat
-    )? (KW_WITH KW_SERDEPROPERTIES serdProp=properties)? (KW_STORED KW_AS fileFormat)? (
-        KW_LOCATION location=stringLiteral
-    )? (
+    : (KW_SORT KW_BY columnAliases)? commentClause? (KW_ROW KW_FORMAT rowFormat)? (
+        KW_WITH KW_SERDEPROPERTIES serdProp=properties
+    )? (KW_STORED KW_AS fileFormat)? (KW_LOCATION location=stringLiteral)? (
         KW_CACHED KW_IN cacheName=qualifiedName (KW_WITH KW_REPLICATION EQ INTEGER_VALUE)?
         | KW_UNCACHED
     )? (KW_TBLPROPERTIES tblProp=properties)?
@@ -588,9 +586,11 @@ assignmentItem
     ;
 
 viewColumns
-    : LPAREN columnNamePathCreate (KW_COMMENT stringLiteral)? (
-        COMMA columnNamePathCreate (KW_COMMENT stringLiteral)?
-    )* RPAREN
+    : LPAREN viewColumnItem? (COMMA viewColumnItem?)* RPAREN
+    ;
+
+viewColumnItem
+    : columnNamePathCreate commentClause?
     ;
 
 queryStatement
@@ -621,19 +621,23 @@ columnSpec
     ;
 
 columnDefinition
-    : columnNamePathCreate type (KW_COMMENT stringLiteral)?
+    : columnNamePathCreate colType=type commentClause?
     ;
 
 kuduTableElement
     : kuduColumnDefinition
     ;
 
 kuduColumnDefinition
-    : columnNamePathCreate type (kuduAttributes kuduAttributes*?)? (KW_COMMENT stringLiteral)? (
+    : columnNamePathCreate colType=type (kuduAttributes kuduAttributes*?)? commentClause? (
         KW_PRIMARY KW_KEY
     )?
     ;
 
+commentClause
+    : KW_COMMENT comment=stringLiteral
+    ;
+
 columnSpecWithKudu
     : columnSpec (kuduAttributes kuduAttributes*?)?
     ;
@@ -838,7 +842,7 @@ sampleType
     ;
 
 aliasedRelation
-    : relationPrimary (KW_AS? identifier columnAliases?)?
+    : relationPrimary (KW_AS? alias=identifier columnAliases?)?
     ;
 
 columnAliases

diff --git a/src/grammar/mysql/MySqlParser.g4 b/src/grammar/mysql/MySqlParser.g4
@@ -442,7 +442,7 @@ functionParameter
     ;
 
 routineOption
-    : KW_COMMENT STRING_LITERAL                                                                  # routineComment
+    : KW_COMMENT comment=STRING_LITERAL                                                          # routineComment
     | KW_LANGUAGE KW_SQL                                                                         # routineLanguage
     | KW_NOT? KW_DETERMINISTIC                                                                   # routineBehavior
     | (KW_CONTAINS KW_SQL | KW_NO KW_SQL | KW_READS KW_SQL KW_DATA | KW_MODIFIES KW_SQL KW_DATA) # routineData
@@ -483,7 +483,7 @@ constraintSymbol
     ;
 
 columnDefinition
-    : dataType columnConstraint*
+    : colType=dataType columnConstraint*
     ;
 
 columnConstraint
@@ -494,7 +494,7 @@ columnConstraint
     | (KW_AUTO_INCREMENT | KW_ON KW_UPDATE currentTimestamp)                       # autoIncrementColumnConstraint
     | KW_PRIMARY? KW_KEY                                                           # primaryKeyColumnConstraint
     | KW_UNIQUE KW_KEY?                                                            # uniqueKeyColumnConstraint
-    | KW_COMMENT STRING_LITERAL                                                    # commentColumnConstraint
+    | KW_COMMENT comment=STRING_LITERAL                                            # commentColumnConstraint
     | KW_COLUMN_FORMAT colformat=(KW_FIXED | KW_DYNAMIC | KW_DEFAULT)              # formatColumnConstraint
     | KW_STORAGE storageval=(KW_DISK | KW_MEMORY | KW_DEFAULT)                     # storageColumnConstraint
     | referenceDefinition                                                          # referenceColumnConstraint
@@ -536,7 +536,7 @@ tableOption
     | KW_DEFAULT? charSet '='? (charsetName | KW_DEFAULT)              # tableOptionCharset
     | (KW_CHECKSUM | KW_PAGE_CHECKSUM) '='? boolValue=('0' | '1')      # tableOptionChecksum
     | KW_DEFAULT? KW_COLLATE '='? collationName                        # tableOptionCollate
-    | KW_COMMENT '='? STRING_LITERAL                                   # tableOptionComment
+    | KW_COMMENT '='? comment=STRING_LITERAL                           # tableOptionComment
     | KW_COMPRESSION '='? (STRING_LITERAL | ID)                        # tableOptionCompression
     | KW_CONNECTION '='? STRING_LITERAL                                # tableOptionConnection
     | (KW_DATA | KW_INDEX) KW_DIRECTORY '='? STRING_LITERAL            # tableOptionDataDirectory

diff --git a/src/grammar/postgresql/PostgreSqlParser.g4 b/src/grammar/postgresql/PostgreSqlParser.g4
@@ -620,7 +620,7 @@ typedtableelement
     ;
 
 column_def
-    : column_name_create typename create_generic_options? (
+    : column_name_create colType=typename create_generic_options? (
         KW_STORAGE (KW_PLAIN | KW_EXTERNAL | KW_EXTENDED | KW_MAIN | KW_DEFAULT | colid)
     )? (KW_COMPRESSION colid)? (opt_collate_clause)? (KW_WITH KW_OPTIONS)? colconstraint*
     ;
@@ -2163,12 +2163,12 @@ table_ref
     ;
 
 alias_clause
-    : KW_AS? colid (OPEN_PAREN name_list CLOSE_PAREN)?
+    : KW_AS? alias=colid (OPEN_PAREN name_list CLOSE_PAREN)?
     ;
 
 func_alias_clause
     : alias_clause
-    | (KW_AS colid? | colid) OPEN_PAREN tablefuncelementlist CLOSE_PAREN
+    | (KW_AS alias=colid? | alias=colid) OPEN_PAREN tablefuncelementlist CLOSE_PAREN
     ;
 
 join_type
@@ -2600,7 +2600,7 @@ func_arg_list
 
 func_arg_expr
     : expression
-    | type_function_name (COLON_EQUALS | EQUALS_GREATER) expression
+    | type_function_name ((COLON_EQUALS | EQUALS_GREATER) expression)?
     ;
 
 array_expr

diff --git a/src/grammar/spark/SparkSqlParser.g4 b/src/grammar/spark/SparkSqlParser.g4
@@ -85,7 +85,7 @@ statement
     | KW_ALTER (KW_TABLE tableName | KW_VIEW viewName) KW_SET KW_TBLPROPERTIES propertyList                                                   # setTableProperties
     | KW_ALTER (KW_TABLE tableName | KW_VIEW viewName) KW_UNSET KW_TBLPROPERTIES (ifExists)? propertyList                                     # unsetTableProperties
     | KW_ALTER KW_TABLE table=tableName (KW_ALTER | KW_CHANGE) KW_COLUMN? column=columnName alterColumnAction?                                # alterTableAlterColumn
-    | KW_ALTER KW_TABLE table=tableName partitionSpec? KW_CHANGE KW_COLUMN? colName=columnName colType colPosition?                           # hiveChangeColumn
+    | KW_ALTER KW_TABLE table=tableName partitionSpec? KW_CHANGE KW_COLUMN? colName=columnName columnType colPosition?                        # hiveChangeColumn
     | KW_ALTER KW_TABLE table=tableName partitionSpec? KW_REPLACE KW_COLUMNS LEFT_PAREN qualifiedColTypeWithPositionSeqForReplace RIGHT_PAREN #
         hiveReplaceColumns
     | KW_ALTER KW_TABLE tableName (partitionSpec)? KW_SET KW_SERDE stringLit (
@@ -153,8 +153,8 @@ statement
     | (KW_DESC | KW_DESCRIBE) KW_DATABASE KW_EXTENDED? namespaceName                                                   # describeNamespace
     | (KW_DESC | KW_DESCRIBE) KW_TABLE? option=(KW_EXTENDED | KW_FORMATTED)? tableName partitionSpec? describeColName? # describeRelation
     | (KW_DESC | KW_DESCRIBE) KW_QUERY? query                                                                          # describeQuery
-    | KW_COMMENT KW_ON namespace namespaceName KW_IS comment                                                           # commentNamespace
-    | KW_COMMENT KW_ON KW_TABLE tableName KW_IS comment                                                                # commentTable
+    | KW_COMMENT KW_ON namespace namespaceName KW_IS commentStr                                                        # commentNamespace
+    | KW_COMMENT KW_ON KW_TABLE tableName KW_IS commentStr                                                             # commentTable
     | KW_REFRESH KW_TABLE tableName                                                                                    # refreshTable
     | KW_REFRESH KW_FUNCTION functionName                                                                              # refreshFunction
     | KW_REFRESH (stringLit | .*?)                                                                                     # refreshResource
@@ -269,7 +269,7 @@ locationSpec
     ;
 
 commentSpec
-    : KW_COMMENT stringLit
+    : KW_COMMENT comment=stringLit
     ;
 
 query
@@ -851,7 +851,7 @@ functionTable
     ;
 
 tableAlias
-    : (KW_AS? strictIdentifier identifierList?)?
+    : (KW_AS? alias=strictIdentifier identifierList?)?
     ;
 
 rowFormat
@@ -905,7 +905,7 @@ partitionFieldList
 
 partitionField
     : transform
-    | colType
+    | columnType
     ;
 
 transform
@@ -1227,10 +1227,10 @@ variableDefaultExpression
     ;
 
 colTypeList
-    : colType (COMMA colType)*
+    : columnType (COMMA columnType)*
     ;
 
-colType
+columnType
     : colName=errorCapturingIdentifier dataType (KW_NOT KW_NULL)? commentSpec?
     ;
 
@@ -1239,7 +1239,7 @@ createOrReplaceTableColTypeList
     ;
 
 createOrReplaceTableColType
-    : colName=columnNameCreate dataType colDefinitionOption*
+    : colName=columnNameCreate colType=dataType colDefinitionOption*
     ;
 
 colDefinitionOption
@@ -1388,7 +1388,7 @@ stringLit
     | DOUBLEQUOTED_STRING
     ;
 
-comment
+commentStr
     : stringLit
     | KW_NULL
     ;

diff --git a/src/grammar/trino/TrinoSql.g4 b/src/grammar/trino/TrinoSql.g4
@@ -84,11 +84,11 @@ statement
     | KW_ALTER KW_SCHEMA schemaRef KW_RENAME KW_TO schemaNameCreate              # renameSchema
     | KW_ALTER KW_SCHEMA schemaRef KW_SET KW_AUTHORIZATION principal             # setSchemaAuthorization
     | KW_CREATE (KW_OR KW_REPLACE)? KW_TABLE (KW_IF KW_NOT KW_EXISTS)? tableNameCreate columnListCreate? (
-        KW_COMMENT string
+        KW_COMMENT comment=string
     )? (KW_WITH properties)? KW_AS (rootQuery | '(' rootQuery ')') (KW_WITH (KW_NO)? KW_DATA)? # createTableAsSelect
     | KW_CREATE (KW_OR KW_REPLACE)? KW_TABLE (KW_IF KW_NOT KW_EXISTS)? tableNameCreate '(' tableElement (
         ',' tableElement
-    )* ')' (KW_COMMENT string)? (KW_WITH properties)?                                                                                         # createTable
+    )* ')' (KW_COMMENT comment=string)? (KW_WITH properties)?                                                                                 # createTable
     | KW_DROP KW_TABLE (KW_IF KW_EXISTS)? tableRef                                                                                            # dropTable
     | KW_INSERT KW_INTO tableRef columnList? rootQuery                                                                                        # insertInto
     | KW_DELETE KW_FROM tableRef (KW_WHERE booleanExpression)?                                                                                # delete
@@ -110,8 +110,8 @@ statement
     | KW_ANALYZE tableRef (KW_WITH properties)? # analyze
     | KW_CREATE (KW_OR KW_REPLACE)? KW_MATERIALIZED KW_VIEW (KW_IF KW_NOT KW_EXISTS)? viewNameCreate (
         KW_GRACE KW_PERIOD interval
-    )? (KW_COMMENT string)? (KW_WITH properties)? KW_AS rootQuery # createMaterializedView
-    | KW_CREATE (KW_OR KW_REPLACE)? KW_VIEW viewNameCreate (KW_COMMENT string)? (
+    )? (KW_COMMENT comment=string)? (KW_WITH properties)? KW_AS rootQuery # createMaterializedView
+    | KW_CREATE (KW_OR KW_REPLACE)? KW_VIEW viewNameCreate (KW_COMMENT comment=string)? (
         KW_SECURITY (KW_DEFINER | KW_INVOKER)
     )? (KW_WITH properties)? KW_AS rootQuery                                                             # createView
     | KW_REFRESH KW_MATERIALIZED KW_VIEW viewRef                                                         # refreshMaterializedView
@@ -214,7 +214,7 @@ tableElement
     ;
 
 columnDefinition
-    : columnNameCreate type (KW_NOT KW_NULL)? (KW_COMMENT string)? (KW_WITH properties)?
+    : columnNameCreate colType=type (KW_NOT KW_NULL)? (KW_COMMENT comment=string)? (KW_WITH properties)?
     ;
 
 likeClause
@@ -330,7 +330,7 @@ setQuantifier
     ;
 
 selectItem
-    : (columnRef | expression) (KW_AS? identifier)?         # selectSingle
+    : (columnRef | expression) (KW_AS? alias=identifier)?   # selectSingle
     | primaryExpression '.' ASTERISK (KW_AS columnAliases)? # selectAll
     | ASTERISK                                              # selectAll
     ;
@@ -425,7 +425,7 @@ variableDefinition
     ;
 
 aliasedRelation
-    : relationPrimary (KW_AS? identifier columnAliases?)?
+    : relationPrimary (KW_AS? alias=identifier columnAliases?)?
     ;
 
 columnListCreate

diff --git a/src/lib/hive/HiveSqlParser.ts b/src/lib/hive/HiveSqlParser.ts
@@ -10836,7 +10836,7 @@ export class HiveSqlParser extends SQLParserBase {
             this.state = 2290;
             localContext._colName = this.columnNameCreate();
             this.state = 2291;
-            this.columnType();
+            localContext._colType = this.columnType();
             this.state = 2293;
             this.errorHandler.sync(this);
             _la = this.tokenStream.LA(1);
@@ -16816,7 +16816,7 @@ export class HiveSqlParser extends SQLParserBase {
                 this.state = 3480;
                 this.function_();
                 this.state = 3481;
-                this.tableAlias();
+                localContext._alias = this.tableAlias();
                 this.state = 3491;
                 this.errorHandler.sync(this);
                 _la = this.tokenStream.LA(1);
@@ -16874,7 +16874,7 @@ export class HiveSqlParser extends SQLParserBase {
                     this.state = 3498;
                     this.function_();
                     this.state = 3499;
-                    this.tableAlias();
+                    localContext._alias = this.tableAlias();
                     this.state = 3509;
                     this.errorHandler.sync(this);
                     _la = this.tokenStream.LA(1);
@@ -16928,7 +16928,7 @@ export class HiveSqlParser extends SQLParserBase {
                     }
 
                     this.state = 3518;
-                    this.tableAlias();
+                    localContext._alias = this.tableAlias();
                     this.state = 3530;
                     this.errorHandler.sync(this);
                     switch (this.interpreter.adaptivePredict(this.tokenStream, 458, this.context) ) {
@@ -34747,16 +34747,17 @@ export class TableConstraintContext extends antlr.ParserRuleContext {
 
 export class ColumnNameTypeConstraintContext extends antlr.ParserRuleContext {
     public _colName?: ColumnNameCreateContext;
+    public _colType?: ColumnTypeContext;
     public _comment?: Token | null;
     public constructor(parent: antlr.ParserRuleContext | null, invokingState: number) {
         super(parent, invokingState);
     }
-    public columnType(): ColumnTypeContext {
-        return this.getRuleContext(0, ColumnTypeContext)!;
-    }
     public columnNameCreate(): ColumnNameCreateContext {
         return this.getRuleContext(0, ColumnNameCreateContext)!;
     }
+    public columnType(): ColumnTypeContext {
+        return this.getRuleContext(0, ColumnTypeContext)!;
+    }
     public columnConstraint(): ColumnConstraintContext | null {
         return this.getRuleContext(0, ColumnConstraintContext);
     }
@@ -38744,6 +38745,7 @@ export class JoinTokenContext extends antlr.ParserRuleContext {
 
 
 export class LateralViewContext extends antlr.ParserRuleContext {
+    public _alias?: TableAliasContext;
     public constructor(parent: antlr.ParserRuleContext | null, invokingState: number) {
         super(parent, invokingState);
     }