Skip to content

Commit 36803b7

Browse files
author
zhaoge
committed
feat(flinksql): collect comment, type attribute for entity
1 parent 5d97aea commit 36803b7

File tree

9 files changed

+1724
-1659
lines changed

9 files changed

+1724
-1659
lines changed

‎src/grammar/flink/FlinkSqlParser.g4

Lines changed: 18 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -156,8 +156,9 @@ createTable
156156
simpleCreateTable
157157
: KW_CREATE KW_TEMPORARY? KW_TABLE ifNotExists? tablePathCreate LR_BRACKET columnOptionDefinition (
158158
COMMA columnOptionDefinition
159-
)* (COMMA watermarkDefinition)? (COMMA tableConstraint)? (COMMA selfDefinitionClause)? RR_BRACKET commentSpec? partitionDefinition? withOption
160-
likeDefinition?
159+
)* (COMMA watermarkDefinition)? (COMMA tableConstraint)? (COMMA selfDefinitionClause)? RR_BRACKET (
160+
KW_COMMENT comment=STRING_LITERAL
161+
)? partitionDefinition? withOption likeDefinition?
161162
;
162163

163164
/*
@@ -175,7 +176,7 @@ columnOptionDefinition
175176
;
176177

177178
physicalColumnDefinition
178-
: columnNameCreate columnType columnConstraint? commentSpec?
179+
: colName=columnNameCreate columnType columnConstraint? (KW_COMMENT comment=STRING_LITERAL)?
179180
;
180181

181182
columnNameCreate
@@ -193,8 +194,8 @@ columnNameList
193194
;
194195

195196
columnType
196-
: typeName=(KW_DATE | KW_BOOLEAN | KW_NULL)
197-
| typeName=(
197+
: colType=(KW_DATE | KW_BOOLEAN | KW_NULL)
198+
| colType=(
198199
KW_CHAR
199200
| KW_VARCHAR
200201
| KW_STRING
@@ -210,12 +211,12 @@ columnType
210211
| KW_TIMESTAMP_LTZ
211212
| KW_DATETIME
212213
) lengthOneDimension?
213-
| typeName=KW_TIMESTAMP lengthOneDimension? ((KW_WITHOUT | KW_WITH) KW_LOCAL? KW_TIME KW_ZONE)?
214-
| typeName=(KW_DECIMAL | KW_DEC | KW_NUMERIC | KW_FLOAT | KW_DOUBLE) lengthTwoOptionalDimension?
215-
| type=(KW_ARRAY | KW_MULTISET) lengthOneTypeDimension?
216-
| type=KW_MAP mapTypeDimension?
217-
| type=KW_ROW rowTypeDimension?
218-
| type=KW_RAW lengthTwoStringDimension?
214+
| colType=KW_TIMESTAMP lengthOneDimension? ((KW_WITHOUT | KW_WITH) KW_LOCAL? KW_TIME KW_ZONE)?
215+
| colType=(KW_DECIMAL | KW_DEC | KW_NUMERIC | KW_FLOAT | KW_DOUBLE) lengthTwoOptionalDimension?
216+
| colType=(KW_ARRAY | KW_MULTISET) lengthOneTypeDimension?
217+
| colType=KW_MAP mapTypeDimension?
218+
| colType=KW_ROW rowTypeDimension?
219+
| colType=KW_RAW lengthTwoStringDimension?
219220
;
220221

221222
lengthOneDimension
@@ -247,20 +248,16 @@ columnConstraint
247248
| KW_NOT? KW_NULL
248249
;
249250

250-
commentSpec
251-
: KW_COMMENT STRING_LITERAL
252-
;
253-
254251
metadataColumnDefinition
255-
: columnNameCreate columnType KW_METADATA (KW_FROM metadataKey)? KW_VIRTUAL?
252+
: colName=columnNameCreate columnType KW_METADATA (KW_FROM metadataKey)? KW_VIRTUAL?
256253
;
257254

258255
metadataKey
259256
: STRING_LITERAL
260257
;
261258

262259
computedColumnDefinition
263-
: columnNameCreate KW_AS computedColumnExpression commentSpec?
260+
: colName=columnNameCreate KW_AS computedColumnExpression (KW_COMMENT comment=STRING_LITERAL)?
264261
;
265262

266263
// 计算表达式
@@ -316,11 +313,13 @@ createCatalog
316313
;
317314

318315
createDatabase
319-
: KW_CREATE KW_DATABASE ifNotExists? databasePathCreate commentSpec? withOption
316+
: KW_CREATE KW_DATABASE ifNotExists? databasePathCreate (KW_COMMENT comment=STRING_LITERAL)? withOption
320317
;
321318

322319
createView
323-
: KW_CREATE KW_TEMPORARY? KW_VIEW ifNotExists? viewPathCreate columnNameList? commentSpec? KW_AS queryStatement
320+
: KW_CREATE KW_TEMPORARY? KW_VIEW ifNotExists? viewPathCreate columnNameList? (
321+
KW_COMMENT comment=STRING_LITERAL
322+
)? KW_AS queryStatement
324323
;
325324

326325
createFunction

‎src/lib/flink/FlinkSqlParser.interp

Lines changed: 1 addition & 2 deletions
Large diffs are not rendered by default.

‎src/lib/flink/FlinkSqlParser.ts

Lines changed: 1556 additions & 1583 deletions
Large diffs are not rendered by default.

‎src/lib/flink/FlinkSqlParserListener.ts

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@ import { LengthOneTypeDimensionContext } from "./FlinkSqlParser.js";
4444
import { MapTypeDimensionContext } from "./FlinkSqlParser.js";
4545
import { RowTypeDimensionContext } from "./FlinkSqlParser.js";
4646
import { ColumnConstraintContext } from "./FlinkSqlParser.js";
47-
import { CommentSpecContext } from "./FlinkSqlParser.js";
4847
import { MetadataColumnDefinitionContext } from "./FlinkSqlParser.js";
4948
import { MetadataKeyContext } from "./FlinkSqlParser.js";
5049
import { ComputedColumnDefinitionContext } from "./FlinkSqlParser.js";
@@ -588,16 +587,6 @@ export class FlinkSqlParserListener implements ParseTreeListener {
588587
* @param ctx the parse tree
589588
*/
590589
exitColumnConstraint?: (ctx: ColumnConstraintContext) => void;
591-
/**
592-
* Enter a parse tree produced by `FlinkSqlParser.commentSpec`.
593-
* @param ctx the parse tree
594-
*/
595-
enterCommentSpec?: (ctx: CommentSpecContext) => void;
596-
/**
597-
* Exit a parse tree produced by `FlinkSqlParser.commentSpec`.
598-
* @param ctx the parse tree
599-
*/
600-
exitCommentSpec?: (ctx: CommentSpecContext) => void;
601590
/**
602591
* Enter a parse tree produced by `FlinkSqlParser.metadataColumnDefinition`.
603592
* @param ctx the parse tree

‎src/lib/flink/FlinkSqlParserVisitor.ts

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@ import { LengthOneTypeDimensionContext } from "./FlinkSqlParser.js";
4444
import { MapTypeDimensionContext } from "./FlinkSqlParser.js";
4545
import { RowTypeDimensionContext } from "./FlinkSqlParser.js";
4646
import { ColumnConstraintContext } from "./FlinkSqlParser.js";
47-
import { CommentSpecContext } from "./FlinkSqlParser.js";
4847
import { MetadataColumnDefinitionContext } from "./FlinkSqlParser.js";
4948
import { MetadataKeyContext } from "./FlinkSqlParser.js";
5049
import { ComputedColumnDefinitionContext } from "./FlinkSqlParser.js";
@@ -447,12 +446,6 @@ export class FlinkSqlParserVisitor<Result> extends AbstractParseTreeVisitor<Resu
447446
* @return the visitor result
448447
*/
449448
visitColumnConstraint?: (ctx: ColumnConstraintContext) => Result;
450-
/**
451-
* Visit a parse tree produced by `FlinkSqlParser.commentSpec`.
452-
* @param ctx the parse tree
453-
* @return the visitor result
454-
*/
455-
visitCommentSpec?: (ctx: CommentSpecContext) => Result;
456449
/**
457450
* Visit a parse tree produced by `FlinkSqlParser.metadataColumnDefinition`.
458451
* @param ctx the parse tree

‎src/parser/common/entityCollector.ts

Lines changed: 96 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
import { ParserRuleContext } from 'antlr4ng';
2-
import { EntityContextType } from './types';
3-
import { WordPosition, TextPosition } from './textAndWord';
4-
import { ctxToText, ctxToWord } from './textAndWord';
1+
import { ParserRuleContext, Token } from 'antlr4ng';
2+
53
import { SimpleStack } from './simpleStack';
4+
import { ctxToText, ctxToWord, TextPosition, tokenToWord, WordPosition } from './textAndWord';
5+
import { EntityContextType } from './types';
66

77
/**
88
* TODO: more stmt type should be supported.
@@ -58,28 +58,62 @@ const baseAlias: BaseAliasContext = {
5858
origin: null,
5959
alias: null,
6060
};
61+
/**
62+
* @key comment 实体的comment统一同comment命名
63+
* @key colType 实体中字段多一个type属性，如果直接用type命名，很容易在编译中重名，前后都会被加上下划线，所以用colType命名
64+
* */
65+
export enum attrName {
66+
comment = '_comment',
67+
colType = '_colType',
68+
}
69+
70+
export const attrNameInRule = {
71+
[attrName.comment]: 'comment',
72+
[attrName.colType]: 'colType',
73+
} as const;
6174

75+
type ParserRuleContextWithAttr = ParserRuleContext & {
76+
[k in attrName]?: Token;
77+
};
6278
export interface EntityContext extends BaseAliasContext {
6379
readonly entityContextType: EntityContextType;
6480
readonly text: string;
6581
readonly position: WordPosition;
6682
readonly belongStmt: StmtContext;
6783
relatedEntities: EntityContext[] | null;
68-
columns: EntityContext[] | null;
84+
columns: ColumnEntityContext[] | null;
85+
comment?: string;
86+
}
87+
88+
export interface FuncEntityContext extends EntityContext {
89+
argMode?: string;
90+
argName?: string;
91+
argType?: string;
92+
}
93+
export interface ColumnEntityContext extends EntityContext {
94+
colType?: string;
95+
comment?: string;
96+
}
97+
98+
interface attrInfo {
99+
needCollectAttr: boolean;
100+
attrList: attrName[];
101+
endContext: string;
69102
}
70103

71104
export function toEntityContext(
72105
ctx: ParserRuleContext,
73106
type: EntityContextType,
74107
input: string,
75108
belongStmt: StmtContext,
109+
attrInfo?: attrInfo,
76110
alias?: BaseAliasContext
77-
): EntityContext | null {
111+
): EntityContext | FuncEntityContext | ColumnEntityContext | null {
78112
const word = ctxToWord(ctx, input);
79113
if (!word) return null;
80114
const { text, ...position } = word;
81115
const finalAlias = Object.assign({}, baseAlias, alias ?? {});
82-
return {
116+
const extraInfo: ColumnEntityContext = {
83117
entityContextType: type,
84118
text,
85119
position,
@@ -88,8 +122,61 @@ export function toEntityContext(
88122
columns: null,
89123
...finalAlias,
90124
};
125+
if (attrInfo?.needCollectAttr) {
126+
for (let k = 0; k < attrInfo?.attrList?.length; k++) {
127+
const attributeName: attrName = attrInfo?.attrList[k];
128+
const attrToken = findAttribute(ctx, attributeName, attrInfo?.endContext);
129+
if (attrToken) {
130+
const attrVal: string = tokenToWord(attrToken, input)?.text;
131+
extraInfo[attrNameInRule[attributeName]] = attrVal;
132+
}
133+
}
134+
}
135+
return extraInfo;
91136
}
92137

138+
export function findAttribute(
139+
ctx: ParserRuleContextWithAttr | null,
140+
keyName: attrName,
141+
endContextName: string
142+
): Token | null {
143+
const parent: ParserRuleContextWithAttr | null = ctx?.parent || null;
144+
let attrVal: Token | null = null;
145+
if (parent?.[keyName]) {
146+
attrVal = parent?.[keyName] || null;
147+
return attrVal;
148+
} else {
149+
if (parent?.constructor?.name !== endContextName) {
150+
attrVal = findAttribute(parent, keyName, endContextName);
151+
}
152+
if (!attrVal) {
153+
if (parent?.children) {
154+
attrVal = findAttributeChildren(parent, keyName);
155+
}
156+
}
157+
}
158+
return attrVal;
159+
}
160+
161+
function findAttributeChildren(
162+
ctx: ParserRuleContextWithAttr | null,
163+
keyName: attrName
164+
): Token | null {
165+
const visitChildren = ctx?.children || [];
166+
let attrVal: Token | null = null;
167+
if (visitChildren.length) {
168+
for (let i = 0; i < visitChildren.length; i++) {
169+
const child = <ParserRuleContextWithAttr | null>visitChildren[i] || null;
170+
if (child?.[keyName]) {
171+
attrVal = child?.[keyName] || null;
172+
return attrVal;
173+
} else {
174+
attrVal = findAttributeChildren(child, keyName);
175+
}
176+
}
177+
}
178+
return attrVal;
179+
}
93180
/**
94181
* @todo: Handle alias, includes column alias, table alias, query as alias and so on.
95182
* @todo: [may be need] Combine the entities in each clause.
@@ -179,13 +266,15 @@ export abstract class EntityCollector {
179266
protected pushEntity(
180267
ctx: ParserRuleContext,
181268
type: EntityContextType,
269+
attrInfo?: attrInfo,
182270
alias?: BaseAliasContext
183271
) {
184272
const entityContext = toEntityContext(
185273
ctx,
186274
type,
187275
this._input,
188276
this._stmtStack.peek(),
277+
attrInfo,
189278
alias
190279
);
191280
if (entityContext) {

‎src/parser/flink/flinkEntityCollector.ts

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import { EntityContextType } from '../common/types';
21
import {
32
CatalogPathContext,
43
CatalogPathCreateContext,
@@ -20,7 +19,8 @@ import {
2019
ViewPathCreateContext,
2120
} from '../../lib/flink/FlinkSqlParser';
2221
import { FlinkSqlParserListener } from '../../lib/flink/FlinkSqlParserListener';
23-
import { StmtContextType, EntityCollector } from '../common/entityCollector';
22+
import { attrName, EntityCollector, StmtContextType } from '../common/entityCollector';
23+
import { EntityContextType } from '../common/types';
2424

2525
export class FlinkEntityCollector extends EntityCollector implements FlinkSqlParserListener {
2626
/** ====== Entity Begin */
@@ -33,7 +33,11 @@ export class FlinkEntityCollector extends EntityCollector implements FlinkSqlPar
3333
}
3434

3535
exitDatabasePathCreate(ctx: DatabasePathCreateContext) {
36-
this.pushEntity(ctx, EntityContextType.DATABASE_CREATE);
36+
this.pushEntity(ctx, EntityContextType.DATABASE_CREATE, {
37+
needCollectAttr: true,
38+
attrList: [attrName.comment],
39+
endContext: 'CreateDatabaseContext',
40+
});
3741
}
3842

3943
exitDatabasePath(ctx: DatabasePathContext) {
@@ -45,19 +49,31 @@ export class FlinkEntityCollector extends EntityCollector implements FlinkSqlPar
4549
}
4650

4751
exitTablePathCreate(ctx: TablePathCreateContext) {
48-
this.pushEntity(ctx, EntityContextType.TABLE_CREATE);
52+
this.pushEntity(ctx, EntityContextType.TABLE_CREATE, {
53+
needCollectAttr: true,
54+
attrList: [attrName.comment],
55+
endContext: 'CreateTableContext',
56+
});
4957
}
5058

5159
exitViewPath(ctx: ViewPathContext) {
5260
this.pushEntity(ctx, EntityContextType.VIEW);
5361
}
5462

5563
exitViewPathCreate(ctx: ViewPathCreateContext) {
56-
this.pushEntity(ctx, EntityContextType.VIEW_CREATE);
64+
this.pushEntity(ctx, EntityContextType.VIEW_CREATE, {
65+
needCollectAttr: true,
66+
attrList: [attrName.comment],
67+
endContext: 'CreateViewContext',
68+
});
5769
}
5870

5971
exitColumnNameCreate(ctx: ColumnNameCreateContext) {
60-
this.pushEntity(ctx, EntityContextType.COLUMN_CREATE);
72+
this.pushEntity(ctx, EntityContextType.COLUMN_CREATE, {
73+
needCollectAttr: true,
74+
attrList: [attrName.comment, attrName.colType],
75+
endContext: 'PhysicalColumnDefinitionContext',
76+
});
6177
}
6278

6379
exitFunctionNameCreate(ctx: FunctionNameCreateContext) {

0 commit comments

Comments
 (0)