Skip to content

Commit 91b7fd2

Browse files
HaydenOrzHaydenOrz
authored
Feat/trino sql (#188)
* refactor: prefix trino lexer rule name with KW_ * test: add commentOtherLine function * feat: optimize trino antlr grammar to adapt to c3 * feat: trinosqlParser supports codeCompletion and spliting * test: trinoSql codeCompletion unit tests
1 parent 12864ee commit 91b7fd2

File tree

16 files changed

+15665
-13959
lines changed

16 files changed

+15665
-13959
lines changed

‎src/grammar/trinosql/TrinoSql.g4

Lines changed: 711 additions & 673 deletions
Large diffs are not rendered by default.

‎src/lib/trinosql/TrinoSql.interp

Lines changed: 247 additions & 234 deletions
Large diffs are not rendered by default.

‎src/lib/trinosql/TrinoSql.tokens

Lines changed: 236 additions & 236 deletions
Large diffs are not rendered by default.

‎src/lib/trinosql/TrinoSqlLexer.interp

Lines changed: 466 additions & 466 deletions
Large diffs are not rendered by default.

‎src/lib/trinosql/TrinoSqlLexer.tokens

Lines changed: 236 additions & 236 deletions
Large diffs are not rendered by default.

‎src/lib/trinosql/TrinoSqlLexer.ts

Lines changed: 328 additions & 312 deletions
Large diffs are not rendered by default.

‎src/lib/trinosql/TrinoSqlListener.ts

Lines changed: 1083 additions & 927 deletions
Large diffs are not rendered by default.

‎src/lib/trinosql/TrinoSqlParser.ts

Lines changed: 11298 additions & 10355 deletions
Large diffs are not rendered by default.

‎src/lib/trinosql/TrinoSqlVisitor.ts

Lines changed: 616 additions & 512 deletions
Large diffs are not rendered by default.

‎src/parser/trinosql.ts

Lines changed: 99 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
import { Token } from 'antlr4ts';
22
import { CandidatesCollection } from 'antlr4-c3';
33
import { TrinoSqlLexer } from '../lib/trinosql/TrinoSqlLexer';
4-
import { TrinoSqlParser, ProgramContext } from '../lib/trinosql/TrinoSqlParser';
4+
import { TrinoSqlParser, ProgramContext, StatementContext } from '../lib/trinosql/TrinoSqlParser';
5+
import { TrinoSqlListener } from '../lib/trinosql/TrinoSqlListener';
56
import BasicParser from './common/basicParser';
6-
import { Suggestions } from './common/basic-parser-types';
7+
import { Suggestions, SyntaxContextType, SyntaxSuggestion } from './common/basic-parser-types';
78

89
export default class TrinoSQL extends BasicParser<TrinoSqlLexer, ProgramContext, TrinoSqlParser> {
910
protected createLexerFormCharStream(charStreams) {
@@ -17,19 +18,110 @@ export default class TrinoSQL extends BasicParser<TrinoSqlLexer, ProgramContext,
1718
}
1819

1920
protected get splitListener() {
20-
return null as any;
21+
return new TrinoSqlSplitListener();
2122
}
2223

23-
protected preferredRules: Set<number> = new Set();
24+
protected preferredRules: Set<number> = new Set([
25+
TrinoSqlParser.RULE_catalogName,
26+
TrinoSqlParser.RULE_catalogNameCreate,
27+
TrinoSqlParser.RULE_schemaName,
28+
TrinoSqlParser.RULE_schemaNameCreate,
29+
TrinoSqlParser.RULE_tableName,
30+
TrinoSqlParser.RULE_tableNameCreate,
31+
TrinoSqlParser.RULE_viewName,
32+
TrinoSqlParser.RULE_viewNameCreate,
33+
TrinoSqlParser.RULE_functionName,
34+
]);
2435

2536
protected processCandidates(
2637
candidates: CandidatesCollection,
2738
allTokens: Token[],
28-
caretTokenIndex: number
39+
caretTokenIndex: number,
40+
tokenIndexOffset: number
2941
): Suggestions<Token> {
42+
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
43+
const keywords: string[] = [];
44+
45+
for (let candidate of candidates.rules) {
46+
const [ruleType, candidateRule] = candidate;
47+
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
48+
const tokenRanges = allTokens.slice(
49+
startTokenIndex,
50+
caretTokenIndex + tokenIndexOffset + 1
51+
);
52+
53+
let syntaxContextType: SyntaxContextType;
54+
switch (ruleType) {
55+
case TrinoSqlParser.RULE_catalogName: {
56+
syntaxContextType = SyntaxContextType.CATALOG;
57+
break;
58+
}
59+
case TrinoSqlParser.RULE_schemaName: {
60+
syntaxContextType = SyntaxContextType.DATABASE;
61+
break;
62+
}
63+
case TrinoSqlParser.RULE_schemaNameCreate: {
64+
syntaxContextType = SyntaxContextType.DATABASE_CREATE;
65+
break;
66+
}
67+
case TrinoSqlParser.RULE_tableName: {
68+
syntaxContextType = SyntaxContextType.TABLE;
69+
break;
70+
}
71+
case TrinoSqlParser.RULE_tableNameCreate: {
72+
syntaxContextType = SyntaxContextType.TABLE_CREATE;
73+
break;
74+
}
75+
case TrinoSqlParser.RULE_viewName: {
76+
syntaxContextType = SyntaxContextType.VIEW;
77+
break;
78+
}
79+
case TrinoSqlParser.RULE_viewNameCreate: {
80+
syntaxContextType = SyntaxContextType.VIEW_CREATE;
81+
break;
82+
}
83+
case TrinoSqlParser.RULE_functionName: {
84+
syntaxContextType = SyntaxContextType.FUNCTION;
85+
break;
86+
}
87+
default:
88+
break;
89+
}
90+
91+
if (syntaxContextType) {
92+
originalSyntaxSuggestions.push({
93+
syntaxContextType,
94+
wordRanges: tokenRanges,
95+
});
96+
}
97+
}
98+
99+
for (let candidate of candidates.tokens) {
100+
const symbolicName = this._parser.vocabulary.getSymbolicName(candidate[0]);
101+
const displayName = this._parser.vocabulary.getDisplayName(candidate[0]);
102+
if (symbolicName && symbolicName.startsWith('KW_')) {
103+
const keyword =
104+
displayName.startsWith("'") && displayName.endsWith("'")
105+
? displayName.slice(1, -1)
106+
: displayName;
107+
keywords.push(keyword);
108+
}
109+
}
30110
return {
31-
syntax: [],
32-
keywords: [],
111+
syntax: originalSyntaxSuggestions,
112+
keywords,
33113
};
34114
}
35115
}
116+
117+
export class TrinoSqlSplitListener implements TrinoSqlListener {
118+
private _statementsContext: StatementContext[] = [];
119+
120+
exitStatement = (ctx: StatementContext) => {
121+
this._statementsContext.push(ctx);
122+
};
123+
124+
get statementsContext() {
125+
return this._statementsContext;
126+
}
127+
}

‎test/helper.ts

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,3 +68,15 @@ export function getReportTableHeader(title: string) {
6868
export function exportReportTable(markdown: string, output: string) {
6969
fs.writeFileSync(path.join(output, 'benchmark.md'), markdown);
7070
}
71+
72+
export function commentOtherLine(sqlContent: string, line: number) {
73+
const slices = sqlContent.split('\n').map((item, index) => {
74+
if (index !== line - 1) {
75+
return '-- ' + item;
76+
} else {
77+
return item;
78+
}
79+
});
80+
81+
return slices.join('\n');
82+
}

‎test/parser/flinksql/suggestion/syntaxSuggestion.test.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ describe('Flink SQL Syntax Suggestion', () => {
5555
const suggestion = syntaxes?.find(
5656
(syn) => syn.syntaxContextType === SyntaxContextType.TABLE
5757
);
58-
console.log(syntaxes);
58+
5959
expect(suggestion).not.toBeUndefined();
6060
expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['cat', '.']);
6161
});

‎test/parser/trinosql/suggestion/fixtures/syntaxSuggestion.sql

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
INSERT INTO db.tb ;
2+
3+
SELECT ids FROM db.;
4+
5+
CREATE TABLE db. VALUES;
6+
7+
DROP TABLE IF EXISTS db.a;
8+
9+
CREATE OR REPLACE VIEW db.v;
10+
11+
DROP VIEW db.v ;
12+
13+
SELECT name, calculate_age(birthday) AS age FROM students;
14+
15+
CREATE SCHEMA db ;
16+
17+
DROP SCHEMA IF EXISTS sch;
18+
19+
SHOW COLUMNS FROM tb ;

‎test/parser/trinosql/suggestion/fixtures/tokenSuggestion.sql

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
ALTER ;
2+
3+
CREATE ;
4+
5+
DEALLOCATE ;
6+
7+
DELETE ;
8+
9+
DESCRIBE ;
10+
11+
DROP ;
12+
13+
INSERT ;

‎test/parser/trinosql/suggestion/syntaxSuggestion.test.ts

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
import fs from 'fs';
2+
import path from 'path';
3+
import { CaretPosition, SyntaxContextType } from '../../../../src/parser/common/basic-parser-types';
4+
import TrinoSQL from '../../../../src/parser/trinosql';
5+
import { commentOtherLine } from '../../../helper';
6+
7+
const syntaxSql = fs.readFileSync(
8+
path.join(__dirname, 'fixtures', 'syntaxSuggestion.sql'),
9+
'utf-8'
10+
);
11+
12+
describe('Trino SQL Syntax Suggestion', () => {
13+
const parser = new TrinoSQL();
14+
15+
test('Validate Syntax SQL', () => {
16+
expect(parser.validate(syntaxSql).length).not.toBe(0);
17+
expect(parser.validate(syntaxSql).length).not.toBe(0);
18+
expect(parser.validate(syntaxSql).length).not.toBe(0);
19+
});
20+
21+
test('Insert table ', () => {
22+
const pos: CaretPosition = {
23+
lineNumber: 1,
24+
column: 18,
25+
};
26+
const syntaxes = parser.getSuggestionAtCaretPosition(
27+
commentOtherLine(syntaxSql, pos.lineNumber),
28+
pos
29+
)?.syntax;
30+
31+
const suggestion = syntaxes?.find(
32+
(syn) => syn.syntaxContextType === SyntaxContextType.TABLE
33+
);
34+
expect(suggestion).not.toBeUndefined();
35+
expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.', 'tb']);
36+
});
37+
38+
test('Select table ', () => {
39+
const pos: CaretPosition = {
40+
lineNumber: 3,
41+
column: 20,
42+
};
43+
const syntaxes =
44+
parser.getSuggestionAtCaretPosition(commentOtherLine(syntaxSql, pos.lineNumber), pos)
45+
?.syntax ?? [];
46+
47+
const suggestion = syntaxes?.find(
48+
(syn) => syn.syntaxContextType === SyntaxContextType.TABLE
49+
);
50+
51+
expect(
52+
syntaxes.some((item) => item.syntaxContextType === SyntaxContextType.VIEW)
53+
).toBeTruthy();
54+
expect(
55+
syntaxes.some((item) => item.syntaxContextType === SyntaxContextType.FUNCTION)
56+
).toBeTruthy();
57+
expect(suggestion).not.toBeUndefined();
58+
expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
59+
});
60+
61+
test('Create table ', () => {
62+
const pos: CaretPosition = {
63+
lineNumber: 5,
64+
column: 17,
65+
};
66+
const syntaxes = parser.getSuggestionAtCaretPosition(
67+
commentOtherLine(syntaxSql, pos.lineNumber),
68+
pos
69+
)?.syntax;
70+
const suggestion = syntaxes?.find(
71+
(syn) => syn.syntaxContextType === SyntaxContextType.TABLE_CREATE
72+
);
73+
74+
expect(suggestion).not.toBeUndefined();
75+
expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
76+
});
77+
78+
test('DROP table ', () => {
79+
const pos: CaretPosition = {
80+
lineNumber: 7,
81+
column: 26,
82+
};
83+
const syntaxes = parser.getSuggestionAtCaretPosition(
84+
commentOtherLine(syntaxSql, pos.lineNumber),
85+
pos
86+
)?.syntax;
87+
const suggestion = syntaxes?.find(
88+
(syn) => syn.syntaxContextType === SyntaxContextType.TABLE
89+
);
90+
91+
expect(suggestion).not.toBeUndefined();
92+
expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.', 'a']);
93+
});
94+
95+
test('Create view ', () => {
96+
const pos: CaretPosition = {
97+
lineNumber: 9,
98+
column: 28,
99+
};
100+
const syntaxes = parser.getSuggestionAtCaretPosition(
101+
commentOtherLine(syntaxSql, pos.lineNumber),
102+
pos
103+
)?.syntax;
104+
const suggestion = syntaxes?.find(
105+
(syn) => syn.syntaxContextType === SyntaxContextType.VIEW_CREATE
106+
);
107+
108+
expect(suggestion).not.toBeUndefined();
109+
expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.', 'v']);
110+
});
111+
112+
test('Drop view ', () => {
113+
const pos: CaretPosition = {
114+
lineNumber: 11,
115+
column: 15,
116+
};
117+
const syntaxes = parser.getSuggestionAtCaretPosition(
118+
commentOtherLine(syntaxSql, pos.lineNumber),
119+
pos
120+
)?.syntax;
121+
const suggestion = syntaxes?.find(
122+
(syn) => syn.syntaxContextType === SyntaxContextType.VIEW
123+
);
124+
125+
expect(suggestion).not.toBeUndefined();
126+
expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.', 'v']);
127+
});
128+
129+
test('Use function', () => {
130+
const pos: CaretPosition = {
131+
lineNumber: 13,
132+
column: 27,
133+
};
134+
const syntaxes = parser.getSuggestionAtCaretPosition(
135+
commentOtherLine(syntaxSql, pos.lineNumber),
136+
pos
137+
)?.syntax;
138+
const suggestion = syntaxes?.find(
139+
(syn) => syn.syntaxContextType === SyntaxContextType.FUNCTION
140+
);
141+
142+
expect(suggestion).not.toBeUndefined();
143+
expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['calculate_age']);
144+
});
145+
146+
test('Create schema', () => {
147+
const pos: CaretPosition = {
148+
lineNumber: 15,
149+
column: 17,
150+
};
151+
const syntaxes = parser.getSuggestionAtCaretPosition(
152+
commentOtherLine(syntaxSql, pos.lineNumber),
153+
pos
154+
)?.syntax;
155+
const suggestion = syntaxes?.find(
156+
(syn) => syn.syntaxContextType === SyntaxContextType.DATABASE_CREATE
157+
);
158+
159+
expect(suggestion).not.toBeUndefined();
160+
expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db']);
161+
});
162+
163+
test('Drop schema', () => {
164+
const pos: CaretPosition = {
165+
lineNumber: 17,
166+
column: 26,
167+
};
168+
const syntaxes = parser.getSuggestionAtCaretPosition(
169+
commentOtherLine(syntaxSql, pos.lineNumber),
170+
pos
171+
)?.syntax;
172+
const suggestion = syntaxes?.find(
173+
(syn) => syn.syntaxContextType === SyntaxContextType.DATABASE
174+
);
175+
176+
expect(suggestion).not.toBeUndefined();
177+
expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['sch']);
178+
});
179+
180+
test('Show Column From', () => {
181+
const pos: CaretPosition = {
182+
lineNumber: 19,
183+
column: 21,
184+
};
185+
const syntaxes =
186+
parser.getSuggestionAtCaretPosition(commentOtherLine(syntaxSql, pos.lineNumber), pos)
187+
?.syntax ?? [];
188+
189+
const suggestion = syntaxes?.find(
190+
(syn) => syn.syntaxContextType === SyntaxContextType.TABLE
191+
);
192+
expect(
193+
syntaxes.some((item) => item.syntaxContextType === SyntaxContextType.VIEW)
194+
).toBeTruthy();
195+
expect(
196+
syntaxes.some((item) => item.syntaxContextType === SyntaxContextType.FUNCTION)
197+
).toBeTruthy();
198+
expect(suggestion).not.toBeUndefined();
199+
expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['tb']);
200+
});
201+
});

0 commit comments

Comments
 (0)