feat: improve errorListener msg (#281)

* feat: add mysql errorListener and commonErrorListener * feat: improve other sql error msg * feat: support i18n for error msg * feat: add all sql errorMsg unit test * feat: update locale file and change i18n funtion name * test: upate error unit test
DTStack · Jul 1, 2024 · 4b7f054 · 4b7f054
1 parent e3eb799
commit 4b7f054
Show file tree

Hide file tree

Showing 27 changed files with 1,310 additions and 9 deletions.
diff --git a/src/locale/locale.ts b/src/locale/locale.ts
@@ -0,0 +1,46 @@
+const zh_CN = {
+    stmtInComplete: '语句不完整',
+    noValidPosition: '在此位置无效',
+    expecting: '，期望',
+    unfinishedMultilineComment: '未完成的多行注释',
+    unfinishedDoubleQuoted: '未完成的双引号字符串字变量',
+    unfinishedSingleQuoted: '未完成的单引号字符串字变量',
+    unfinishedTickQuoted: '未完成的反引号引用字符串字变量',
+    noValidInput: '没有有效的输入',
+    newObj: '一个新的对象',
+    existingObj: '一个存在的对象',
+    new: '一个新的',
+    existing: '一个存在的',
+    orKeyword: '或者一个关键字',
+    keyword: '一个关键字',
+    missing: '缺少',
+    at: '在',
+    or: '或者',
+};
+
+const en_US: typeof zh_CN = {
+    stmtInComplete: 'Statement is incomplete',
+    noValidPosition: 'is not valid at this position',
+    expecting: ', expecting ',
+    unfinishedMultilineComment: 'Unfinished multiline comment',
+    unfinishedDoubleQuoted: 'Unfinished double quoted string literal',
+    unfinishedSingleQuoted: 'Unfinished single quoted string literal',
+    unfinishedTickQuoted: 'Unfinished back tick quoted string literal',
+    noValidInput: 'is no valid input at all',
+    newObj: 'a new object',
+    existingObj: 'an existing object',
+    new: 'a new ',
+    existing: 'an existing ',
+    orKeyword: ' or a keyword',
+    keyword: 'a keyword',
+    missing: 'missing ',
+    at: ' at ',
+    or: ' or ',
+};
+
+const i18n = {
+    zh_CN,
+    en_US,
+};
+
+export { i18n };
diff --git a/src/parser/common/basicSQL.ts b/src/parser/common/basicSQL.ts
@@ -8,13 +8,14 @@ import {
     ParseTreeWalker,
     ParseTreeListener,
     PredictionMode,
+    ANTLRErrorListener,
 } from 'antlr4ng';
 import { CandidatesCollection, CodeCompletionCore } from 'antlr4-c3';
 import { SQLParserBase } from '../../lib/SQLParserBase';
 import { findCaretTokenIndex } from './findCaretTokenIndex';
 import { ctxToText, tokenToWord, WordRange, TextSlice } from './textAndWord';
-import { CaretPosition, Suggestions, SyntaxSuggestion } from './types';
-import { ParseError, ErrorListener, ParseErrorListener } from './parseErrorListener';
+import { CaretPosition, LOCALE_TYPE, Suggestions, SyntaxSuggestion } from './types';
+import { ParseError, ErrorListener } from './parseErrorListener';
 import { ErrorStrategy } from './errorStrategy';
 import type { SplitListener } from './splitListener';
 import type { EntityCollector } from './entityCollector';
@@ -78,6 +79,11 @@ export abstract class BasicSQL<
      */
     protected abstract get splitListener(): SplitListener<ParserRuleContext>;
 
+    /**
+     * Get a new errorListener instance.
+     */
+    protected abstract createErrorListener(errorListener: ErrorListener): ANTLRErrorListener;
+
     /**
      * Get a new entityCollector instance.
      */
@@ -86,6 +92,8 @@ export abstract class BasicSQL<
         caretTokenIndex?: number
     ): EntityCollector;
 
+    public locale: LOCALE_TYPE = 'en_US';
+
     /**
      * Create an antlr4 lexer from input.
      * @param input string
@@ -95,7 +103,7 @@ export abstract class BasicSQL<
         const lexer = this.createLexerFromCharStream(charStreams);
         if (errorListener) {
             lexer.removeErrorListeners();
-            lexer.addErrorListener(new ParseErrorListener(errorListener));
+            lexer.addErrorListener(this.createErrorListener(errorListener));
         }
         return lexer;
     }
@@ -111,7 +119,7 @@ export abstract class BasicSQL<
         parser.interpreter.predictionMode = PredictionMode.SLL;
         if (errorListener) {
             parser.removeErrorListeners();
-            parser.addErrorListener(new ParseErrorListener(errorListener));
+            parser.addErrorListener(this.createErrorListener(errorListener));
         }
 
         return parser;
@@ -142,7 +150,7 @@ export abstract class BasicSQL<
         this._lexer = this.createLexerFromCharStream(this._charStreams);
 
         this._lexer.removeErrorListeners();
-        this._lexer.addErrorListener(new ParseErrorListener(this._errorListener));
+        this._lexer.addErrorListener(this.createErrorListener(this._errorListener));
 
         this._tokenStream = new CommonTokenStream(this._lexer);
         /**
@@ -178,7 +186,7 @@ export abstract class BasicSQL<
         this._parsedInput = input;
 
         parser.removeErrorListeners();
-        parser.addErrorListener(new ParseErrorListener(this._errorListener));
+        parser.addErrorListener(this.createErrorListener(this._errorListener));
 
         this._parseTree = parser.program();
 

diff --git a/src/parser/common/parseErrorListener.ts b/src/parser/common/parseErrorListener.ts
@@ -4,7 +4,14 @@ import {
     ANTLRErrorListener,
     RecognitionException,
     ATNSimulator,
+    LexerNoViableAltException,
+    Lexer,
+    Parser,
+    InputMismatchException,
+    NoViableAltException,
 } from 'antlr4ng';
+import { LOCALE_TYPE } from './types';
+import { transform } from './transform';
 
 /**
  * Converted from {@link SyntaxError}.
@@ -39,10 +46,12 @@ export interface SyntaxError {
  */
 export type ErrorListener = (parseError: ParseError, originalError: SyntaxError) => void;
 
-export class ParseErrorListener implements ANTLRErrorListener {
+export abstract class ParseErrorListener implements ANTLRErrorListener {
     private _errorListener: ErrorListener;
+    private locale: LOCALE_TYPE;
 
-    constructor(errorListener: ErrorListener) {
+    constructor(errorListener: ErrorListener, locale: LOCALE_TYPE = 'en_US') {
+        this.locale = locale;
         this._errorListener = errorListener;
     }
 
@@ -52,6 +61,8 @@ export class ParseErrorListener implements ANTLRErrorListener {
 
     reportContextSensitivity() {}
 
+    protected abstract getExpectedText(parser: Parser, token: Token): string;
+
     syntaxError(
         recognizer: Recognizer<ATNSimulator>,
         offendingSymbol: Token | null,
@@ -60,6 +71,87 @@ export class ParseErrorListener implements ANTLRErrorListener {
         msg: string,
         e: RecognitionException
     ) {
+        let message = '';
+        // If not undefined then offendingSymbol is of type Token.
+        if (offendingSymbol) {
+            let token = offendingSymbol as Token;
+            const parser = recognizer as Parser;
+
+            // judge token is EOF
+            const isEof = token.type === Token.EOF;
+            if (isEof) {
+                token = parser.tokenStream.get(token.tokenIndex - 1);
+            }
+            const wrongText = token.text ?? '';
+
+            const isInComplete = isEof && wrongText !== ' ';
+
+            const expectedText = isInComplete ? '' : this.getExpectedText(parser, token);
+
+            if (!e) {
+                // handle missing or unwanted tokens.
+                message = msg;
+                if (msg.includes('extraneous')) {
+                    message = `'${wrongText}' {noValidPosition}${
+                        expectedText.length ? `{expecting}${expectedText}` : ''
+                    }`;
+                }
+                if (msg.includes('missing')) {
+                    const regex = /missing\s+'([^']+)'/;
+                    const match = msg.match(regex);
+                    message = `{missing}`;
+                    if (match) {
+                        const missKeyword = match[1];
+                        message += `'${missKeyword}'`;
+                    } else {
+                        message += `{keyword}`;
+                    }
+                    message += `{at}'${wrongText}'`;
+                }
+            } else {
+                // handle mismatch exception or no viable alt exception
+                if (e instanceof InputMismatchException || e instanceof NoViableAltException) {
+                    if (isEof) {
+                        message = `{stmtInComplete}`;
+                    } else {
+                        message = `'${wrongText}' {noValidPosition}`;
+                    }
+                    if (expectedText.length > 0) {
+                        message += `{expecting}${expectedText}`;
+                    }
+                } else {
+                    message = msg;
+                }
+            }
+        } else {
+            // No offending symbol, which indicates this is a lexer error.
+            if (e instanceof LexerNoViableAltException) {
+                const lexer = recognizer as Lexer;
+                const input = lexer.inputStream;
+                let text = lexer.getErrorDisplay(
+                    input.getText(lexer._tokenStartCharIndex, input.index)
+                );
+                switch (text[0]) {
+                    case '/':
+                        message = '{unfinishedMultilineComment}';
+                        break;
+                    case '"':
+                        message = '{unfinishedDoubleQuoted}';
+                        break;
+                    case "'":
+                        message = '{unfinishedSingleQuoted}';
+                        break;
+                    case '`':
+                        message = '{unfinishedTickQuoted}';
+                        break;
+
+                    default:
+                        message = '"' + text + '" {noValidInput}';
+                        break;
+                }
+            }
+        }
+        message = transform(message, this.locale);
         let endCol = charPositionInLine + 1;
         if (offendingSymbol && offendingSymbol.text !== null) {
             endCol = charPositionInLine + offendingSymbol.text.length;
@@ -71,7 +163,7 @@ export class ParseErrorListener implements ANTLRErrorListener {
                     endLine: line,
                     startColumn: charPositionInLine + 1,
                     endColumn: endCol + 1,
-                    message: msg,
+                    message,
                 },
                 {
                     e,

diff --git a/src/parser/common/transform.ts b/src/parser/common/transform.ts
@@ -0,0 +1,17 @@
+import { LOCALE_TYPE } from './types';
+import { i18n } from '../../locale/locale';
+
+/**
+ * transform message to locale language
+ * @param message error msg
+ * @param locale language setting
+ */
+function transform(message: string, locale: LOCALE_TYPE) {
+    const regex = /{([^}]+)}/g;
+    return message.replace(
+        regex,
+        (_, key: keyof (typeof i18n)[typeof locale]) => i18n[locale][key] || ''
+    );
+}
+
+export { transform };
diff --git a/src/parser/common/types.ts b/src/parser/common/types.ts
@@ -67,3 +67,5 @@ export interface Suggestions<T = WordRange> {
      */
     readonly keywords: string[];
 }
+
+export type LOCALE_TYPE = 'zh_CN' | 'en_US';
diff --git a/src/parser/flink/flinkErrorListener.ts b/src/parser/flink/flinkErrorListener.ts
@@ -0,0 +1,76 @@
+import { CodeCompletionCore } from 'antlr4-c3';
+import { ErrorListener, ParseErrorListener } from '../common/parseErrorListener';
+import { Parser, Token } from 'antlr4ng';
+import { FlinkSqlParser } from '../../lib/flink/FlinkSqlParser';
+import { LOCALE_TYPE } from '../common/types';
+
+export class FlinkErrorListener extends ParseErrorListener {
+    private preferredRules: Set<number>;
+
+    private objectNames: Map<number, string> = new Map([
+        [FlinkSqlParser.RULE_catalogPath, 'catalog'],
+        [FlinkSqlParser.RULE_catalogPathCreate, 'catalog'],
+        [FlinkSqlParser.RULE_databasePath, 'database'],
+        [FlinkSqlParser.RULE_databasePathCreate, 'database'],
+        [FlinkSqlParser.RULE_tablePath, 'table'],
+        [FlinkSqlParser.RULE_tablePathCreate, 'table'],
+        [FlinkSqlParser.RULE_viewPath, 'view'],
+        [FlinkSqlParser.RULE_viewPathCreate, 'view'],
+        [FlinkSqlParser.RULE_functionName, 'function'],
+        [FlinkSqlParser.RULE_functionNameCreate, 'function'],
+        [FlinkSqlParser.RULE_columnName, 'column'],
+        [FlinkSqlParser.RULE_columnNameCreate, 'column'],
+    ]);
+
+    constructor(errorListener: ErrorListener, preferredRules: Set<number>, locale: LOCALE_TYPE) {
+        super(errorListener, locale);
+        this.preferredRules = preferredRules;
+    }
+
+    public getExpectedText(parser: Parser, token: Token) {
+        let expectedText = '';
+
+        let currentContext = parser.context ?? undefined;
+        while (currentContext?.parent) {
+            currentContext = currentContext.parent;
+        }
+
+        const core = new CodeCompletionCore(parser);
+        core.preferredRules = this.preferredRules;
+        const candidates = core.collectCandidates(token.tokenIndex, currentContext);
+
+        if (candidates.rules.size) {
+            const result: string[] = [];
+            // get expectedText as collect rules first
+            for (const candidate of candidates.rules) {
+                const [ruleType] = candidate;
+                const name = this.objectNames.get(ruleType);
+                switch (ruleType) {
+                    case FlinkSqlParser.RULE_databasePath:
+                    case FlinkSqlParser.RULE_tablePath:
+                    case FlinkSqlParser.RULE_viewPath:
+                    case FlinkSqlParser.RULE_functionName:
+                    case FlinkSqlParser.RULE_columnName:
+                    case FlinkSqlParser.RULE_catalogPath: {
+                        result.push(`{existing}${name}`);
+                        break;
+                    }
+                    case FlinkSqlParser.RULE_databasePathCreate:
+                    case FlinkSqlParser.RULE_tablePathCreate:
+                    case FlinkSqlParser.RULE_functionNameCreate:
+                    case FlinkSqlParser.RULE_viewPathCreate:
+                    case FlinkSqlParser.RULE_columnNameCreate:
+                    case FlinkSqlParser.RULE_catalogPathCreate: {
+                        result.push(`{new}${name}`);
+                        break;
+                    }
+                }
+            }
+            expectedText = result.join('{or}');
+        }
+        if (candidates.tokens.size) {
+            expectedText += expectedText ? '{orKeyword}' : '{keyword}';
+        }
+        return expectedText;
+    }
+}
diff --git a/src/parser/flink/index.ts b/src/parser/flink/index.ts
@@ -7,6 +7,8 @@ import { BasicSQL } from '../common/basicSQL';
 import { StmtContextType } from '../common/entityCollector';
 import { FlinkSqlSplitListener } from './flinkSplitListener';
 import { FlinkEntityCollector } from './flinkEntityCollector';
+import { ErrorListener } from '../common/parseErrorListener';
+import { FlinkErrorListener } from './flinkErrorListener';
 
 export { FlinkSqlSplitListener, FlinkEntityCollector };
 
@@ -37,6 +39,10 @@ export class FlinkSQL extends BasicSQL<FlinkSqlLexer, ProgramContext, FlinkSqlPa
         return new FlinkSqlSplitListener();
     }
 
+    protected createErrorListener(_errorListener: ErrorListener) {
+        return new FlinkErrorListener(_errorListener, this.preferredRules, this.locale);
+    }
+
     protected createEntityCollector(input: string, caretTokenIndex?: number) {
         return new FlinkEntityCollector(input, caretTokenIndex);
     }