diff --git a/.vscodeignore b/.vscodeignore index 659bcd4..f3c5c0f 100644 --- a/.vscodeignore +++ b/.vscodeignore @@ -1,3 +1,6 @@ +.vscode/ +.gitignore/ +.git/ **/*.ts **/tsconfig.json **/tslint.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c7d9a6..4f5c0cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,24 @@ ## Release +## 0.2.0 27/04/2020 +### Added +- Added lex/yacc parsers and language services +- Added feature: basic diagnostics support (lex/yacc) +- Added feature: rename symbol support (lex/yacc) +- Added feature: find references support (lex/yacc) + +### Changed +- Architectural changing, now uses language service pattern +- Better completion handling/detection (lex/yacc) +- General optimization + - parsing time 3x less than before + - binary search to detect C code region, computation time reduced from O(n) to O(log(n)) +- Updated README.md + +### Bug Fixes +- Minor bug fixes related to completion handling/detection + ## 0.1.2 ### Added - Added recognition of start condition block (lex) @@ -43,7 +61,6 @@ - Fixed a bug on wrong detection of tokens (yacc) - Fixed recognition of comment in %type, keywords, keywords-block, rules (yacc) - ## 0.0.7 ### Added - Added comment highlight (lex) diff --git a/README.md b/README.md index 8ba10f0..4fdaded 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,32 @@ This is yet an another syntax highlighter for lex/yacc and flex/bison. This extension provides full syntax highlight for these languages and also for the embedded language C/C++. +This extension also supports some basic language features such as: +- Code diagnostic +- Auto-completion +- Hover feature +- Go to Definition +- Find references +- Rename Symbol + +## Preview + +### Completion for lex +![](images/lex_define.gif) + +### Completion for yacc +![](images/yacc_symbol.gif) + +### Diagnostic + +![](images/redefinition.png) + +### More examples + +You can find more previews here [previews](images/README.md). + +## Notice + Since 1.43.0 VSCode enabled a new feature called Semantic Highlighting, this extension supports it. By default, only the built-in themes has semantic highlighting enabled, so if you are using a 3rd party theme for example [Dracula](https://github.com/dracula/visual-studio-code/) which doesn't support the semantic coloring yet, you have to add these lines to your `settings.json` file to have the feature enabled. @@ -24,28 +50,9 @@ On left enabled, on right disabled ![](images/semantic_comparison.png) -### Completion features - -### Auto-Completion for keywords, declared definitions in lex/flex - -![](images/lex_define.gif) - -![](images/lex_rule.gif) - -### Auto-Completion for keywords, declared union types in yacc/bison - -![](images/yacc_token.gif) - - -### Auto-Completion for symbols in yacc/bison -![](images/yacc_symbol.gif) - -### Auto-Completion for symbol type in yacc/bison -![](images/yacc_type.gif) - ## Requirements -VSCode 1.44.0+ +VSCode ^1.44 ## Contributors diff --git a/images/README.md b/images/README.md new file mode 100644 index 0000000..8387609 --- /dev/null +++ b/images/README.md @@ -0,0 +1,24 @@ +# Feature examples +## Diagnostics + +![](not_decl.png) +![](redefinition.png) + +## Hover +![](hover.png) + +## Auto-Completion for keywords, declared definitions in lex/flex + +![](lex_define.gif) + +![](lex_rule.gif) + +## Auto-Completion for keywords, declared union types in yacc/bison + +![](yacc_token.gif) + +## Auto-Completion for symbols in yacc/bison +![](yacc_symbol.gif) + +## Auto-Completion for symbol type in yacc/bison +![](yacc_type.gif) \ No newline at end of file diff --git a/images/hover.png b/images/hover.png new file mode 100644 index 0000000..0f5ab57 Binary files /dev/null and b/images/hover.png differ diff --git a/images/missing_decl.png b/images/missing_decl.png new file mode 100644 index 0000000..cb456ec Binary files /dev/null and b/images/missing_decl.png differ diff --git a/images/not_decl.png b/images/not_decl.png new file mode 100644 index 0000000..389f3d0 Binary files /dev/null and b/images/not_decl.png differ diff --git a/images/redefinition.png b/images/redefinition.png new file mode 100644 index 0000000..a1c5405 Binary files /dev/null and b/images/redefinition.png differ diff --git a/package-lock.json b/package-lock.json index 271bce3..7b65954 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { - "name": "lex-yacc-lang", - "version": "0.0.1", + "name": "yash", + "version": "0.2.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index fceae9e..f01b7c7 100644 --- a/package.json +++ b/package.json @@ -2,10 +2,11 @@ "name": "yash", "displayName": "Yash", "description": "Yet another syntax highlighter for lex/yacc & flex/bison.", - "version": "0.1.2", + "version": "0.2.0", "engines": { "vscode": "^1.44.0" }, + "license": "MIT", "publisher": "daohong-emilio", "icon": "assets/logo.png", "repository": { diff --git a/src/documentCache.ts b/src/documentCache.ts new file mode 100644 index 0000000..8ac94e6 --- /dev/null +++ b/src/documentCache.ts @@ -0,0 +1,88 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. + * + * Modified to adapt the project + *--------------------------------------------------------------------------------------------*/ + +import { TextDocument } from 'vscode'; + +export interface DocumentCache { + get(document: TextDocument): T; + onDocumentRemoved(document: TextDocument): void; + dispose(): void; +} + +export function CreateDocumentCache(maxEntries: number, cleanupIntervalTimeInSec: number, parse: (document: TextDocument) => T): DocumentCache { + let languageModels: { [uri: string]: { version: number, languageId: string, cTime: number, languageModel: T } } = {}; + let nModels = 0; + + let cleanupInterval: NodeJS.Timer | undefined = undefined; + if (cleanupIntervalTimeInSec > 0) { + cleanupInterval = setInterval(() => { + const cutoffTime = Date.now() - cleanupIntervalTimeInSec * 1000; + const uris = Object.keys(languageModels); + for (const uri of uris) { + const languageModelInfo = languageModels[uri]; + if (languageModelInfo.cTime < cutoffTime) { + delete languageModels[uri]; + nModels--; + } + } + }, cleanupIntervalTimeInSec * 1000); + } + + return { + get(document: TextDocument): T { + const version = document.version; + const languageId = document.languageId; + const languageModelInfo = languageModels[document.uri.toString()]; + if (languageModelInfo && languageModelInfo.version === version && languageModelInfo.languageId === languageId) { + languageModelInfo.cTime = Date.now(); + return languageModelInfo.languageModel; + } + + const t0 = Date.now(); + const languageModel = parse(document); + const t1 = Date.now(); + console.log(`Parsing time ${t1 - t0}`); + languageModels[document.uri.toString()] = { languageModel, version, languageId, cTime: Date.now() }; + if (!languageModelInfo) { + nModels++; + } + + if (nModels === maxEntries) { + let oldestTime = Number.MAX_VALUE; + let oldestUri = null; + for (const uri in languageModels) { + const languageModelInfo = languageModels[uri]; + if (languageModelInfo.cTime < oldestTime) { + oldestUri = uri; + oldestTime = languageModelInfo.cTime; + } + } + if (oldestUri) { + delete languageModels[oldestUri]; + nModels--; + } + } + return languageModel; + + }, + onDocumentRemoved(document: TextDocument) { + const uri = document.uri.toString(); + if (languageModels[uri]) { + delete languageModels[uri]; + nModels--; + } + }, + dispose() { + if (typeof cleanupInterval !== 'undefined') { + clearInterval(cleanupInterval); + cleanupInterval = undefined; + languageModels = {}; + nModels = 0; + } + } + }; +} diff --git a/src/extension.ts b/src/extension.ts index 7e7da64..609f9a6 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -1,742 +1,131 @@ import * as vscode from 'vscode'; +import { newSemanticTokenProvider } from './modes/semanticProvider'; +import { getLanguageModes } from './modes/languageModes'; +import { runSafe, formatError } from './runner'; +const pendingValidationRequests: { [uri: string]: NodeJS.Timer } = {}; +const validationDelayMs = 500; -const tokenTypes = new Map(); -const tokenModifiers = new Map(); +const languageModes = getLanguageModes({ yacc: true, lex: true }) +const semanticProvider = newSemanticTokenProvider(languageModes) -const legend = (function () { - const tokenTypesLegend = [ - 'comment', 'string', 'keyword', 'number', 'regexp', 'operator', 'namespace', - 'type', 'struct', 'class', 'interface', 'enum', 'typeParameter', 'function', - 'member', 'macro', 'variable', 'parameter', 'property', 'label' - ]; - tokenTypesLegend.forEach((tokenType, index) => tokenTypes.set(tokenType, index)); - - const tokenModifiersLegend = [ - 'declaration', 'documentation', 'readonly', 'static', 'abstract', 'deprecated', - 'modification', 'async' - ]; - tokenModifiersLegend.forEach((tokenModifier, index) => tokenModifiers.set(tokenModifier, index)); - - return new vscode.SemanticTokensLegend(tokenTypesLegend, tokenModifiersLegend); -})(); +const selector: vscode.DocumentSelector = [{ scheme: 'file', language: 'yacc' }, { scheme: 'file', language: 'lex' }] +const diagnostics = vscode.languages.createDiagnosticCollection(); export function activate(context: vscode.ExtensionContext) { - const yacc = new YaccSemanticProvider(); - context.subscriptions.push(vscode.languages.registerCompletionItemProvider('yacc', yacc, '%', '<')); - context.subscriptions.push(vscode.languages.registerDocumentSemanticTokensProvider('yacc', yacc, legend)); - context.subscriptions.push(vscode.languages.registerHoverProvider('yacc', yacc)); - context.subscriptions.push(vscode.languages.registerDefinitionProvider('yacc', yacc)); - - const lex = new LexSemanticProvider(); - context.subscriptions.push(vscode.languages.registerCompletionItemProvider('lex', lex, '%', '{', '<')); - context.subscriptions.push(vscode.languages.registerDocumentSemanticTokensProvider('lex', lex, legend)); - context.subscriptions.push(vscode.languages.registerHoverProvider('lex', lex)); - context.subscriptions.push(vscode.languages.registerDefinitionProvider('lex', lex)); -} - -function getEqualLengthSpaces(str: string) { - return str.replace(/[^\n]*/mg, (m) => { - return ' '.repeat(m.length); - }); -} - -interface IParsedToken { - line: number; - startCharacter: number; - length: number; - tokenType: string; - tokenModifiers: string[]; -} - -abstract class SemanticAnalyzer implements vscode.DocumentSemanticTokensProvider, vscode.CompletionItemProvider, vscode.HoverProvider, vscode.DefinitionProvider { - protected keywords: string[]; - protected invalidRegions: vscode.Range[] = []; - protected startingLine = -1; - protected endingLine = -1; + context.subscriptions.push(diagnostics); - constructor(keywords: string[]) { - this.keywords = keywords; - } - - async provideDefinition(document: vscode.TextDocument, position: vscode.Position, token: vscode.CancellationToken): Promise { - for (let i = 0; i < this.invalidRegions.length; i++) { - const range = this.invalidRegions[i]; - if (range.contains(position)) { - return []; - } - } - - return this._provideDefinition(document, position); - } - - async provideHover(document: vscode.TextDocument, position: vscode.Position, token: vscode.CancellationToken): Promise { - for (let i = 0; i < this.invalidRegions.length; i++) { - const range = this.invalidRegions[i]; - if (range.contains(position)) { - return { contents: [] }; - } - } - - return this._provideHover(document, position); - } - - async provideCompletionItems(document: vscode.TextDocument, position: vscode.Position, token: vscode.CancellationToken, context: vscode.CompletionContext): Promise { - for (let i = 0; i < this.invalidRegions.length; i++) { - const range = this.invalidRegions[i]; - if (range.contains(position)) { - return []; - } - } - - if (context.triggerKind === vscode.CompletionTriggerKind.TriggerCharacter) { - return this._handleTrigger(document, position, context.triggerCharacter); - } - - return this._rulesCompletion(document, position); - } - - async provideDocumentSemanticTokens(document: vscode.TextDocument, token: vscode.CancellationToken): Promise { - const allTokens = this._parseText(document); - const builder = new vscode.SemanticTokensBuilder(); - allTokens.forEach((token) => { - builder.push(token.line, token.startCharacter, token.length, this._encodeTokenType(token.tokenType), this._encodeTokenModifiers(token.tokenModifiers)); - }); - return builder.build() - } - - private _encodeTokenType(tokenType: string): number { - if (!tokenTypes.has(tokenType)) { - return 0; - } - return tokenTypes.get(tokenType)!; - } - - private _encodeTokenModifiers(strTokenModifiers: string[]): number { - let result = 0; - for (let i = 0; i < strTokenModifiers.length; i++) { - const tokenModifier = strTokenModifiers[i]; - if (tokenModifiers.has(tokenModifier)) { - result = result | (1 << tokenModifiers.get(tokenModifier)!); - } - } - return result; - } - - protected _keywordCompletions(document: vscode.TextDocument, position: vscode.Position): vscode.CompletionItem[] | vscode.CompletionList { - if (position.line > this.startingLine) { - return []; - } - return this.keywords.map((keyword) => { - const completion = new vscode.CompletionItem(keyword, vscode.CompletionItemKind.Constructor); - completion.detail = "keyword"; - return completion; - }); - } - - protected abstract _buildHoverMsg(code?: string, lang?: string, info?: string): vscode.MarkdownString; - - protected abstract _provideDefinition(document: vscode.TextDocument, position: vscode.Position): vscode.Location | vscode.Location[] | vscode.LocationLink[]; - protected abstract _provideHover(document: vscode.TextDocument, position: vscode.Position): vscode.Hover; - - protected abstract _handleTrigger(document: vscode.TextDocument, position: vscode.Position, character: string | undefined): vscode.CompletionItem[] | vscode.CompletionList; - protected abstract _rulesCompletion(document: vscode.TextDocument, position: vscode.Position): vscode.CompletionItem[] | vscode.CompletionList; - protected abstract _parseText(document: vscode.TextDocument): IParsedToken[]; -} - -interface ISymbolDefinition { - name: string; - snippet?: string; - position: vscode.Position; -}; - -enum LineType { - Definition, - Rule -}; - -class YaccSemanticProvider extends SemanticAnalyzer { - private symbols: Map = new Map(); - private tokens: Map = new Map(); - private types: Map = new Map(); - private rulesSection: [LineType, string][] = []; - - constructor() { - super(['type', 'option', 'token', 'left', 'right', 'define', 'output', - 'precedence', 'nterm', 'destructor', 'union', 'code', 'printer', - 'parse-param', 'lex-param', 'pure-parser', 'expect', 'name-prefix', 'locations', 'nonassoc']); - } - - protected _buildHoverMsg(code?: string, lang?: string, info?: string): vscode.MarkdownString { - const msg = new vscode.MarkdownString(); - if (code !== undefined) { - msg.appendCodeblock(code, lang !== undefined ? lang : 'yacc'); - if (info !== undefined) - msg.appendMarkdown('---\n'); - } - - if (info !== undefined) - msg.appendMarkdown(info); - return msg; - } - - protected _provideDefinition(document: vscode.TextDocument, position: vscode.Position): vscode.Location | vscode.Location[] | vscode.LocationLink[] { - const word = document.getText(document.getWordRangeAtPosition(position)); - - if (this.symbols.has(word)) { - const pos = this.symbols.get(word)!; - return new vscode.Location(document.uri, pos.position); - } - - if (this.tokens.has(word)) { - const pos = this.tokens.get(word)!; - return new vscode.Location(document.uri, pos.position); - } - - if (this.types.has(word)) { - const pos = this.types.get(word)!; - return new vscode.Location(document.uri, pos.position); - } - - return []; - } - - protected _provideHover(document: vscode.TextDocument, position: vscode.Position): vscode.Hover { - const word = document.getText(document.getWordRangeAtPosition(position)); - if (this.symbols.has(word)) { - const symbol = this.symbols.get(word)!; - return { contents: [this._buildHoverMsg(symbol.snippet)] } - } - - if (this.tokens.has(word)) { - const symbol = this.tokens.get(word)!; - return { contents: [this._buildHoverMsg(symbol.snippet)] } - } - - if (this.types.has(word)) { - const symbol = this.types.get(word)!; - return { contents: [this._buildHoverMsg(symbol.snippet)] } - } - - return { contents: [] }; - } - - protected _handleTrigger(document: vscode.TextDocument, position: vscode.Position, character: string | undefined): vscode.CompletionItem[] | vscode.CompletionList { - let line = document.lineAt(position).text.substr(0, position.character); - if (character === '%') { - if (line.startsWith('%') && !line.startsWith('%%')) - return this._keywordCompletions(document, position); - } else if (character === '<') { - if (line.match(/^%(?:type|token)\s*<.*/)) { - return this._typeParamCompletions(document, position); - } - } - - return []; - } - - protected _rulesCompletion(document: vscode.TextDocument, position: vscode.Position): vscode.CompletionItem[] | vscode.CompletionList { - let line = document.lineAt(position).text.substr(0, position.character); - /** - * Result suggestion on defining type - */ - if (line.match(/^%type\s*<.*>[\sa-zA-Z0-9_]*$/)) { - var completions: vscode.CompletionItem[] = []; - this.symbols.forEach((value, key) => { - const completion = new vscode.CompletionItem(key, vscode.CompletionItemKind.Class); - completion.detail = "symbol" - completions.push(completion); - }) - return completions; - } - - if (line.match(/^%(?:type|token)\s*<[^>\n]+$/)) { - return this._typeParamCompletions(document, position); - } - - /** - * Token and result suggestion only inside the rules section - */ - if (position.line > this.startingLine && position.line < this.endingLine && position.character > 1) { - const startLine = position.line - this.startingLine; - var localLine = position.line - this.startingLine; - var rule; - var parens = 0; - var braces = 0; - // TODO: make a validation function - do { - /* Loop until we get the definition line */ - rule = this.rulesSection[localLine]; - if (localLine == startLine) - line = rule[1].slice(0, position.character - 1); - else - line = rule[1]; - let has = false; - for (let i = line.length - 1; i > 0; i--) { - const ch = line[i]; - switch (ch) { - case '[': - parens++; - // if found not closed [] - if (parens === 1) i = 0; - break; - case ']': - parens--; - break; - case '{': - braces++; - break; - case '}': - // if found } before | - braces--; - i = 0; - break; - case '|': - case ':': - has = true; - i = 0; - break; - case ';': - /* If not inside a valid declaration */ - i = 0; - localLine = -1; - default: - break; - } - } - - const good = (parens === 0) && (braces === 0); - if (!good) { - break; - } - - if (has && good) { - var completions: vscode.CompletionItem[] = []; - var completion: vscode.CompletionItem; - this.symbols.forEach((value, key) => { - completion = new vscode.CompletionItem(key, vscode.CompletionItemKind.Class) - completion.detail = "symbol"; - completions.push(completion); - }) - this.tokens.forEach((value, key) => { - completion = new vscode.CompletionItem(key, vscode.CompletionItemKind.Field) - completion.detail = "token"; - completions.push(completion); - }) - return completions; + context.subscriptions.push(vscode.languages.registerCompletionItemProvider(selector, { + async provideCompletionItems(document, position, token): Promise { + return runSafe(() => { + const mode = languageModes.getMode(document.languageId); + if (!mode || !mode.doComplete) { + return { isIncomplete: true, items: [] }; } - localLine--; - } while (rule[0] === LineType.Rule && localLine > 0); + const doComplete = mode.doComplete!; + console.log('complete') + return doComplete(document, position); + }, null, `Error while computing completion for ${document.uri.toString()}`, token); } - return []; - } - - private _typeParamCompletions(document: vscode.TextDocument, position: vscode.Position): vscode.CompletionItem[] | vscode.CompletionList { - if (position.line > this.startingLine) { - return []; - } - - var completion: vscode.CompletionItem; - var completions: vscode.CompletionItem[] = []; - this.types.forEach((value, key) => { - completion = new vscode.CompletionItem(key, vscode.CompletionItemKind.TypeParameter) - completion.detail = "type" - completions.push(completion); - }) - return completions; - } - - protected _parseText(document: vscode.TextDocument): IParsedToken[] { - let text = document.getText(); - this.startingLine = -1; - this.endingLine = document.lineCount; - this.tokens.clear(); - this.symbols.clear(); - this.invalidRegions = []; - this.rulesSection = []; - - const r: IParsedToken[] = []; - text = text.replace(/"(?:[^"\\\n]|\\.)*"|'(?:[^'\\\n]|\\.)*'|\/\*[\s\S]*?\*\//mg, getEqualLengthSpaces); - - const lines = text.split(/\r\n|\r|\n/); + }, '%', '<', '{')); - let brackets = 0; - let currentPos: vscode.Position | undefined = undefined; - let tokenContinue = false; - let tokenType = ''; - let symbolContinue = false; - let symbolType = ''; - let unionFound = false; - let rulesText: string = ''; - const rules: string[] = []; - - for (let i = 0; i < lines.length; i++) { - const line = lines[i]; - if (line.startsWith('%%')) { - if (this.startingLine === -1) - this.startingLine = i; - else { - /** - * Stop on end rules section - */ - this.endingLine = i; - break; - } - } - - if (line.startsWith('%')) { - // save tokens - if (line.startsWith('%token')) { - const type = line.match(/(<.*>)/); - if (type) tokenType = ' ' + type[1]; - else tokenType = ''; - const tokenSymbols = line.slice(6).replace(/<.*>/, "").trim().split(" "); - tokenSymbols.forEach(token => { - if (token.length > 0) { - this.tokens.set(token, { name: token, snippet: '%token' + tokenType + ' ' + token, position: new vscode.Position(i, 0) }); - } - }); - tokenContinue = true; - continue; - } else { - tokenType = '' - tokenContinue = false; - } - - if (line.startsWith('%type')) { - const type = line.match(/(<.*>)/); - if (type) symbolType = ' ' + type[1]; - else symbolType = ''; - const symbols = line.slice(5).replace(/<.*>/, "").trim().split(" "); - symbols.forEach(symbol => { - if (symbol.length > 0) { - this.symbols.set(symbol, { name: symbol, snippet: '%type' + symbolType + ' ' + symbol, position: new vscode.Position(i, 0) }); - } - }); - symbolContinue = true; - continue; - } else { - symbolType = '' - symbolContinue = false; + context.subscriptions.push(vscode.languages.registerHoverProvider(selector, { + async provideHover(document, position, token): Promise { + return runSafe(() => { + const mode = languageModes.getMode(document.languageId); + if (!mode || !mode.doHover) { + return null; } - - if (line.startsWith('%union')) { - unionFound = true; - } - } else { - // continue saving tokens - if (tokenContinue) { - const tokenSymbols = line.trim().split(" "); - tokenSymbols.forEach(token => { - if (token.length > 0) { - this.tokens.set(token, { name: token, snippet: '%token' + tokenType + ' ' + token, position: new vscode.Position(i, 0) }); - } - }); - continue; - } - - if (symbolContinue) { - const symbols = line.trim().split(" "); - symbols.forEach(symbol => { - if (symbol.length > 0) { - this.symbols.set(symbol, { name: symbol, snippet: '%type' + symbolType + ' ' + symbol, position: new vscode.Position(i, 0) }); - } - }); - continue; - } - } - - if (unionFound) { - const type = /^(.*[ \t\f*&])([a-zA-Z0-9_]+)\s*;$/.exec(line); - if (type !== null) { - const snippet = type[1].replace(/\s*/g, "") + " " + type[2]; - this.types.set(type[2], { name: type[2], snippet: snippet, position: new vscode.Position(i, type.index) }); - } - } - - /** - * Finding nested C code block - */ - for (let j = 0; j < line.length; j++) { - const ch = line[j]; - switch (ch) { - case '{': - brackets++; - if (currentPos === undefined) { - currentPos = new vscode.Position(i, j); - } - if (this.startingLine !== -1) { - rulesText += ch; - } - break; - case '}': - brackets--; - if (brackets === 0) { - this.invalidRegions.push(new vscode.Range(currentPos!, new vscode.Position(i, j))); - currentPos = undefined; - if (unionFound) - unionFound = false; - } - if (this.startingLine !== -1) { - rulesText += ch; - } - break; - default: - if (this.startingLine !== -1) { - /** - * Clear out C code and save yacc code - */ - if (brackets === 0) - rulesText += ch; - else - rulesText += ' '; - } - break; - } - } - - if (this.startingLine !== -1) { - rules.push(rulesText); - rulesText = ''; - } + return mode.doHover(document, position); + }, null, `Error while computing hover for ${document.uri.toString()}`, token); } + })); - /** - * Find all symbols - */ - for (let i = 0; i < rules.length; i++) { - const ruleMatcher = /^[a-zA-Z0-9_]+/; - const rule = ruleMatcher.exec(rules[i]); - if (rule !== null) { - this.rulesSection.push([LineType.Definition, rules[i]]); - const symbol = this.symbols.get(rule[0]); - if (symbol !== undefined) { - this.symbols.set(rule[0], { name: rule[0], snippet: symbol.snippet, position: new vscode.Position(this.startingLine + i, 1) }); - } else { - this.symbols.set(rule[0], { name: rule[0], snippet: "%type " + rule[0], position: new vscode.Position(this.startingLine + i, 1) }); + context.subscriptions.push(vscode.languages.registerDefinitionProvider(selector, { + async provideDefinition(document, position, token): Promise { + return runSafe(() => { + const mode = languageModes.getMode(document.languageId); + if (!mode || !mode.findDefinition) { + return null; } - } else { - this.rulesSection.push([LineType.Rule, rules[i]]); - } + return mode.findDefinition(document, position); + }, null, `Error while computing find definition for ${document.uri.toString()}`, token); } + })); - /** - * Highlight symbols in rules section - */ - const matcher = /[a-zA-Z0-9_]+/g; - for (let i = 0; i < rules.length; i++) { - const line = rules[i]; - var match; - while ((match = matcher.exec(line)) != null) { - const word = match[0]; - if (this.symbols.has(word)) { - r.push({ - line: this.startingLine + i, - startCharacter: match.index, - length: word.length, - tokenType: "class", - tokenModifiers: [] - }); + context.subscriptions.push(vscode.languages.registerRenameProvider(selector, { + async provideRenameEdits(document, position, newName, token): Promise { + return runSafe(() => { + const mode = languageModes.getMode(document.languageId); + if (!mode || !mode.doRename) { + return null; } - } - } - return r; - } -} - -class LexSemanticProvider extends SemanticAnalyzer { - private defines: Map = new Map(); - private states: Map = new Map(); - - constructor() { - super(['array', 'pointer', 'option', 's', 'x']); - } - - protected _buildHoverMsg(code?: string, lang?: string, info?: string): vscode.MarkdownString { - const msg = new vscode.MarkdownString(); - if (code !== undefined) { - msg.appendCodeblock(code, lang !== undefined ? lang : 'lex'); - if (info !== undefined) - msg.appendMarkdown('---\n'); + return mode.doRename(document, position, newName); + }, null, `Error while computing find definition for ${document.uri.toString()}`, token); } + })); - if (info !== undefined) - msg.appendMarkdown(info); - return msg; - } - - protected _provideDefinition(document: vscode.TextDocument, position: vscode.Position): vscode.Location | vscode.Location[] | vscode.LocationLink[] { - const word = document.getText(document.getWordRangeAtPosition(position)); - - if (this.defines.has(word)) { - const pos = this.defines.get(word)!; - return new vscode.Location(document.uri, pos.position); - } - - if (this.states.has(word)) { - const pos = this.states.get(word)!; - return new vscode.Location(document.uri, pos.position); + context.subscriptions.push(vscode.languages.registerReferenceProvider(selector, { + async provideReferences(document, position, context, token): Promise { + return runSafe(() => { + const mode = languageModes.getMode(document.languageId); + if (!mode || !mode.findReferences) { + return null; + } + return mode.findReferences(document, position); + }, null, `Error while computing find references for ${document.uri.toString()}`, token); } + })); - return []; - } - - protected _provideHover(document: vscode.TextDocument, position: vscode.Position): vscode.Hover { - const word = document.getText(document.getWordRangeAtPosition(position)); - if (this.defines.has(word)) { - const symbol = this.defines.get(word)!; - return { contents: [this._buildHoverMsg(symbol.snippet)] }; + context.subscriptions.push(vscode.languages.registerDocumentSemanticTokensProvider('yacc', { + async provideDocumentSemanticTokens(document, token): Promise { + return runSafe(() => { + return semanticProvider.getSemanticTokens(document); + }, null, `Error while computing semantic tokens for ${document.uri.toString()}`, token); } + }, new vscode.SemanticTokensLegend(semanticProvider.legend.types, semanticProvider.legend.modifiers))); - if (this.states.has(word)) { - const symbol = this.states.get(word)!; - return { contents: [this._buildHoverMsg(symbol.snippet)] }; - } + // The content of a text document has changed. + context.subscriptions.push(vscode.workspace.onDidChangeTextDocument(change => { + triggerValidation(change.document); + })); - return { contents: [] }; - } + // A document has closed: clear all diagnostics + context.subscriptions.push(vscode.workspace.onDidCloseTextDocument(document => { + cleanPendingValidation(document); + diagnostics.set(document.uri, []); + })); - protected _handleTrigger(document: vscode.TextDocument, position: vscode.Position, character: string | undefined): vscode.CompletionItem[] | vscode.CompletionList { - var completion: vscode.CompletionItem; - const completions: vscode.CompletionItem[] = []; - let line = document.lineAt(position).text.substr(0, position.character); - if (character === '%') { - if (line.startsWith('%') && !line.startsWith('%%')) - return this._keywordCompletions(document, position); - } else if (character === '{') { - var ok = false; - if (position.line < this.startingLine) { - // if before rules zone, definition need to be on the right - ok = line.match(/^[a-zA-Z0-9_]+/) !== null; - } else if (position.line < this.endingLine) { - // if inside rules zone - const res = line.match(/^(?:{[a-zA-Z0-9_]*}?)+/); - if (res) { - ok = res[0].length === position.character; - } - } - if (ok) { - this.defines.forEach((value, key) => { - completion = new vscode.CompletionItem(key, vscode.CompletionItemKind.Class); - completion.detail = "definition"; - completions.push(completion); - }) - } - } else if (character === '<' && position.line > this.startingLine && position.line < this.endingLine) { - if (line.match(/^ { - completion = new vscode.CompletionItem(key, vscode.CompletionItemKind.Class); - completion.detail = "initial state"; - completions.push(completion); - }) - } + context.subscriptions.push(vscode.window.onDidChangeActiveTextEditor(editor => { + if (editor) { + triggerValidation(editor.document); } + })); - return completions; + if (vscode.window.activeTextEditor) { + triggerValidation(vscode.window.activeTextEditor.document); } +} - protected _rulesCompletion(document: vscode.TextDocument, position: vscode.Position): vscode.CompletionItem[] | vscode.CompletionList { - const line = document.lineAt(position).text.substr(0, position.character); - const res = line.match(/^(?:{[a-zA-Z0-9_]*}?)+/); - console.log(line) - if (res) { - if (res[0].length === position.character) { - const completions: vscode.CompletionItem[] = []; - this.defines.forEach((value, key) => { - const completion = new vscode.CompletionItem(key, vscode.CompletionItemKind.Class); - completion.detail = "definition"; - completions.push(completion); - }) - - return completions; - } - } - return []; +function cleanPendingValidation(textDocument: vscode.TextDocument): void { + const request = pendingValidationRequests[textDocument.uri.toString()]; + if (request) { + clearTimeout(request); + delete pendingValidationRequests[textDocument.uri.toString()]; } +} - protected _parseText(document: vscode.TextDocument): IParsedToken[] { - let text = document.getText(); - this.startingLine = -1; - this.endingLine = document.lineCount; - this.defines.clear(); - this.invalidRegions = []; - - const r: IParsedToken[] = []; - text = text.replace(/"\/\*[\s\S]*?\*\//mg, getEqualLengthSpaces); - - const lines = text.split(/\r\n|\r|\n/); - - let brackets = 0; - let currentPos: vscode.Position | undefined = undefined; - let isBlock = false; - for (let i = 0; i < lines.length; i++) { - const line = lines[i]; - if (line.startsWith('%%')) { - if (this.startingLine === -1) - this.startingLine = i; - else { - /** - * Stop on end rules section - */ - this.endingLine = i; - break; - } - } - - if (this.startingLine === -1) { - const defined = line.match(/^[a-zA-Z0-9_]+/); - if (defined !== null) { - this.defines.set(defined[0], { name: defined[0], snippet: line.trim(), position: new vscode.Position(i, 0) }); - } - } - - if (line.startsWith('%x') || line.startsWith('%s')) { - const tokenSymbols = line.slice(2).trim().split(" "); - tokenSymbols.forEach(token => { - if (token.length > 0) { - this.states.set(token, { name: token, snippet: line.trim(), position: new vscode.Position(i, 0) }); - } - }); - continue; - } - - if (line.match(/^%(top)?/)) { - isBlock = true; - } - - if (this.startingLine !== -1 || isBlock) { - let j = 0; - const defines = /^(?:<[a-zA-Z0-9_,]+>)?(?:{[a-zA-Z0-9_]+})+\s+/.exec(line); - if (defines) { - j = defines[0].length; - } - - /** - * Finding nested C code block - */ - for (; j < line.length; j++) { - const ch = line[j]; - switch (ch) { - case '{': - brackets++; - if (currentPos === undefined) { - currentPos = new vscode.Position(i, j); - } - break; - case '}': - brackets--; - if (brackets === 0) { - this.invalidRegions.push(new vscode.Range(currentPos!, new vscode.Position(i, j))); - currentPos = undefined; - if (isBlock) isBlock = false; - } - break; - default: - break; - } - } - } - } +function triggerValidation(textDocument: vscode.TextDocument): void { + cleanPendingValidation(textDocument); + pendingValidationRequests[textDocument.uri.toString()] = setTimeout(() => { + delete pendingValidationRequests[textDocument.uri.toString()]; + validateTextDocument(textDocument); + }, validationDelayMs); +} - return r; +async function validateTextDocument(document: vscode.TextDocument) { + const mode = languageModes.getMode(document.languageId); + if (!mode || !mode.doValidation) { + return null; } + diagnostics.set(document.uri, mode.doValidation(document)); } \ No newline at end of file diff --git a/src/languages/lexLanguageService.ts b/src/languages/lexLanguageService.ts new file mode 100644 index 0000000..20f573d --- /dev/null +++ b/src/languages/lexLanguageService.ts @@ -0,0 +1,34 @@ +import { createScanner } from './parser/lexScanner'; +import { parse, LexDocument } from './parser/lexParser'; +import { Scanner } from './lexLanguageTypes'; +import { TextDocument, Position, WorkspaceEdit, Hover, CompletionList, CompletionItem, Definition, Location, Diagnostic } from 'vscode'; +import { doLEXCompletion } from './services/lexCompletions'; +import { doLEXHover } from './services/lexHover'; +import { doLEXFindDefinition } from './services/lexDefinition'; +import { doLEXRename } from './services/lexRename'; +import { doLEXFindReferences } from './services/lexReferences'; +import { doLEXValidation } from './services/lexValidation'; + +export interface LanguageService { + createScanner(input: string, initialOffset?: number): Scanner; + parseLexDocument(document: TextDocument): LexDocument; + doValidation: (document: TextDocument, lexDocument: LexDocument) => Diagnostic[]; + doComplete(document: TextDocument, position: Position, lexDocument: LexDocument): CompletionItem[] | CompletionList; + doHover(document: TextDocument, position: Position, lexDocument: LexDocument): Hover | null; + findDefinition(document: TextDocument, position: Position, lexDocument: LexDocument): Definition | null; + findReferences(document: TextDocument, position: Position, lexDocument: LexDocument): Location[]; + doRename(document: TextDocument, position: Position, newName: string, lexDocument: LexDocument): WorkspaceEdit | null; +} + +export function getLanguageService(): LanguageService { + return { + createScanner, + parseLexDocument: document => parse(document.getText()), + doValidation: (document, lexDocument) => doLEXValidation(document, lexDocument), + doComplete: (document, position, lexDocument) => doLEXCompletion(document, position, lexDocument), + doHover: (document, position, lexDocument) => doLEXHover(document, position, lexDocument), + findDefinition: (document, position, lexDocument) => doLEXFindDefinition(document, position, lexDocument), + findReferences: (document, position, lexDocument) => doLEXFindReferences(document, position, lexDocument), + doRename: (document, position, newName, lexDocument) => doLEXRename(document, position, newName, lexDocument) + }; +} \ No newline at end of file diff --git a/src/languages/lexLanguageTypes.ts b/src/languages/lexLanguageTypes.ts new file mode 100644 index 0000000..28a1d04 --- /dev/null +++ b/src/languages/lexLanguageTypes.ts @@ -0,0 +1,70 @@ +export enum ProblemType { + Information, + Warning, + Error +}; + +export interface ProblemRelated { + readonly message: string; + readonly offset: number; + readonly end: number; +}; + +export interface Problem { + readonly message: string; + readonly offset: number; + readonly end: number; + readonly type: ProblemType; + readonly related?: ProblemRelated; +}; + +export enum TokenType { + Component, + Literal, + Bar, + Percent, + Option, + RulesTag, + StartComment, + EndComment, + Comment, + StartAction, + EndAction, + Action, + StartCode, + EndCode, + Code, + StartPredefined, + EndPredefined, + Predefined, + StartStates, + EndStates, + States, + Invalid, + Unknown, + Divider, + EOL, + EOS +} + +export enum ScannerState { + WithinContent, + WithinComment, + WithinCode, + WithinAction, + WithinPredefined, + WithinStates +} + +export interface Scanner { + scan(): TokenType; + getTokenType(): TokenType; + getTokenOffset(): number; + getTokenLength(): number; + getTokenEnd(): number; + getTokenText(): string; + getTokenError(): string | undefined; + getScannerState(): ScannerState; + disableMultiLineBrackets(): void; + enableMultiLineBrackets(): void; +} \ No newline at end of file diff --git a/src/languages/parser/lexParser.ts b/src/languages/parser/lexParser.ts new file mode 100644 index 0000000..6dcec99 --- /dev/null +++ b/src/languages/parser/lexParser.ts @@ -0,0 +1,310 @@ +import { createScanner } from "./lexScanner"; +import { TokenType, ProblemType, ProblemRelated } from "../lexLanguageTypes"; +import { binarySearch } from "./utils"; +import { Problem } from "../lexLanguageTypes"; + +const _CHX = 'x'.charCodeAt(0); +const _CHS = 's'.charCodeAt(0); + +export interface ISymbol { + offset: number; + length: number; + end: number; + name: string; + used: boolean; + definition: [number, number]; + references: [number, number][]; +}; + +export interface LexDocument { + readonly embedded: Code[]; + readonly rulesRange: [number, number]; + readonly defines: { [name: string]: ISymbol }; + readonly states: { [name: string]: ISymbol }; + readonly components: ISymbol[]; + readonly problems: Problem[]; + + getEmbeddedCode(offset: number): Code | undefined; +}; + +export enum NodeType { + Token, + Type, + Rule, + Embedded +}; + +export interface Code { + offset: number; + length: number; + end: number; +}; + + +enum ParserState { + WaitingDecl, + WaitingDef, + WaitingOptionParams, + WaitingRule, + WaitingAction, + WithinRules, + WithinCode, +}; + +export function parse(text: string): LexDocument { + const scanner = createScanner(text); + const embedded: Code[] = []; + const rulesRange: [number, number] = [-1, -1]; + const defines: { [name: string]: ISymbol } = {}; + const states: { [name: string]: ISymbol } = {}; + const components: ISymbol[] = []; + const problems: Problem[] = []; + + const document: LexDocument = { + embedded, + rulesRange, + defines, + states, + components, + problems, + getEmbeddedCode(offset: number) { + return binarySearch(embedded, offset, (code, offset) => offset < code.offset ? 1 : (offset > code.end ? -1 : 0)) + } + }; + + function addProblem(message: string, offset: number, end: number, severity: ProblemType, related?: ProblemRelated) { + document.problems.push({ + offset: offset, + end: end, + message: message, + type: severity, + related: related + }); + } + + function addSymbol(symbols: { [name: string]: ISymbol }, name: string, offset: number, end: number) { + const old = symbols[name]; + if (old) { + addProblem(`Symbol was already declared.`, offset, end, ProblemType.Error, { + offset: old.offset, + end: old.end, + message: "Was declared here." + }); + } else { + symbols[name] = { + offset: offset, + length: end - offset, + end: end, + name: name, + used: false, + definition: [offset, end], + references: [[offset, end]] + }; + } + } + + let end = -2; + let state = ParserState.WaitingDecl; + let type = ''; + let token = scanner.scan(); + let offset = 0; + let codeOffset = 0; + let tokenText = ''; + let acceptingStates = false; + while (end < 0 && token !== TokenType.EOS) { + offset = scanner.getTokenOffset(); + switch (token) { + case TokenType.StartCode: + codeOffset = offset; + token = scanner.scan(); + continue; + case TokenType.EndCode: + document.embedded.push({ + offset: codeOffset, + length: scanner.getTokenEnd() - codeOffset, + end: scanner.getTokenEnd() + }); + token = scanner.scan(); + continue; + case TokenType.Code: + token = scanner.scan(); + continue; + case TokenType.StartComment: + case TokenType.EndComment: + case TokenType.Comment: + token = scanner.scan(); + continue; + } + switch (state) { + case ParserState.WaitingDecl: + switch (token) { + case TokenType.Component: + addSymbol(document.defines, scanner.getTokenText(), scanner.getTokenOffset(), scanner.getTokenEnd()); + scanner.disableMultiLineBrackets(); + state = ParserState.WaitingDef; + break; + case TokenType.Option: + state = ParserState.WaitingOptionParams; + const ch = scanner.getTokenText().charCodeAt(1); + if (ch === _CHS || ch === _CHX) { + acceptingStates = true; + } + break; + case TokenType.RulesTag: + state = ParserState.WaitingRule; + end++; + document.rulesRange[0] = offset; + break; + case TokenType.Divider: + addProblem("No white spaces are allowed at the beginning of the line.", scanner.getTokenOffset(), scanner.getTokenEnd(), ProblemType.Error); + break; + } + break; + case ParserState.WaitingDef: + switch (token) { + case TokenType.EOL: + state = ParserState.WaitingDecl; + scanner.enableMultiLineBrackets(); + break; + case TokenType.Action: + tokenText = scanner.getTokenText(); + if (/^[a-zA-Z]\w*$/.test(tokenText)) + document.components.push({ + name: tokenText, + offset: offset, + length: scanner.getTokenLength(), + end: scanner.getTokenEnd(), + used: true, + definition: [-1, -1], + references: [[offset, scanner.getTokenEnd()]] + }); + break; + } + break; + case ParserState.WaitingOptionParams: + switch (token) { + case TokenType.EOL: + state = ParserState.WaitingDecl; + acceptingStates = false; + break; + case TokenType.Component: + if (acceptingStates) + addSymbol(document.states, scanner.getTokenText(), scanner.getTokenOffset(), scanner.getTokenEnd()); + break; + } + break; + case ParserState.WaitingRule: + switch (token) { + case TokenType.Literal: + case TokenType.Component: + break; + case TokenType.Predefined: + break; + case TokenType.States: + tokenText = scanner.getTokenText(); + const matcher = /\w+/g; + var match; + while ((match = matcher.exec(tokenText)) !== null) { + const start = offset + match.index; + const end = offset + match.index + match[0].length; + document.components.push({ + name: match[0], + offset: start, + length: match[0].length, + end: end, + used: true, + definition: [-1, -1], + references: [[start, end]] + }); + } + break; + case TokenType.Action: + tokenText = scanner.getTokenText(); + if (/^\w+$/.test(tokenText)) + document.components.push({ + name: scanner.getTokenText(), + offset: offset, + length: scanner.getTokenLength(), + end: scanner.getTokenEnd(), + used: true, + definition: [-1, -1], + references: [[offset, scanner.getTokenEnd()]] + }); + break; + case TokenType.Divider: + state = ParserState.WaitingAction; + break; + case TokenType.RulesTag: + end++; + document.rulesRange[1] = offset; + break; + } + break; + case ParserState.WaitingAction: + switch (token) { + case TokenType.EOL: + case TokenType.Bar: + state = ParserState.WaitingRule; + break; + case TokenType.StartAction: + codeOffset = offset; + break; + case TokenType.EndAction: + document.embedded.push({ + offset: codeOffset, + length: scanner.getTokenEnd() - codeOffset, + end: scanner.getTokenEnd() + }); + break; + } + break; + } + token = scanner.scan(); + } + + for (let i = 0; i < document.components.length; i++) { + const component = document.components[i]; + let symbol: ISymbol; + if ((symbol = document.defines[component.name])) { + component.definition = symbol.definition; + component.references = symbol.references; + symbol.references.push([component.offset, component.end]); + symbol.used = true; + } else if ((symbol = document.states[component.name])) { + component.definition = symbol.definition; + component.references = symbol.references; + symbol.references.push([component.offset, component.end]); + symbol.used = true; + } else { + addProblem('Symbol not declared.', + component.offset, + component.end, + ProblemType.Error + ) + } + } + + Object.keys(document.defines).forEach(key => { + const component = document.defines[key]; + if (!component.used) { + addProblem('Definition declared but never used.', + component.offset, + component.end, + ProblemType.Warning + ) + } + }); + + Object.keys(document.states).forEach(key => { + const component = document.states[key]; + if (!component.used) { + addProblem('Definition declared but never used.', + component.offset, + component.end, + ProblemType.Warning + ) + } + }); + + return document; +} \ No newline at end of file diff --git a/src/languages/parser/lexScanner.ts b/src/languages/parser/lexScanner.ts new file mode 100644 index 0000000..18a5206 --- /dev/null +++ b/src/languages/parser/lexScanner.ts @@ -0,0 +1,223 @@ +import { TokenType, ScannerState, Scanner } from '../lexLanguageTypes' +import { MultiLineStream, _FSL, _AST, _NWL, _PCS, _BOP, _LAN, _BAR, _WSP, _DQO, _SQO, _BCL, _RAN } from './utils'; + +export function createScanner(input: string, initialOffset = 0, initialState: ScannerState = ScannerState.WithinContent): Scanner { + const stream = new MultiLineStream(input, initialOffset); + let state = initialState; + let tokenOffset: number = 0; + let tokenType: TokenType = TokenType.Unknown; + let tokenError: string | undefined; + let multiLineBracket: boolean = true; + + function nextComponent(): string { + return stream.advanceIfRegExp(/^[a-zA-Z]\w*/); + } + + function nextLiteral(): string { + return stream.advanceIfRegExp(/^("(?:[^"\\\n]|\\.)*"|'(?:[^'\\\n]|\\.)*')/); + } + + function finishToken(offset: number, type: TokenType, errorMessage?: string): TokenType { + tokenType = type; + tokenOffset = offset; + tokenError = errorMessage; + return type; + } + + function disableBrackets() { + multiLineBracket = false; + } + + function enableBrackets() { + multiLineBracket = true; + } + + function scan(): TokenType { + const offset = stream.pos(); + const oldState = state; + const token = internalScan(); + if (token !== TokenType.EOS && offset === stream.pos()) { + console.log('Scanner.scan has not advanced at offset ' + offset + ', state before: ' + oldState + ' after: ' + state); + stream.advance(1); + return finishToken(offset, TokenType.Unknown); + } + return token; + } + + function internalScan(): TokenType { + let white = false; + switch (state) { + case ScannerState.WithinAction: + case ScannerState.WithinCode: + case ScannerState.WithinComment: + stream.skipWhitespace(); + break; + default: + white = stream.skipWitheSpaceWithoutNewLine(); + } + + const offset = stream.pos(); + if (stream.eos()) { + return finishToken(offset, TokenType.EOS); + } + + if (white) { + return finishToken(offset, TokenType.Divider); + } + + switch (state) { + case ScannerState.WithinContent: + const ch = stream.nextChar(); + switch (ch) { + case _FSL: // / + if (stream.advanceIfChar(_AST)) { // /* + state = ScannerState.WithinComment; + return finishToken(offset, TokenType.StartComment); + } + if (stream.advanceIfChar(_FSL)) { // // + stream.advanceUntilChar(_NWL); + return finishToken(offset, TokenType.Comment); + } + break; + case _PCS: // % + if (stream.advanceIfChar(_PCS)) { // %% + return finishToken(offset, TokenType.RulesTag); + } + if (stream.advanceIfChar(_BOP)) { // %{ + state = ScannerState.WithinCode; + return finishToken(offset, TokenType.StartCode); + } + + if (stream.advanceIfRegExp(/^[\w-]+/)) { + return finishToken(offset, TokenType.Option); + } + + return finishToken(offset, TokenType.Percent); + case _LAN: // < + if (stream.advanceIfChar(_LAN)) { // < + state = ScannerState.WithinPredefined; + return finishToken(offset, TokenType.StartPredefined); + } + state = ScannerState.WithinStates; + return finishToken(offset, TokenType.StartStates); + case _BAR: // | + return finishToken(offset, TokenType.Bar); + case _BOP: // { + state = ScannerState.WithinAction; + return finishToken(offset, TokenType.StartAction); + case _WSP: // ' ' + stream.advanceUntilChar(_NWL); + return finishToken(offset, TokenType.Invalid); + case _NWL: // \n + return finishToken(offset, TokenType.EOL); + case _DQO: // " + case _SQO: // ' + stream.goBack(1); + const literal = nextLiteral() + if (literal.length > 0) { + return finishToken(offset, TokenType.Literal); + } + stream.advance(1); + return finishToken(offset, TokenType.Unknown); + } + + stream.goBack(1); + + const component = nextComponent(); + if (component.length > 0) { + return finishToken(offset, TokenType.Component); + } + + stream.advance(1); + return finishToken(offset, TokenType.Unknown); + case ScannerState.WithinComment: + if (stream.advanceIfChars([_AST, _FSL])) { // */ + state = ScannerState.WithinContent; + return finishToken(offset, TokenType.EndComment); + } + stream.advanceUntilChars([_AST, _FSL]); // */ + return finishToken(offset, TokenType.Comment); + case ScannerState.WithinCode: + if (stream.advanceIfChars([_PCS, _BCL])) { // %} + state = ScannerState.WithinContent; + return finishToken(offset, TokenType.EndCode); + } + stream.advanceUntilChars([_PCS, _BCL]); + return finishToken(offset, TokenType.Code); + case ScannerState.WithinAction: + if (stream.advanceIfChar(_BCL)) { // } + state = ScannerState.WithinContent; + return finishToken(offset, TokenType.EndAction); + } + var exit = false; + var brackets = 1; + while (!exit && brackets > 0) { + const ch = stream.nextChar(); + switch (ch) { + case _BOP: + brackets++; + break; + case _BCL: + brackets--; + break; + case _FSL: // / + if (stream.advanceIfChar(_AST)) { // /* + stream.advanceUntilChars([_AST, _FSL]); + stream.advance(2); + } else if (stream.advanceIfChar(_FSL)) { // // + stream.advanceUntilChar(_NWL); + stream.advance(1); + } + break; + case _SQO: // ' + case _DQO: // " + stream.goBack(1); + if (!nextLiteral()) // skip string if not skip character + stream.advance(1); + break; + case _NWL: + if (!multiLineBracket) { + exit = true; + state = ScannerState.WithinContent; + } + break; + } + if (ch === 0) break; + } + if (brackets > 0) { + return finishToken(offset, TokenType.Unknown, "Code not closed!"); + } + if (!exit) + stream.goBack(1); + return finishToken(offset, TokenType.Action); + case ScannerState.WithinPredefined: + if (stream.advanceIfChars([_RAN, _RAN])) { // >> + state = ScannerState.WithinContent; + return finishToken(offset, TokenType.EndPredefined); + } + stream.advanceUntilChars([_RAN, _RAN]); + return finishToken(offset, TokenType.Predefined); + case ScannerState.WithinStates: + if (stream.advanceIfChar(_RAN)) { // > + state = ScannerState.WithinContent; + return finishToken(offset, TokenType.EndStates); + } + stream.advanceUntilChar(_RAN); + return finishToken(offset, TokenType.States); + } + state = ScannerState.WithinContent; + return finishToken(offset, TokenType.Unknown, "invalid symbol found"); + } + return { + scan, + getTokenType: () => tokenType, + getTokenOffset: () => tokenOffset, + getTokenLength: () => stream.pos() - tokenOffset, + getTokenEnd: () => stream.pos(), + getTokenText: () => stream.getSource().substring(tokenOffset, stream.pos()), + getScannerState: () => state, + getTokenError: () => tokenError, + enableMultiLineBrackets: () => enableBrackets(), + disableMultiLineBrackets: () => disableBrackets() + }; +} diff --git a/src/languages/parser/utils.ts b/src/languages/parser/utils.ts new file mode 100644 index 0000000..5a6f309 --- /dev/null +++ b/src/languages/parser/utils.ts @@ -0,0 +1,182 @@ +export function binarySearch(array: T[], key: number, comparator: (op1: T, op2: number) => number): T | undefined { + let low = 0, + high = array.length - 1; + + while (low <= high) { + const mid = ((low + high) / 2) | 0; + const comp = comparator(array[mid], key); + if (comp < 0) { + low = mid + 1; + } else if (comp > 0) { + high = mid - 1; + } else { + return array[mid]; + } + } + return undefined; +} + +export const _NWL = '\n'.charCodeAt(0); +export const _CAR = '\r'.charCodeAt(0); +export const _LFD = '\f'.charCodeAt(0); +export const _TAB = '\t'.charCodeAt(0); +export const _WSP = ' '.charCodeAt(0); + +export const _LAN = '<'.charCodeAt(0); +export const _RAN = '>'.charCodeAt(0); +export const _FSL = '/'.charCodeAt(0); +export const _AST = '*'.charCodeAt(0); +export const _COL = ':'.charCodeAt(0); +export const _BAR = '|'.charCodeAt(0); +export const _BOP = '{'.charCodeAt(0); +export const _BCL = '}'.charCodeAt(0); +export const _PCS = '%'.charCodeAt(0); +export const _DOT = '.'.charCodeAt(0); +export const _DQO = '"'.charCodeAt(0); +export const _SQO = '\''.charCodeAt(0); +export const _SBO = '['.charCodeAt(0); +export const _SBC = ']'.charCodeAt(0); +export const _SCL = ';'.charCodeAt(0); + +/** + * Imported form Microsoft's vscode-html-languageservice's scanner. + * + * Thank you Microsoft. + */ +export class MultiLineStream { + private source: string; + private len: number; + private position: number; + + constructor(source: string, position: number) { + this.source = source; + this.len = source.length; + this.position = position; + } + + public eos(): boolean { + return this.len <= this.position; + } + + public getSource(): string { + return this.source; + } + + public pos(): number { + return this.position; + } + + public goBackTo(pos: number): void { + this.position = pos; + } + + public goBack(n: number): void { + this.position -= n; + } + + public advance(n: number): void { + this.position += n; + } + + public goToEnd(): void { + this.position = this.source.length; + } + + public nextChar(): number { + return this.source.charCodeAt(this.position++) || 0; + } + + public peekChar(n: number = 0): number { + return this.source.charCodeAt(this.position + n) || 0; + } + + public advanceIfChar(ch: number): boolean { + if (ch === this.source.charCodeAt(this.position)) { + this.position++; + return true; + } + return false; + } + + public advanceIfChars(ch: number[]): boolean { + let i: number; + if (this.position + ch.length > this.source.length) { + return false; + } + for (i = 0; i < ch.length; i++) { + if (this.source.charCodeAt(this.position + i) !== ch[i]) { + return false; + } + } + this.advance(i); + return true; + } + + public advanceIfRegExp(regex: RegExp): string { + const str = this.source.substr(this.position); + const match = str.match(regex); + if (match) { + this.position = this.position + match.index! + match[0].length; + return match[0]; + } + return ''; + } + + public advanceUntilRegExp(regex: RegExp): string { + const str = this.source.substr(this.position); + const match = str.match(regex); + if (match) { + this.position = this.position + match.index!; + return match[0]; + } else { + this.goToEnd(); + } + return ''; + } + + public advanceUntilChar(ch: number): boolean { + while (this.position < this.source.length) { + if (this.source.charCodeAt(this.position) === ch) { + return true; + } + this.advance(1); + } + return false; + } + + public advanceUntilChars(ch: number[]): boolean { + while (this.position + ch.length <= this.source.length) { + let i = 0; + for (; i < ch.length && this.source.charCodeAt(this.position + i) === ch[i]; i++) { + } + if (i === ch.length) { + return true; + } + this.advance(1); + } + this.goToEnd(); + return false; + } + + public skipWhitespace(): boolean { + const n = this.advanceWhileChar(ch => { + return ch === _WSP || ch === _TAB || ch === _NWL || ch === _LFD || ch === _CAR; + }); + return n > 0; + } + + public skipWitheSpaceWithoutNewLine(): boolean { + const n = this.advanceWhileChar(ch => { + return ch === _WSP || ch === _TAB || ch === _LFD; + }); + return n > 0; + } + + public advanceWhileChar(condition: (ch: number) => boolean): number { + const posNow = this.position; + while (this.position < this.len && condition(this.source.charCodeAt(this.position))) { + this.position++; + } + return this.position - posNow; + } +} diff --git a/src/languages/parser/yaccParser.ts b/src/languages/parser/yaccParser.ts new file mode 100644 index 0000000..16321f0 --- /dev/null +++ b/src/languages/parser/yaccParser.ts @@ -0,0 +1,390 @@ +import { createScanner } from './yaccScanner'; +import { TokenType, ProblemType, Problem, ProblemRelated, tokenTypes } from '../yaccLanguageTypes'; +import { binarySearch } from './utils'; +import { SemanticTokenData, SemanticTokenModifier, SemanticTokenType } from '../semanticTokens'; +import { Position } from 'vscode'; + +const predefined: { [name: string]: boolean } = {}; +predefined['UPLUS'] = true; +predefined['UMINUS'] = true; +predefined['POSTFIXOP'] = true; + +enum ParserState { + WaitingToken, + WaitingSymbol, + WaitingRule, + WaitingUnion, + Normal +}; + +export interface ISymbol { + terminal: boolean; + offset: number; + length: number; + end: number; + name: string; + type: string; + used: boolean; + definition: [number, number]; + references: [number, number][]; +}; + +export interface YACCDocument { + readonly embedded: Node[]; + readonly nodes: Node[]; + readonly types: { [name: string]: ISymbol }; + readonly tokens: { [name: string]: ISymbol }; + readonly symbols: { [name: string]: ISymbol }; + readonly components: ISymbol[]; + readonly rulesRange: [number, number]; + readonly problems: Problem[]; + + getNodeByOffset(offset: number): Node | undefined; + getEmbeddedNode(offset: number): Node | undefined; + getSemanticTokens(getPos: (offset: number) => Position): SemanticTokenData[]; +}; + +export enum NodeType { + Token, + Type, + Rule, + Embedded +}; + +export interface Node { + nodeType: NodeType; + offset: number; + length: number; + end: number; + + typeOffset?: number; + typeEnd?: number; + + actions?: string[]; +}; + +export function parse(text: string): YACCDocument { + const scanner = createScanner(text); + + const embedded: Node[] = []; + const nodes: Node[] = []; + const types: { [name: string]: ISymbol } = {}; + const tokens: { [name: string]: ISymbol } = {}; + const symbols: { [name: string]: ISymbol } = {}; + const components: ISymbol[] = []; + const rulesRange: [number, number] = [-1, -1]; + const problems: Problem[] = []; + const document: YACCDocument = { + embedded, + nodes, + types, + tokens, + symbols, + components, + rulesRange, + problems, + + getNodeByOffset(offset: number): Node | undefined { + return binarySearch(this.nodes, offset, (node, offset) => offset < node.offset ? 1 : (offset > node.end ? -1 : 0)) + }, + getEmbeddedNode(offset: number): Node | undefined { + return binarySearch(this.embedded, offset, (node, offset) => offset < node.offset ? 1 : (offset > node.end ? -1 : 0)) + }, + getSemanticTokens(getPos: (offset: number) => Position): SemanticTokenData[] { + const r: SemanticTokenData[] = []; + for (let i = 0; i < this.components.length; i++) { + const component = this.components[i]; + if (!component.terminal) { + r.push({ + start: getPos(component.offset), + length: component.length, + typeIdx: SemanticTokenType.class, + modifierSet: SemanticTokenModifier._ + }); + } else { + r.push({ + start: getPos(component.offset), + length: component.length, + typeIdx: SemanticTokenType.parameter, + modifierSet: SemanticTokenModifier._ + }); + } + } + return r; + } + }; + + function addProblem(message: string, offset: number, end: number, severity: ProblemType, related?: ProblemRelated) { + document.problems.push({ + offset: offset, + end: end, + message: message, + type: severity, + related: related + }); + } + + function addSymbolToMap(symbols: { [name: string]: ISymbol }, terminal: boolean, offset: number, end: number, name: string, type: string) { + const old = symbols[name]; + if (old) { + addProblem(`Symbol was already declared/defined.`, offset, end, ProblemType.Error, { + offset: old.offset, + end: old.end, + message: "Was declared/defined here." + }); + } else { + symbols[name] = { + terminal: terminal, + offset: offset, + length: end - offset, + end: end, + name: name, + type: type, + used: false, + definition: [offset, end], + references: [[offset, end]] + }; + } + } + let end = -2; + let state = ParserState.Normal; + let type = ''; + let token = scanner.scan(); + let offset = 0; + let actionOffset = 0; + let tokenText = ''; + let lastNode: Node | undefined; + let lastToken = token; + while (end < 0 && token !== TokenType.EOS) { + offset = scanner.getTokenOffset(); + switch (token) { + case TokenType.StartAction: // save the offset of the action zone + actionOffset = offset; + break; + case TokenType.EndAction: // save the action region + document.embedded.push({ nodeType: NodeType.Embedded, offset: actionOffset, length: scanner.getTokenLength(), end: scanner.getTokenEnd() }); + break; + case TokenType.Action: + switch (state) { + case ParserState.WaitingUnion: // if we are inside union, extract type information + tokenText = scanner.getTokenText(); + const typeMatcher = /(.*[ \t\f*&])([a-zA-Z0-9_]+)\s*;/g; + var res; + while ((res = typeMatcher.exec(tokenText)) !== null) { + const typeOffset = offset + res.index; + const typeEnd = offset + res.index + res[0].length; + addSymbolToMap(document.types, true, typeOffset, typeEnd, res[2], res[1].replace(/\s*/g, "")); + } + state = ParserState.Normal; + break; + case ParserState.WaitingRule: // if we are inside a rule, save the code + if (lastNode && lastNode.actions) { + lastNode.actions.push(scanner.getTokenText()); + } + break; + } + break; + case TokenType.Option: + // save the last node + if (state !== ParserState.WaitingRule && lastNode !== undefined) { + lastNode.end = offset - 1; + lastNode.length = lastNode.end - lastNode.offset; + document.nodes.push(lastNode); + type = ''; + lastNode = undefined; + state = ParserState.Normal; + } + tokenText = scanner.getTokenText(); + switch (tokenText) { + case '%union': + state = ParserState.WaitingUnion; + break; + case '%token': + lastNode = { nodeType: NodeType.Token, offset: offset, length: -1, end: -1 } + state = ParserState.WaitingToken; + break; + case '%type': + lastNode = { nodeType: NodeType.Type, offset: offset, length: -1, end: -1 } + state = ParserState.WaitingSymbol; + break; + default: + break; + } + break; + case TokenType.StartType: + type = '' + if (lastNode) + lastNode.typeOffset = scanner.getTokenOffset(); + break; + case TokenType.EndType: + if (lastNode) + lastNode.typeEnd = scanner.getTokenOffset(); + break; + case TokenType.TypeValue: + // extract the type inside the tag <[type]> + type = scanner.getTokenText(); + const t = document.types[type]; + if (t) { + t.references.push([scanner.getTokenOffset(), scanner.getTokenEnd()]); + } else { + addProblem(`Type was not declared in the %union.`, scanner.getTokenOffset(), scanner.getTokenEnd(), ProblemType.Error); + } + break; + case TokenType.RulesTag: + // start of the rule section + if (lastNode !== undefined) { + lastNode.end = offset - 1; + lastNode.length = lastNode.end - lastNode.offset; + document.nodes.push(lastNode); + lastNode = undefined; + type = ''; + } + document.rulesRange[end === -2 ? 0 : 1] = offset; + end++; + state = ParserState.WaitingRule; + break; + case TokenType.Word: + const word = scanner.getTokenText(); + switch (state) { + case ParserState.Normal: + break; + case ParserState.WaitingToken: + addSymbolToMap(document.tokens, true, offset, scanner.getTokenEnd(), word, type); + break; + case ParserState.WaitingSymbol: + addSymbolToMap(document.symbols, true, offset, scanner.getTokenEnd(), word, type); + break; + case ParserState.WaitingRule: + document.components.push({ + terminal: true, + offset: offset, + length: scanner.getTokenLength(), + end: scanner.getTokenEnd(), + name: scanner.getTokenText(), + type: '', + used: true, + definition: [-1, -1], + references: [[offset, scanner.getTokenEnd()]] + }); + break; + default: + addProblem(`Unexpected symbol ${word}`, offset, scanner.getTokenEnd(), ProblemType.Error); + } + break; + case TokenType.Colon: + switch (state) { + case ParserState.WaitingRule: // we maybe found a new non-terminal symbol definition + if (lastToken !== TokenType.Word) { + addProblem(`Unexpected ':' you can only declare a non-terminal with a word.`, scanner.getTokenOffset(), scanner.getTokenEnd(), ProblemType.Error); + break; + } + const nonTerminal = document.components.pop(); // the last symbol was not part of last rule + if (nonTerminal !== undefined) { // I think the array will never be empty, but check for sanity + if (lastNode !== undefined) { // Last rule finished + lastNode.end = nonTerminal.offset - 1; + lastNode.length = lastNode.end - lastNode.offset; + document.nodes.push(lastNode); + } + nonTerminal.terminal = false; // this will not be a terminal + nonTerminal.definition = [nonTerminal.offset, nonTerminal.end]; // is defined here + const symbol = document.symbols[nonTerminal.name]; + if (symbol !== undefined) { // if the symbol was previously declared with %type ... + if (!symbol.terminal) { // there is a redefinition of the symbol + addProblem(`Non-terminal symbol was already declared.`, nonTerminal.offset, nonTerminal.end, ProblemType.Error, { + offset: symbol.offset, + end: symbol.end, + message: "Was declared here." + }); + } + nonTerminal.references.push(symbol.references[0]); // add %type reference + nonTerminal.type = symbol.type; // assign the type from %type + symbol.references = nonTerminal.references; // update also the old references + } + const token = document.tokens[nonTerminal.name]; + if (token !== undefined) { // if the symbol was already declared as a token + addProblem(`Symbol was already declared as a token.`, nonTerminal.offset, nonTerminal.end, ProblemType.Error, { + offset: token.offset, + end: token.end, + message: "Was declared here." + }); + } + document.symbols[nonTerminal.name] = nonTerminal; // update symbol table + lastNode = { nodeType: NodeType.Rule, offset: nonTerminal.offset, length: -1, end: -1, actions: [] } + } + break; + default: + addProblem(`Unexpected : character`, scanner.getTokenOffset(), scanner.getTokenEnd(), ProblemType.Error); + break; + } + break; + case TokenType.SemiColon: + case TokenType.StartComment: + case TokenType.EndComment: + case TokenType.Comment: + case TokenType.Param: + case TokenType.Literal: + break; + case TokenType.Bar: + if (state !== ParserState.WaitingRule) { + addProblem(`Unexpected | symbol.`, scanner.getTokenOffset(), scanner.getTokenEnd(), ProblemType.Error); + } + break; + default: + // TODO: better problem detection with unexpected symbols + if (state === ParserState.WaitingRule) + addProblem(`Unknown symbol ${scanner.getTokenText()}.`, scanner.getTokenOffset(), scanner.getTokenEnd(), ProblemType.Error); + break; + } + lastToken = token; + token = scanner.scan(); + } + + for (let i = 0; i < document.components.length; i++) { + const component = document.components[i]; + let symbol: ISymbol; + if ((symbol = document.symbols[component.name])) { + component.terminal = false; + component.definition = symbol.definition; + component.type = symbol.type; + component.references = symbol.references; + symbol.references.push([component.offset, component.end]); + } else if ((symbol = document.tokens[component.name])) { + component.definition = symbol.definition; + component.type = symbol.type; + component.references = symbol.references; + symbol.references.push([component.offset, component.end]); + symbol.used = true; + } else if (!predefined[component.name]) { + document.problems.push({ + offset: component.offset, + end: component.end, + message: 'Symbol was not declared.', + type: ProblemType.Error + }); + } + } + + Object.keys(document.tokens).forEach(key => { + const component = document.tokens[key]; + if (!component.used) { + addProblem('Token declared but never used.', + component.offset, + component.end, + ProblemType.Warning + ) + } + }); + + Object.keys(document.symbols).forEach(key => { + if (document.symbols[key].definition[0] < document.rulesRange[0]) { + addProblem('Non-terminal symbol type defined but never declared.', + document.symbols[key].offset, + document.symbols[key].end, + ProblemType.Warning + ) + delete document.symbols[key]; + } + }); + + return document +} \ No newline at end of file diff --git a/src/languages/parser/yaccScanner.ts b/src/languages/parser/yaccScanner.ts new file mode 100644 index 0000000..e075ae5 --- /dev/null +++ b/src/languages/parser/yaccScanner.ts @@ -0,0 +1,201 @@ +import { TokenType, ScannerState, Scanner } from '../yaccLanguageTypes' + +import { + MultiLineStream, _FSL, _AST, _NWL, _BAR, _COL, _BOP, _BCL, _DOT, _PCS, _LAN, _DQO, _SQO, _RAN, _SBO, _SCL +} from './utils' + +export function createScanner(input: string, initialOffset = 0, initialState: ScannerState = ScannerState.WithinContent): Scanner { + const stream = new MultiLineStream(input, initialOffset); + let state = initialState; + let tokenOffset: number = 0; + let tokenType: TokenType = TokenType.Unknown; + let tokenError: string | undefined; + + function nextWord(): string { + return stream.advanceIfRegExp(/^[a-zA-Z]\w*/); + } + + function nextLiteral(): string { + return stream.advanceIfRegExp(/^("(?:[^"\\\n]|\\.)*"|'(?:[^'\\\n]|\\.)*')/); + } + + function nextParam(): string { + return stream.advanceIfRegExp(/^\[[a-zA-Z]\w*\]/); + } + + function finishToken(offset: number, type: TokenType, errorMessage?: string): TokenType { + tokenType = type; + tokenOffset = offset; + tokenError = errorMessage; + return type; + } + + function scan(): TokenType { + const offset = stream.pos(); + const oldState = state; + const token = internalScan(); + if (token !== TokenType.EOS && offset === stream.pos()) { + console.log('Scanner.scan has not advanced at offset ' + offset + ', state before: ' + oldState + ' after: ' + state); + stream.advance(1); + return finishToken(offset, TokenType.Unknown); + } + return token; + } + + function internalScan(): TokenType { + stream.skipWhitespace(); + const offset = stream.pos(); + if (stream.eos()) { + return finishToken(offset, TokenType.EOS); + } + + switch (state) { + case ScannerState.WithinContent: + const ch = stream.nextChar(); + switch (ch) { + case _FSL: // / + if (stream.advanceIfChar(_AST)) { // /* + state = ScannerState.WithinComment; + return finishToken(offset, TokenType.StartComment); + } + if (stream.advanceIfChar(_FSL)) { // // + stream.advanceUntilChar(_NWL); + return finishToken(offset, TokenType.Comment); + } + break; + case _BAR: // | + return finishToken(offset, TokenType.Bar); + case _COL: // : + return finishToken(offset, TokenType.Colon); + case _BOP: // { + state = ScannerState.WithinCode; + return finishToken(offset, TokenType.StartAction); + case _BCL: // } + return finishToken(offset, TokenType.EndAction); + case _DOT: // . + return finishToken(offset, TokenType.Dot); + case _PCS: // % + if (stream.advanceIfChar(_PCS)) { // %% + return finishToken(offset, TokenType.RulesTag); + } + if (stream.advanceIfChar(_BOP)) { // %{ + state = ScannerState.WithinCode; + return finishToken(offset, TokenType.StartAction); + } + + if (stream.advanceIfRegExp(/^[\w-]+/)) { + return finishToken(offset, TokenType.Option); + } + return finishToken(offset, TokenType.Percent); + case _LAN: // < + state = ScannerState.WithinTypeValue; + return finishToken(offset, TokenType.StartType); + case _DQO: // " + case _SQO: // ' + stream.goBack(1); + const literal = nextLiteral() + if (literal.length > 0) { + return finishToken(offset, TokenType.Literal); + } + stream.advance(1); + return finishToken(offset, TokenType.Unknown); + case _SBO: // [ + stream.goBack(1); + const param = nextParam() + if (param.length > 0) { + return finishToken(offset, TokenType.Param); + } + stream.advance(1); + return finishToken(offset, TokenType.Unknown); + case _SCL: + return finishToken(offset, TokenType.SemiColon); + } + + stream.goBack(1); + + const literal = nextLiteral() + if (literal.length > 0) { + return finishToken(offset, TokenType.Literal); + } + + const component = nextWord(); + if (component.length > 0) { + return finishToken(offset, TokenType.Word); + } + + stream.advance(1); + return finishToken(offset, TokenType.Unknown); + case ScannerState.WithinTypeValue: + if (stream.advanceIfChar(_RAN)) { // > + state = ScannerState.WithinContent; + return finishToken(offset, TokenType.EndType); + } + + const typeValue = nextWord(); + if (typeValue.length > 0) { + return finishToken(offset, TokenType.TypeValue); + } + + state = ScannerState.WithinContent; + stream.goBack(1); + return finishToken(offset, TokenType.Unknown); + case ScannerState.WithinComment: + if (stream.advanceIfChars([_AST, _FSL])) { // */ + state = ScannerState.WithinContent; + return finishToken(offset, TokenType.EndComment); + } + stream.advanceUntilChars([_AST, _FSL]); // */ + return finishToken(offset, TokenType.Comment); + case ScannerState.WithinCode: + if (stream.advanceIfChar(_BCL)) { // } + state = ScannerState.WithinContent; + return finishToken(offset, TokenType.EndAction); + } + var brackets = 1; + while (brackets > 0) { + const ch = stream.nextChar(); + switch (ch) { + case _BOP: + brackets++; + break; + case _BCL: + brackets--; + break; + case _FSL: // / + if (stream.advanceIfChar(_AST)) { // /* + stream.advanceUntilChars([_AST, _FSL]); + stream.advance(2); + } else if (stream.advanceIfChar(_FSL)) { // // + stream.advanceUntilChar(_NWL); + stream.advance(1); + } + break; + case _SQO: // ' + case _DQO: // " + stream.goBack(1); + if(!nextLiteral()) // skip string if not skip character + stream.advance(1); + break; + } + if (ch === 0) break; + } + if (brackets > 0) { + return finishToken(offset, TokenType.Unknown, "Code not closed!"); + } + stream.goBack(1); + return finishToken(offset, TokenType.Action); + } + state = ScannerState.WithinContent; + return finishToken(offset, TokenType.Unknown, "invalid symbol found"); + } + return { + scan, + getTokenType: () => tokenType, + getTokenOffset: () => tokenOffset, + getTokenLength: () => stream.pos() - tokenOffset, + getTokenEnd: () => stream.pos(), + getTokenText: () => stream.getSource().substring(tokenOffset, stream.pos()), + getScannerState: () => state, + getTokenError: () => tokenError + }; +} diff --git a/src/languages/semanticTokens.ts b/src/languages/semanticTokens.ts new file mode 100644 index 0000000..4d4cf98 --- /dev/null +++ b/src/languages/semanticTokens.ts @@ -0,0 +1,16 @@ +import { Position } from 'vscode'; + +export interface SemanticTokenData { + start: Position; + length: number; + typeIdx: number; + modifierSet: number; +} + +export const enum SemanticTokenType { + class, enum, interface, namespace, typeParameter, type, parameter, variable, property, function, member, _ +} + +export const enum SemanticTokenModifier { + declaration, static, async, readonly, _ +} diff --git a/src/languages/services/lexCompletions.ts b/src/languages/services/lexCompletions.ts new file mode 100644 index 0000000..4ca0295 --- /dev/null +++ b/src/languages/services/lexCompletions.ts @@ -0,0 +1,72 @@ +import { TextDocument, CompletionList, CompletionItem, CompletionItemKind, Position, Range } from 'vscode'; +import { LexDocument } from '../parser/lexParser'; +import { createScanner } from '../parser/lexScanner'; +import { TokenType } from '../lexLanguageTypes'; + +const keywords = ['array', 'pointer', 'option', 's', 'x']; + +export function doLEXCompletion(document: TextDocument, position: Position, lexDocument: LexDocument): CompletionItem[] | CompletionList { + const offset = document.offsetAt(position); + const text = document.getText(); + const embedded = lexDocument.getEmbeddedCode(offset); + if (embedded !== undefined) { + return []; + } + + const scanner = createScanner(text, offset - 1); + if (scanner.scan() === TokenType.Percent) { + if (position.character === 1 && offset < lexDocument.rulesRange[0]) + return keywords.map((keyword) => { + const completion = new CompletionItem(keyword); + completion.detail = "keyword"; + completion.kind = CompletionItemKind.Constructor; + return completion; + }); + return []; + } + + const line = document.lineAt(position.line).text.substring(0, position.character); + const result: CompletionItem[] = []; + if (offset < lexDocument.rulesRange[0]) { + // if before rules zone, definition need to be on the right + const ok = line.match(/^\w+.*({\w*}?)+/); + if (ok) { + Object.keys(lexDocument.defines).forEach((key) => { + const completion = new CompletionItem(key); + completion.detail = "definition"; + completion.kind = CompletionItemKind.Class; + result.push(completion); + }) + } + } else if (offset < lexDocument.rulesRange[1]) { + const res = line.match(/^(?:{[a-zA-Z0-9_]*}?)+$/); + if (res) { + if (res[0].length >= position.character) { + Object.keys(lexDocument.defines).forEach((key) => { + const completion = new CompletionItem(key); + completion.detail = "definition"; + completion.kind = CompletionItemKind.Class; + result.push(completion); + }) + } + } else { + if (line.match(/^<[\w,]*>(?:{[a-zA-Z0-9_]*}?)+$/)) { + Object.keys(lexDocument.defines).forEach((key) => { + const completion = new CompletionItem(key); + completion.detail = "definition"; + completion.kind = CompletionItemKind.Class; + result.push(completion); + }) + } else if (line.match(/^<[\w,]*$/)) { // TODO: fix completion for {} after <> + + Object.keys(lexDocument.states).forEach((key) => { + const completion = new CompletionItem(key); + completion.detail = "initial state"; + completion.kind = CompletionItemKind.Class; + result.push(completion); + }) + } + } + } + return result; +} \ No newline at end of file diff --git a/src/languages/services/lexDefinition.ts b/src/languages/services/lexDefinition.ts new file mode 100644 index 0000000..ddd5eb0 --- /dev/null +++ b/src/languages/services/lexDefinition.ts @@ -0,0 +1,18 @@ +import { TextDocument, Position, Definition, Location, Range } from 'vscode'; +import { LexDocument, ISymbol } from "../parser/lexParser"; + +export function doLEXFindDefinition(document: TextDocument, position: Position, lexDocument: LexDocument): Definition | null { + const offset = document.offsetAt(position); + const node = lexDocument.getEmbeddedCode(offset); + if (node) { + return null; + } + + const word = document.getText(document.getWordRangeAtPosition(position)); + var symbol: ISymbol | undefined = lexDocument.defines[word] || lexDocument.states[word]; + let location: Location | null = null; + if (symbol) { + location = new Location(document.uri, new Range(document.positionAt(symbol.definition[0]), document.positionAt(symbol.definition[1]))); + } + return location; +} \ No newline at end of file diff --git a/src/languages/services/lexHover.ts b/src/languages/services/lexHover.ts new file mode 100644 index 0000000..12213d1 --- /dev/null +++ b/src/languages/services/lexHover.ts @@ -0,0 +1,20 @@ +import { TextDocument, Hover, Position, MarkdownString } from 'vscode'; +import { LexDocument, ISymbol } from "../parser/lexParser"; +import { createMarkedCodeString } from "./utils"; + +export function doLEXHover(document: TextDocument, position: Position, lexDocument: LexDocument): Hover | null { + const offset = document.offsetAt(position); + const node = lexDocument.getEmbeddedCode(offset); + if (node) { + return null; + } + + const word = document.getText(document.getWordRangeAtPosition(position)); + var symbol: ISymbol = lexDocument.defines[word] || lexDocument.states[word]; + if (symbol) { + const line = document.lineAt(document.positionAt(symbol.offset)).text; + return { contents: [createMarkedCodeString(line, 'lex')] }; + } + + return null; +} \ No newline at end of file diff --git a/src/languages/services/lexReferences.ts b/src/languages/services/lexReferences.ts new file mode 100644 index 0000000..6586701 --- /dev/null +++ b/src/languages/services/lexReferences.ts @@ -0,0 +1,18 @@ +import { TextDocument, Position, Location, Range } from 'vscode'; +import { LexDocument, ISymbol } from "../parser/lexParser"; + +export function doLEXFindReferences(document: TextDocument, position: Position, lexDocument: LexDocument): Location[] { + const offset = document.offsetAt(position); + const node = lexDocument.getEmbeddedCode(offset); + if (node) { + return []; + } + + const word = document.getText(document.getWordRangeAtPosition(position)); + var symbol: ISymbol | undefined = lexDocument.defines[word] || lexDocument.states[word]; + let location: Location[] = []; + symbol?.references.forEach(reference => { + location.push(new Location(document.uri, new Range(document.positionAt(reference[0]), document.positionAt(reference[1])))); + }) + return location; +} \ No newline at end of file diff --git a/src/languages/services/lexRename.ts b/src/languages/services/lexRename.ts new file mode 100644 index 0000000..1f6f145 --- /dev/null +++ b/src/languages/services/lexRename.ts @@ -0,0 +1,18 @@ +import { TextDocument, Position, Range, WorkspaceEdit, TextEdit } from 'vscode'; +import { LexDocument, ISymbol } from "../parser/lexParser" + +export function doLEXRename(document: TextDocument, position: Position, newName: string, lexDocument: LexDocument): WorkspaceEdit | null { + const offset = document.offsetAt(position); + const node = lexDocument.getEmbeddedCode(offset); + if (node) { + return null; + } + + const word = document.getText(document.getWordRangeAtPosition(position)); + var symbol: ISymbol | undefined = lexDocument.defines[word] || lexDocument.states[word]; + const edits = new WorkspaceEdit(); + symbol?.references.forEach(reference => { + edits.replace(document.uri, new Range(document.positionAt(reference[0]), document.positionAt(reference[1])), newName); + }) + return edits; +} \ No newline at end of file diff --git a/src/languages/services/lexValidation.ts b/src/languages/services/lexValidation.ts new file mode 100644 index 0000000..d6cefd6 --- /dev/null +++ b/src/languages/services/lexValidation.ts @@ -0,0 +1,31 @@ +import { TextDocument, Diagnostic, Range, DiagnosticSeverity, DiagnosticRelatedInformation, Location } from "vscode"; +import { ProblemType } from "../yaccLanguageTypes"; +import { LexDocument } from "../parser/lexParser"; + +export function doLEXValidation(document: TextDocument, lexDocument: LexDocument): Diagnostic[] { + const diags: Diagnostic[] = []; + lexDocument.problems.forEach(problem => { + const range = new Range(document.positionAt(problem.offset), document.positionAt(problem.end)); + let severity: DiagnosticSeverity = DiagnosticSeverity.Information; + switch (problem.type) { + case ProblemType.Error: + severity = DiagnosticSeverity.Error; + break; + case ProblemType.Information: + severity = DiagnosticSeverity.Information; + break; + case ProblemType.Warning: + severity = DiagnosticSeverity.Warning; + break; + } + const diag = new Diagnostic(range, problem.message, severity); + if (problem.related) { + diag.relatedInformation = [new DiagnosticRelatedInformation( + new Location(document.uri, new Range(document.positionAt(problem.related.offset), document.positionAt(problem.related.end))), + problem.related.message + )]; + } + diags.push(diag); + }); + return diags; +} \ No newline at end of file diff --git a/src/languages/services/utils.ts b/src/languages/services/utils.ts new file mode 100644 index 0000000..e0e2d97 --- /dev/null +++ b/src/languages/services/utils.ts @@ -0,0 +1,7 @@ +import { MarkdownString } from 'vscode'; + +export function createMarkedCodeString(code: string, languageId: string): MarkdownString { + const str = new MarkdownString(); + str.appendCodeblock(code, languageId); + return str; +} \ No newline at end of file diff --git a/src/languages/services/yaccCompletions.ts b/src/languages/services/yaccCompletions.ts new file mode 100644 index 0000000..ab47582 --- /dev/null +++ b/src/languages/services/yaccCompletions.ts @@ -0,0 +1,77 @@ +import { TextDocument, CompletionList, CompletionItem, CompletionItemKind, Position } from 'vscode'; +import { YACCDocument, NodeType } from '../parser/yaccParser'; +import { createScanner } from '../parser/yaccScanner'; +import { TokenType } from '../yaccLanguageTypes'; + +const keywords = ['type', 'option', 'token', 'left', 'right', 'define', 'output', + 'precedence', 'nterm', 'destructor', 'union', 'code', 'printer', + 'parse-param', 'lex-param', 'pure-parser', 'expect', 'name-prefix', 'locations', 'nonassoc']; + +export function doYACCComplete(document: TextDocument, position: Position, yaccDocument: YACCDocument): CompletionItem[] | CompletionList { + const offset = document.offsetAt(position); + const text = document.getText(); + const embedded = yaccDocument.getEmbeddedNode(offset); + if (embedded !== undefined) { + return []; + } + + const scanner = createScanner(text, offset - 1); + if (scanner.scan() === TokenType.Percent) { + if (position.character === 1 && offset < yaccDocument.rulesRange[0]) + return keywords.map((keyword) => { + const completion = new CompletionItem(keyword); + completion.detail = "keyword"; + completion.kind = CompletionItemKind.Constructor; + return completion; + }); + return []; + } + + const node = yaccDocument.getNodeByOffset(offset); + if (node === undefined) { + return []; + } + + var completion: CompletionItem; + const result: CompletionItem[] = []; + switch (node.nodeType) { + case NodeType.Token: + case NodeType.Type: + if (node.typeOffset && offset > node.typeOffset) { + if (!node.typeEnd || offset <= node.typeEnd) { + Object.keys(yaccDocument.types).forEach((type) => { + completion = new CompletionItem(type) + completion.detail = "type" + completion.kind = CompletionItemKind.TypeParameter; + result.push(completion); + }) + break; + } + } + if (node.nodeType === NodeType.Type) + Object.keys(yaccDocument.symbols).forEach((symbol) => { + completion = new CompletionItem(symbol) + completion.detail = "symbol"; + completion.kind = CompletionItemKind.Class; + result.push(completion); + }); + break; + case NodeType.Rule: + Object.keys(yaccDocument.symbols).forEach((symbol) => { + completion = new CompletionItem(symbol) + completion.detail = "symbol"; + completion.kind = CompletionItemKind.Class; + result.push(completion); + }) + Object.keys(yaccDocument.tokens).forEach((token) => { + completion = new CompletionItem(token) + completion.detail = "token"; + completion.kind = CompletionItemKind.Field; + result.push(completion); + }) + break; + default: + break; + } + return result; +} \ No newline at end of file diff --git a/src/languages/services/yaccDefinitions.ts b/src/languages/services/yaccDefinitions.ts new file mode 100644 index 0000000..140ffb5 --- /dev/null +++ b/src/languages/services/yaccDefinitions.ts @@ -0,0 +1,18 @@ +import { TextDocument, Position, Definition, Location, Range } from 'vscode'; +import { YACCDocument, ISymbol } from "../parser/yaccParser"; + +export function doYACCFindDefinition(document: TextDocument, position: Position, yaccDocument: YACCDocument): Definition | null { + const offset = document.offsetAt(position); + const node = yaccDocument.getEmbeddedNode(offset); + if (node) { + return null; + } + + const word = document.getText(document.getWordRangeAtPosition(position)); + var symbol: ISymbol | undefined = yaccDocument.types[word] || yaccDocument.symbols[word] || yaccDocument.tokens[word]; + let location: Location | null = null; + if (symbol) { + location = new Location(document.uri, new Range(document.positionAt(symbol.definition[0]), document.positionAt(symbol.definition[1]))); + } + return location; +} \ No newline at end of file diff --git a/src/languages/services/yaccHover.ts b/src/languages/services/yaccHover.ts new file mode 100644 index 0000000..36afcc6 --- /dev/null +++ b/src/languages/services/yaccHover.ts @@ -0,0 +1,32 @@ +import { TextDocument, Hover, Position, MarkedString, MarkdownString } from 'vscode'; +import { YACCDocument, ISymbol } from "../parser/yaccParser"; +import { createMarkedCodeString } from "./utils"; + +export function doYACCHover(document: TextDocument, position: Position, yaccDocument: YACCDocument): Hover | null { + const offset = document.offsetAt(position); + const node = yaccDocument.getEmbeddedNode(offset); + if (node) { + return null; + } + + const word = document.getText(document.getWordRangeAtPosition(position)); + + var message: MarkdownString | undefined = undefined; + var symbol: ISymbol; + if ((symbol = yaccDocument.types[word])) { + message = createMarkedCodeString(`${symbol.type} ${symbol.name}`, 'yacc'); + } + + if ((symbol = yaccDocument.symbols[word])) { + message = createMarkedCodeString(`%type <${symbol.type ? symbol.type : '?'}> ${symbol.name}`, 'yacc'); + } + + if ((symbol = yaccDocument.tokens[word])) { + message = createMarkedCodeString(`%token <${symbol.type ? symbol.type : '?'}> ${symbol.name}`, 'yacc'); + } + + if (message) + return { contents: [message] }; + + return null; +} \ No newline at end of file diff --git a/src/languages/services/yaccReferences.ts b/src/languages/services/yaccReferences.ts new file mode 100644 index 0000000..245a3c2 --- /dev/null +++ b/src/languages/services/yaccReferences.ts @@ -0,0 +1,18 @@ +import { TextDocument, Position, Location, Range } from 'vscode'; +import { YACCDocument, ISymbol } from "../parser/yaccParser"; + +export function doYACCFindReferences(document: TextDocument, position: Position, yaccDocument: YACCDocument): Location[] { + const offset = document.offsetAt(position); + const node = yaccDocument.getEmbeddedNode(offset); + if (node) { + return []; + } + + const word = document.getText(document.getWordRangeAtPosition(position)); + var symbol: ISymbol | undefined = yaccDocument.types[word] || yaccDocument.symbols[word] || yaccDocument.tokens[word]; + let location: Location[] = []; + symbol?.references.forEach(reference => { + location.push(new Location(document.uri, new Range(document.positionAt(reference[0]), document.positionAt(reference[1])))); + }) + return location; +} \ No newline at end of file diff --git a/src/languages/services/yaccRename.ts b/src/languages/services/yaccRename.ts new file mode 100644 index 0000000..ef287eb --- /dev/null +++ b/src/languages/services/yaccRename.ts @@ -0,0 +1,18 @@ +import { TextDocument, Position, Range, WorkspaceEdit, TextEdit } from 'vscode'; +import { YACCDocument, ISymbol } from "../parser/yaccParser"; + +export function doYACCRename(document: TextDocument, position: Position, newName: string, yaccDocument: YACCDocument): WorkspaceEdit | null { + const offset = document.offsetAt(position); + const node = yaccDocument.getEmbeddedNode(offset); + if (node) { + return null; + } + + const word = document.getText(document.getWordRangeAtPosition(position)); + var symbol: ISymbol | undefined = yaccDocument.types[word] || yaccDocument.symbols[word] || yaccDocument.tokens[word]; + const edits = new WorkspaceEdit(); + symbol?.references.forEach(reference => { + edits.replace(document.uri, new Range(document.positionAt(reference[0]), document.positionAt(reference[1])), newName); + }) + return edits; +} \ No newline at end of file diff --git a/src/languages/services/yaccValidation.ts b/src/languages/services/yaccValidation.ts new file mode 100644 index 0000000..24e16ec --- /dev/null +++ b/src/languages/services/yaccValidation.ts @@ -0,0 +1,31 @@ +import { TextDocument, Diagnostic, Range, DiagnosticSeverity, DiagnosticRelatedInformation, Location } from "vscode"; +import { YACCDocument } from "../parser/yaccParser"; +import { ProblemType } from "../yaccLanguageTypes"; + +export function doYACCValidation(document: TextDocument, yaccDocument: YACCDocument): Diagnostic[] { + const diags: Diagnostic[] = []; + yaccDocument.problems.forEach(problem => { + const range = new Range(document.positionAt(problem.offset), document.positionAt(problem.end)); + let severity: DiagnosticSeverity = DiagnosticSeverity.Information; + switch (problem.type) { + case ProblemType.Error: + severity = DiagnosticSeverity.Error; + break; + case ProblemType.Information: + severity = DiagnosticSeverity.Information; + break; + case ProblemType.Warning: + severity = DiagnosticSeverity.Warning; + break; + } + const diag = new Diagnostic(range, problem.message, severity); + if (problem.related) { + diag.relatedInformation = [new DiagnosticRelatedInformation( + new Location(document.uri, new Range(document.positionAt(problem.related.offset), document.positionAt(problem.related.end))), + problem.related.message + )]; + } + diags.push(diag); + }); + return diags; +} \ No newline at end of file diff --git a/src/languages/yaccLanguageServices.ts b/src/languages/yaccLanguageServices.ts new file mode 100644 index 0000000..a81ee0d --- /dev/null +++ b/src/languages/yaccLanguageServices.ts @@ -0,0 +1,37 @@ +import { createScanner } from './parser/yaccScanner'; +import { parse, YACCDocument } from './parser/yaccParser'; +import { Scanner } from './yaccLanguageTypes'; +import { TextDocument, Position, Diagnostic, WorkspaceEdit, Hover, CompletionList, CompletionItem, Range, Definition, Location } from 'vscode'; +import { doYACCComplete } from './services/yaccCompletions'; +import { doYACCHover } from './services/yaccHover'; +import { SemanticTokenData } from './semanticTokens'; +import { doYACCFindDefinition } from './services/yaccDefinitions'; +import { doYACCFindReferences } from './services/yaccReferences'; +import { doYACCRename } from './services/yaccRename'; +import { doYACCValidation } from './services/yaccValidation'; + +export interface LanguageService { + createScanner(input: string, initialOffset?: number): Scanner; + parseYACCDocument(document: TextDocument): YACCDocument; + doComplete(document: TextDocument, position: Position, yaccDocument: YACCDocument): CompletionItem[] | CompletionList; + doValidation: (document: TextDocument, yaccDocument: YACCDocument) => Diagnostic[]; + getSemanticTokens(Document: TextDocument, yaccDocument: YACCDocument): SemanticTokenData[]; + doHover(document: TextDocument, position: Position, yaccDocument: YACCDocument): Hover | null; + findDefinition(document: TextDocument, position: Position, yaccDocument: YACCDocument): Definition | null; + findReferences(document: TextDocument, position: Position, yaccDocument: YACCDocument): Location[]; + doRename(document: TextDocument, position: Position, newName: string, yaccDocument: YACCDocument): WorkspaceEdit | null; +} + +export function getLanguageService(): LanguageService { + return { + createScanner, + parseYACCDocument: document => parse(document.getText()), + doComplete: (document, position, yaccDocument) => doYACCComplete(document, position, yaccDocument), + doValidation: (document, yaccDocument) => doYACCValidation(document, yaccDocument), + getSemanticTokens: (document, yaccDocument) => yaccDocument.getSemanticTokens(document.positionAt.bind(document)), + doHover: (document, position, yaccDocument) => doYACCHover(document, position, yaccDocument), + findDefinition: (document, position, yaccDocument) => doYACCFindDefinition(document, position, yaccDocument), + findReferences: (document, position, yaccDocument) => doYACCFindReferences(document, position, yaccDocument), + doRename: (document, position, newName, yaccDocument) => doYACCRename(document, position, newName, yaccDocument) + }; +} diff --git a/src/languages/yaccLanguageTypes.ts b/src/languages/yaccLanguageTypes.ts new file mode 100644 index 0000000..623b532 --- /dev/null +++ b/src/languages/yaccLanguageTypes.ts @@ -0,0 +1,83 @@ +import { SemanticTokenType, SemanticTokenModifier } from "./semanticTokens"; + +export enum ProblemType { + Information, + Warning, + Error +}; + +export interface ProblemRelated { + readonly message: string; + readonly offset: number; + readonly end: number; +}; + +export interface Problem { + readonly message: string; + readonly offset: number; + readonly end: number; + readonly type: ProblemType; + readonly related?: ProblemRelated; +}; + +export enum TokenType { + Word, + Literal, + Bar, + Dot, + Colon, + SemiColon, + Percent, + Param, + Option, + RulesTag, + StartType, + EndType, + TypeValue, + StartComment, + EndComment, + Comment, + StartAction, + EndAction, + Action, + Unknown, + EOS +} + +export enum ScannerState { + WithinContent, + WithinComment, + WithinCode, + WithinUnion, + WithinTypeValue +} + +export interface Scanner { + scan(): TokenType; + getTokenType(): TokenType; + getTokenOffset(): number; + getTokenLength(): number; + getTokenEnd(): number; + getTokenText(): string; + getTokenError(): string | undefined; + getScannerState(): ScannerState; +} + +export const tokenTypes: string[] = []; +tokenTypes[SemanticTokenType.class] = 'class'; +tokenTypes[SemanticTokenType.enum] = 'enum'; +tokenTypes[SemanticTokenType.interface] = 'interface'; +tokenTypes[SemanticTokenType.namespace] = 'namespace'; +tokenTypes[SemanticTokenType.typeParameter] = 'typeParameter'; +tokenTypes[SemanticTokenType.type] = 'type'; +tokenTypes[SemanticTokenType.parameter] = 'parameter'; +tokenTypes[SemanticTokenType.variable] = 'variable'; +tokenTypes[SemanticTokenType.property] = 'property'; +tokenTypes[SemanticTokenType.function] = 'function'; +tokenTypes[SemanticTokenType.member] = 'member'; + +export const tokenModifiers: string[] = []; +tokenModifiers[SemanticTokenModifier.async] = 'async'; +tokenModifiers[SemanticTokenModifier.declaration] = 'declaration'; +tokenModifiers[SemanticTokenModifier.readonly] = 'readonly'; +tokenModifiers[SemanticTokenModifier.static] = 'static'; \ No newline at end of file diff --git a/src/modes/languageModes.ts b/src/modes/languageModes.ts new file mode 100644 index 0000000..609f4cb --- /dev/null +++ b/src/modes/languageModes.ts @@ -0,0 +1,103 @@ +import { getLanguageService as getYACCLanguageService } from '../languages/yaccLanguageServices'; +import { getLanguageService as getLEXLanguageService } from '../languages/lexLanguageService'; +import { SemanticTokenData } from '../languages/semanticTokens'; + +import { + TextDocument, + Position, + SelectionRange, + CompletionList, + CompletionItem, + Diagnostic, + Hover, + Definition, + Location, + WorkspaceEdit, + SignatureHelp, + DocumentHighlight, + FormattingOptions, + TextEdit +} from 'vscode' + +import { DocumentCache } from '../documentCache'; +import { getYACCMode } from './yaccMode'; +import { getLEXMode } from './lexMode'; + +export interface LanguageMode { + getId(): string; + getSelectionRange?: (document: TextDocument, position: Position) => SelectionRange; + doValidation?: (document: TextDocument) => Diagnostic[]; + doComplete?: (document: TextDocument, position: Position) => CompletionList | CompletionItem[]; + doResolve?: (document: TextDocument, item: CompletionItem) => CompletionItem; + doHover?: (document: TextDocument, position: Position) => Hover | null; + doSignatureHelp?: (document: TextDocument, position: Position) => SignatureHelp | null; + doRename?: (document: TextDocument, position: Position, newName: string) => WorkspaceEdit | null; + doOnTypeRename?: (document: TextDocument, position: Position) => Range[] | null; + findDocumentHighlight?: (document: TextDocument, position: Position) => DocumentHighlight[]; + // findDocumentSymbols?: (document: TextDocument) => SymbolInformation[]; + // findDocumentLinks?: (document: TextDocument, documentContext: DocumentContext) => DocumentLink[]; + findDefinition?: (document: TextDocument, position: Position) => Definition | null; + findReferences?: (document: TextDocument, position: Position) => Location[]; + format?: (document: TextDocument, range: Range, options: FormattingOptions, ) => TextEdit[]; + // findDocumentColors?: (document: TextDocument) => ColorInformation[]; + // getColorPresentations?: (document: TextDocument, color: Color, range: Range) => ColorPresentation[]; + // doAutoClose?: (document: TextDocument, position: Position) => string | null; + // getFoldingRanges?: (document: TextDocument) => FoldingRange[]; + onDocumentRemoved(document: TextDocument): void; + getSemanticTokens?(document: TextDocument): SemanticTokenData[]; + getSemanticTokenLegend?(): { types: string[], modifiers: string[] }; + dispose(): void; +} + +export interface LanguageModes { + getAllModes(): LanguageMode[]; + getMode(languageId: string): LanguageMode | undefined; + onDocumentRemoved(document: TextDocument): void; + dispose(): void; +} + +export function getLanguageModes(supportedLanguages: { [languageId: string]: boolean; }): LanguageModes { + const yaccLanguageService = getYACCLanguageService(); + const lexLanguageService = getLEXLanguageService(); + + let modelCaches: DocumentCache[] = []; + + let modes = Object.create(null); + if (supportedLanguages['yacc']) { + modes['yacc'] = getYACCMode(yaccLanguageService); + } + + if (supportedLanguages['lex']) { + modes['lex'] = getLEXMode(lexLanguageService); + } + + return { + getAllModes(): LanguageMode[] { + let result = []; + for (let languageId in modes) { + let mode = modes[languageId]; + if (mode) { + result.push(mode); + } + } + return result; + }, + getMode(languageId: string): LanguageMode { + return modes[languageId]; + }, + onDocumentRemoved(document: TextDocument) { + modelCaches.forEach(mc => mc.onDocumentRemoved(document)); + for (let mode in modes) { + modes[mode].onDocumentRemoved(document); + } + }, + dispose(): void { + modelCaches.forEach(mc => mc.dispose()); + modelCaches = []; + for (let mode in modes) { + modes[mode].dispose(); + } + modes = {}; + } + }; +} diff --git a/src/modes/lexMode.ts b/src/modes/lexMode.ts new file mode 100644 index 0000000..c5a6d2c --- /dev/null +++ b/src/modes/lexMode.ts @@ -0,0 +1,44 @@ +import { LanguageService as LEXLanguageService } from '../languages/lexLanguageService'; +import { LanguageMode } from './languageModes'; +import { TextDocument, CompletionList, CompletionItem, Position, Hover, Location, Definition, WorkspaceEdit, Diagnostic } from 'vscode'; +import { CreateDocumentCache } from '../documentCache'; +import { LexDocument } from '../languages/parser/lexParser'; + +export function getLEXMode(lexLanguageService: LEXLanguageService): LanguageMode { + const cache = CreateDocumentCache(10, 60, document => lexLanguageService.parseLexDocument(document)); + return { + getId() { + return 'lex'; + }, + doValidation(document: TextDocument): Diagnostic[] { + const lex = cache.get(document); + return lexLanguageService.doValidation(document, lex); + }, + doComplete(document: TextDocument, position: Position): CompletionList | CompletionItem[] { + const lex = cache.get(document); + return lexLanguageService.doComplete(document, position, lex); + }, + doHover(document: TextDocument, position: Position): Hover | null { + const lex = cache.get(document); + return lexLanguageService.doHover(document, position, lex); + }, + findDefinition(document: TextDocument, position: Position): Definition | null { + const lex = cache.get(document); + return lexLanguageService.findDefinition(document, position, lex); + }, + findReferences(document: TextDocument, position: Position): Location[] { + const lex = cache.get(document); + return lexLanguageService.findReferences(document, position, lex); + }, + doRename(document: TextDocument, position: Position, newName: string): WorkspaceEdit | null { + const lex = cache.get(document); + return lexLanguageService.doRename(document, position, newName, lex); + }, + onDocumentRemoved(document: TextDocument) { + cache.onDocumentRemoved(document); + }, + dispose() { + cache.dispose(); + } + }; +} \ No newline at end of file diff --git a/src/modes/semanticProvider.ts b/src/modes/semanticProvider.ts new file mode 100644 index 0000000..d25bbec --- /dev/null +++ b/src/modes/semanticProvider.ts @@ -0,0 +1,95 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. + * + * Modified to adapt the project + *--------------------------------------------------------------------------------------------*/ + +import { SemanticTokenData } from '../languages/semanticTokens'; +import { TextDocument, Range, SemanticTokens, SemanticTokensBuilder } from 'vscode'; +import { LanguageModes } from './languageModes'; + +export interface SemanticTokenProvider { + readonly legend: { types: string[]; modifiers: string[] }; + getSemanticTokens(document: TextDocument): SemanticTokens; +} + +interface LegendMapping { + types: number[] | undefined; + modifiers: number[] | undefined; +} + +export function newSemanticTokenProvider(languageModes: LanguageModes): SemanticTokenProvider { + const legend = { types: [], modifiers: [] }; + const legendMappings: { [modeId: string]: LegendMapping } = {}; + + for (let mode of languageModes.getAllModes()) { + if (mode.getSemanticTokenLegend && mode.getSemanticTokens) { + const modeLegend = mode.getSemanticTokenLegend(); + legendMappings[mode.getId()] = { types: createMapping(modeLegend.types, legend.types), modifiers: createMapping(modeLegend.modifiers, legend.modifiers) }; + } + } + + return { + legend, + getSemanticTokens(document: TextDocument, ranges?: Range[]): SemanticTokens { + const builder = new SemanticTokensBuilder(); + for (let mode of languageModes.getAllModes()) { + if (mode.getSemanticTokens) { + const mapping = legendMappings[mode.getId()]; + const tokens = mode.getSemanticTokens(document); + applyTypesMapping(tokens, mapping.types); + applyModifiersMapping(tokens, mapping.modifiers); + tokens.forEach(token => { + builder.push(token.start.line, token.start.character, token.length, token.typeIdx, token.modifierSet); + }); + } + } + return builder.build(); + } + }; +} + +function createMapping(origLegend: string[], newLegend: string[]): number[] | undefined { + const mapping: number[] = []; + let needsMapping = false; + for (let origIndex = 0; origIndex < origLegend.length; origIndex++) { + const entry = origLegend[origIndex]; + let newIndex = newLegend.indexOf(entry); + if (newIndex === -1) { + newIndex = newLegend.length; + newLegend.push(entry); + } + mapping.push(newIndex); + needsMapping = needsMapping || (newIndex !== origIndex); + } + return needsMapping ? mapping : undefined; +} + +function applyTypesMapping(tokens: SemanticTokenData[], typesMapping: number[] | undefined): void { + if (typesMapping) { + for (let token of tokens) { + token.typeIdx = typesMapping[token.typeIdx]; + } + } +} + +function applyModifiersMapping(tokens: SemanticTokenData[], modifiersMapping: number[] | undefined): void { + if (modifiersMapping) { + for (let token of tokens) { + let modifierSet = token.modifierSet; + if (modifierSet) { + let index = 0; + let result = 0; + while (modifierSet > 0) { + if ((modifierSet & 1) !== 0) { + result = result + (1 << modifiersMapping[index]); + } + index++; + modifierSet = modifierSet >> 1; + } + token.modifierSet = result; + } + } + } +} \ No newline at end of file diff --git a/src/modes/yaccMode.ts b/src/modes/yaccMode.ts new file mode 100644 index 0000000..887ef54 --- /dev/null +++ b/src/modes/yaccMode.ts @@ -0,0 +1,53 @@ +import { LanguageService as YACCLanguageService } from '../languages/yaccLanguageServices'; +import { tokenTypes, tokenModifiers } from '../languages/yaccLanguageTypes'; +import { LanguageMode } from './languageModes'; +import { TextDocument, CompletionList, CompletionItem, Position, Hover, Definition, Location, WorkspaceEdit, Diagnostic } from 'vscode'; +import { SemanticTokenData } from '../languages/semanticTokens'; +import { CreateDocumentCache } from '../documentCache'; +import { YACCDocument } from '../languages/parser/yaccParser'; + +export function getYACCMode(yaccLanguageService: YACCLanguageService): LanguageMode { + const cache = CreateDocumentCache(10, 60, document => yaccLanguageService.parseYACCDocument(document)); + return { + getId() { + return 'yacc'; + }, + doValidation(document: TextDocument): Diagnostic[] { + const yacc = cache.get(document); + return yaccLanguageService.doValidation(document, yacc); + }, + doComplete(document: TextDocument, position: Position): CompletionList | CompletionItem[] { + const yacc = cache.get(document); + return yaccLanguageService.doComplete(document, position, yacc); + }, + doHover(document: TextDocument, position: Position): Hover | null { + const yacc = cache.get(document); + return yaccLanguageService.doHover(document, position, yacc); + }, + findDefinition(document: TextDocument, position: Position): Definition | null { + const yacc = cache.get(document); + return yaccLanguageService.findDefinition(document, position, yacc); + }, + findReferences(document: TextDocument, position: Position): Location[] { + const yacc = cache.get(document); + return yaccLanguageService.findReferences(document, position, yacc); + }, + doRename(document: TextDocument, position: Position, newName: string): WorkspaceEdit | null { + const yacc = cache.get(document); + return yaccLanguageService.doRename(document, position, newName, yacc); + }, + getSemanticTokens(document: TextDocument): SemanticTokenData[] { + const yacc = cache.get(document); + return yaccLanguageService.getSemanticTokens(document, yacc); + }, + getSemanticTokenLegend() { + return { types: tokenTypes, modifiers: tokenModifiers }; + }, + onDocumentRemoved(document: TextDocument) { + cache.onDocumentRemoved(document); + }, + dispose() { + cache.dispose(); + } + }; +} \ No newline at end of file diff --git a/src/runner.ts b/src/runner.ts new file mode 100644 index 0000000..65a42e9 --- /dev/null +++ b/src/runner.ts @@ -0,0 +1,70 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. + * + * Modified to adapt the project + *--------------------------------------------------------------------------------------------*/ + +import { CancellationToken } from 'vscode'; + +export function formatError(message: string, err: any): string { + if (err instanceof Error) { + let error = err; + return `${message}: ${error.message}\n${error.stack}`; + } else if (typeof err === 'string') { + return `${message}: ${err}`; + } else if (err) { + return `${message}: ${err.toString()}`; + } + return message; +} + +export function runSafeAsync(func: () => Thenable, errorVal: T, errorMessage: string, token: CancellationToken): Thenable { + return new Promise((resolve) => { + setImmediate(async () => { + if (token.isCancellationRequested) { + resolve(cancelValue()); + } + return func().then(result => { + if (token.isCancellationRequested) { + resolve(cancelValue()); + return; + } else { + resolve(result); + } + }, e => { + console.error(formatError(errorMessage, e)); + resolve(errorVal); + }); + }); + }); +} + +export function runSafe(func: () => T, errorVal: T, errorMessage: string, token: CancellationToken): Thenable { + return new Promise((resolve) => { + setImmediate(() => { + if (token.isCancellationRequested) { + resolve(cancelValue()); + } else { + try { + let result = func(); + if (token.isCancellationRequested) { + resolve(cancelValue()); + return; + } else { + resolve(result); + } + + } catch (e) { + console.error(formatError(errorMessage, e)); + resolve(errorVal); + } + } + }); + }); +} + +function cancelValue() { + console.log("Request cancelled..."); + return undefined; +} \ No newline at end of file