From 3dbf1172ccaef6ea100a51e071540e2d56f07d6d Mon Sep 17 00:00:00 2001 From: Ziwei Wang Date: Tue, 5 Dec 2023 13:25:56 -0500 Subject: [PATCH 1/4] Refactor: optimize the extraction of symbols in string content. No more storing the symbols as they are only needed occasionally --- server/src/connectionHandlers/onDefinition.ts | 17 ++--- server/src/tree-sitter/analyzer.ts | 73 ++++++++----------- 2 files changed, 37 insertions(+), 53 deletions(-) diff --git a/server/src/connectionHandlers/onDefinition.ts b/server/src/connectionHandlers/onDefinition.ts index cde6753d..5b31479a 100644 --- a/server/src/connectionHandlers/onDefinition.ts +++ b/server/src/connectionHandlers/onDefinition.ts @@ -72,16 +72,15 @@ export function onDefinitionHandler (textDocumentPositionParams: TextDocumentPos } // Symbols in string content if (analyzer.isStringContent(documentUri, position.line, position.character)) { - const allSymbolsFoundAtPosition = analyzer.getSymbolInStringContentForPosition(documentUri, position.line, position.character) - if (allSymbolsFoundAtPosition !== undefined) { - allSymbolsFoundAtPosition.forEach((symbol) => { - definitions.push({ - uri: symbol.location.uri, - range: { start: { line: 0, character: 0 }, end: { line: 0, character: 0 } } - }) + const allSymbolsAtPosition = analyzer.getSymbolsInStringContent(documentUri, position.line, position.character) + + allSymbolsAtPosition.forEach((symbol) => { + definitions.push({ + uri: symbol.location.uri, + range: { start: { line: 0, character: 0 }, end: { line: 0, character: 0 } } }) - return definitions - } + }) + return definitions } } diff --git a/server/src/tree-sitter/analyzer.ts b/server/src/tree-sitter/analyzer.ts index b80e6f0d..2e1ce7f2 100644 --- a/server/src/tree-sitter/analyzer.ts +++ b/server/src/tree-sitter/analyzer.ts @@ -34,7 +34,6 @@ interface AnalyzedDocument { embeddedRegions: EmbeddedRegions tree: Parser.Tree extraSymbols?: GlobalDeclarations[] // symbols from the include files - symbolsInStringContent?: SymbolInformation[] } export default class Analyzer { @@ -74,7 +73,6 @@ export default class Analyzer { const tree = this.parser.parse(fileContent) const globalDeclarations = getGlobalDeclarations({ tree, uri }) - const symbolsInStringContent = this.getSymbolsInStringContent(tree, uri) const embeddedRegions = getEmbeddedRegionsFromNode(tree, uri) /* eslint-disable-next-line prefer-const */ let extraSymbols: GlobalDeclarations[] = [] @@ -85,8 +83,7 @@ export default class Analyzer { globalDeclarations, embeddedRegions, tree, - extraSymbols, - symbolsInStringContent + extraSymbols } let debouncedExecuteAnalyzation = this.debouncedExecuteAnalyzation @@ -516,34 +513,35 @@ export default class Analyzer { /** * Extract symbols from the string content of the tree */ - public getSymbolsInStringContent (tree: Parser.Tree, uri: string): SymbolInformation[] { - const symbolInformation: SymbolInformation[] = [] + public getSymbolsInStringContent (uri: string, line: number, character: number): SymbolInformation[] { + const allSymbolsAtPosition: SymbolInformation[] = [] const wholeWordRegex = /(? { - if (n.type === 'string_content') { - const splittedStringContent = n.text.split(/\n/g) - for (let i = 0; i < splittedStringContent.length; i++) { - const line = splittedStringContent[i] - for (const match of line.matchAll(wholeWordRegex)) { - if (match !== undefined && uri !== undefined) { - const start = { - line: n.startPosition.row + i, - character: match.index !== undefined ? match.index + n.startPosition.column : 0 - } - const end = { - line: n.startPosition.row + i, - character: match.index !== undefined ? match.index + n.startPosition.column + match[0].length : 0 - } - if (i > 0) { - start.character = match.index ?? 0 - end.character = (match.index ?? 0) + match[0].length - } + const n = this.nodeAtPoint(uri, line, character) + if (n?.type === 'string_content') { + const splittedStringContent = n.text.split(/\n/g) + for (let i = 0; i < splittedStringContent.length; i++) { + const lineText = splittedStringContent[i] + for (const match of lineText.matchAll(wholeWordRegex)) { + if (match !== undefined && uri !== undefined) { + const start = { + line: n.startPosition.row + i, + character: match.index !== undefined ? match.index + n.startPosition.column : 0 + } + const end = { + line: n.startPosition.row + i, + character: match.index !== undefined ? match.index + n.startPosition.column + match[0].length : 0 + } + if (i > 0) { + start.character = match.index ?? 0 + end.character = (match.index ?? 0) + match[0].length + } + if (this.positionIsInRange(line, character, { start, end })) { const foundRecipe = bitBakeProjectScannerClient.bitbakeScanResult._recipes.find((recipe) => { return recipe.name === match[0] }) if (foundRecipe !== undefined) { if (foundRecipe?.path !== undefined) { - symbolInformation.push({ + allSymbolsAtPosition.push({ name: match[0], kind: SymbolKind.Variable, location: { @@ -557,7 +555,7 @@ export default class Analyzer { } if (foundRecipe?.appends !== undefined && foundRecipe.appends.length > 0) { foundRecipe.appends.forEach((append) => { - symbolInformation.push({ + allSymbolsAtPosition.push({ name: append.name, kind: SymbolKind.Variable, location: { @@ -575,26 +573,13 @@ export default class Analyzer { } } } - return true - }) + } - return symbolInformation + return allSymbolsAtPosition } - public getSymbolInStringContentForPosition (uri: string, line: number, column: number): SymbolInformation[] | undefined { - const analyzedDocument = this.uriToAnalyzedDocument[uri] - if (analyzedDocument?.symbolsInStringContent !== undefined) { - const { symbolsInStringContent } = analyzedDocument - const allSymbolsFoundAtPosition: SymbolInformation[] = [] // recipe + appends - for (const symbol of symbolsInStringContent) { - const { location: { range } } = symbol - if (line === range.start.line && column >= range.start.character && column <= range.end.character) { - allSymbolsFoundAtPosition.push(symbol) - } - } - return allSymbolsFoundAtPosition - } - return undefined + public positionIsInRange (line: number, character: number, range: Range): boolean { + return line === range.start.line && character >= range.start.character && character <= range.end.character } } From 76dd5992cc1427ca71195cbf230dd56a1bc4b4d7 Mon Sep 17 00:00:00 2001 From: Ziwei Wang Date: Tue, 5 Dec 2023 17:58:27 -0500 Subject: [PATCH 2/4] Chore: maintain a reference of workspaceFolders in analyzer --- server/src/server.ts | 1 + server/src/tree-sitter/analyzer.ts | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/server.ts b/server/src/server.ts index f90295aa..827e0873 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -61,6 +61,7 @@ connection.onInitialize(async (params: InitializeParams): Promise { logger.debug('[On scanReady] Analyzing the current document again...') diff --git a/server/src/tree-sitter/analyzer.ts b/server/src/tree-sitter/analyzer.ts index 2e1ce7f2..5983c909 100644 --- a/server/src/tree-sitter/analyzer.ts +++ b/server/src/tree-sitter/analyzer.ts @@ -13,7 +13,8 @@ import { type Diagnostic, type SymbolInformation, type Range, - SymbolKind + SymbolKind, + type WorkspaceFolder } from 'vscode-languageserver' import type Parser from 'web-tree-sitter' import { TextDocument } from 'vscode-languageserver-textdocument' @@ -40,6 +41,7 @@ export default class Analyzer { private parser?: Parser private uriToAnalyzedDocument: Record = {} private debouncedExecuteAnalyzation?: ReturnType + public workspaceFolders: WorkspaceFolder[] | undefined | null = [] public getDocumentTexts (uri: string): string[] | undefined { return this.uriToAnalyzedDocument[uri]?.document.getText().split(/\r?\n/g) From 5bbe950d94662b1a839f3c317cf5a8ef83767174 Mon Sep 17 00:00:00 2001 From: Ziwei Wang Date: Tue, 5 Dec 2023 18:02:08 -0500 Subject: [PATCH 3/4] Feat: Go to definition for uris in the string content when they prefix with file:// --- server/src/connectionHandlers/onDefinition.ts | 45 +++++++++- server/src/tree-sitter/analyzer.ts | 89 +++++++++++-------- 2 files changed, 94 insertions(+), 40 deletions(-) diff --git a/server/src/connectionHandlers/onDefinition.ts b/server/src/connectionHandlers/onDefinition.ts index 5b31479a..9a1a4759 100644 --- a/server/src/connectionHandlers/onDefinition.ts +++ b/server/src/connectionHandlers/onDefinition.ts @@ -10,6 +10,7 @@ import { type DirectiveStatementKeyword } from '../lib/src/types/directiveKeywor import { bitBakeProjectScannerClient } from '../BitbakeProjectScannerClient' import path, { type ParsedPath } from 'path' import { type ElementInfo } from '../lib/src/types/BitbakeScanResult' +import fs from 'fs' export function onDefinitionHandler (textDocumentPositionParams: TextDocumentPositionParams): Definition | null { const { textDocument: { uri: documentUri }, position } = textDocumentPositionParams @@ -72,7 +73,29 @@ export function onDefinitionHandler (textDocumentPositionParams: TextDocumentPos } // Symbols in string content if (analyzer.isStringContent(documentUri, position.line, position.character)) { - const allSymbolsAtPosition = analyzer.getSymbolsInStringContent(documentUri, position.line, position.character) + const wholeWordRegex = /(?.*)\b/g + const [uriAtPosition] = analyzer.getSymbolsInStringContent(documentUri, position.line, position.character, uriRegex) + if (uriAtPosition !== undefined) { + const { workspaceFolders } = analyzer + if (workspaceFolders !== undefined && workspaceFolders !== null) { + for (const workspaceFolder of workspaceFolders) { + const filePath = findFileInDirectory(workspaceFolder.uri.replace('file://', ''), uriAtPosition.name) + if (filePath !== null) { + definitions.push( + { + uri: 'file://' + filePath, + range: { start: { line: 0, character: 0 }, end: { line: 0, character: 0 } } + } + ) + break + } + } + return definitions + } + } + + const allSymbolsAtPosition = analyzer.getSymbolsInStringContent(documentUri, position.line, position.character, wholeWordRegex) allSymbolsAtPosition.forEach((symbol) => { definitions.push({ @@ -80,6 +103,7 @@ export function onDefinitionHandler (textDocumentPositionParams: TextDocumentPos range: { start: { line: 0, character: 0 }, end: { line: 0, character: 0 } } }) }) + return definitions } } @@ -132,3 +156,22 @@ function createDefinitionLocationForPathInfo (path: ParsedPath): Location { return location } + +function findFileInDirectory (dir: string, fileName: string): string | null { + try { + const filePaths = fs.readdirSync(dir).map(name => path.join(dir, name)) + for (const filePath of filePaths) { + if (fs.statSync(filePath).isDirectory()) { + const result = findFileInDirectory(filePath, fileName) + if (result !== null) return result + } else if (path.basename(filePath) === fileName) { + return filePath + } + } + } catch { + logger.debug(`[findFileInDirectory] ${dir} not found`) + return null + } + + return null +} diff --git a/server/src/tree-sitter/analyzer.ts b/server/src/tree-sitter/analyzer.ts index 5983c909..d5375dac 100644 --- a/server/src/tree-sitter/analyzer.ts +++ b/server/src/tree-sitter/analyzer.ts @@ -515,61 +515,72 @@ export default class Analyzer { /** * Extract symbols from the string content of the tree */ - public getSymbolsInStringContent (uri: string, line: number, character: number): SymbolInformation[] { + public getSymbolsInStringContent (uri: string, line: number, character: number, regex: RegExp): SymbolInformation[] { const allSymbolsAtPosition: SymbolInformation[] = [] - const wholeWordRegex = /(? 0) { - start.character = match.index ?? 0 - end.character = (match.index ?? 0) + match[0].length + for (const match of lineText.matchAll(regex)) { + const matchedUri = match.groups?.uri + const start = { + line: n.startPosition.row + i, + character: match.index !== undefined ? match.index + n.startPosition.column : 0 + } + const end = { + line: n.startPosition.row + i, + character: match.index !== undefined ? match.index + n.startPosition.column + match[0].length : 0 + } + if (i > 0) { + start.character = match.index ?? 0 + end.character = (match.index ?? 0) + match[0].length + } + if (this.positionIsInRange(line, character, { start, end })) { + if (matchedUri !== undefined) { + return [{ + name: matchedUri, + kind: SymbolKind.String, + location: { + range: { + start, + end + }, + uri + } + }] } - if (this.positionIsInRange(line, character, { start, end })) { - const foundRecipe = bitBakeProjectScannerClient.bitbakeScanResult._recipes.find((recipe) => { - return recipe.name === match[0] - }) - if (foundRecipe !== undefined) { - if (foundRecipe?.path !== undefined) { + const foundRecipe = bitBakeProjectScannerClient.bitbakeScanResult._recipes.find((recipe) => { + return recipe.name === match[0] + }) + if (foundRecipe !== undefined) { + if (foundRecipe?.path !== undefined) { + allSymbolsAtPosition.push({ + name: match[0], + kind: SymbolKind.Variable, + location: { + range: { + start, + end + }, + uri: 'file://' + foundRecipe.path.dir + '/' + foundRecipe.path.base + } + }) + } + if (foundRecipe?.appends !== undefined && foundRecipe.appends.length > 0) { + foundRecipe.appends.forEach((append) => { allSymbolsAtPosition.push({ - name: match[0], + name: append.name, kind: SymbolKind.Variable, location: { range: { start, end }, - uri: 'file://' + foundRecipe.path.dir + '/' + foundRecipe.path.base + uri: 'file://' + append.dir + '/' + append.base } }) - } - if (foundRecipe?.appends !== undefined && foundRecipe.appends.length > 0) { - foundRecipe.appends.forEach((append) => { - allSymbolsAtPosition.push({ - name: append.name, - kind: SymbolKind.Variable, - location: { - range: { - start, - end - }, - uri: 'file://' + append.dir + '/' + append.base - } - }) - }) - } + }) } } } From 6453d5a18d6a35eb45bcdb3ebf3f496d0370f751 Mon Sep 17 00:00:00 2001 From: Ziwei Wang Date: Tue, 5 Dec 2023 18:05:08 -0500 Subject: [PATCH 4/4] Test: Add tests for go to definition for uris in strings --- server/src/__tests__/definition.test.ts | 40 +++++++++++++++++++++- server/src/__tests__/fixtures/directive.bb | 5 ++- server/src/__tests__/fixtures/fixtures.ts | 2 +- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/server/src/__tests__/definition.test.ts b/server/src/__tests__/definition.test.ts index 045ccfa6..e85fa638 100644 --- a/server/src/__tests__/definition.test.ts +++ b/server/src/__tests__/definition.test.ts @@ -6,7 +6,7 @@ import { analyzer } from '../tree-sitter/analyzer' import { generateParser } from '../tree-sitter/parser' import { onDefinitionHandler } from '../connectionHandlers/onDefinition' -import { FIXTURE_DOCUMENT, DUMMY_URI, FIXTURE_URI } from './fixtures/fixtures' +import { FIXTURE_DOCUMENT, DUMMY_URI, FIXTURE_URI, FIXTURE_FOLDER } from './fixtures/fixtures' import path from 'path' import { bitBakeProjectScannerClient } from '../BitbakeProjectScannerClient' @@ -238,4 +238,42 @@ describe('on definition', () => { expect(shouldNotWork).toEqual([]) }) + + it('provides go to definition for uris found in the string content', async () => { + await analyzer.analyze({ + uri: DUMMY_URI, + document: FIXTURE_DOCUMENT.DIRECTIVE + }) + + analyzer.workspaceFolders = [{ uri: FIXTURE_FOLDER, name: 'test' }] + + const shouldWork1 = onDefinitionHandler({ + textDocument: { + uri: DUMMY_URI + }, + position: { + line: 12, + character: 13 + } + }) + + const shouldWork2 = onDefinitionHandler({ + textDocument: { + uri: DUMMY_URI + }, + position: { + line: 13, + character: 10 + } + }) + + expect(shouldWork1).toEqual([ + { + uri: FIXTURE_URI.FOO_INC, + range: { start: { line: 0, character: 0 }, end: { line: 0, character: 0 } } + } + ]) + + expect(shouldWork2).toEqual(shouldWork1) + }) }) diff --git a/server/src/__tests__/fixtures/directive.bb b/server/src/__tests__/fixtures/directive.bb index ee555f6e..ee5c7e12 100644 --- a/server/src/__tests__/fixtures/directive.bb +++ b/server/src/__tests__/fixtures/directive.bb @@ -8,4 +8,7 @@ FOO = '${APPEND}' SYMBOL_IN_STRING = 'hover is a package ${FOO} \ parentFolder/hover should also be seen as symbol \ this hover too, other words should not. \ - ' \ No newline at end of file + ' + +SRC_URI = 'file://foo.inc \ + file://foo.inc' \ No newline at end of file diff --git a/server/src/__tests__/fixtures/fixtures.ts b/server/src/__tests__/fixtures/fixtures.ts index f85f4424..f791f1e1 100644 --- a/server/src/__tests__/fixtures/fixtures.ts +++ b/server/src/__tests__/fixtures/fixtures.ts @@ -12,7 +12,7 @@ import path from 'path' import fs from 'fs' import { TextDocument } from 'vscode-languageserver-textdocument' -const FIXTURE_FOLDER = path.join(__dirname, './') +export const FIXTURE_FOLDER = path.join(__dirname, './') type FIXTURE_URI_KEY = keyof typeof FIXTURE_URI