From 340c5738d6f500864d47e8a7f1b2788271d86cb4 Mon Sep 17 00:00:00 2001 From: Alex Ross Date: Fri, 24 Jan 2025 18:52:00 +0100 Subject: [PATCH] Fix range end position in piece tree buffer (#238658) * Fix range end position in piece tree buffer also fix refresh ranges in token guess * Simplify first tree sitter tokens * Compute newRange end for changes in tree sitter * Fix tests --- .../pieceTreeTextBuffer.ts | 18 +++------------ src/vs/editor/common/model/textModel.ts | 1 - src/vs/editor/common/model/tokenStore.ts | 22 +++++++++---------- .../model/treeSitterTokenStoreService.ts | 9 ++++---- .../treeSitter/treeSitterParserService.ts | 20 ++++++++--------- src/vs/editor/common/textModelEvents.ts | 5 ----- .../widget/observableCodeEditor.test.ts | 16 +++++++------- src/vs/monaco.d.ts | 4 ---- .../test/browser/extHostDocumentData.test.ts | 6 ----- .../extHostDocumentSaveParticipant.test.ts | 2 -- .../browser/treeSitterTokenizationFeature.ts | 21 +++++------------- 11 files changed, 42 insertions(+), 82 deletions(-) diff --git a/src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.ts b/src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.ts index 1b812d383bc10..42d1995e36794 100644 --- a/src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.ts +++ b/src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.ts @@ -27,10 +27,6 @@ export interface IValidatedEditOperation { isAutoWhitespaceEdit: boolean; } -export interface IValidatedEditOperationWithReverseRange extends IValidatedEditOperation { - reverseRange: Range; -} - interface IReverseSingleEditOperation extends IValidEditOperation { sortIndex: number; } @@ -327,7 +323,7 @@ export class PieceTreeTextBuffer extends Disposable implements ITextBuffer { } // Delta encode operations - const reverseRanges = PieceTreeTextBuffer._getInverseEditRanges(operations); + const reverseRanges = (computeUndoEdits || recordTrimAutoWhitespace ? PieceTreeTextBuffer._getInverseEditRanges(operations) : []); const newTrimAutoWhitespaceCandidates: { lineNumber: number; oldContent: string }[] = []; if (recordTrimAutoWhitespace) { for (let i = 0; i < operations.length; i++) { @@ -377,19 +373,12 @@ export class PieceTreeTextBuffer extends Disposable implements ITextBuffer { } } - const operationsWithReverseRanges: IValidatedEditOperationWithReverseRange[] = new Array(operations.length); - for (let i = 0; i < operations.length; i++) { - operationsWithReverseRanges[i] = { - ...operations[i], - reverseRange: reverseRanges[i] - }; - } this._mightContainRTL = mightContainRTL; this._mightContainUnusualLineTerminators = mightContainUnusualLineTerminators; this._mightContainNonBasicASCII = mightContainNonBasicASCII; - const contentChanges = this._doApplyEdits(operationsWithReverseRanges); + const contentChanges = this._doApplyEdits(operations); let trimAutoWhitespaceLineNumbers: number[] | null = null; if (recordTrimAutoWhitespace && newTrimAutoWhitespaceCandidates.length > 0) { @@ -487,7 +476,7 @@ export class PieceTreeTextBuffer extends Disposable implements ITextBuffer { }; } - private _doApplyEdits(operations: IValidatedEditOperationWithReverseRange[]): IInternalModelContentChange[] { + private _doApplyEdits(operations: IValidatedEditOperation[]): IInternalModelContentChange[] { operations.sort(PieceTreeTextBuffer._sortOpsDescending); const contentChanges: IInternalModelContentChange[] = []; @@ -520,7 +509,6 @@ export class PieceTreeTextBuffer extends Disposable implements ITextBuffer { contentChanges.push({ range: contentChangeRange, rangeLength: op.rangeLength, - rangeEndPosition: op.reverseRange.getEndPosition(), text: op.text, rangeOffset: op.rangeOffset, forceMoveMarkers: op.forceMoveMarkers diff --git a/src/vs/editor/common/model/textModel.ts b/src/vs/editor/common/model/textModel.ts index cac9f3b882ed4..e54c9ebd69da2 100644 --- a/src/vs/editor/common/model/textModel.ts +++ b/src/vs/editor/common/model/textModel.ts @@ -460,7 +460,6 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati range: range, rangeOffset: rangeOffset, rangeLength: rangeLength, - rangeEndPosition: rangeEndPosition, text: text, }], eol: this._buffer.getEOL(), diff --git a/src/vs/editor/common/model/tokenStore.ts b/src/vs/editor/common/model/tokenStore.ts index 9ad6e415308c5..862b300c1a43f 100644 --- a/src/vs/editor/common/model/tokenStore.ts +++ b/src/vs/editor/common/model/tokenStore.ts @@ -210,15 +210,15 @@ export class TokenStore implements IDisposable { this._root = this.createFromUpdates(tokens, true); } - private createFromUpdates(tokens: TokenUpdate[], isNew?: boolean): Node { + private createFromUpdates(tokens: TokenUpdate[], needsRefresh?: boolean): Node { let newRoot: Node = { length: tokens[0].length, token: tokens[0].token, height: 0, - needsRefresh: isNew + needsRefresh }; for (let j = 1; j < tokens.length; j++) { - newRoot = append(newRoot, { length: tokens[j].length, token: tokens[j].token, height: 0, needsRefresh: isNew }); + newRoot = append(newRoot, { length: tokens[j].length, token: tokens[j].token, height: 0, needsRefresh }); } return newRoot; } @@ -227,22 +227,22 @@ export class TokenStore implements IDisposable { * * @param tokens tokens are in sequence in the document. */ - update(length: number, tokens: TokenUpdate[]) { + update(length: number, tokens: TokenUpdate[], needsRefresh?: boolean) { if (tokens.length === 0) { return; } - this.replace(length, tokens[0].startOffsetInclusive, tokens); + this.replace(length, tokens[0].startOffsetInclusive, tokens, needsRefresh); } delete(length: number, startOffset: number) { - this.replace(length, startOffset, []); + this.replace(length, startOffset, [], true); } /** * * @param tokens tokens are in sequence in the document. */ - private replace(length: number, updateOffsetStart: number, tokens: TokenUpdate[]) { + private replace(length: number, updateOffsetStart: number, tokens: TokenUpdate[], needsRefresh?: boolean) { const firstUnchangedOffsetAfterUpdate = updateOffsetStart + length; // Find the last unchanged node preceding the update const precedingNodes: Node[] = []; @@ -260,7 +260,7 @@ export class TokenStore implements IDisposable { continue; } else if (isLeaf(node.node) && (currentOffset < updateOffsetStart)) { // We have a partial preceding node - precedingNodes.push({ length: updateOffsetStart - currentOffset, token: node.node.token, height: 0 }); + precedingNodes.push({ length: updateOffsetStart - currentOffset, token: node.node.token, height: 0, needsRefresh: needsRefresh || node.node.needsRefresh }); // Node could also be postceeding, so don't continue } @@ -274,7 +274,7 @@ export class TokenStore implements IDisposable { continue; } else if (isLeaf(node.node) && (currentOffset + node.node.length >= firstUnchangedOffsetAfterUpdate)) { // we have a partial postceeding node - postcedingNodes.push({ length: currentOffset + node.node.length - firstUnchangedOffsetAfterUpdate, token: node.node.token, height: 0 }); + postcedingNodes.push({ length: currentOffset + node.node.length - firstUnchangedOffsetAfterUpdate, token: node.node.token, height: 0, needsRefresh: needsRefresh || node.node.needsRefresh }); continue; } @@ -290,7 +290,7 @@ export class TokenStore implements IDisposable { let allNodes: Node[]; if (tokens.length > 0) { - allNodes = precedingNodes.concat(this.createFromUpdates(tokens), postcedingNodes); + allNodes = precedingNodes.concat(this.createFromUpdates(tokens, needsRefresh), postcedingNodes); } else { allNodes = precedingNodes.concat(postcedingNodes); } @@ -445,7 +445,7 @@ export class TokenStore implements IDisposable { const indent = ' '.repeat(depth); if (isLeaf(node)) { - result.push(`${indent}Leaf(length: ${node.length}, token: ${node.token})\n`); + result.push(`${indent}Leaf(length: ${node.length}, token: ${node.token}, refresh: ${node.needsRefresh})\n`); } else { result.push(`${indent}List(length: ${node.length})\n`); // Push children in reverse order so they get processed left-to-right diff --git a/src/vs/editor/common/model/treeSitterTokenStoreService.ts b/src/vs/editor/common/model/treeSitterTokenStoreService.ts index 56302824adae9..4295bab91ac81 100644 --- a/src/vs/editor/common/model/treeSitterTokenStoreService.ts +++ b/src/vs/editor/common/model/treeSitterTokenStoreService.ts @@ -48,17 +48,18 @@ class TreeSitterTokenizationStoreService implements ITreeSitterTokenizationStore storeInfo.guessVersion = e.versionId; for (const change of e.changes) { - storeInfo.store.markForRefresh(change.rangeOffset, change.rangeOffset + change.rangeLength); if (change.text.length > change.rangeLength) { const oldToken = storeInfo.store.getTokenAt(change.rangeOffset)!; // Insert. Just grow the token at this position to include the insert. const newToken = { startOffsetInclusive: oldToken.startOffsetInclusive, length: oldToken.length + change.text.length - change.rangeLength, token: oldToken.token }; - storeInfo.store.update(oldToken.length, [newToken]); + storeInfo.store.update(oldToken.length, [newToken], true); } else if (change.text.length < change.rangeLength) { // Delete. Delete the tokens at the corresponding range. const deletedCharCount = change.rangeLength - change.text.length; storeInfo.store.delete(deletedCharCount, change.rangeOffset); } + const refreshLength = change.rangeLength > change.text.length ? change.rangeLength : change.text.length; + storeInfo.store.markForRefresh(change.rangeOffset, change.rangeOffset + refreshLength); } })); disposables.add(model.onWillDispose(() => { @@ -105,8 +106,8 @@ class TreeSitterTokenizationStoreService implements ITreeSitterTokenizationStore existingTokens.accurateVersion = version; for (const update of updates) { - const lastToken = update.newTokens[update.newTokens.length - 1]; - const oldRangeLength = (existingTokens.guessVersion >= version) ? (lastToken.startOffsetInclusive + lastToken.length - update.newTokens[0].startOffsetInclusive) : update.oldRangeLength; + const lastToken = update.newTokens.length > 0 ? update.newTokens[update.newTokens.length - 1] : undefined; + const oldRangeLength = ((existingTokens.guessVersion >= version) && lastToken) ? (lastToken.startOffsetInclusive + lastToken.length - update.newTokens[0].startOffsetInclusive) : update.oldRangeLength; existingTokens.store.update(oldRangeLength, update.newTokens); } } diff --git a/src/vs/editor/common/services/treeSitter/treeSitterParserService.ts b/src/vs/editor/common/services/treeSitter/treeSitterParserService.ts index 11175de6a0aae..2d0ac90468eed 100644 --- a/src/vs/editor/common/services/treeSitter/treeSitterParserService.ts +++ b/src/vs/editor/common/services/treeSitter/treeSitterParserService.ts @@ -24,6 +24,7 @@ import { PromiseResult } from '../../../../base/common/observable.js'; import { Range } from '../../core/range.js'; import { Position } from '../../core/position.js'; import { LimitedQueue } from '../../../../base/common/async.js'; +import { TextLength } from '../../core/textLength.js'; const EDITOR_TREESITTER_TELEMETRY = 'editor.experimental.treeSitterTelemetry'; const MODULE_LOCATION_SUBPATH = `@vscode/tree-sitter-wasm/wasm`; @@ -350,22 +351,19 @@ export class TreeSitterParseResult implements IDisposable, ITreeSitterParseResul private _applyEdits(changes: IModelContentChange[], version: number) { for (const change of changes) { - this._tree?.edit({ + const originalTextLength = TextLength.ofRange(Range.lift(change.range)); + const newTextLength = TextLength.ofText(change.text); + const summedTextLengths = change.text.length === 0 ? newTextLength : originalTextLength.add(newTextLength); + const edit = { startIndex: change.rangeOffset, oldEndIndex: change.rangeOffset + change.rangeLength, newEndIndex: change.rangeOffset + change.text.length, startPosition: { row: change.range.startLineNumber - 1, column: change.range.startColumn - 1 }, oldEndPosition: { row: change.range.endLineNumber - 1, column: change.range.endColumn - 1 }, - newEndPosition: { row: change.rangeEndPosition.lineNumber - 1, column: change.rangeEndPosition.column - 1 } - }); - this._lastFullyParsedWithEdits?.edit({ - startIndex: change.rangeOffset, - oldEndIndex: change.rangeOffset + change.rangeLength, - newEndIndex: change.rangeOffset + change.text.length, - startPosition: { row: change.range.startLineNumber - 1, column: change.range.startColumn - 1 }, - oldEndPosition: { row: change.range.endLineNumber - 1, column: change.range.endColumn - 1 }, - newEndPosition: { row: change.rangeEndPosition.lineNumber - 1, column: change.rangeEndPosition.column - 1 } - }); + newEndPosition: { row: change.range.startLineNumber + summedTextLengths.lineCount - 1, column: summedTextLengths.columnCount } + }; + this._tree?.edit(edit); + this._lastFullyParsedWithEdits?.edit(edit); } this._editVersion = version; } diff --git a/src/vs/editor/common/textModelEvents.ts b/src/vs/editor/common/textModelEvents.ts index 8cc19576d3ee2..c35d0472106ba 100644 --- a/src/vs/editor/common/textModelEvents.ts +++ b/src/vs/editor/common/textModelEvents.ts @@ -3,7 +3,6 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { Position } from './core/position.js'; import { IRange } from './core/range.js'; import { Selection } from './core/selection.js'; import { IModelDecoration, InjectedTextOptions } from './model.js'; @@ -46,10 +45,6 @@ export interface IModelContentChange { * The length of the range that got replaced. */ readonly rangeLength: number; - /** - * The new end position of the range that got replaced. - */ - readonly rangeEndPosition: Position; /** * The new text for the range. */ diff --git a/src/vs/editor/test/browser/widget/observableCodeEditor.test.ts b/src/vs/editor/test/browser/widget/observableCodeEditor.test.ts index 501d92ea85caf..7bb14f1387139 100644 --- a/src/vs/editor/test/browser/widget/observableCodeEditor.test.ts +++ b/src/vs/editor/test/browser/widget/observableCodeEditor.test.ts @@ -82,9 +82,9 @@ suite("CodeEditorWidget", () => { assert.deepStrictEqual(log.getAndClearEntries(), [ 'handle change: editor.onDidType "abc"', - 'handle change: editor.versionId {"changes":[{"range":"[1,1 -> 1,1]","rangeLength":0,"rangeEndPosition":{"lineNumber":1,"column":2},"text":"a","rangeOffset":0}],"eol":"\\n","versionId":2}', - 'handle change: editor.versionId {"changes":[{"range":"[1,2 -> 1,2]","rangeLength":0,"rangeEndPosition":{"lineNumber":1,"column":3},"text":"b","rangeOffset":1}],"eol":"\\n","versionId":3}', - 'handle change: editor.versionId {"changes":[{"range":"[1,3 -> 1,3]","rangeLength":0,"rangeEndPosition":{"lineNumber":1,"column":4},"text":"c","rangeOffset":2}],"eol":"\\n","versionId":4}', + 'handle change: editor.versionId {"changes":[{"range":"[1,1 -> 1,1]","rangeLength":0,"text":"a","rangeOffset":0}],"eol":"\\n","versionId":2}', + 'handle change: editor.versionId {"changes":[{"range":"[1,2 -> 1,2]","rangeLength":0,"text":"b","rangeOffset":1}],"eol":"\\n","versionId":3}', + 'handle change: editor.versionId {"changes":[{"range":"[1,3 -> 1,3]","rangeLength":0,"text":"c","rangeOffset":2}],"eol":"\\n","versionId":4}', 'handle change: editor.selections {"selection":"[1,4 -> 1,4]","modelVersionId":4,"oldSelections":["[1,1 -> 1,1]"],"oldModelVersionId":1,"source":"keyboard","reason":0}', 'running derived: selection: [1,4 -> 1,4], value: 4', ]); @@ -96,9 +96,9 @@ suite("CodeEditorWidget", () => { assert.deepStrictEqual(log.getAndClearEntries(), [ 'handle change: editor.onDidType "abc"', - 'handle change: editor.versionId {"changes":[{"range":"[1,1 -> 1,1]","rangeLength":0,"rangeEndPosition":{"lineNumber":1,"column":2},"text":"a","rangeOffset":0}],"eol":"\\n","versionId":2}', - 'handle change: editor.versionId {"changes":[{"range":"[1,2 -> 1,2]","rangeLength":0,"rangeEndPosition":{"lineNumber":1,"column":3},"text":"b","rangeOffset":1}],"eol":"\\n","versionId":3}', - 'handle change: editor.versionId {"changes":[{"range":"[1,3 -> 1,3]","rangeLength":0,"rangeEndPosition":{"lineNumber":1,"column":4},"text":"c","rangeOffset":2}],"eol":"\\n","versionId":4}', + 'handle change: editor.versionId {"changes":[{"range":"[1,1 -> 1,1]","rangeLength":0,"text":"a","rangeOffset":0}],"eol":"\\n","versionId":2}', + 'handle change: editor.versionId {"changes":[{"range":"[1,2 -> 1,2]","rangeLength":0,"text":"b","rangeOffset":1}],"eol":"\\n","versionId":3}', + 'handle change: editor.versionId {"changes":[{"range":"[1,3 -> 1,3]","rangeLength":0,"text":"c","rangeOffset":2}],"eol":"\\n","versionId":4}', 'handle change: editor.selections {"selection":"[1,4 -> 1,4]","modelVersionId":4,"oldSelections":["[1,1 -> 1,1]"],"oldModelVersionId":1,"source":"keyboard","reason":0}', 'running derived: selection: [1,4 -> 1,4], value: 4', ]); @@ -134,7 +134,7 @@ suite("CodeEditorWidget", () => { ">>> before get", "<<< after get", 'handle change: editor.onDidType "a"', - 'handle change: editor.versionId {"changes":[{"range":"[1,1 -> 1,1]","rangeLength":0,"rangeEndPosition":{"lineNumber":1,"column":2},"text":"a","rangeOffset":0}],"eol":"\\n","versionId":2}', + 'handle change: editor.versionId {"changes":[{"range":"[1,1 -> 1,1]","rangeLength":0,"text":"a","rangeOffset":0}],"eol":"\\n","versionId":2}', 'handle change: editor.selections {"selection":"[1,2 -> 1,2]","modelVersionId":2,"oldSelections":["[1,1 -> 1,1]"],"oldModelVersionId":1,"source":"keyboard","reason":0}', "running derived: selection: [1,2 -> 1,2], value: 2", ]); @@ -172,7 +172,7 @@ suite("CodeEditorWidget", () => { "running derived: selection: [1,2 -> 1,2], value: 2", "<<< after get", 'handle change: editor.onDidType "a"', - 'handle change: editor.versionId {"changes":[{"range":"[1,1 -> 1,1]","rangeLength":0,"rangeEndPosition":{"lineNumber":1,"column":2},"text":"a","rangeOffset":0}],"eol":"\\n","versionId":2}', + 'handle change: editor.versionId {"changes":[{"range":"[1,1 -> 1,1]","rangeLength":0,"text":"a","rangeOffset":0}],"eol":"\\n","versionId":2}', 'handle change: editor.selections {"selection":"[1,2 -> 1,2]","modelVersionId":2,"oldSelections":["[1,1 -> 1,1]"],"oldModelVersionId":1,"source":"keyboard","reason":0}', "running derived: selection: [1,2 -> 1,2], value: 2", ]); diff --git a/src/vs/monaco.d.ts b/src/vs/monaco.d.ts index 4cb6641940357..f64b9dda5d55a 100644 --- a/src/vs/monaco.d.ts +++ b/src/vs/monaco.d.ts @@ -2914,10 +2914,6 @@ declare namespace monaco.editor { * The length of the range that got replaced. */ readonly rangeLength: number; - /** - * The new end position of the range that got replaced. - */ - readonly rangeEndPosition: Position; /** * The new text for the range. */ diff --git a/src/vs/workbench/api/test/browser/extHostDocumentData.test.ts b/src/vs/workbench/api/test/browser/extHostDocumentData.test.ts index 1c05433c1170e..926b8fdc6d29a 100644 --- a/src/vs/workbench/api/test/browser/extHostDocumentData.test.ts +++ b/src/vs/workbench/api/test/browser/extHostDocumentData.test.ts @@ -104,7 +104,6 @@ suite('ExtHostDocumentData', () => { range: { startLineNumber: 1, startColumn: 1, endLineNumber: 1, endColumn: 1 }, rangeOffset: undefined!, rangeLength: undefined!, - rangeEndPosition: undefined!, text: '\t ' }], eol: undefined!, @@ -164,7 +163,6 @@ suite('ExtHostDocumentData', () => { range: { startLineNumber: 1, startColumn: 3, endLineNumber: 1, endColumn: 6 }, rangeOffset: undefined!, rangeLength: undefined!, - rangeEndPosition: undefined!, text: '' }], eol: undefined!, @@ -185,7 +183,6 @@ suite('ExtHostDocumentData', () => { range: { startLineNumber: 1, startColumn: 3, endLineNumber: 1, endColumn: 6 }, rangeOffset: undefined!, rangeLength: undefined!, - rangeEndPosition: undefined!, text: 'is could be' }], eol: undefined!, @@ -206,7 +203,6 @@ suite('ExtHostDocumentData', () => { range: { startLineNumber: 1, startColumn: 3, endLineNumber: 1, endColumn: 6 }, rangeOffset: undefined!, rangeLength: undefined!, - rangeEndPosition: undefined!, text: 'is could be\na line with number' }], eol: undefined!, @@ -230,7 +226,6 @@ suite('ExtHostDocumentData', () => { range: { startLineNumber: 1, startColumn: 3, endLineNumber: 2, endColumn: 6 }, rangeOffset: undefined!, rangeLength: undefined!, - rangeEndPosition: undefined!, text: '' }], eol: undefined!, @@ -421,7 +416,6 @@ suite('ExtHostDocumentData updates line mapping', () => { range: range, rangeOffset: undefined!, rangeLength: undefined!, - rangeEndPosition: undefined!, text: text }], eol: eol!, diff --git a/src/vs/workbench/api/test/browser/extHostDocumentSaveParticipant.test.ts b/src/vs/workbench/api/test/browser/extHostDocumentSaveParticipant.test.ts index b5b3933c529a2..965bb16223ed4 100644 --- a/src/vs/workbench/api/test/browser/extHostDocumentSaveParticipant.test.ts +++ b/src/vs/workbench/api/test/browser/extHostDocumentSaveParticipant.test.ts @@ -294,7 +294,6 @@ suite('ExtHostDocumentSaveParticipant', () => { documents.$acceptModelChanged(resource, { changes: [{ range: { startLineNumber: 1, startColumn: 1, endLineNumber: 1, endColumn: 1 }, - rangeEndPosition: undefined!, rangeOffset: undefined!, rangeLength: undefined!, text: 'bar' @@ -330,7 +329,6 @@ suite('ExtHostDocumentSaveParticipant', () => { changes: [{ range, text, - rangeEndPosition: undefined!, rangeOffset: undefined!, rangeLength: undefined!, }], diff --git a/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts b/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts index f168a28f28975..a1a3b86b81834 100644 --- a/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts +++ b/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts @@ -136,26 +136,17 @@ export class TreeSitterTokenizationSupport extends Disposable implements ITreeSi })); } - private _createEmptyTokens(captures: { tree: ITreeSitterParseResult | undefined; captures: QueryCapture[] }, modelEndOffset: number) { + private _createEmptyTokens(textModel: ITextModel) { const languageId = this._languageIdCodec.encodeLanguageId(this._languageId); const emptyToken = this._emptyToken(languageId); - const capturedTokens = this._createTokensFromCaptures(captures.tree, captures.captures, 0, modelEndOffset); - if (!capturedTokens) { - return; - } - const emptyTokens: EndOffsetToken[] = capturedTokens.endOffsets.map(capture => ({ endOffset: capture.endOffset, metadata: emptyToken })); - return this._rangeTokensAsUpdates(0, emptyTokens); + const modelEndOffset = textModel.getValueLength(); + + const emptyTokens: TokenUpdate[] = [{ token: emptyToken, length: modelEndOffset, startOffsetInclusive: 0 }]; + return emptyTokens; } private _firstTreeUpdate(textModel: ITextModel, versionId: number, ranges: { readonly fromLineNumber: number; readonly toLineNumber: number }[]) { - const modelEndOffset = textModel.getValueLength(); - const editorEndPosition = textModel.getPositionAt(modelEndOffset); - const captures = this._getTreeAndCaptures(new Range(1, 1, editorEndPosition.lineNumber, editorEndPosition.column), textModel); - if (captures.captures.length === 0) { - return; - } - // Make empty tokens to populate the store - const tokens: TokenUpdate[] = this._createEmptyTokens(captures, modelEndOffset) ?? []; + const tokens: TokenUpdate[] = this._createEmptyTokens(textModel); this._tokenizationStoreService.setTokens(textModel, tokens); this._setViewPortTokens(textModel, versionId); }