From 5080e25fa3df49df1e3fff5d9a143dcd685f297d Mon Sep 17 00:00:00 2001 From: Kyle Butts Date: Wed, 3 Jul 2024 16:33:05 -0500 Subject: [PATCH 1/5] Fix R code cell parser and add `# %%` support (#3709) - R parser would end cells after first blank line; this is now removed - `# %%` is now recognized as a valid cell demarcation in R - cellDecorationSetting now matches python (only highlights currently active cell - Tests added to test for these changes --- extensions/positron-code-cells/src/parser.ts | 14 ++--- .../src/test/codeLenses.test.ts | 52 ++++++++++-------- .../src/test/folding.test.ts | 53 +++++++++++-------- .../src/test/parser.test.ts | 12 ++--- 4 files changed, 77 insertions(+), 54 deletions(-) diff --git a/extensions/positron-code-cells/src/parser.ts b/extensions/positron-code-cells/src/parser.ts index 179d8f96710..acee66e6fee 100644 --- a/extensions/positron-code-cells/src/parser.ts +++ b/extensions/positron-code-cells/src/parser.ts @@ -64,8 +64,10 @@ function getJupyterMarkdownCellText(cell: Cell, document: vscode.TextDocument): return `%%markdown\n${text}\n\n`; } -const pythonIsCellStartRegExp = new RegExp(/^\s*#\s*%%/); -const pythonMarkdownRegExp = new RegExp(/^\s*#\s*%%[^[]*\[markdown\]/); +// Spaces can not occur before # +const pythonIsCellStartRegExp = new RegExp(/^#\s*%%/); +const pythonMarkdownRegExp = new RegExp(/^#\s*%%[^[]*\[markdown\]/); +const rIsCellStartRegExp = new RegExp(/^#[\s*%%|+]/); // TODO: Expose an API to let extensions register parsers const pythonCellParser: CellParser = { @@ -81,12 +83,12 @@ const pythonCellParser: CellParser = { }; const rCellParser: CellParser = { - isCellStart: (line) => line.startsWith('#+'), - isCellEnd: (line) => line.trim() === '', + isCellStart: (line) => rIsCellStartRegExp.test(line), + isCellEnd: (_line) => false, getCellType: (_line) => CellType.Code, getCellText: getCellText, - newCell: () => '\n\n#+', - cellDecorationSetting: () => CellDecorationSetting.All, + newCell: () => '\n#+\n', + cellDecorationSetting: () => CellDecorationSetting.Current, }; const parsers: Map = new Map([ diff --git a/extensions/positron-code-cells/src/test/codeLenses.test.ts b/extensions/positron-code-cells/src/test/codeLenses.test.ts index 47181fb787c..c3c277b51e6 100644 --- a/extensions/positron-code-cells/src/test/codeLenses.test.ts +++ b/extensions/positron-code-cells/src/test/codeLenses.test.ts @@ -11,45 +11,55 @@ import { closeAllEditors } from './utils'; suite('CodeLenses', () => { teardown(closeAllEditors); - test('Provides Python cell code lenses', async () => { - const language = 'python'; - const content = `#%% + const content = `# %% testing1 -#%% + testing2 -#%% -testing3`; - const document = await vscode.workspace.openTextDocument({ language, content }); + +# %% +testing3 + +# %% +testing4`; + const content_with_plus = content.replaceAll("# %%", "#+"); + + test('Provides Python cell code lenses', async () => { const provider = new CellCodeLensProvider(); + const language = 'python'; + const document = await vscode.workspace.openTextDocument({ language: language, content: content }); const codeLenses = await provider.provideCodeLenses(document); assert.ok(codeLenses, 'No code lenses provided'); verifyCodeLenses(codeLenses, [ - new vscode.Range(0, 0, 1, 8), - new vscode.Range(2, 0, 3, 8), - new vscode.Range(4, 0, 5, 8) + new vscode.Range(0, 0, 4, 0), + new vscode.Range(5, 0, 7, 0), + new vscode.Range(8, 0, 9, 8) ]); }); test('Provides R cell code lenses', async () => { - const language = 'r'; - const content = `#+ -testing1 -#+ -testing2 -#+ -testing3`; - const document = await vscode.workspace.openTextDocument({ language, content }); const provider = new CellCodeLensProvider(); + const language = 'r'; + const document = await vscode.workspace.openTextDocument({ language: language, content: content }); const codeLenses = await provider.provideCodeLenses(document); assert.ok(codeLenses, 'No code lenses provided'); verifyCodeLenses(codeLenses, [ - new vscode.Range(0, 0, 1, 8), - new vscode.Range(2, 0, 3, 8), - new vscode.Range(4, 0, 5, 8) + new vscode.Range(0, 0, 4, 0), + new vscode.Range(5, 0, 7, 0), + new vscode.Range(8, 0, 9, 8) + ]); + + const document2 = await vscode.workspace.openTextDocument({ language: language, content: content_with_plus }); + const codeLenses2 = await provider.provideCodeLenses(document2); + + assert.ok(codeLenses2, 'No code lenses provided'); + verifyCodeLenses(codeLenses2, [ + new vscode.Range(0, 0, 4, 0), + new vscode.Range(5, 0, 7, 0), + new vscode.Range(8, 0, 9, 8) ]); }); }); diff --git a/extensions/positron-code-cells/src/test/folding.test.ts b/extensions/positron-code-cells/src/test/folding.test.ts index 12d846fc50b..ee361fd926b 100644 --- a/extensions/positron-code-cells/src/test/folding.test.ts +++ b/extensions/positron-code-cells/src/test/folding.test.ts @@ -11,45 +11,56 @@ import { closeAllEditors } from './utils'; suite('Folding', () => { teardown(closeAllEditors); - test('Provides Python cell folding ranges', async () => { - const language = 'python'; - const content = `#%% + const content = `# %% testing1 -#%% + testing2 -#%% -testing3`; - const document = await vscode.workspace.openTextDocument({ language, content }); + +# %% +testing3 + +# %% +testing4`; + const content_with_plus = content.replaceAll("# %%", "#+"); + + + test('Provides Python cell folding ranges', async () => { const provider = new CellFoldingRangeProvider(); + const language = 'python'; + const document = await vscode.workspace.openTextDocument({ language, content }); const foldingRanges = await provider.provideFoldingRanges(document); assert.ok(foldingRanges, 'No folding ranges provided'); assert.deepStrictEqual(foldingRanges, [ - new vscode.FoldingRange(0, 1), - new vscode.FoldingRange(2, 3), - new vscode.FoldingRange(4, 5), + new vscode.FoldingRange(0, 4), + new vscode.FoldingRange(5, 7), + new vscode.FoldingRange(8, 9), ], 'Incorrect folding ranges'); }); test('Provides R cell folding ranges', async () => { - const language = 'r'; - const content = `#+ -testing1 -#+ -testing2 -#+ -testing3`; - const document = await vscode.workspace.openTextDocument({ language, content }); const provider = new CellFoldingRangeProvider(); + const language = 'r'; + const document = await vscode.workspace.openTextDocument({ language: language, content: content }); const foldingRanges = await provider.provideFoldingRanges(document); assert.ok(foldingRanges, 'No folding ranges provided'); assert.deepStrictEqual(foldingRanges, [ - new vscode.FoldingRange(0, 1), - new vscode.FoldingRange(2, 3), - new vscode.FoldingRange(4, 5), + new vscode.FoldingRange(0, 4), + new vscode.FoldingRange(5, 7), + new vscode.FoldingRange(8, 9), + ], 'Incorrect folding ranges'); + + const document2 = await vscode.workspace.openTextDocument({ language: language, content: content_with_plus }); + const foldingRanges2 = await provider.provideFoldingRanges(document2); + + assert.ok(foldingRanges2, 'No folding ranges provided'); + assert.deepStrictEqual(foldingRanges2, [ + new vscode.FoldingRange(0, 4), + new vscode.FoldingRange(5, 7), + new vscode.FoldingRange(8, 9), ], 'Incorrect folding ranges'); }); }); diff --git a/extensions/positron-code-cells/src/test/parser.test.ts b/extensions/positron-code-cells/src/test/parser.test.ts index 45253526fc9..cada04c57d4 100644 --- a/extensions/positron-code-cells/src/test/parser.test.ts +++ b/extensions/positron-code-cells/src/test/parser.test.ts @@ -99,9 +99,9 @@ And a [link](target)`; suite('R Parser', () => { const language = 'r'; - const codeCellBody = '123\n456'; + const codeCellBody = '\n123\n456'; const codeCell1 = `#+\n${codeCellBody}`; - const codeCell2 = `#+\n789\n012`; + const codeCell2 = `# %%\n789\n\n012`; const parser = getParser(language); @@ -123,8 +123,8 @@ And a [link](target)`; const content = [codeCell1, codeCell2].join('\n\n'); const document = await vscode.workspace.openTextDocument({ language, content }); assert.deepStrictEqual(parseCells(document), [ - { range: new vscode.Range(0, 0, 2, 3), type: CellType.Code }, - { range: new vscode.Range(4, 0, 6, 3), type: CellType.Code } + { range: new vscode.Range(0, 0, 4, 0), type: CellType.Code }, + { range: new vscode.Range(5, 0, 8, 3), type: CellType.Code } ]); }); @@ -150,11 +150,11 @@ And a [link](target)`; }); test('New cell', async () => { - assert.strictEqual(parser?.newCell(), '\n\n#+'); + assert.strictEqual(parser?.newCell(), '\n#+\n'); }); test('Cell decoration setting', async () => { - assert.strictEqual(parser?.cellDecorationSetting(), CellDecorationSetting.All); + assert.strictEqual(parser?.cellDecorationSetting(), CellDecorationSetting.Current); }); }); }); From bd2107b15e94b4a354a44c892acd0ade94d1a02b Mon Sep 17 00:00:00 2001 From: Kyle F Butts Date: Fri, 5 Jul 2024 11:22:22 -0400 Subject: [PATCH 2/5] Update extensions/positron-code-cells/src/parser.ts Co-authored-by: Wasim Lorgat Signed-off-by: Kyle F Butts --- extensions/positron-code-cells/src/parser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/positron-code-cells/src/parser.ts b/extensions/positron-code-cells/src/parser.ts index acee66e6fee..78db60685ea 100644 --- a/extensions/positron-code-cells/src/parser.ts +++ b/extensions/positron-code-cells/src/parser.ts @@ -67,7 +67,7 @@ function getJupyterMarkdownCellText(cell: Cell, document: vscode.TextDocument): // Spaces can not occur before # const pythonIsCellStartRegExp = new RegExp(/^#\s*%%/); const pythonMarkdownRegExp = new RegExp(/^#\s*%%[^[]*\[markdown\]/); -const rIsCellStartRegExp = new RegExp(/^#[\s*%%|+]/); +const rIsCellStartRegExp = new RegExp(/^#(\s*%%|+)/); // TODO: Expose an API to let extensions register parsers const pythonCellParser: CellParser = { From 6cab4265b18d48aa227737585ede853b5a1c7a20 Mon Sep 17 00:00:00 2001 From: Kyle Butts Date: Sat, 6 Jul 2024 09:55:55 -0500 Subject: [PATCH 3/5] Fix regex expression --- extensions/positron-code-cells/src/parser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/positron-code-cells/src/parser.ts b/extensions/positron-code-cells/src/parser.ts index 78db60685ea..5db440ecfe6 100644 --- a/extensions/positron-code-cells/src/parser.ts +++ b/extensions/positron-code-cells/src/parser.ts @@ -67,7 +67,7 @@ function getJupyterMarkdownCellText(cell: Cell, document: vscode.TextDocument): // Spaces can not occur before # const pythonIsCellStartRegExp = new RegExp(/^#\s*%%/); const pythonMarkdownRegExp = new RegExp(/^#\s*%%[^[]*\[markdown\]/); -const rIsCellStartRegExp = new RegExp(/^#(\s*%%|+)/); +const rIsCellStartRegExp = new RegExp(/^#(\s*%%|\s*\+|\s*-)/); // TODO: Expose an API to let extensions register parsers const pythonCellParser: CellParser = { From 57a6248fb3e1b6d1286adc1f437125536d32957b Mon Sep 17 00:00:00 2001 From: Kyle Butts Date: Sat, 6 Jul 2024 11:02:16 -0500 Subject: [PATCH 4/5] Fix regex (for real) --- extensions/positron-code-cells/src/parser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/positron-code-cells/src/parser.ts b/extensions/positron-code-cells/src/parser.ts index 5db440ecfe6..0835f678759 100644 --- a/extensions/positron-code-cells/src/parser.ts +++ b/extensions/positron-code-cells/src/parser.ts @@ -67,7 +67,7 @@ function getJupyterMarkdownCellText(cell: Cell, document: vscode.TextDocument): // Spaces can not occur before # const pythonIsCellStartRegExp = new RegExp(/^#\s*%%/); const pythonMarkdownRegExp = new RegExp(/^#\s*%%[^[]*\[markdown\]/); -const rIsCellStartRegExp = new RegExp(/^#(\s*%%|\s*\+|\s*-)/); +const rIsCellStartRegExp = new RegExp(/^#\s+(%%|\+)/); // TODO: Expose an API to let extensions register parsers const pythonCellParser: CellParser = { From 07490d81e7946bf8d028fd68b0b0ed06940a99fd Mon Sep 17 00:00:00 2001 From: Kyle Butts Date: Mon, 8 Jul 2024 08:11:00 -0500 Subject: [PATCH 5/5] Fix regex expression --- extensions/positron-code-cells/src/parser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/positron-code-cells/src/parser.ts b/extensions/positron-code-cells/src/parser.ts index 0835f678759..92ae1273b84 100644 --- a/extensions/positron-code-cells/src/parser.ts +++ b/extensions/positron-code-cells/src/parser.ts @@ -67,7 +67,7 @@ function getJupyterMarkdownCellText(cell: Cell, document: vscode.TextDocument): // Spaces can not occur before # const pythonIsCellStartRegExp = new RegExp(/^#\s*%%/); const pythonMarkdownRegExp = new RegExp(/^#\s*%%[^[]*\[markdown\]/); -const rIsCellStartRegExp = new RegExp(/^#\s+(%%|\+)/); +const rIsCellStartRegExp = new RegExp(/^#\s*(%%|\+)/); // TODO: Expose an API to let extensions register parsers const pythonCellParser: CellParser = {