From 9113dde9bef9fd864ccd8a39534a0c4b7a7a92d6 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Mon, 25 Nov 2024 11:49:54 +0000 Subject: [PATCH 01/12] feat!: add new outputs node --- packages/myst-cli/src/process/mdast.ts | 1 + packages/myst-cli/src/process/notebook.ts | 22 +- packages/myst-cli/src/transforms/code.spec.ts | 32 ++- packages/myst-cli/src/transforms/code.ts | 17 +- .../src/transforms/crossReferences.ts | 59 ++--- .../myst-cli/src/transforms/outputs.spec.ts | 49 ++-- packages/myst-cli/src/transforms/outputs.ts | 234 ++++++++++-------- packages/myst-directives/src/code.ts | 9 +- packages/myst-execute/src/execute.ts | 19 +- packages/myst-execute/tests/execute.yml | 209 ++++++---------- packages/myst-execute/tests/run.spec.ts | 1 + packages/myst-spec-ext/src/types.ts | 12 +- packages/myst-transforms/src/blocks.spec.ts | 12 +- packages/myst-transforms/src/blocks.ts | 23 +- .../notebook-fig-embed/outputs/index.json | 101 ++++++++ 15 files changed, 454 insertions(+), 346 deletions(-) diff --git a/packages/myst-cli/src/process/mdast.ts b/packages/myst-cli/src/process/mdast.ts index e06b70aab..8a68ca756 100644 --- a/packages/myst-cli/src/process/mdast.ts +++ b/packages/myst-cli/src/process/mdast.ts @@ -450,5 +450,6 @@ export async function finalizeMdast( postData.widgets = cache.$getMdast(file)?.pre.widgets; updateFileInfoFromFrontmatter(session, file, frontmatter); } + logMessagesFromVFile(session, vfile); } diff --git a/packages/myst-cli/src/process/notebook.ts b/packages/myst-cli/src/process/notebook.ts index ce8323f39..59257a5dc 100644 --- a/packages/myst-cli/src/process/notebook.ts +++ b/packages/myst-cli/src/process/notebook.ts @@ -165,17 +165,21 @@ export async function processNotebookFull( value: ensureString(cell.source), }; - // Embed outputs in an output block - const output: { type: 'output'; id: string; data: IOutput[] } = { - type: 'output', + const outputsChildren = (cell.outputs as IOutput[]).map((output) => { + // Embed outputs in an output block + const result = { + type: 'output', + jupyter_data: output, + children: [], + }; + return result; + }); + const outputs = { + type: 'outputs', id: nanoid(), - data: [], + children: outputsChildren, }; - - if (cell.outputs && (cell.outputs as IOutput[]).length > 0) { - output.data = cell.outputs as IOutput[]; - } - return acc.concat(blockParent(cell, [code, output])); + return acc.concat(blockParent(cell, [code, outputs])); } return acc; }, diff --git a/packages/myst-cli/src/transforms/code.spec.ts b/packages/myst-cli/src/transforms/code.spec.ts index 0807f5740..27d50958f 100644 --- a/packages/myst-cli/src/transforms/code.spec.ts +++ b/packages/myst-cli/src/transforms/code.spec.ts @@ -162,7 +162,10 @@ function build_mdast(tags: string[], has_output: boolean) { ], }; if (has_output) { - mdast.children[0].children.push({ type: 'output' }); + mdast.children[0].children.push({ + type: 'outputs', + children: [{ type: 'output', children: [] }], + }); } return mdast; } @@ -261,7 +264,7 @@ describe('propagateBlockDataToCode', () => { const mdast = build_mdast([tag], has_output); propagateBlockDataToCode(new Session(), new VFile(), mdast); let result = ''; - const outputNode = mdast.children[0].children[1]; + const outputsNode = mdast.children[0].children[1]; switch (target) { case 'cell': result = mdast.children[0].visibility; @@ -270,12 +273,14 @@ describe('propagateBlockDataToCode', () => { result = mdast.children[0].children[0].visibility; break; case 'output': - if (!has_output && target == 'output') { - expect(outputNode).toEqual(undefined); + if (!has_output) { + expect(outputsNode).toEqual(undefined); continue; } - result = outputNode.visibility; + result = outputsNode.visibility; break; + default: + throw new Error(); } expect(result).toEqual(action); } @@ -290,13 +295,13 @@ describe('propagateBlockDataToCode', () => { propagateBlockDataToCode(new Session(), new VFile(), mdast); const blockNode = mdast.children[0]; const codeNode = mdast.children[0].children[0]; - const outputNode = mdast.children[0].children[1]; + const outputsNode = mdast.children[0].children[1]; expect(blockNode.visibility).toEqual(action); expect(codeNode.visibility).toEqual(action); if (has_output) { - expect(outputNode.visibility).toEqual(action); + expect(outputsNode.visibility).toEqual(action); } else { - expect(outputNode).toEqual(undefined); + expect(outputsNode).toEqual(undefined); } } } @@ -313,7 +318,8 @@ describe('propagateBlockDataToCode', () => { executable: true, }, { - type: 'output', + type: 'outputs', + children: [], }, ], data: { @@ -323,10 +329,10 @@ describe('propagateBlockDataToCode', () => { ], }; propagateBlockDataToCode(new Session(), new VFile(), mdast); - const outputNode = mdast.children[0].children[1]; - expect(outputNode.children?.length).toEqual(1); - expect(outputNode.children[0].type).toEqual('image'); - expect(outputNode.children[0].placeholder).toBeTruthy(); + const outputsNode = mdast.children[0].children[1]; + expect(outputsNode.children?.length).toEqual(1); + expect(outputsNode.children[0].type).toEqual('image'); + expect(outputsNode.children[0].placeholder).toBeTruthy(); }); it('placeholder passes with no output', async () => { const mdast: any = { diff --git a/packages/myst-cli/src/transforms/code.ts b/packages/myst-cli/src/transforms/code.ts index 782401a8f..b36303a15 100644 --- a/packages/myst-cli/src/transforms/code.ts +++ b/packages/myst-cli/src/transforms/code.ts @@ -1,6 +1,6 @@ import type { GenericNode, GenericParent } from 'myst-common'; import { NotebookCellTags, RuleId, fileError, fileWarn } from 'myst-common'; -import type { Image, Output } from 'myst-spec-ext'; +import type { Image, Outputs } from 'myst-spec-ext'; import { select, selectAll } from 'unist-util-select'; import yaml from 'js-yaml'; import type { VFile } from 'vfile'; @@ -156,10 +156,9 @@ export function propagateBlockDataToCode(session: ISession, vfile: VFile, mdast: const blocks = selectAll('block', mdast) as GenericNode[]; blocks.forEach((block) => { if (!block.data) return; - const outputNode = select('output', block) as Output | null; - if (block.data.placeholder && outputNode) { - if (!outputNode.children) outputNode.children = []; - outputNode.children.push({ + const outputsNode = select('outputs', block) as Outputs | null; + if (block.data.placeholder && outputsNode) { + outputsNode.children.push({ type: 'image', placeholder: true, url: block.data.placeholder as string, @@ -195,10 +194,10 @@ export function propagateBlockDataToCode(session: ISession, vfile: VFile, mdast: if (codeNode) codeNode.visibility = 'remove'; break; case NotebookCellTags.hideOutput: - if (outputNode) outputNode.visibility = 'hide'; + if (outputsNode) outputsNode.visibility = 'hide'; break; case NotebookCellTags.removeOutput: - if (outputNode) outputNode.visibility = 'remove'; + if (outputsNode) outputsNode.visibility = 'remove'; break; default: session.log.debug(`tag '${tag}' is not valid in code-cell tags'`); @@ -206,7 +205,7 @@ export function propagateBlockDataToCode(session: ISession, vfile: VFile, mdast: }); if (!block.visibility) block.visibility = 'show'; if (codeNode && !codeNode.visibility) codeNode.visibility = 'show'; - if (outputNode && !outputNode.visibility) outputNode.visibility = 'show'; + if (outputsNode && !outputsNode.visibility) outputsNode.visibility = 'show'; }); } @@ -233,7 +232,7 @@ export function transformLiftCodeBlocksInJupytext(mdast: GenericParent) { child.type === 'block' && child.children?.length === 2 && child.children?.[0].type === 'code' && - child.children?.[1].type === 'output' + child.children?.[1].type === 'outputs' ) { newBlocks.push(child as GenericParent); newBlocks.push({ type: 'block', children: [] }); diff --git a/packages/myst-cli/src/transforms/crossReferences.ts b/packages/myst-cli/src/transforms/crossReferences.ts index ea9146a91..ea7596645 100644 --- a/packages/myst-cli/src/transforms/crossReferences.ts +++ b/packages/myst-cli/src/transforms/crossReferences.ts @@ -39,36 +39,39 @@ async function fetchMystData( urlSource: string | undefined, vfile: VFile, ) { - let note: string; - if (dataUrl) { - const filename = mystDataFilename(dataUrl); - const cacheData = loadFromCache(session, filename, { maxAge: XREF_MAX_AGE }); - if (cacheData) { - return JSON.parse(cacheData) as MystData; - } - try { - const resp = await session.fetch(dataUrl); - if (resp.ok) { - const data = (await resp.json()) as MystData; - writeToCache(session, filename, JSON.stringify(data)); - return data; - } - } catch { - // data is unset + const onError = (note: string): undefined => { + fileWarn( + vfile, + `Unable to resolve link text from external MyST reference: ${urlSource ?? dataUrl ?? ''}`, + { + ruleId: RuleId.mystLinkValid, + note, + }, + ); + }; + if (!dataUrl) { + return onError('Data source URL unavailable'); + } + + const filename = mystDataFilename(dataUrl); + const cacheData = loadFromCache(session, filename, { maxAge: XREF_MAX_AGE }); + if (cacheData) { + return JSON.parse(cacheData) as MystData; + } + let data: MystData; + try { + const resp = await session.fetch(dataUrl); + if (!resp.ok) { + throw new Error('Could not fetch data from external project URL'); } - note = 'Could not load data from external project'; - } else { - note = 'Data source URL unavailable'; + data = (await resp.json()) as MystData; + } catch { + return onError('Could not load data from external project'); + // data is unset } - fileWarn( - vfile, - `Unable to resolve link text from external MyST reference: ${urlSource ?? dataUrl ?? ''}`, - { - ruleId: RuleId.mystLinkValid, - note, - }, - ); - return; + + writeToCache(session, filename, JSON.stringify(data)); + return data; } export async function fetchMystLinkData(session: ISession, node: Link, vfile: VFile) { diff --git a/packages/myst-cli/src/transforms/outputs.spec.ts b/packages/myst-cli/src/transforms/outputs.spec.ts index f15bdd674..361df3552 100644 --- a/packages/myst-cli/src/transforms/outputs.spec.ts +++ b/packages/myst-cli/src/transforms/outputs.spec.ts @@ -20,9 +20,14 @@ describe('reduceOutputs', () => { ], }, { - type: 'output', - id: 'abc123', - data: [], + type: 'outputs', + children: [ + { + type: 'output', + id: 'abc123', + jupyter_data: null, + }, + ], }, ], }, @@ -49,18 +54,21 @@ describe('reduceOutputs', () => { ], }, { - type: 'output', + type: 'outputs', id: 'abc123', - data: [ + children: [ { - output_type: 'display_data', - execution_count: 3, - metadata: {}, - data: { - 'application/octet-stream': { - content_type: 'application/octet-stream', - hash: 'def456', - path: '/my/path/def456.png', + type: 'output', + jupyter_data: { + output_type: 'display_data', + execution_count: 3, + metadata: {}, + data: { + 'application/octet-stream': { + content_type: 'application/octet-stream', + hash: 'def456', + path: '/my/path/def456.png', + }, }, }, }, @@ -91,14 +99,19 @@ describe('reduceOutputs', () => { ], }, { - type: 'output', + type: 'outputs', id: 'abc123', - data: [], children: [ { - type: 'image', - placeholder: true, - url: 'placeholder.png', + type: 'output', + jupyter_data: null, + children: [ + { + type: 'image', + placeholder: true, + url: 'placeholder.png', + }, + ], }, ], }, diff --git a/packages/myst-cli/src/transforms/outputs.ts b/packages/myst-cli/src/transforms/outputs.ts index 7046450e7..92990523d 100644 --- a/packages/myst-cli/src/transforms/outputs.ts +++ b/packages/myst-cli/src/transforms/outputs.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; import { dirname, join, relative } from 'node:path'; import { computeHash } from 'myst-cli-utils'; -import type { Image, SourceFileKind } from 'myst-spec-ext'; +import type { Image, SourceFileKind, Output } from 'myst-spec-ext'; import { liftChildren, fileError, RuleId, fileWarn } from 'myst-common'; import type { GenericNode, GenericParent } from 'myst-common'; import type { ProjectSettings } from 'myst-frontmatter'; @@ -34,17 +34,28 @@ export async function transformOutputsToCache( kind: SourceFileKind, opts?: { minifyMaxCharacters?: number }, ) { - const outputs = selectAll('output', mdast) as GenericNode[]; - if (!outputs.length) return; + const outputsNodes = selectAll('outputs', mdast) as GenericNode[]; const cache = castSession(session); await Promise.all( - outputs - .filter((output) => output.visibility !== 'remove') + outputsNodes + // Ignore outputs that are hidden + .filter((outputs) => outputs.visibility !== 'remove') + // Pull out children + .map((outputs) => outputs.children as Output[]) + .flat() + // Filter outputs with no data + // TODO: can this ever occur? + .filter((output) => (output as any).jupyter_data !== undefined) + // Minify output data .map(async (output) => { - output.data = await minifyCellOutput(output.data as IOutput[], cache.$outputs, { - computeHash, - maxCharacters: opts?.minifyMaxCharacters, - }); + [(output as any).jupyter_data] = await minifyCellOutput( + [(output as any).jupyter_data] as IOutput[], + cache.$outputs, + { + computeHash, + maxCharacters: opts?.minifyMaxCharacters, + }, + ); }), ); } @@ -75,9 +86,10 @@ export function transformFilterOutputStreams( const blockRemoveStderr = tags.includes('remove-stderr'); const blockRemoveStdout = tags.includes('remove-stdout'); const outputs = selectAll('output', block) as GenericNode[]; - // There should be only one output in the block - outputs.forEach((output) => { - output.data = output.data.filter((data: IStream | MinifiedMimeOutput) => { + outputs + .filter((output) => { + const data = output.jupyter_data; + if ( (stderr !== 'show' || blockRemoveStderr) && data.output_type === 'stream' && @@ -96,7 +108,7 @@ export function transformFilterOutputStreams( }, ); } - return !doRemove; + return doRemove; } if ( (stdout !== 'show' || blockRemoveStdout) && @@ -116,7 +128,7 @@ export function transformFilterOutputStreams( }, ); } - return !doRemove; + return doRemove; } if ( mpl !== 'show' && @@ -141,12 +153,15 @@ export function transformFilterOutputStreams( }, ); } - return !doRemove; + return doRemove; } - return true; + return false; + }) + .forEach((output) => { + output.type = '__delete__'; }); - }); }); + remove(mdast, { cascade: false }, '__delete__'); } function writeCachedOutputToFile( @@ -192,16 +207,19 @@ export function transformOutputsToFile( const outputs = selectAll('output', mdast) as GenericNode[]; const cache = castSession(session); - outputs.forEach((node) => { - walkOutputs(node.data, (obj) => { - const { hash } = obj; - if (!hash || !cache.$outputs[hash]) return undefined; - obj.path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { - ...opts, - node, + outputs + .filter((output) => !!output.jupyter_data) + .forEach((node) => { + // TODO: output-refactoring -- drop to single output in future + walkOutputs([node.jupyter_data], (obj) => { + const { hash } = obj; + if (!hash || !cache.$outputs[hash]) return undefined; + obj.path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { + ...opts, + node, + }); }); }); - }); } /** @@ -233,87 +251,99 @@ export function reduceOutputs( writeFolder: string, opts?: { altOutputFolder?: string; vfile?: VFile }, ) { - const outputs = selectAll('output', mdast) as GenericNode[]; + const outputsNodes = selectAll('outputs', mdast) as GenericNode[]; const cache = castSession(session); - outputs.forEach((node) => { - if (node.visibility === 'remove' || node.visibility === 'hide') { - // Hidden nodes should not show up in simplified outputs for static export - node.type = '__delete__'; - return; - } - if (!node.data?.length && !node.children?.length) { - node.type = '__delete__'; - return; - } - node.type = '__lift__'; - if (node.children?.length) return; - const selectedOutputs: { content_type: string; hash: string }[] = []; - node.data.forEach((output: MinifiedOutput) => { - let selectedOutput: { content_type: string; hash: string } | undefined; - walkOutputs([output], (obj: any) => { - const { output_type, content_type, hash } = obj; - if (!hash) return undefined; - if (!selectedOutput || isPreferredOutputType(content_type, selectedOutput.content_type)) { - if (['error', 'stream'].includes(output_type)) { - selectedOutput = { content_type: 'text/plain', hash }; - } else if (typeof content_type === 'string') { - if ( - content_type.startsWith('image/') || - content_type === 'text/plain' || - content_type === 'text/html' - ) { - selectedOutput = { content_type, hash }; + outputsNodes.forEach((outputsNode) => { + const outputs = outputsNode.children as GenericNode[]; + + outputs.forEach((outputNode) => { + if (outputNode.visibility === 'remove' || outputNode.visibility === 'hide') { + // Hidden nodes should not show up in simplified outputs for static export + outputNode.type = '__delete__'; + return; + } + if (!outputNode.jupyter_data && !outputNode.children?.length) { + outputNode.type = '__delete__'; + return; + } + // Lift the `output` node into `Outputs` + outputNode.type = '__lift__'; + + // If the output already has children, we don't need to do anything + if (outputNode.children?.length) { + return; + } + + // Find a preferred IOutput type to render into the AST + const selectedOutputs: { content_type: string; hash: string }[] = []; + if (outputNode.jupyter_data) { + const output = outputNode.jupyter_data; + + let selectedOutput: { content_type: string; hash: string } | undefined; + walkOutputs([output], (obj: any) => { + const { output_type, content_type, hash } = obj; + if (!hash) return undefined; + if (!selectedOutput || isPreferredOutputType(content_type, selectedOutput.content_type)) { + if (['error', 'stream'].includes(output_type)) { + selectedOutput = { content_type: 'text/plain', hash }; + } else if (typeof content_type === 'string') { + if ( + content_type.startsWith('image/') || + content_type === 'text/plain' || + content_type === 'text/html' + ) { + selectedOutput = { content_type, hash }; + } } } - } - }); - if (selectedOutput) selectedOutputs.push(selectedOutput); - }); - const children: (Image | GenericNode)[] = selectedOutputs - .map((output): Image | GenericNode | GenericNode[] | undefined => { - const { content_type, hash } = output ?? {}; - if (!hash || !cache.$outputs[hash]) return undefined; - if (content_type === 'text/html') { - const htmlTree = { - type: 'root', - children: [ - { - type: 'html', - value: cache.$outputs[hash][0], - }, - ], - }; - htmlTransform(htmlTree); - if ((selectAll('image', htmlTree) as GenericNode[]).find((htmlImage) => !htmlImage.url)) { - return undefined; + }); + if (selectedOutput) selectedOutputs.push(selectedOutput); + } + const children: (Image | GenericNode)[] = selectedOutputs + .map((output): Image | GenericNode | GenericNode[] | undefined => { + const { content_type, hash } = output ?? {}; + if (!hash || !cache.$outputs[hash]) return undefined; + if (content_type === 'text/html') { + const htmlTree = { + type: 'root', + children: [ + { + type: 'html', + value: cache.$outputs[hash][0], + }, + ], + }; + htmlTransform(htmlTree); + return htmlTree.children; + } else if (content_type.startsWith('image/')) { + const path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { + ...opts, + node: outputNode, + }); + if (!path) return undefined; + const relativePath = relative(dirname(file), path); + return { + type: 'image', + data: { type: 'output' }, + url: relativePath, + urlSource: relativePath, + }; + } else if (content_type === 'text/plain' && cache.$outputs[hash]) { + const [content] = cache.$outputs[hash]; + return { + type: 'code', + data: { type: 'output' }, + value: stripAnsi(content), + }; } - return htmlTree.children; - } else if (content_type.startsWith('image/')) { - const path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { - ...opts, - node, - }); - if (!path) return undefined; - const relativePath = relative(dirname(file), path); - return { - type: 'image', - data: { type: 'output' }, - url: relativePath, - urlSource: relativePath, - }; - } else if (content_type === 'text/plain' && cache.$outputs[hash]) { - const [content] = cache.$outputs[hash]; - return { - type: 'code', - data: { type: 'output' }, - value: stripAnsi(content), - }; - } - return undefined; - }) - .flat() - .filter((output): output is Image | GenericNode => !!output); - node.children = children; + return undefined; + }) + .flat() + .filter((output): output is Image | GenericNode => !!output); + outputNode.children = children; + }); + // Lift the `outputs` node + outputsNode.type = '__lift__'; }); remove(mdast, '__delete__'); liftChildren(mdast, '__lift__'); diff --git a/packages/myst-directives/src/code.ts b/packages/myst-directives/src/code.ts index ebf7d739a..ea29c7150 100644 --- a/packages/myst-directives/src/code.ts +++ b/packages/myst-directives/src/code.ts @@ -209,15 +209,14 @@ export const codeCellDirective: DirectiveSpec = { executable: true, value: (data.body ?? '') as string, }; - const output = { - type: 'output', - id: nanoid(), - data: [], + const outputs = { + type: 'outputs', + children: [], }; const block: GenericNode = { type: 'block', kind: NotebookCell.code, - children: [code, output], + children: [code, outputs], data: {}, }; addCommonDirectiveOptions(data, block); diff --git a/packages/myst-execute/src/execute.ts b/packages/myst-execute/src/execute.ts index e97dcfad4..ab2d5bc1a 100644 --- a/packages/myst-execute/src/execute.ts +++ b/packages/myst-execute/src/execute.ts @@ -2,7 +2,7 @@ import { select, selectAll } from 'unist-util-select'; import type { Logger } from 'myst-cli-utils'; import type { PageFrontmatter, KernelSpec } from 'myst-frontmatter'; import type { Kernel, KernelMessage, Session, SessionManager } from '@jupyterlab/services'; -import type { Block, Code, InlineExpression, Output } from 'myst-spec-ext'; +import type { Block, Code, InlineExpression, Output, Outputs } from 'myst-spec-ext'; import type { IOutput } from '@jupyterlab/nbformat'; import type { GenericNode, GenericParent, IExpressionResult, IExpressionError } from 'myst-common'; import { NotebookCell, NotebookCellTags, fileError } from 'myst-common'; @@ -166,7 +166,7 @@ type CodeBlock = Block & { * @param node node to test */ function isCellBlock(node: GenericNode): node is CodeBlock { - return node.type === 'block' && select('code', node) !== null && select('output', node) !== null; + return node.type === 'block' && select('code', node) !== null && select('outputs', node) !== null; } /** @@ -282,14 +282,17 @@ function applyComputedOutputsToNodes( const thisResult = computedResult.shift(); if (isCellBlock(matchedNode)) { - // Pull out output to set data - const output = select('output', matchedNode) as unknown as { data: IOutput[] }; - // Set the output array to empty if we don't have a result (e.g. due to a kernel error) - output.data = thisResult === undefined ? [] : (thisResult as IOutput[]); + const rawOutputData = (thisResult as IOutput[]) ?? []; + // Pull out outputs to set data + const outputs = select('outputs', matchedNode) as Outputs; + // Ensure that whether this fails or succeeds, we write to `children` (e.g. due to a kernel error) + outputs.children = rawOutputData.map((data) => { + return { type: 'output', children: [], jupyter_data: data as any }; + }); } else if (isInlineExpression(matchedNode)) { + const rawOutputData = thisResult as Record | undefined; // Set data of expression to the result, or empty if we don't have one - matchedNode.result = // TODO: FIXME .data - thisResult === undefined ? undefined : (thisResult as unknown as Record); + matchedNode.result = rawOutputData; } else { // This should never happen throw new Error('Node must be either code block or inline expression.'); diff --git a/packages/myst-execute/tests/execute.yml b/packages/myst-execute/tests/execute.yml index f768992aa..664d53067 100644 --- a/packages/myst-execute/tests/execute.yml +++ b/packages/myst-execute/tests/execute.yml @@ -27,22 +27,18 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 + children: - type: code lang: python executable: true value: print('abc') - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output + - type: outputs + children: + - type: output + children: [] + after: type: root children: @@ -54,27 +50,22 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 + children: - type: code lang: python executable: true value: print('abc') - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - output_type: stream - name: stdout - text: | - abc + - type: outputs + children: + - type: output + children: [] + jupyter_data: + output_type: stream + name: stdout + text: | + abc - title: tree with inline expression is evaluated before: type: root @@ -115,100 +106,72 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: print('abc') - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + - type: outputs + children: + - type: output + jupyter_data: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + - type: outputs + children: + - type: output + jupyter_data: after: type: root children: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: print('abc') - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - output_type: stream - name: stdout - text: | - abc + - type: outputs + children: + - type: output + jupyter_data: + output_type: stream + name: stdout + text: | + abc - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - output_type: error - # Note this traceback can be different on various machines - # Not including it means we still validate an error, just don't care about the traceback - # traceback: - # - "\e[0;31m---------------------------------------------------------------------------\e[0m" - # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" - # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" - # - "\e[0;31mValueError\e[0m: " - ename: ValueError - evalue: '' + - type: outputs + children: + - type: output + jupyter_data: + output_type: error + # Note this traceback can be different on various machines + # Not including it means we still validate an error, just don't care about the traceback + # traceback: + # - "\e[0;31m---------------------------------------------------------------------------\e[0m" + # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" + # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" + # - "\e[0;31mValueError\e[0m: " + ename: ValueError + evalue: '' - title: tree with bad executable code and `raises-exception` is evaluated and passes before: type: root @@ -216,58 +179,44 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 tags: raises-exception - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + - type: outputs + children: + - type: output + jupyter_data: after: type: root children: - type: block kind: notebook-code data: - id: nb-cell-0 tags: raises-exception - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - output_type: error - # Note this traceback can be different on various machines - # Not including it means we still validate an error, just don't care about the traceback - # traceback: - # - "\e[0;31m---------------------------------------------------------------------------\e[0m" - # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" - # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" - # - "\e[0;31mValueError\e[0m: " - ename: ValueError - evalue: '' + - type: outputs + children: + - type: output + jupyter_data: + output_type: error + # Note this traceback can be different on various machines + # Not including it means we still validate an error, just don't care about the traceback + # traceback: + # - "\e[0;31m---------------------------------------------------------------------------\e[0m" + # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" + # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" + # - "\e[0;31mValueError\e[0m: " + ename: ValueError + evalue: '' - title: tree with bad executable code and `skip-execution` is not evaluated before: type: root @@ -275,45 +224,31 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 tags: skip-execution - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + - type: outputs + children: + - type: output + jupyter_data: after: type: root children: - type: block kind: notebook-code data: - id: nb-cell-0 tags: skip-execution - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + - type: outputs + children: + - type: output + jupyter_data: diff --git a/packages/myst-execute/tests/run.spec.ts b/packages/myst-execute/tests/run.spec.ts index e6a5d159f..ab4f0bbfb 100644 --- a/packages/myst-execute/tests/run.spec.ts +++ b/packages/myst-execute/tests/run.spec.ts @@ -96,6 +96,7 @@ casesList.forEach(({ title, cases }) => { expect.arrayContaining([expect.stringMatching(throws)]), ); } + console.log(JSON.stringify(after, null, 2)); expect(before).toMatchObject(after); }, { timeout: 30_000 }, diff --git a/packages/myst-spec-ext/src/types.ts b/packages/myst-spec-ext/src/types.ts index b58ef8f9b..ae2c38090 100644 --- a/packages/myst-spec-ext/src/types.ts +++ b/packages/myst-spec-ext/src/types.ts @@ -252,10 +252,16 @@ export type Container = Omit & { export type Output = Node & Target & { type: 'output'; - id?: string; - data?: any[]; // MinifiedOutput[] + children: (FlowContent | ListContent | PhrasingContent)[]; + jupyter_data: any; // TODO: set this to IOutput + }; + +export type Outputs = Node & + Target & { + type: 'outputs'; + children: (Output | FlowContent | ListContent | PhrasingContent)[]; // Support placeholders in addition to outputs visibility?: Visibility; - children?: (FlowContent | ListContent | PhrasingContent)[]; + id?: string; }; export type Aside = Node & diff --git a/packages/myst-transforms/src/blocks.spec.ts b/packages/myst-transforms/src/blocks.spec.ts index 5ce0fecd4..3d868033e 100644 --- a/packages/myst-transforms/src/blocks.spec.ts +++ b/packages/myst-transforms/src/blocks.spec.ts @@ -120,8 +120,10 @@ describe('Test blockMetadataTransform', () => { test('label is propagated to outputs', async () => { const mdast = u('root', [ u('block', { meta: '{"label": "My_Label", "key": "value"}' }, [ - u('output', 'We know what we are'), - u('output', 'but know not what we may be.'), + u('outputs', [ + u('output', 'We know what we are'), + u('output', 'but know not what we may be.'), + ]), ]), ]) as any; blockMetadataTransform(mdast, new VFile()); @@ -136,8 +138,10 @@ describe('Test blockMetadataTransform', () => { data: { key: 'value' }, }, [ - u('output', { identifier: 'my_label-output-0' }, 'We know what we are'), - u('output', { identifier: 'my_label-output-1' }, 'but know not what we may be.'), + u('outputs', { identifier: 'my_label-output' }, [ + u('output', { identifier: 'my_label-output-0' }, 'We know what we are'), + u('output', { identifier: 'my_label-output-1' }, 'but know not what we may be.'), + ]), ], ), ]), diff --git a/packages/myst-transforms/src/blocks.ts b/packages/myst-transforms/src/blocks.ts index 20e994146..49f4a5cf1 100644 --- a/packages/myst-transforms/src/blocks.ts +++ b/packages/myst-transforms/src/blocks.ts @@ -1,7 +1,7 @@ import type { VFile } from 'vfile'; import type { Plugin } from 'unified'; import type { Node } from 'myst-spec'; -import { selectAll } from 'unist-util-select'; +import { selectAll, select } from 'unist-util-select'; import type { GenericNode, GenericParent } from 'myst-common'; import { NotebookCell, RuleId, fileError, normalizeLabel } from 'myst-common'; import type { Code } from 'myst-spec-ext'; @@ -89,15 +89,18 @@ export function blockMetadataTransform(mdast: GenericParent, file: VFile) { child.identifier = `${block.identifier}-code-${index}`; } }); - const outputChildren = selectAll('output', block) as GenericNode[]; - outputChildren.forEach((child, index) => { - if (child.identifier) return; - if (outputChildren.length === 1) { - child.identifier = `${block.identifier}-output`; - } else { - child.identifier = `${block.identifier}-output-${index}`; - } - }); + const outputsNode = select('outputs', block) as GenericNode | undefined; + if (outputsNode && !outputsNode.identifier) { + // Label outputs node + outputsNode.identifier = `${block.identifier}-output`; + // Enumerate outputs + const outputs = selectAll('output', outputsNode) as GenericNode[]; + outputs.forEach((outputNode, index) => { + if (outputNode && !outputNode.identifier) { + outputNode.identifier = `${block.identifier}-output-${index}`; + } + }); + } } }); } diff --git a/packages/mystmd/tests/notebook-fig-embed/outputs/index.json b/packages/mystmd/tests/notebook-fig-embed/outputs/index.json index efa957528..ea43e455b 100644 --- a/packages/mystmd/tests/notebook-fig-embed/outputs/index.json +++ b/packages/mystmd/tests/notebook-fig-embed/outputs/index.json @@ -55,6 +55,30 @@ "children": [ { "type": "output", + "_future_ast": { + "children": [ + { + "jupyter_data": { + "data": { + "text/html": { + "content_type": "text/html", + "hash": "a16fcedcd26437c820ccfc05d1f48a57", + "path": "/a16fcedcd26437c820ccfc05d1f48a57.html" + }, + "text/plain": { + "content": "alt.Chart(...)", + "content_type": "text/plain" + } + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + }, + "type": "output" + } + ], + "type": "outputs" + }, "data": [ { "output_type": "execute_result", @@ -144,6 +168,30 @@ }, { "type": "output", + "_future_ast": { + "children": [ + { + "jupyter_data": { + "data": { + "text/html": { + "content_type": "text/html", + "hash": "a16fcedcd26437c820ccfc05d1f48a57", + "path": "/a16fcedcd26437c820ccfc05d1f48a57.html" + }, + "text/plain": { + "content": "alt.Chart(...)", + "content_type": "text/plain" + } + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + }, + "type": "output" + } + ], + "type": "outputs" + }, "data": [ { "output_type": "execute_result", @@ -190,6 +238,32 @@ "children": [ { "type": "output", + "_future_ast": { + "children": [ + { + "children": [], + "jupyter_data": { + "data": { + "text/html": { + "content_type": "text/html", + "hash": "a85dae82213ec48ea05754a345bab5ce", + "path": "https://cdn.curvenote.com/01944f0b-783c-7ea2-8d19-f59b784cd49e/public/a85dae82213ec48ea05754a345bab5ce.html" + }, + "text/plain": { + "content": "alt.VConcatChart(...)", + "content_type": "text/plain" + } + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + }, + "type": "output" + } + ], + "type": "outputs", + "visibility": "show" + }, "data": [ { "output_type": "execute_result", @@ -286,6 +360,33 @@ }, { "type": "output", + "_future_ast": { + "children": [ + { + "children": [], + "jupyter_data": { + "data": { + "text/html": { + "content_type": "text/html", + "hash": "a85dae82213ec48ea05754a345bab5ce", + "path": "https://cdn.curvenote.com/01944f0b-783c-7ea2-8d19-f59b784cd49e/public/a85dae82213ec48ea05754a345bab5ce.html" + }, + "text/plain": { + "content": "alt.VConcatChart(...)", + "content_type": "text/plain" + } + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + }, + "type": "output" + } + ], + "type": "outputs", + "visibility": "show" + }, + "data": [ { "output_type": "execute_result", From 6870f572add8e6272117627a24571be5c8474176 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 7 Mar 2025 11:05:36 +0000 Subject: [PATCH 02/12] chore: add changeset --- .changeset/ten-bats-warn.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .changeset/ten-bats-warn.md diff --git a/.changeset/ten-bats-warn.md b/.changeset/ten-bats-warn.md new file mode 100644 index 000000000..00fc05d9b --- /dev/null +++ b/.changeset/ten-bats-warn.md @@ -0,0 +1,10 @@ +--- +"mystmd": minor +"myst-directives": patch +"myst-transforms": patch +"myst-spec-ext": patch +"myst-execute": patch +"myst-cli": patch +--- + +Add support for new Outputs node From da805f2fcc8fcc393dbbc21a255df8c8605caf84 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 7 Mar 2025 11:05:58 +0000 Subject: [PATCH 03/12] fix: drop style change --- packages/myst-cli/src/process/mdast.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/myst-cli/src/process/mdast.ts b/packages/myst-cli/src/process/mdast.ts index 8a68ca756..e06b70aab 100644 --- a/packages/myst-cli/src/process/mdast.ts +++ b/packages/myst-cli/src/process/mdast.ts @@ -450,6 +450,5 @@ export async function finalizeMdast( postData.widgets = cache.$getMdast(file)?.pre.widgets; updateFileInfoFromFrontmatter(session, file, frontmatter); } - logMessagesFromVFile(session, vfile); } From 2e6297de4b0fc09fdef48797c7044379405c3ab9 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 7 Mar 2025 15:14:46 +0000 Subject: [PATCH 04/12] Update packages/myst-execute/tests/run.spec.ts --- packages/myst-execute/tests/run.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/myst-execute/tests/run.spec.ts b/packages/myst-execute/tests/run.spec.ts index ab4f0bbfb..e6a5d159f 100644 --- a/packages/myst-execute/tests/run.spec.ts +++ b/packages/myst-execute/tests/run.spec.ts @@ -96,7 +96,6 @@ casesList.forEach(({ title, cases }) => { expect.arrayContaining([expect.stringMatching(throws)]), ); } - console.log(JSON.stringify(after, null, 2)); expect(before).toMatchObject(after); }, { timeout: 30_000 }, From 705a058b18bc2b7ff5542c75b0aa3e14a397e28b Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 7 Mar 2025 15:16:02 +0000 Subject: [PATCH 05/12] Update packages/mystmd/tests/notebook-fig-embed/outputs/index.json --- .../notebook-fig-embed/outputs/index.json | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/packages/mystmd/tests/notebook-fig-embed/outputs/index.json b/packages/mystmd/tests/notebook-fig-embed/outputs/index.json index ea43e455b..c542b9b01 100644 --- a/packages/mystmd/tests/notebook-fig-embed/outputs/index.json +++ b/packages/mystmd/tests/notebook-fig-embed/outputs/index.json @@ -55,30 +55,6 @@ "children": [ { "type": "output", - "_future_ast": { - "children": [ - { - "jupyter_data": { - "data": { - "text/html": { - "content_type": "text/html", - "hash": "a16fcedcd26437c820ccfc05d1f48a57", - "path": "/a16fcedcd26437c820ccfc05d1f48a57.html" - }, - "text/plain": { - "content": "alt.Chart(...)", - "content_type": "text/plain" - } - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - }, - "type": "output" - } - ], - "type": "outputs" - }, "data": [ { "output_type": "execute_result", From 1c6e810f4c04a8c9c851213cc3fedc91bb3ebfc2 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 7 Mar 2025 15:19:28 +0000 Subject: [PATCH 06/12] Update packages/mystmd/tests/notebook-fig-embed/outputs/index.json --- .../notebook-fig-embed/outputs/index.json | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/packages/mystmd/tests/notebook-fig-embed/outputs/index.json b/packages/mystmd/tests/notebook-fig-embed/outputs/index.json index c542b9b01..6f3e9c299 100644 --- a/packages/mystmd/tests/notebook-fig-embed/outputs/index.json +++ b/packages/mystmd/tests/notebook-fig-embed/outputs/index.json @@ -144,30 +144,6 @@ }, { "type": "output", - "_future_ast": { - "children": [ - { - "jupyter_data": { - "data": { - "text/html": { - "content_type": "text/html", - "hash": "a16fcedcd26437c820ccfc05d1f48a57", - "path": "/a16fcedcd26437c820ccfc05d1f48a57.html" - }, - "text/plain": { - "content": "alt.Chart(...)", - "content_type": "text/plain" - } - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - }, - "type": "output" - } - ], - "type": "outputs" - }, "data": [ { "output_type": "execute_result", From 71728e85388dc257f658f9adcd81f002966dfec0 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 7 Mar 2025 15:19:48 +0000 Subject: [PATCH 07/12] Update packages/mystmd/tests/notebook-fig-embed/outputs/index.json --- .../notebook-fig-embed/outputs/index.json | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/packages/mystmd/tests/notebook-fig-embed/outputs/index.json b/packages/mystmd/tests/notebook-fig-embed/outputs/index.json index 6f3e9c299..5c9c3061d 100644 --- a/packages/mystmd/tests/notebook-fig-embed/outputs/index.json +++ b/packages/mystmd/tests/notebook-fig-embed/outputs/index.json @@ -190,32 +190,6 @@ "children": [ { "type": "output", - "_future_ast": { - "children": [ - { - "children": [], - "jupyter_data": { - "data": { - "text/html": { - "content_type": "text/html", - "hash": "a85dae82213ec48ea05754a345bab5ce", - "path": "https://cdn.curvenote.com/01944f0b-783c-7ea2-8d19-f59b784cd49e/public/a85dae82213ec48ea05754a345bab5ce.html" - }, - "text/plain": { - "content": "alt.VConcatChart(...)", - "content_type": "text/plain" - } - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - }, - "type": "output" - } - ], - "type": "outputs", - "visibility": "show" - }, "data": [ { "output_type": "execute_result", From 48936eb541980ff06df37d5fc8e4e30dd6aae1a8 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 7 Mar 2025 15:20:07 +0000 Subject: [PATCH 08/12] Update packages/mystmd/tests/notebook-fig-embed/outputs/index.json --- .../notebook-fig-embed/outputs/index.json | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/packages/mystmd/tests/notebook-fig-embed/outputs/index.json b/packages/mystmd/tests/notebook-fig-embed/outputs/index.json index 5c9c3061d..efa957528 100644 --- a/packages/mystmd/tests/notebook-fig-embed/outputs/index.json +++ b/packages/mystmd/tests/notebook-fig-embed/outputs/index.json @@ -286,33 +286,6 @@ }, { "type": "output", - "_future_ast": { - "children": [ - { - "children": [], - "jupyter_data": { - "data": { - "text/html": { - "content_type": "text/html", - "hash": "a85dae82213ec48ea05754a345bab5ce", - "path": "https://cdn.curvenote.com/01944f0b-783c-7ea2-8d19-f59b784cd49e/public/a85dae82213ec48ea05754a345bab5ce.html" - }, - "text/plain": { - "content": "alt.VConcatChart(...)", - "content_type": "text/plain" - } - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - }, - "type": "output" - } - ], - "type": "outputs", - "visibility": "show" - }, - "data": [ { "output_type": "execute_result", From 1e402cb2847b74401175a43463122067b8db24c9 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 7 Mar 2025 15:27:23 +0000 Subject: [PATCH 09/12] fix: only label first code and outputs --- packages/myst-transforms/src/blocks.ts | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/packages/myst-transforms/src/blocks.ts b/packages/myst-transforms/src/blocks.ts index 49f4a5cf1..73c4aa5b9 100644 --- a/packages/myst-transforms/src/blocks.ts +++ b/packages/myst-transforms/src/blocks.ts @@ -80,17 +80,12 @@ export function blockMetadataTransform(mdast: GenericParent, file: VFile) { } } if (block.identifier) { - const codeChildren = selectAll('code', block) as Code[]; - codeChildren.forEach((child, index) => { - if (child.identifier) return; - if (codeChildren.length === 1) { - child.identifier = `${block.identifier}-code`; - } else { - child.identifier = `${block.identifier}-code-${index}`; - } - }); + const codeNode = selectAll('code', block) as any as Code | null; + if (codeNode !== null && !codeNode.identifier) { + codeNode.identifier = `${block.identifier}-code`; + } const outputsNode = select('outputs', block) as GenericNode | undefined; - if (outputsNode && !outputsNode.identifier) { + if (outputsNode !== undefined && !outputsNode.identifier) { // Label outputs node outputsNode.identifier = `${block.identifier}-output`; // Enumerate outputs From 9190e11eb0313f07e3e411da3255196d5b3b8d02 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 7 Mar 2025 15:50:40 +0000 Subject: [PATCH 10/12] fix: handle IDs --- packages/myst-execute/src/execute.ts | 5 +- packages/myst-execute/tests/execute.yml | 168 ++++++++++++++++++++---- 2 files changed, 142 insertions(+), 31 deletions(-) diff --git a/packages/myst-execute/src/execute.ts b/packages/myst-execute/src/execute.ts index ab2d5bc1a..07a531714 100644 --- a/packages/myst-execute/src/execute.ts +++ b/packages/myst-execute/src/execute.ts @@ -286,8 +286,9 @@ function applyComputedOutputsToNodes( // Pull out outputs to set data const outputs = select('outputs', matchedNode) as Outputs; // Ensure that whether this fails or succeeds, we write to `children` (e.g. due to a kernel error) - outputs.children = rawOutputData.map((data) => { - return { type: 'output', children: [], jupyter_data: data as any }; + outputs.children = rawOutputData.map((data, index) => { + const identifier = outputs.identifier ? `${outputs.identifier}-${index}` : undefined; + return { type: 'output', children: [], jupyter_data: data as any, identifier }; }); } else if (isInlineExpression(matchedNode)) { const rawOutputData = thisResult as Record | undefined; diff --git a/packages/myst-execute/tests/execute.yml b/packages/myst-execute/tests/execute.yml index 664d53067..f1816e5af 100644 --- a/packages/myst-execute/tests/execute.yml +++ b/packages/myst-execute/tests/execute.yml @@ -27,16 +27,24 @@ cases: - type: block kind: notebook-code data: - + id: nb-cell-0 + identifier: nb-cell-0 + label: nb-cell-0 + html_id: nb-cell-0 children: - type: code lang: python executable: true value: print('abc') enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code - type: outputs + identifier: nb-cell-0-output + html_id: nb-cell-0-output children: - type: output + identifier: nb-cell-0-output-0 children: [] after: @@ -50,22 +58,86 @@ cases: - type: block kind: notebook-code data: + id: nb-cell-0 + identifier: nb-cell-0 + label: nb-cell-0 + html_id: nb-cell-0 + children: + - type: code + lang: python + executable: true + value: print('abc') + enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code + - type: outputs + identifier: nb-cell-0-output + html_id: nb-cell-0-output + children: + - type: output + children: [] + identifier: nb-cell-0-output-0 + jupyter_data: + output_type: stream + name: stdout + text: | + abc + - title: output without identifier is given one + before: + type: root + children: + - type: block + children: + - type: block + kind: notebook-code + identifier: nb-cell-0 + label: nb-cell-0 + html_id: nb-cell-0 + children: + - type: code + lang: python + executable: true + value: print('abc') + enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code + - type: outputs + identifier: nb-cell-0-output + html_id: nb-cell-0-output + children: + - type: output + children: [] + after: + type: root + children: + - type: block + children: + - type: block + kind: notebook-code + identifier: nb-cell-0 + label: nb-cell-0 + html_id: nb-cell-0 children: - type: code lang: python executable: true value: print('abc') enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code - type: outputs + identifier: nb-cell-0-output + html_id: nb-cell-0-output children: - type: output children: [] + identifier: nb-cell-0-output-0 jupyter_data: - output_type: stream - name: stdout - text: | - abc + output_type: stream + name: stdout + text: | + abc - title: tree with inline expression is evaluated before: type: root @@ -112,9 +184,14 @@ cases: executable: true value: print('abc') enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code - type: outputs + identifier: nb-cell-0-output + html_id: nb-cell-0-output children: - type: output + identifier: nb-cell-0-output-0 jupyter_data: - type: block kind: notebook-code @@ -125,9 +202,14 @@ cases: executable: true value: raise ValueError enumerator: 1 + identifier: nb-cell-1-code + html_id: nb-cell-1-code - type: outputs + identifier: nb-cell-1-output + html_id: nb-cell-1-output children: - type: output + identifier: nb-cell-1-output-0 jupyter_data: after: type: root @@ -141,14 +223,19 @@ cases: executable: true value: print('abc') enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code - type: outputs + identifier: nb-cell-0-output + html_id: nb-cell-0-output children: - type: output + identifier: nb-cell-0-output-0 jupyter_data: - output_type: stream - name: stdout - text: | - abc + output_type: stream + name: stdout + text: | + abc - type: block kind: notebook-code data: @@ -158,20 +245,25 @@ cases: executable: true value: raise ValueError enumerator: 1 + identifier: nb-cell-1-code + html_id: nb-cell-1-code - type: outputs + identifier: nb-cell-1-output + html_id: nb-cell-1-output children: - type: output + identifier: nb-cell-1-output-0 jupyter_data: - output_type: error - # Note this traceback can be different on various machines - # Not including it means we still validate an error, just don't care about the traceback - # traceback: - # - "\e[0;31m---------------------------------------------------------------------------\e[0m" - # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" - # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" - # - "\e[0;31mValueError\e[0m: " - ename: ValueError - evalue: '' + output_type: error + # Note this traceback can be different on various machines + # Not including it means we still validate an error, just don't care about the traceback + # traceback: + # - "\e[0;31m---------------------------------------------------------------------------\e[0m" + # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" + # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" + # - "\e[0;31mValueError\e[0m: " + ename: ValueError + evalue: '' - title: tree with bad executable code and `raises-exception` is evaluated and passes before: type: root @@ -186,9 +278,14 @@ cases: executable: true value: raise ValueError enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code - type: outputs + identifier: nb-cell-0-output + html_id: nb-cell-0-output children: - type: output + identifier: nb-cell-0-output-0 jupyter_data: after: type: root @@ -203,20 +300,25 @@ cases: executable: true value: raise ValueError enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code - type: outputs + identifier: nb-cell-0-output + html_id: nb-cell-0-output children: - type: output + identifier: nb-cell-0-output-0 jupyter_data: - output_type: error - # Note this traceback can be different on various machines - # Not including it means we still validate an error, just don't care about the traceback - # traceback: - # - "\e[0;31m---------------------------------------------------------------------------\e[0m" - # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" - # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" - # - "\e[0;31mValueError\e[0m: " - ename: ValueError - evalue: '' + output_type: error + # Note this traceback can be different on various machines + # Not including it means we still validate an error, just don't care about the traceback + # traceback: + # - "\e[0;31m---------------------------------------------------------------------------\e[0m" + # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" + # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" + # - "\e[0;31mValueError\e[0m: " + ename: ValueError + evalue: '' - title: tree with bad executable code and `skip-execution` is not evaluated before: type: root @@ -231,9 +333,13 @@ cases: executable: true value: raise ValueError enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code - type: outputs + identifier: nb-cell-0-output children: - type: output + identifier: nb-cell-0-output-0 jupyter_data: after: type: root @@ -248,7 +354,11 @@ cases: executable: true value: raise ValueError enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code - type: outputs + identifier: nb-cell-0-output children: - type: output + identifier: nb-cell-0-output-0 jupyter_data: From 51842f183a2389f724410a64e1e7069fa7136823 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 7 Mar 2025 16:16:48 +0000 Subject: [PATCH 11/12] fix: return type --- packages/myst-transforms/src/blocks.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/myst-transforms/src/blocks.ts b/packages/myst-transforms/src/blocks.ts index 73c4aa5b9..5c220cb36 100644 --- a/packages/myst-transforms/src/blocks.ts +++ b/packages/myst-transforms/src/blocks.ts @@ -84,8 +84,8 @@ export function blockMetadataTransform(mdast: GenericParent, file: VFile) { if (codeNode !== null && !codeNode.identifier) { codeNode.identifier = `${block.identifier}-code`; } - const outputsNode = select('outputs', block) as GenericNode | undefined; - if (outputsNode !== undefined && !outputsNode.identifier) { + const outputsNode = select('outputs', block) as GenericNode | null; + if (outputsNode !== null && !outputsNode.identifier) { // Label outputs node outputsNode.identifier = `${block.identifier}-output`; // Enumerate outputs From a4d6245cb731866a416a5bc08909e44a8b8d843e Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 7 Mar 2025 16:20:02 +0000 Subject: [PATCH 12/12] =?UTF-8?q?fix:=20selectAll=20=E2=86=92=20select?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/myst-transforms/src/blocks.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/myst-transforms/src/blocks.ts b/packages/myst-transforms/src/blocks.ts index 5c220cb36..730565326 100644 --- a/packages/myst-transforms/src/blocks.ts +++ b/packages/myst-transforms/src/blocks.ts @@ -80,7 +80,7 @@ export function blockMetadataTransform(mdast: GenericParent, file: VFile) { } } if (block.identifier) { - const codeNode = selectAll('code', block) as any as Code | null; + const codeNode = select('code', block) as any as Code | null; if (codeNode !== null && !codeNode.identifier) { codeNode.identifier = `${block.identifier}-code`; }