diff --git a/.changeset/ten-bats-warn.md b/.changeset/ten-bats-warn.md new file mode 100644 index 000000000..00fc05d9b --- /dev/null +++ b/.changeset/ten-bats-warn.md @@ -0,0 +1,10 @@ +--- +"mystmd": minor +"myst-directives": patch +"myst-transforms": patch +"myst-spec-ext": patch +"myst-execute": patch +"myst-cli": patch +--- + +Add support for new Outputs node diff --git a/packages/myst-cli/src/process/notebook.ts b/packages/myst-cli/src/process/notebook.ts index ce8323f39..59257a5dc 100644 --- a/packages/myst-cli/src/process/notebook.ts +++ b/packages/myst-cli/src/process/notebook.ts @@ -165,17 +165,21 @@ export async function processNotebookFull( value: ensureString(cell.source), }; - // Embed outputs in an output block - const output: { type: 'output'; id: string; data: IOutput[] } = { - type: 'output', + const outputsChildren = (cell.outputs as IOutput[]).map((output) => { + // Embed outputs in an output block + const result = { + type: 'output', + jupyter_data: output, + children: [], + }; + return result; + }); + const outputs = { + type: 'outputs', id: nanoid(), - data: [], + children: outputsChildren, }; - - if (cell.outputs && (cell.outputs as IOutput[]).length > 0) { - output.data = cell.outputs as IOutput[]; - } - return acc.concat(blockParent(cell, [code, output])); + return acc.concat(blockParent(cell, [code, outputs])); } return acc; }, diff --git a/packages/myst-cli/src/transforms/code.spec.ts b/packages/myst-cli/src/transforms/code.spec.ts index 0807f5740..27d50958f 100644 --- a/packages/myst-cli/src/transforms/code.spec.ts +++ b/packages/myst-cli/src/transforms/code.spec.ts @@ -162,7 +162,10 @@ function build_mdast(tags: string[], has_output: boolean) { ], }; if (has_output) { - mdast.children[0].children.push({ type: 'output' }); + mdast.children[0].children.push({ + type: 'outputs', + children: [{ type: 'output', children: [] }], + }); } return mdast; } @@ -261,7 +264,7 @@ describe('propagateBlockDataToCode', () => { const mdast = build_mdast([tag], has_output); propagateBlockDataToCode(new Session(), new VFile(), mdast); let result = ''; - const outputNode = mdast.children[0].children[1]; + const outputsNode = mdast.children[0].children[1]; switch (target) { case 'cell': result = mdast.children[0].visibility; @@ -270,12 +273,14 @@ describe('propagateBlockDataToCode', () => { result = mdast.children[0].children[0].visibility; break; case 'output': - if (!has_output && target == 'output') { - expect(outputNode).toEqual(undefined); + if (!has_output) { + expect(outputsNode).toEqual(undefined); continue; } - result = outputNode.visibility; + result = outputsNode.visibility; break; + default: + throw new Error(); } expect(result).toEqual(action); } @@ -290,13 +295,13 @@ describe('propagateBlockDataToCode', () => { propagateBlockDataToCode(new Session(), new VFile(), mdast); const blockNode = mdast.children[0]; const codeNode = mdast.children[0].children[0]; - const outputNode = mdast.children[0].children[1]; + const outputsNode = mdast.children[0].children[1]; expect(blockNode.visibility).toEqual(action); expect(codeNode.visibility).toEqual(action); if (has_output) { - expect(outputNode.visibility).toEqual(action); + expect(outputsNode.visibility).toEqual(action); } else { - expect(outputNode).toEqual(undefined); + expect(outputsNode).toEqual(undefined); } } } @@ -313,7 +318,8 @@ describe('propagateBlockDataToCode', () => { executable: true, }, { - type: 'output', + type: 'outputs', + children: [], }, ], data: { @@ -323,10 +329,10 @@ describe('propagateBlockDataToCode', () => { ], }; propagateBlockDataToCode(new Session(), new VFile(), mdast); - const outputNode = mdast.children[0].children[1]; - expect(outputNode.children?.length).toEqual(1); - expect(outputNode.children[0].type).toEqual('image'); - expect(outputNode.children[0].placeholder).toBeTruthy(); + const outputsNode = mdast.children[0].children[1]; + expect(outputsNode.children?.length).toEqual(1); + expect(outputsNode.children[0].type).toEqual('image'); + expect(outputsNode.children[0].placeholder).toBeTruthy(); }); it('placeholder passes with no output', async () => { const mdast: any = { diff --git a/packages/myst-cli/src/transforms/code.ts b/packages/myst-cli/src/transforms/code.ts index 782401a8f..b36303a15 100644 --- a/packages/myst-cli/src/transforms/code.ts +++ b/packages/myst-cli/src/transforms/code.ts @@ -1,6 +1,6 @@ import type { GenericNode, GenericParent } from 'myst-common'; import { NotebookCellTags, RuleId, fileError, fileWarn } from 'myst-common'; -import type { Image, Output } from 'myst-spec-ext'; +import type { Image, Outputs } from 'myst-spec-ext'; import { select, selectAll } from 'unist-util-select'; import yaml from 'js-yaml'; import type { VFile } from 'vfile'; @@ -156,10 +156,9 @@ export function propagateBlockDataToCode(session: ISession, vfile: VFile, mdast: const blocks = selectAll('block', mdast) as GenericNode[]; blocks.forEach((block) => { if (!block.data) return; - const outputNode = select('output', block) as Output | null; - if (block.data.placeholder && outputNode) { - if (!outputNode.children) outputNode.children = []; - outputNode.children.push({ + const outputsNode = select('outputs', block) as Outputs | null; + if (block.data.placeholder && outputsNode) { + outputsNode.children.push({ type: 'image', placeholder: true, url: block.data.placeholder as string, @@ -195,10 +194,10 @@ export function propagateBlockDataToCode(session: ISession, vfile: VFile, mdast: if (codeNode) codeNode.visibility = 'remove'; break; case NotebookCellTags.hideOutput: - if (outputNode) outputNode.visibility = 'hide'; + if (outputsNode) outputsNode.visibility = 'hide'; break; case NotebookCellTags.removeOutput: - if (outputNode) outputNode.visibility = 'remove'; + if (outputsNode) outputsNode.visibility = 'remove'; break; default: session.log.debug(`tag '${tag}' is not valid in code-cell tags'`); @@ -206,7 +205,7 @@ export function propagateBlockDataToCode(session: ISession, vfile: VFile, mdast: }); if (!block.visibility) block.visibility = 'show'; if (codeNode && !codeNode.visibility) codeNode.visibility = 'show'; - if (outputNode && !outputNode.visibility) outputNode.visibility = 'show'; + if (outputsNode && !outputsNode.visibility) outputsNode.visibility = 'show'; }); } @@ -233,7 +232,7 @@ export function transformLiftCodeBlocksInJupytext(mdast: GenericParent) { child.type === 'block' && child.children?.length === 2 && child.children?.[0].type === 'code' && - child.children?.[1].type === 'output' + child.children?.[1].type === 'outputs' ) { newBlocks.push(child as GenericParent); newBlocks.push({ type: 'block', children: [] }); diff --git a/packages/myst-cli/src/transforms/crossReferences.ts b/packages/myst-cli/src/transforms/crossReferences.ts index ea9146a91..ea7596645 100644 --- a/packages/myst-cli/src/transforms/crossReferences.ts +++ b/packages/myst-cli/src/transforms/crossReferences.ts @@ -39,36 +39,39 @@ async function fetchMystData( urlSource: string | undefined, vfile: VFile, ) { - let note: string; - if (dataUrl) { - const filename = mystDataFilename(dataUrl); - const cacheData = loadFromCache(session, filename, { maxAge: XREF_MAX_AGE }); - if (cacheData) { - return JSON.parse(cacheData) as MystData; - } - try { - const resp = await session.fetch(dataUrl); - if (resp.ok) { - const data = (await resp.json()) as MystData; - writeToCache(session, filename, JSON.stringify(data)); - return data; - } - } catch { - // data is unset + const onError = (note: string): undefined => { + fileWarn( + vfile, + `Unable to resolve link text from external MyST reference: ${urlSource ?? dataUrl ?? ''}`, + { + ruleId: RuleId.mystLinkValid, + note, + }, + ); + }; + if (!dataUrl) { + return onError('Data source URL unavailable'); + } + + const filename = mystDataFilename(dataUrl); + const cacheData = loadFromCache(session, filename, { maxAge: XREF_MAX_AGE }); + if (cacheData) { + return JSON.parse(cacheData) as MystData; + } + let data: MystData; + try { + const resp = await session.fetch(dataUrl); + if (!resp.ok) { + throw new Error('Could not fetch data from external project URL'); } - note = 'Could not load data from external project'; - } else { - note = 'Data source URL unavailable'; + data = (await resp.json()) as MystData; + } catch { + return onError('Could not load data from external project'); + // data is unset } - fileWarn( - vfile, - `Unable to resolve link text from external MyST reference: ${urlSource ?? dataUrl ?? ''}`, - { - ruleId: RuleId.mystLinkValid, - note, - }, - ); - return; + + writeToCache(session, filename, JSON.stringify(data)); + return data; } export async function fetchMystLinkData(session: ISession, node: Link, vfile: VFile) { diff --git a/packages/myst-cli/src/transforms/outputs.spec.ts b/packages/myst-cli/src/transforms/outputs.spec.ts index f15bdd674..361df3552 100644 --- a/packages/myst-cli/src/transforms/outputs.spec.ts +++ b/packages/myst-cli/src/transforms/outputs.spec.ts @@ -20,9 +20,14 @@ describe('reduceOutputs', () => { ], }, { - type: 'output', - id: 'abc123', - data: [], + type: 'outputs', + children: [ + { + type: 'output', + id: 'abc123', + jupyter_data: null, + }, + ], }, ], }, @@ -49,18 +54,21 @@ describe('reduceOutputs', () => { ], }, { - type: 'output', + type: 'outputs', id: 'abc123', - data: [ + children: [ { - output_type: 'display_data', - execution_count: 3, - metadata: {}, - data: { - 'application/octet-stream': { - content_type: 'application/octet-stream', - hash: 'def456', - path: '/my/path/def456.png', + type: 'output', + jupyter_data: { + output_type: 'display_data', + execution_count: 3, + metadata: {}, + data: { + 'application/octet-stream': { + content_type: 'application/octet-stream', + hash: 'def456', + path: '/my/path/def456.png', + }, }, }, }, @@ -91,14 +99,19 @@ describe('reduceOutputs', () => { ], }, { - type: 'output', + type: 'outputs', id: 'abc123', - data: [], children: [ { - type: 'image', - placeholder: true, - url: 'placeholder.png', + type: 'output', + jupyter_data: null, + children: [ + { + type: 'image', + placeholder: true, + url: 'placeholder.png', + }, + ], }, ], }, diff --git a/packages/myst-cli/src/transforms/outputs.ts b/packages/myst-cli/src/transforms/outputs.ts index 7046450e7..92990523d 100644 --- a/packages/myst-cli/src/transforms/outputs.ts +++ b/packages/myst-cli/src/transforms/outputs.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; import { dirname, join, relative } from 'node:path'; import { computeHash } from 'myst-cli-utils'; -import type { Image, SourceFileKind } from 'myst-spec-ext'; +import type { Image, SourceFileKind, Output } from 'myst-spec-ext'; import { liftChildren, fileError, RuleId, fileWarn } from 'myst-common'; import type { GenericNode, GenericParent } from 'myst-common'; import type { ProjectSettings } from 'myst-frontmatter'; @@ -34,17 +34,28 @@ export async function transformOutputsToCache( kind: SourceFileKind, opts?: { minifyMaxCharacters?: number }, ) { - const outputs = selectAll('output', mdast) as GenericNode[]; - if (!outputs.length) return; + const outputsNodes = selectAll('outputs', mdast) as GenericNode[]; const cache = castSession(session); await Promise.all( - outputs - .filter((output) => output.visibility !== 'remove') + outputsNodes + // Ignore outputs that are hidden + .filter((outputs) => outputs.visibility !== 'remove') + // Pull out children + .map((outputs) => outputs.children as Output[]) + .flat() + // Filter outputs with no data + // TODO: can this ever occur? + .filter((output) => (output as any).jupyter_data !== undefined) + // Minify output data .map(async (output) => { - output.data = await minifyCellOutput(output.data as IOutput[], cache.$outputs, { - computeHash, - maxCharacters: opts?.minifyMaxCharacters, - }); + [(output as any).jupyter_data] = await minifyCellOutput( + [(output as any).jupyter_data] as IOutput[], + cache.$outputs, + { + computeHash, + maxCharacters: opts?.minifyMaxCharacters, + }, + ); }), ); } @@ -75,9 +86,10 @@ export function transformFilterOutputStreams( const blockRemoveStderr = tags.includes('remove-stderr'); const blockRemoveStdout = tags.includes('remove-stdout'); const outputs = selectAll('output', block) as GenericNode[]; - // There should be only one output in the block - outputs.forEach((output) => { - output.data = output.data.filter((data: IStream | MinifiedMimeOutput) => { + outputs + .filter((output) => { + const data = output.jupyter_data; + if ( (stderr !== 'show' || blockRemoveStderr) && data.output_type === 'stream' && @@ -96,7 +108,7 @@ export function transformFilterOutputStreams( }, ); } - return !doRemove; + return doRemove; } if ( (stdout !== 'show' || blockRemoveStdout) && @@ -116,7 +128,7 @@ export function transformFilterOutputStreams( }, ); } - return !doRemove; + return doRemove; } if ( mpl !== 'show' && @@ -141,12 +153,15 @@ export function transformFilterOutputStreams( }, ); } - return !doRemove; + return doRemove; } - return true; + return false; + }) + .forEach((output) => { + output.type = '__delete__'; }); - }); }); + remove(mdast, { cascade: false }, '__delete__'); } function writeCachedOutputToFile( @@ -192,16 +207,19 @@ export function transformOutputsToFile( const outputs = selectAll('output', mdast) as GenericNode[]; const cache = castSession(session); - outputs.forEach((node) => { - walkOutputs(node.data, (obj) => { - const { hash } = obj; - if (!hash || !cache.$outputs[hash]) return undefined; - obj.path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { - ...opts, - node, + outputs + .filter((output) => !!output.jupyter_data) + .forEach((node) => { + // TODO: output-refactoring -- drop to single output in future + walkOutputs([node.jupyter_data], (obj) => { + const { hash } = obj; + if (!hash || !cache.$outputs[hash]) return undefined; + obj.path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { + ...opts, + node, + }); }); }); - }); } /** @@ -233,87 +251,99 @@ export function reduceOutputs( writeFolder: string, opts?: { altOutputFolder?: string; vfile?: VFile }, ) { - const outputs = selectAll('output', mdast) as GenericNode[]; + const outputsNodes = selectAll('outputs', mdast) as GenericNode[]; const cache = castSession(session); - outputs.forEach((node) => { - if (node.visibility === 'remove' || node.visibility === 'hide') { - // Hidden nodes should not show up in simplified outputs for static export - node.type = '__delete__'; - return; - } - if (!node.data?.length && !node.children?.length) { - node.type = '__delete__'; - return; - } - node.type = '__lift__'; - if (node.children?.length) return; - const selectedOutputs: { content_type: string; hash: string }[] = []; - node.data.forEach((output: MinifiedOutput) => { - let selectedOutput: { content_type: string; hash: string } | undefined; - walkOutputs([output], (obj: any) => { - const { output_type, content_type, hash } = obj; - if (!hash) return undefined; - if (!selectedOutput || isPreferredOutputType(content_type, selectedOutput.content_type)) { - if (['error', 'stream'].includes(output_type)) { - selectedOutput = { content_type: 'text/plain', hash }; - } else if (typeof content_type === 'string') { - if ( - content_type.startsWith('image/') || - content_type === 'text/plain' || - content_type === 'text/html' - ) { - selectedOutput = { content_type, hash }; + outputsNodes.forEach((outputsNode) => { + const outputs = outputsNode.children as GenericNode[]; + + outputs.forEach((outputNode) => { + if (outputNode.visibility === 'remove' || outputNode.visibility === 'hide') { + // Hidden nodes should not show up in simplified outputs for static export + outputNode.type = '__delete__'; + return; + } + if (!outputNode.jupyter_data && !outputNode.children?.length) { + outputNode.type = '__delete__'; + return; + } + // Lift the `output` node into `Outputs` + outputNode.type = '__lift__'; + + // If the output already has children, we don't need to do anything + if (outputNode.children?.length) { + return; + } + + // Find a preferred IOutput type to render into the AST + const selectedOutputs: { content_type: string; hash: string }[] = []; + if (outputNode.jupyter_data) { + const output = outputNode.jupyter_data; + + let selectedOutput: { content_type: string; hash: string } | undefined; + walkOutputs([output], (obj: any) => { + const { output_type, content_type, hash } = obj; + if (!hash) return undefined; + if (!selectedOutput || isPreferredOutputType(content_type, selectedOutput.content_type)) { + if (['error', 'stream'].includes(output_type)) { + selectedOutput = { content_type: 'text/plain', hash }; + } else if (typeof content_type === 'string') { + if ( + content_type.startsWith('image/') || + content_type === 'text/plain' || + content_type === 'text/html' + ) { + selectedOutput = { content_type, hash }; + } } } - } - }); - if (selectedOutput) selectedOutputs.push(selectedOutput); - }); - const children: (Image | GenericNode)[] = selectedOutputs - .map((output): Image | GenericNode | GenericNode[] | undefined => { - const { content_type, hash } = output ?? {}; - if (!hash || !cache.$outputs[hash]) return undefined; - if (content_type === 'text/html') { - const htmlTree = { - type: 'root', - children: [ - { - type: 'html', - value: cache.$outputs[hash][0], - }, - ], - }; - htmlTransform(htmlTree); - if ((selectAll('image', htmlTree) as GenericNode[]).find((htmlImage) => !htmlImage.url)) { - return undefined; + }); + if (selectedOutput) selectedOutputs.push(selectedOutput); + } + const children: (Image | GenericNode)[] = selectedOutputs + .map((output): Image | GenericNode | GenericNode[] | undefined => { + const { content_type, hash } = output ?? {}; + if (!hash || !cache.$outputs[hash]) return undefined; + if (content_type === 'text/html') { + const htmlTree = { + type: 'root', + children: [ + { + type: 'html', + value: cache.$outputs[hash][0], + }, + ], + }; + htmlTransform(htmlTree); + return htmlTree.children; + } else if (content_type.startsWith('image/')) { + const path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { + ...opts, + node: outputNode, + }); + if (!path) return undefined; + const relativePath = relative(dirname(file), path); + return { + type: 'image', + data: { type: 'output' }, + url: relativePath, + urlSource: relativePath, + }; + } else if (content_type === 'text/plain' && cache.$outputs[hash]) { + const [content] = cache.$outputs[hash]; + return { + type: 'code', + data: { type: 'output' }, + value: stripAnsi(content), + }; } - return htmlTree.children; - } else if (content_type.startsWith('image/')) { - const path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, { - ...opts, - node, - }); - if (!path) return undefined; - const relativePath = relative(dirname(file), path); - return { - type: 'image', - data: { type: 'output' }, - url: relativePath, - urlSource: relativePath, - }; - } else if (content_type === 'text/plain' && cache.$outputs[hash]) { - const [content] = cache.$outputs[hash]; - return { - type: 'code', - data: { type: 'output' }, - value: stripAnsi(content), - }; - } - return undefined; - }) - .flat() - .filter((output): output is Image | GenericNode => !!output); - node.children = children; + return undefined; + }) + .flat() + .filter((output): output is Image | GenericNode => !!output); + outputNode.children = children; + }); + // Lift the `outputs` node + outputsNode.type = '__lift__'; }); remove(mdast, '__delete__'); liftChildren(mdast, '__lift__'); diff --git a/packages/myst-directives/src/code.ts b/packages/myst-directives/src/code.ts index ebf7d739a..ea29c7150 100644 --- a/packages/myst-directives/src/code.ts +++ b/packages/myst-directives/src/code.ts @@ -209,15 +209,14 @@ export const codeCellDirective: DirectiveSpec = { executable: true, value: (data.body ?? '') as string, }; - const output = { - type: 'output', - id: nanoid(), - data: [], + const outputs = { + type: 'outputs', + children: [], }; const block: GenericNode = { type: 'block', kind: NotebookCell.code, - children: [code, output], + children: [code, outputs], data: {}, }; addCommonDirectiveOptions(data, block); diff --git a/packages/myst-execute/src/execute.ts b/packages/myst-execute/src/execute.ts index e97dcfad4..07a531714 100644 --- a/packages/myst-execute/src/execute.ts +++ b/packages/myst-execute/src/execute.ts @@ -2,7 +2,7 @@ import { select, selectAll } from 'unist-util-select'; import type { Logger } from 'myst-cli-utils'; import type { PageFrontmatter, KernelSpec } from 'myst-frontmatter'; import type { Kernel, KernelMessage, Session, SessionManager } from '@jupyterlab/services'; -import type { Block, Code, InlineExpression, Output } from 'myst-spec-ext'; +import type { Block, Code, InlineExpression, Output, Outputs } from 'myst-spec-ext'; import type { IOutput } from '@jupyterlab/nbformat'; import type { GenericNode, GenericParent, IExpressionResult, IExpressionError } from 'myst-common'; import { NotebookCell, NotebookCellTags, fileError } from 'myst-common'; @@ -166,7 +166,7 @@ type CodeBlock = Block & { * @param node node to test */ function isCellBlock(node: GenericNode): node is CodeBlock { - return node.type === 'block' && select('code', node) !== null && select('output', node) !== null; + return node.type === 'block' && select('code', node) !== null && select('outputs', node) !== null; } /** @@ -282,14 +282,18 @@ function applyComputedOutputsToNodes( const thisResult = computedResult.shift(); if (isCellBlock(matchedNode)) { - // Pull out output to set data - const output = select('output', matchedNode) as unknown as { data: IOutput[] }; - // Set the output array to empty if we don't have a result (e.g. due to a kernel error) - output.data = thisResult === undefined ? [] : (thisResult as IOutput[]); + const rawOutputData = (thisResult as IOutput[]) ?? []; + // Pull out outputs to set data + const outputs = select('outputs', matchedNode) as Outputs; + // Ensure that whether this fails or succeeds, we write to `children` (e.g. due to a kernel error) + outputs.children = rawOutputData.map((data, index) => { + const identifier = outputs.identifier ? `${outputs.identifier}-${index}` : undefined; + return { type: 'output', children: [], jupyter_data: data as any, identifier }; + }); } else if (isInlineExpression(matchedNode)) { + const rawOutputData = thisResult as Record | undefined; // Set data of expression to the result, or empty if we don't have one - matchedNode.result = // TODO: FIXME .data - thisResult === undefined ? undefined : (thisResult as unknown as Record); + matchedNode.result = rawOutputData; } else { // This should never happen throw new Error('Node must be either code block or inline expression.'); diff --git a/packages/myst-execute/tests/execute.yml b/packages/myst-execute/tests/execute.yml index f768992aa..f1816e5af 100644 --- a/packages/myst-execute/tests/execute.yml +++ b/packages/myst-execute/tests/execute.yml @@ -36,13 +36,17 @@ cases: lang: python executable: true value: print('abc') - identifier: nb-cell-0-code enumerator: 1 + identifier: nb-cell-0-code html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM + - type: outputs identifier: nb-cell-0-output html_id: nb-cell-0-output + children: + - type: output + identifier: nb-cell-0-output-0 + children: [] + after: type: root children: @@ -63,18 +67,77 @@ cases: lang: python executable: true value: print('abc') + enumerator: 1 identifier: nb-cell-0-code + html_id: nb-cell-0-code + - type: outputs + identifier: nb-cell-0-output + html_id: nb-cell-0-output + children: + - type: output + children: [] + identifier: nb-cell-0-output-0 + jupyter_data: + output_type: stream + name: stdout + text: | + abc + - title: output without identifier is given one + before: + type: root + children: + - type: block + children: + - type: block + kind: notebook-code + identifier: nb-cell-0 + label: nb-cell-0 + html_id: nb-cell-0 + children: + - type: code + lang: python + executable: true + value: print('abc') enumerator: 1 + identifier: nb-cell-0-code html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM + - type: outputs + identifier: nb-cell-0-output + html_id: nb-cell-0-output + children: + - type: output + children: [] + + after: + type: root + children: + - type: block + children: + - type: block + kind: notebook-code + identifier: nb-cell-0 + label: nb-cell-0 + html_id: nb-cell-0 + children: + - type: code + lang: python + executable: true + value: print('abc') + enumerator: 1 + identifier: nb-cell-0-code + html_id: nb-cell-0-code + - type: outputs identifier: nb-cell-0-output html_id: nb-cell-0-output - data: - - output_type: stream - name: stdout - text: | - abc + children: + - type: output + children: [] + identifier: nb-cell-0-output-0 + jupyter_data: + output_type: stream + name: stdout + text: | + abc - title: tree with inline expression is evaluated before: type: root @@ -115,100 +178,92 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: print('abc') - identifier: nb-cell-0-code enumerator: 1 + identifier: nb-cell-0-code html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM + - type: outputs identifier: nb-cell-0-output html_id: nb-cell-0-output - data: + children: + - type: output + identifier: nb-cell-0-output-0 + jupyter_data: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + identifier: nb-cell-1-code + html_id: nb-cell-1-code + - type: outputs + identifier: nb-cell-1-output + html_id: nb-cell-1-output + children: + - type: output + identifier: nb-cell-1-output-0 + jupyter_data: after: type: root children: - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: print('abc') - identifier: nb-cell-0-code enumerator: 1 + identifier: nb-cell-0-code html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM + - type: outputs identifier: nb-cell-0-output html_id: nb-cell-0-output - data: - - output_type: stream - name: stdout - text: | - abc + children: + - type: output + identifier: nb-cell-0-output-0 + jupyter_data: + output_type: stream + name: stdout + text: | + abc - type: block kind: notebook-code data: - id: nb-cell-0 - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 - html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM - identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: - - output_type: error - # Note this traceback can be different on various machines - # Not including it means we still validate an error, just don't care about the traceback - # traceback: - # - "\e[0;31m---------------------------------------------------------------------------\e[0m" - # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" - # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" - # - "\e[0;31mValueError\e[0m: " - ename: ValueError - evalue: '' + identifier: nb-cell-1-code + html_id: nb-cell-1-code + - type: outputs + identifier: nb-cell-1-output + html_id: nb-cell-1-output + children: + - type: output + identifier: nb-cell-1-output-0 + jupyter_data: + output_type: error + # Note this traceback can be different on various machines + # Not including it means we still validate an error, just don't care about the traceback + # traceback: + # - "\e[0;31m---------------------------------------------------------------------------\e[0m" + # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" + # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" + # - "\e[0;31mValueError\e[0m: " + ename: ValueError + evalue: '' - title: tree with bad executable code and `raises-exception` is evaluated and passes before: type: root @@ -216,58 +271,54 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 tags: raises-exception - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 + identifier: nb-cell-0-code html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM + - type: outputs identifier: nb-cell-0-output html_id: nb-cell-0-output - data: + children: + - type: output + identifier: nb-cell-0-output-0 + jupyter_data: after: type: root children: - type: block kind: notebook-code data: - id: nb-cell-0 tags: raises-exception - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 + identifier: nb-cell-0-code html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM + - type: outputs identifier: nb-cell-0-output html_id: nb-cell-0-output - data: - - output_type: error - # Note this traceback can be different on various machines - # Not including it means we still validate an error, just don't care about the traceback - # traceback: - # - "\e[0;31m---------------------------------------------------------------------------\e[0m" - # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" - # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" - # - "\e[0;31mValueError\e[0m: " - ename: ValueError - evalue: '' + children: + - type: output + identifier: nb-cell-0-output-0 + jupyter_data: + output_type: error + # Note this traceback can be different on various machines + # Not including it means we still validate an error, just don't care about the traceback + # traceback: + # - "\e[0;31m---------------------------------------------------------------------------\e[0m" + # - "\e[0;31mValueError\e[0m Traceback (most recent call last)" + # - "Cell \e[0;32mIn[2], line 1\e[0m\n\e[0;32m----> 1\e[0m \e[38;5;28;01mraise\e[39;00m \e[38;5;167;01mValueError\e[39;00m\n" + # - "\e[0;31mValueError\e[0m: " + ename: ValueError + evalue: '' - title: tree with bad executable code and `skip-execution` is not evaluated before: type: root @@ -275,45 +326,39 @@ cases: - type: block kind: notebook-code data: - id: nb-cell-0 tags: skip-execution - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 + identifier: nb-cell-0-code html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM + - type: outputs identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + children: + - type: output + identifier: nb-cell-0-output-0 + jupyter_data: after: type: root children: - type: block kind: notebook-code data: - id: nb-cell-0 tags: skip-execution - identifier: nb-cell-0 - label: nb-cell-0 - html_id: nb-cell-0 children: - type: code lang: python executable: true value: raise ValueError - identifier: nb-cell-0-code enumerator: 1 + identifier: nb-cell-0-code html_id: nb-cell-0-code - - type: output - id: T7FMDqDm8dM2bOT1tKeeM + - type: outputs identifier: nb-cell-0-output - html_id: nb-cell-0-output - data: + children: + - type: output + identifier: nb-cell-0-output-0 + jupyter_data: diff --git a/packages/myst-spec-ext/src/types.ts b/packages/myst-spec-ext/src/types.ts index b58ef8f9b..ae2c38090 100644 --- a/packages/myst-spec-ext/src/types.ts +++ b/packages/myst-spec-ext/src/types.ts @@ -252,10 +252,16 @@ export type Container = Omit & { export type Output = Node & Target & { type: 'output'; - id?: string; - data?: any[]; // MinifiedOutput[] + children: (FlowContent | ListContent | PhrasingContent)[]; + jupyter_data: any; // TODO: set this to IOutput + }; + +export type Outputs = Node & + Target & { + type: 'outputs'; + children: (Output | FlowContent | ListContent | PhrasingContent)[]; // Support placeholders in addition to outputs visibility?: Visibility; - children?: (FlowContent | ListContent | PhrasingContent)[]; + id?: string; }; export type Aside = Node & diff --git a/packages/myst-transforms/src/blocks.spec.ts b/packages/myst-transforms/src/blocks.spec.ts index 5ce0fecd4..3d868033e 100644 --- a/packages/myst-transforms/src/blocks.spec.ts +++ b/packages/myst-transforms/src/blocks.spec.ts @@ -120,8 +120,10 @@ describe('Test blockMetadataTransform', () => { test('label is propagated to outputs', async () => { const mdast = u('root', [ u('block', { meta: '{"label": "My_Label", "key": "value"}' }, [ - u('output', 'We know what we are'), - u('output', 'but know not what we may be.'), + u('outputs', [ + u('output', 'We know what we are'), + u('output', 'but know not what we may be.'), + ]), ]), ]) as any; blockMetadataTransform(mdast, new VFile()); @@ -136,8 +138,10 @@ describe('Test blockMetadataTransform', () => { data: { key: 'value' }, }, [ - u('output', { identifier: 'my_label-output-0' }, 'We know what we are'), - u('output', { identifier: 'my_label-output-1' }, 'but know not what we may be.'), + u('outputs', { identifier: 'my_label-output' }, [ + u('output', { identifier: 'my_label-output-0' }, 'We know what we are'), + u('output', { identifier: 'my_label-output-1' }, 'but know not what we may be.'), + ]), ], ), ]), diff --git a/packages/myst-transforms/src/blocks.ts b/packages/myst-transforms/src/blocks.ts index 20e994146..730565326 100644 --- a/packages/myst-transforms/src/blocks.ts +++ b/packages/myst-transforms/src/blocks.ts @@ -1,7 +1,7 @@ import type { VFile } from 'vfile'; import type { Plugin } from 'unified'; import type { Node } from 'myst-spec'; -import { selectAll } from 'unist-util-select'; +import { selectAll, select } from 'unist-util-select'; import type { GenericNode, GenericParent } from 'myst-common'; import { NotebookCell, RuleId, fileError, normalizeLabel } from 'myst-common'; import type { Code } from 'myst-spec-ext'; @@ -80,24 +80,22 @@ export function blockMetadataTransform(mdast: GenericParent, file: VFile) { } } if (block.identifier) { - const codeChildren = selectAll('code', block) as Code[]; - codeChildren.forEach((child, index) => { - if (child.identifier) return; - if (codeChildren.length === 1) { - child.identifier = `${block.identifier}-code`; - } else { - child.identifier = `${block.identifier}-code-${index}`; - } - }); - const outputChildren = selectAll('output', block) as GenericNode[]; - outputChildren.forEach((child, index) => { - if (child.identifier) return; - if (outputChildren.length === 1) { - child.identifier = `${block.identifier}-output`; - } else { - child.identifier = `${block.identifier}-output-${index}`; - } - }); + const codeNode = select('code', block) as any as Code | null; + if (codeNode !== null && !codeNode.identifier) { + codeNode.identifier = `${block.identifier}-code`; + } + const outputsNode = select('outputs', block) as GenericNode | null; + if (outputsNode !== null && !outputsNode.identifier) { + // Label outputs node + outputsNode.identifier = `${block.identifier}-output`; + // Enumerate outputs + const outputs = selectAll('output', outputsNode) as GenericNode[]; + outputs.forEach((outputNode, index) => { + if (outputNode && !outputNode.identifier) { + outputNode.identifier = `${block.identifier}-output-${index}`; + } + }); + } } }); }