🤖 Store execution outputs in Outputs

.changeset/ten-bats-warn.md

@@ -0,0 +1,10 @@
+---
+"mystmd": minor
+"myst-directives": patch
+"myst-transforms": patch
+"myst-spec-ext": patch
+"myst-execute": patch
+"myst-cli": patch
+---
+
+Add support for new Outputs node

packages/myst-cli/src/process/notebook.ts

@@ -165,17 +165,21 @@ export async function processNotebookFull(
           value: ensureString(cell.source),
         };
 
-        // Embed outputs in an output block
-        const output: { type: 'output'; id: string; data: IOutput[] } = {
-          type: 'output',
+        const outputsChildren = (cell.outputs as IOutput[]).map((output) => {
+          // Embed outputs in an output block
+          const result = {
+            type: 'output',
+            jupyter_data: output,
+            children: [],
+          };
+          return result;
+        });
+        const outputs = {
+          type: 'outputs',
           id: nanoid(),
-          data: [],
+          children: outputsChildren,
         };
-
-        if (cell.outputs && (cell.outputs as IOutput[]).length > 0) {
-          output.data = cell.outputs as IOutput[];
-        }
-        return acc.concat(blockParent(cell, [code, output]));
+        return acc.concat(blockParent(cell, [code, outputs]));
       }
       return acc;
     },

packages/myst-cli/src/transforms/code.spec.ts

@@ -162,7 +162,10 @@ function build_mdast(tags: string[], has_output: boolean) {
     ],
   };
   if (has_output) {
-    mdast.children[0].children.push({ type: 'output' });
+    mdast.children[0].children.push({
+      type: 'outputs',
+      children: [{ type: 'output', children: [] }],
+    });
   }
   return mdast;
 }
@@ -261,7 +264,7 @@ describe('propagateBlockDataToCode', () => {
           const mdast = build_mdast([tag], has_output);
           propagateBlockDataToCode(new Session(), new VFile(), mdast);
           let result = '';
-          const outputNode = mdast.children[0].children[1];
+          const outputsNode = mdast.children[0].children[1];
           switch (target) {
             case 'cell':
               result = mdast.children[0].visibility;
@@ -270,12 +273,14 @@ describe('propagateBlockDataToCode', () => {
               result = mdast.children[0].children[0].visibility;
               break;
             case 'output':
-              if (!has_output && target == 'output') {
-                expect(outputNode).toEqual(undefined);
+              if (!has_output) {
+                expect(outputsNode).toEqual(undefined);
                 continue;
               }
-              result = outputNode.visibility;
+              result = outputsNode.visibility;
               break;
+            default:
+              throw new Error();
           }
           expect(result).toEqual(action);
         }
@@ -290,13 +295,13 @@ describe('propagateBlockDataToCode', () => {
         propagateBlockDataToCode(new Session(), new VFile(), mdast);
         const blockNode = mdast.children[0];
         const codeNode = mdast.children[0].children[0];
-        const outputNode = mdast.children[0].children[1];
+        const outputsNode = mdast.children[0].children[1];
         expect(blockNode.visibility).toEqual(action);
         expect(codeNode.visibility).toEqual(action);
         if (has_output) {
-          expect(outputNode.visibility).toEqual(action);
+          expect(outputsNode.visibility).toEqual(action);
         } else {
-          expect(outputNode).toEqual(undefined);
+          expect(outputsNode).toEqual(undefined);
         }
       }
     }
@@ -313,7 +318,8 @@ describe('propagateBlockDataToCode', () => {
               executable: true,
             },
             {
-              type: 'output',
+              type: 'outputs',
+              children: [],
             },
           ],
           data: {
@@ -323,10 +329,10 @@ describe('propagateBlockDataToCode', () => {
       ],
     };
     propagateBlockDataToCode(new Session(), new VFile(), mdast);
-    const outputNode = mdast.children[0].children[1];
-    expect(outputNode.children?.length).toEqual(1);
-    expect(outputNode.children[0].type).toEqual('image');
-    expect(outputNode.children[0].placeholder).toBeTruthy();
+    const outputsNode = mdast.children[0].children[1];
+    expect(outputsNode.children?.length).toEqual(1);
+    expect(outputsNode.children[0].type).toEqual('image');
+    expect(outputsNode.children[0].placeholder).toBeTruthy();
   });
   it('placeholder passes with no output', async () => {
     const mdast: any = {

packages/myst-cli/src/transforms/code.ts

@@ -1,6 +1,6 @@
 import type { GenericNode, GenericParent } from 'myst-common';
 import { NotebookCellTags, RuleId, fileError, fileWarn } from 'myst-common';
-import type { Image, Output } from 'myst-spec-ext';
+import type { Image, Outputs } from 'myst-spec-ext';
 import { select, selectAll } from 'unist-util-select';
 import yaml from 'js-yaml';
 import type { VFile } from 'vfile';
@@ -156,10 +156,9 @@ export function propagateBlockDataToCode(session: ISession, vfile: VFile, mdast:
   const blocks = selectAll('block', mdast) as GenericNode[];
   blocks.forEach((block) => {
     if (!block.data) return;
-    const outputNode = select('output', block) as Output | null;
-    if (block.data.placeholder && outputNode) {
-      if (!outputNode.children) outputNode.children = [];
-      outputNode.children.push({
+    const outputsNode = select('outputs', block) as Outputs | null;
+    if (block.data.placeholder && outputsNode) {
+      outputsNode.children.push({
         type: 'image',
         placeholder: true,
         url: block.data.placeholder as string,
@@ -195,18 +194,18 @@ export function propagateBlockDataToCode(session: ISession, vfile: VFile, mdast:
           if (codeNode) codeNode.visibility = 'remove';
           break;
         case NotebookCellTags.hideOutput:
-          if (outputNode) outputNode.visibility = 'hide';
+          if (outputsNode) outputsNode.visibility = 'hide';
           break;
         case NotebookCellTags.removeOutput:
-          if (outputNode) outputNode.visibility = 'remove';
+          if (outputsNode) outputsNode.visibility = 'remove';
           break;
         default:
           session.log.debug(`tag '${tag}' is not valid in code-cell tags'`);
       }
     });
     if (!block.visibility) block.visibility = 'show';
     if (codeNode && !codeNode.visibility) codeNode.visibility = 'show';
-    if (outputNode && !outputNode.visibility) outputNode.visibility = 'show';
+    if (outputsNode && !outputsNode.visibility) outputsNode.visibility = 'show';
   });
 }
 
@@ -233,7 +232,7 @@ export function transformLiftCodeBlocksInJupytext(mdast: GenericParent) {
         child.type === 'block' &&
         child.children?.length === 2 &&
         child.children?.[0].type === 'code' &&
-        child.children?.[1].type === 'output'
+        child.children?.[1].type === 'outputs'
       ) {
         newBlocks.push(child as GenericParent);
         newBlocks.push({ type: 'block', children: [] });

packages/myst-cli/src/transforms/crossReferences.ts

@@ -39,36 +39,39 @@ async function fetchMystData(
   urlSource: string | undefined,
   vfile: VFile,
 ) {
-  let note: string;
-  if (dataUrl) {
-    const filename = mystDataFilename(dataUrl);
-    const cacheData = loadFromCache(session, filename, { maxAge: XREF_MAX_AGE });
-    if (cacheData) {
-      return JSON.parse(cacheData) as MystData;
-    }
-    try {
-      const resp = await session.fetch(dataUrl);
-      if (resp.ok) {
-        const data = (await resp.json()) as MystData;
-        writeToCache(session, filename, JSON.stringify(data));
-        return data;
-      }
-    } catch {
-      // data is unset
+  const onError = (note: string): undefined => {
+    fileWarn(
+      vfile,
+      `Unable to resolve link text from external MyST reference: ${urlSource ?? dataUrl ?? ''}`,
+      {
+        ruleId: RuleId.mystLinkValid,
+        note,
+      },
+    );
+  };
+  if (!dataUrl) {
+    return onError('Data source URL unavailable');
+  }
+
+  const filename = mystDataFilename(dataUrl);
+  const cacheData = loadFromCache(session, filename, { maxAge: XREF_MAX_AGE });
+  if (cacheData) {
+    return JSON.parse(cacheData) as MystData;
+  }
+  let data: MystData;
+  try {
+    const resp = await session.fetch(dataUrl);
+    if (!resp.ok) {
+      throw new Error('Could not fetch data from external project URL');
     }
-    note = 'Could not load data from external project';
-  } else {
-    note = 'Data source URL unavailable';
+    data = (await resp.json()) as MystData;
+  } catch {
+    return onError('Could not load data from external project');
+    // data is unset
   }
-  fileWarn(
-    vfile,
-    `Unable to resolve link text from external MyST reference: ${urlSource ?? dataUrl ?? ''}`,
-    {
-      ruleId: RuleId.mystLinkValid,
-      note,
-    },
-  );
-  return;
+
+  writeToCache(session, filename, JSON.stringify(data));
+  return data;
 }
 
 export async function fetchMystLinkData(session: ISession, node: Link, vfile: VFile) {

packages/myst-cli/src/transforms/outputs.spec.ts

@@ -20,9 +20,14 @@ describe('reduceOutputs', () => {
               ],
             },
             {
-              type: 'output',
-              id: 'abc123',
-              data: [],
+              type: 'outputs',
+              children: [
+                {
+                  type: 'output',
+                  id: 'abc123',
+                  jupyter_data: null,
+                },
+              ],
             },
           ],
         },
@@ -49,18 +54,21 @@ describe('reduceOutputs', () => {
               ],
             },
             {
-              type: 'output',
+              type: 'outputs',
               id: 'abc123',
-              data: [
+              children: [
                 {
-                  output_type: 'display_data',
-                  execution_count: 3,
-                  metadata: {},
-                  data: {
-                    'application/octet-stream': {
-                      content_type: 'application/octet-stream',
-                      hash: 'def456',
-                      path: '/my/path/def456.png',
+                  type: 'output',
+                  jupyter_data: {
+                    output_type: 'display_data',
+                    execution_count: 3,
+                    metadata: {},
+                    data: {
+                      'application/octet-stream': {
+                        content_type: 'application/octet-stream',
+                        hash: 'def456',
+                        path: '/my/path/def456.png',
+                      },
                     },
                   },
                 },
@@ -91,14 +99,19 @@ describe('reduceOutputs', () => {
               ],
             },
             {
-              type: 'output',
+              type: 'outputs',
               id: 'abc123',
-              data: [],
               children: [
                 {
-                  type: 'image',
-                  placeholder: true,
-                  url: 'placeholder.png',
+                  type: 'output',
+                  jupyter_data: null,
+                  children: [
+                    {
+                      type: 'image',
+                      placeholder: true,
+                      url: 'placeholder.png',
+                    },
+                  ],
                 },
               ],
             },