Further Enhancements to the Module System

.babelrc

@@ -1,3 +1,3 @@
 {
-  "presets": ["es2015", "stage-2"]
+  "presets": ["@babel/preset-env"]
 }

.eslintrc.json

@@ -9,6 +9,11 @@
     "plugin:@typescript-eslint/recommended-requiring-type-checking",
     "prettier"
   ],
+  "ignorePatterns": [
+    "**/__tests__/**",
+    "**/__mocks__/**",
+    "jest.setup.ts"
+  ],
   "parser": "@typescript-eslint/parser",
   "parserOptions": {
     "project": "tsconfig.json",

.gitignore

@@ -6,6 +6,7 @@ node_modules
 dist/
 .idea/
 coverage/
+yarn-error.log
 
 # emacs backup files
 *~

package.json

@@ -33,9 +33,9 @@
   "dependencies": {
     "@babel/parser": "^7.19.4",
     "@joeychenofficial/alt-ergo-modified": "^2.4.0",
-    "@ts-morph/bootstrap": "^0.18.0",
+    "@ts-morph/bootstrap": "^0.20.0",
     "@types/estree": "0.0.52",
-    "acorn": "^8.8.2",
+    "acorn": "^8.10.0",
     "acorn-class-fields": "^1.0.0",
     "acorn-loose": "^8.0.0",
     "acorn-walk": "^8.0.0",
@@ -44,7 +44,7 @@
     "js-base64": "^3.7.5",
     "lodash": "^4.17.20",
     "node-getopt": "^0.3.2",
-    "source-map": "0.7.3",
+    "source-map": "0.7.4",
     "xmlhttprequest-ts": "^1.0.1"
   },
   "scripts": {
@@ -67,7 +67,7 @@
     "@types/jest": "^29.0.0",
     "@types/lodash.assignin": "^4.2.6",
     "@types/lodash.clonedeep": "^4.5.6",
-    "@types/node": "^17.0.5",
+    "@types/node": "^20.4.5",
     "@types/offscreencanvas": "^2019.7.0",
     "@typescript-eslint/eslint-plugin": "^4.4.1",
     "@typescript-eslint/parser": "^4.4.1",

src/__tests__/environment.ts

@@ -18,7 +18,10 @@ test('Function params and body identifiers are in different environment', () =>
   const context = mockContext(Chapter.SOURCE_4)
   context.prelude = null // hide the unneeded prelude
   const parsed = parse(code, context)
-  const it = evaluate(parsed as any as Program, context, false, false)
+  const it = evaluate(parsed as any as Program, context, {
+    loadTabs: false,
+    wrapModules: false
+  })
   const stepsToComment = 13 // manually counted magic number
   for (let i = 0; i < stepsToComment; i += 1) {
     it.next()

src/ec-evaluator/__tests__/__snapshots__/ec-evaluator-errors.ts.snap

@@ -792,19 +792,6 @@ Object {
 }
 `;
 
-exports[`Importing unknown variables throws UndefinedImport error: expectParsedError 1`] = `
-Object {
-  "alertResult": Array [],
-  "code": "import { foo1 } from 'one_module';",
-  "displayResult": Array [],
-  "numErrors": 1,
-  "parsedErrors": "'one_module' does not contain a definition for 'foo1'",
-  "result": undefined,
-  "resultStatus": "error",
-  "visualiseListResult": Array [],
-}
-`;
-
 exports[`In a block, every going-to-be-defined variable in the block cannot be accessed until it has been defined in the block.: expectParsedError 1`] = `
 Object {
   "alertResult": Array [],

src/ec-evaluator/__tests__/ec-evaluator-errors.ts

@@ -1,6 +1,4 @@
 /* tslint:disable:max-line-length */
-import * as _ from 'lodash'
-
 import { Chapter, Variant } from '../../types'
 import { stripIndent } from '../../utils/formatters'
 import {
@@ -10,20 +8,6 @@ import {
   expectResult
 } from '../../utils/testing'
 
-jest.spyOn(_, 'memoize').mockImplementation(func => func as any)
-
-const mockXMLHttpRequest = (xhr: Partial<XMLHttpRequest> = {}) => {
-  const xhrMock: Partial<XMLHttpRequest> = {
-    open: jest.fn(() => {}),
-    send: jest.fn(() => {}),
-    status: 200,
-    responseText: 'Hello World!',
-    ...xhr
-  }
-  jest.spyOn(window, 'XMLHttpRequest').mockImplementationOnce(() => xhrMock as XMLHttpRequest)
-  return xhrMock
-}
-
 const undefinedVariable = stripIndent`
 im_undefined;
 `
@@ -1016,35 +1000,3 @@ test('Shadowed variables may not be assigned to until declared in the current sc
     optionEC3
   ).toMatchInlineSnapshot(`"Line 3: Name variable not declared."`)
 })
-
-test('Importing unknown variables throws UndefinedImport error', () => {
-  // for getModuleFile
-  mockXMLHttpRequest({
-    responseText: `{
-    "one_module": {
-      "tabs": []
-    },
-    "another_module": {
-      "tabs": []
-    }
-  }`
-  })
-
-  // for bundle body
-  mockXMLHttpRequest({
-    responseText: `
-      require => {
-        return {
-          foo: () => 'foo',
-        }
-      }
-    `
-  })
-
-  return expectParsedError(
-    stripIndent`
-    import { foo1 } from 'one_module';
-  `,
-    optionEC
-  ).toMatchInlineSnapshot("\"'one_module' does not contain a definition for 'foo1'\"")
-})

src/ec-evaluator/__tests__/ec-evaluator.ts

@@ -2,22 +2,8 @@ import { Chapter, Variant } from '../../types'
 import { stripIndent } from '../../utils/formatters'
 import { expectResult } from '../../utils/testing'
 
-// jest.mock('lodash', () => ({
-//   ...jest.requireActual('lodash'),
-//   memoize: jest.fn(func => func)
-// }))
-
-const mockXMLHttpRequest = (xhr: Partial<XMLHttpRequest> = {}) => {
-  const xhrMock: Partial<XMLHttpRequest> = {
-    open: jest.fn(() => {}),
-    send: jest.fn(() => {}),
-    status: 200,
-    responseText: 'Hello World!',
-    ...xhr
-  }
-  jest.spyOn(window, 'XMLHttpRequest').mockImplementationOnce(() => xhrMock as XMLHttpRequest)
-  return xhrMock
-}
+jest.mock('../../modules/moduleLoaderAsync')
+jest.mock('../../modules/moduleLoader')
 
 const optionEC = { variant: Variant.EXPLICIT_CONTROL }
 const optionEC3 = { chapter: Chapter.SOURCE_3, variant: Variant.EXPLICIT_CONTROL }
@@ -316,29 +302,6 @@ test('streams can be created and functions with no return statements are still e
 })
 
 test('Imports are properly handled', () => {
-  // for getModuleFile
-  mockXMLHttpRequest({
-    responseText: `{
-    "one_module": {
-      "tabs": []
-    },
-    "another_module": {
-      "tabs": []
-    }
-  }`
-  })
-
-  // for bundle body
-  mockXMLHttpRequest({
-    responseText: `
-      require => {
-        return {
-          foo: () => 'foo',
-        }
-      }
-    `
-  })
-
   return expectResult(
     stripIndent`
     import { foo } from 'one_module';

src/ec-evaluator/interpreter.ts

@@ -6,20 +6,23 @@
  */
 
 /* tslint:disable:max-classes-per-file */
-import * as es from 'estree'
-import { partition, uniqueId } from 'lodash'
+import { uniqueId } from 'lodash'
 
 import { IOptions } from '..'
 import { UNKNOWN_LOCATION } from '../constants'
 import * as errors from '../errors/errors'
 import { RuntimeSourceError } from '../errors/runtimeSourceError'
 import Closure from '../interpreter/closure'
-import { UndefinedImportError } from '../modules/errors'
-import { loadModuleBundle, loadModuleTabs } from '../modules/moduleLoader'
-import { ModuleFunctions } from '../modules/moduleTypes'
+import { loadModuleBundle } from '../modules/moduleLoader'
+import type { ImportTransformOptions } from '../modules/moduleTypes'
+import { initModuleContext } from '../modules/utils'
 import { checkEditorBreakpoints } from '../stdlib/inspector'
-import { Context, ContiguousArrayElements, Result, Value } from '../types'
-import * as ast from '../utils/astCreator'
+import type { Context, ContiguousArrayElements, Result, Value } from '../types'
+import { arrayMapFrom } from '../utils/arrayMap'
+import assert from '../utils/assert'
+import * as ast from '../utils/ast/astCreator'
+import { isImportDeclaration } from '../utils/ast/typeGuards'
+import type * as es from '../utils/ast/types'
 import { evaluateBinaryExpression, evaluateUnaryExpression } from '../utils/operators'
 import * as rttc from '../utils/rttc'
 import * as instr from './instrCreator'
@@ -129,7 +132,11 @@
  * @param context The context to evaluate the program in.
  * @returns The result of running the ECE machine.
  */
-export function evaluate(program: es.Program, context: Context, options: IOptions): Value {
+export async function evaluate(
+  program: es.Program,
+  context: Context,
+  options: IOptions
+): Promise<Value> {
   try {
     context.runtime.isRunning = true
     context.runtime.agenda = new Agenda(program)
@@ -172,47 +179,77 @@
 function evaluateImports(
   program: es.Program,
   context: Context,
-  loadTabs: boolean,
-  checkImports: boolean
+  { loadTabs, wrapModules }: ImportTransformOptions
 ) {
-  const [importNodes] = partition(program.body, ({ type }) => type === 'ImportDeclaration') as [
-    es.ImportDeclaration[],
-    es.Statement[]
-  ]
-  const moduleFunctions: Record<string, ModuleFunctions> = {}
+  const importNodes = program.body.filter(isImportDeclaration)
 
   try {
-    for (const node of importNodes) {
-      const moduleName = node.source.value
-      if (typeof moduleName !== 'string') {
-        throw new Error(`ImportDeclarations should have string sources, got ${moduleName}`)
-      }
+    const importNodeMap = arrayMapFrom(
+      importNodes.map(node => {
+        const moduleName = node.source.value
+        assert(
+          typeof moduleName === 'string',
+          `ImportDeclarations should have string sources, got ${moduleName}`
+        )
+        return [moduleName, node]
+      })
+    )
 
-      if (!(moduleName in moduleFunctions)) {
-        context.moduleContexts[moduleName] = {
-          state: null,
-          tabs: loadTabs ? loadModuleTabs(moduleName, node) : null
+    const environment = currentEnvironment(context)
+    importNodeMap.entries().forEach(([moduleName, nodes]) => {
+      initModuleContext(moduleName, context, loadTabs, nodes[0])
+      const functions = loadModuleBundle(moduleName, context, wrapModules, nodes[0])
+      for (const node of nodes) {
+        for (const spec of node.specifiers) {
+          let importedName: string
+          switch (spec.type) {
+            case 'ImportSpecifier': {
+              importedName = spec.imported.name
+              break
+            }
+            case 'ImportDefaultSpecifier': {
+              importedName = 'default'
+              break
+            }
+            case 'ImportNamespaceSpecifier': {
+              throw new Error('Namespace imports are not supported!')
+            }
+          }
+
+          declareIdentifier(context, spec.local.name, node, environment)
+          defineVariable(context, spec.local.name, functions[importedName], true, node)
         }
-        moduleFunctions[moduleName] = loadModuleBundle(moduleName, context, node)
       }
+    })
 
-      const functions = moduleFunctions[moduleName]
-      const environment = currentEnvironment(context)
-      for (const spec of node.specifiers) {
-        if (spec.type !== 'ImportSpecifier') {
-          throw new Error(`Only ImportSpecifiers are supported, got: ${spec.type}`)
-        }
-
-        if (checkImports && !(spec.imported.name in functions)) {
-          throw new UndefinedImportError(spec.imported.name, moduleName, node)
-        }
-
-        declareIdentifier(context, spec.local.name, node, environment)
-        defineVariable(context, spec.local.name, functions[spec.imported.name], true, node)
-      }
-    }
+    // await importNodeMap.forEachAsync(async (moduleName, nodes) => {
+    //   await initModuleContextAsync(moduleName, context, loadTabs, nodes[0])
+    //   const functions = await loadModuleBundleAsync(moduleName, context, wrapModules, nodes[0])
+
+    //   for (const node of nodes) {
+    //     for (const spec of node.specifiers) {
+    //       let importedName: string
+    //       switch (spec.type) {
+    //         case 'ImportSpecifier': {
+    //           importedName = spec.imported.name
+    //           break
+    //         }
+    //         case 'ImportDefaultSpecifier': {
+    //           importedName = 'default'
+    //           break
+    //         }
+    //         case 'ImportNamespaceSpecifier': {
+    //           throw new Error('Namespace imports are not supported!')
+    //         }
+    //       }
+
+    //       declareIdentifier(context, spec.local.name, node, environment)
+    //       defineVariable(context, spec.local.name, functions[importedName], true, node)
+    //     }
+    //   }
+    // })
   } catch (error) {
-    // console.log(error)
+    // console.error(error)
     handleRuntimeError(context, error)
   }
 }
@@ -224,16 +261,15 @@
  * @param value The value of ec evaluating the program.
  * @returns The corresponding promise.
  */
-export function ECEResultPromise(context: Context, value: Value): Promise<Result> {
-  return new Promise((resolve, reject) => {
-    if (value instanceof ECEBreak) {
-      resolve({ status: 'suspended-ec-eval', context })
-    } else if (value instanceof ECError) {
-      resolve({ status: 'error' })
-    } else {
-      resolve({ status: 'finished', context, value })
-    }
-  })
+export async function ECEResultPromise(context: Context, promise: Promise<Value>): Promise<Result> {
+  const value = await promise
+  if (value instanceof ECEBreak) {
+    return { status: 'suspended-ec-eval', context }
+  } else if (value instanceof ECError) {
+    return { status: 'error' }
+  } else {
+    return { status: 'finished', context, value }
+  }
 }
 
 /**
@@ -335,7 +371,10 @@
     if (hasDeclarations(command) || hasImportDeclarations(command)) {
       const environment = createBlockEnvironment(context, 'programEnvironment')
       pushEnvironment(context, environment)
-      evaluateImports(command as unknown as es.Program, context, true, true)
+      evaluateImports(command as unknown as es.Program, context, {
+        loadTabs: true,
+        wrapModules: true
+      })
       declareFunctionsAndVariables(context, command, environment)
     }
 
@@ -451,7 +490,7 @@
     }
   },
 
   IfStatement: function (command: es.IfStatement, context: Context, agenda: Agenda, stash: Stash) {
     agenda.push(...reduceConditional(command))
   },
 
@@ -523,7 +562,7 @@
     command: es.ContinueStatement,
     context: Context,
     agenda: Agenda,
     stash: Stash
   ) {
     agenda.push(instr.contInstr(command))
   },
@@ -532,7 +571,7 @@
     command: es.BreakStatement,
     context: Context,
     agenda: Agenda,
     stash: Stash
   ) {
     agenda.push(instr.breakInstr(command))
   },
@@ -578,7 +617,7 @@
     command: es.MemberExpression,
     context: Context,
     agenda: Agenda,
     stash: Stash
   ) {
     agenda.push(instr.arrAccInstr(command))
     agenda.push(command.property)
@@ -589,7 +628,7 @@
     command: es.ConditionalExpression,
     context: Context,
     agenda: Agenda,
     stash: Stash
   ) {
     agenda.push(...reduceConditional(command))
   },
@@ -660,7 +699,7 @@
    * Instructions
    */
 
   [InstrType.RESET]: function (command: Instr, context: Context, agenda: Agenda, stash: Stash) {
     // Keep pushing reset instructions until marker is found.
     const cmdNext: AgendaItem | undefined = agenda.pop()
     if (cmdNext && (isNode(cmdNext) || cmdNext.instrType !== InstrType.MARKER)) {
@@ -947,7 +986,7 @@
     stash.push(value)
   },
 
   [InstrType.CONTINUE]: function (command: Instr, context: Context, agenda: Agenda, stash: Stash) {
     const next = agenda.pop() as AgendaItem
     if (isInstr(next) && next.instrType == InstrType.CONTINUE_MARKER) {
       // Encountered continue mark, stop popping
@@ -962,7 +1001,7 @@
 
   [InstrType.CONTINUE_MARKER]: function () {},
 
   [InstrType.BREAK]: function (command: Instr, context: Context, agenda: Agenda, stash: Stash) {
     const next = agenda.pop() as AgendaItem
     if (isInstr(next) && next.instrType == InstrType.BREAK_MARKER) {
       // Encountered break mark, stop popping