From 16ba5ad09c23cd526b9c5236d12a0cc8b223f44f Mon Sep 17 00:00:00 2001 From: Lee Yi Date: Wed, 21 Jun 2023 18:27:11 -0700 Subject: [PATCH 01/50] Update configuration files --- .gitignore | 1 + jest.setup.ts | 16 ++++++++++++++++ package.json | 3 +++ tsconfig.json | 2 ++ 4 files changed, 22 insertions(+) create mode 100644 jest.setup.ts diff --git a/.gitignore b/.gitignore index 7bb67a701..914a029d2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ node_modules dist/ .idea/ coverage/ +yarn-error.log # emacs backup files *~ diff --git a/jest.setup.ts b/jest.setup.ts new file mode 100644 index 000000000..a8f733d04 --- /dev/null +++ b/jest.setup.ts @@ -0,0 +1,16 @@ +import { SourceMapConsumer } from "source-map" + +jest.mock('lodash', () => ({ + ...jest.requireActual('lodash'), + memoize: jest.fn((x: any) => x), +})) + +jest.mock('./src/modules/moduleLoaderAsync') +jest.mock('./src/modules/moduleLoader') + +// @ts-ignore +SourceMapConsumer.initialize({ + 'lib/mappings.wasm': 'https://unpkg.com/source-map@0.7.3/lib/mappings.wasm' +}) + +global.fetch = jest.fn() \ No newline at end of file diff --git a/package.json b/package.json index e0db7316f..5add1481c 100644 --- a/package.json +++ b/package.json @@ -115,6 +115,9 @@ "/src/scm-slang", "/src/py-slang/" ], + "setupFilesAfterEnv": [ + "/jest.setup.ts" + ], "reporters": [ "default", [ diff --git a/tsconfig.json b/tsconfig.json index 2bd9aa318..fc3fdf3bf 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -29,6 +29,8 @@ "exclude": [ "src/stdlib/metacircular-interpreter", "src/stdlib/**/*.js", + "src/__tests__/**", + "jest.*.*s", "node_modules", "dist", "sicp_publish" From c1a51dd6c71dbcf2b709cd9bb7d72579da044845 Mon Sep 17 00:00:00 2001 From: Lee Yi Date: Wed, 21 Jun 2023 18:31:59 -0700 Subject: [PATCH 02/50] Add async module loading code and tests --- src/modules/__tests__/moduleLoader.ts | 17 +- src/modules/__tests__/moduleLoaderAsync.ts | 176 +++++++++++++++++++++ src/modules/moduleLoaderAsync.ts | 102 ++++++++++++ src/modules/moduleTypes.ts | 44 +++++- 4 files changed, 322 insertions(+), 17 deletions(-) create mode 100644 src/modules/__tests__/moduleLoaderAsync.ts create mode 100644 src/modules/moduleLoaderAsync.ts diff --git a/src/modules/__tests__/moduleLoader.ts b/src/modules/__tests__/moduleLoader.ts index d191d8fac..9d7471ba9 100644 --- a/src/modules/__tests__/moduleLoader.ts +++ b/src/modules/__tests__/moduleLoader.ts @@ -1,11 +1,9 @@ import { createEmptyContext } from '../../createContext' -import { ModuleConnectionError, ModuleInternalError } from '../../errors/moduleErrors' import { Variant } from '../../types' import { stripIndent } from '../../utils/formatters' -import * as moduleLoader from '../moduleLoader' +import { ModuleConnectionError, ModuleInternalError } from '../errors' -// Mock memoize function from lodash -jest.mock('lodash', () => ({ memoize: jest.fn(func => func) })) +const moduleLoader = jest.requireActual('../moduleLoader') /** * Mock XMLHttpRequest from jsdom environment @@ -80,7 +78,7 @@ describe('Testing modules/moduleLoader.ts in a jsdom environment', () => { const sampleResponse = `(function () {'use strict'; function index(_params) { return { }; } return index; })();` const correctUrl = moduleLoader.MODULES_STATIC_URL + `/bundles/${validModuleBundle}.js` const mockedXMLHttpRequest = mockXMLHttpRequest({ responseText: sampleResponse }) - const response = moduleLoader.memoizedGetModuleFile(validModuleBundle, 'bundle') + const response = moduleLoader.memoizedGetBundle(validModuleBundle) expect(mockedXMLHttpRequest.open).toHaveBeenCalledTimes(1) expect(mockedXMLHttpRequest.open).toHaveBeenCalledWith('GET', correctUrl, false) expect(mockedXMLHttpRequest.send).toHaveBeenCalledTimes(1) @@ -93,7 +91,7 @@ describe('Testing modules/moduleLoader.ts in a jsdom environment', () => { const sampleResponse = `(function (React) {});` const correctUrl = moduleLoader.MODULES_STATIC_URL + `/tabs/${validModuleTab}.js` const mockedXMLHttpRequest = mockXMLHttpRequest({ responseText: sampleResponse }) - const response = moduleLoader.memoizedGetModuleFile(validModuleTab, 'tab') + const response = moduleLoader.memoizedGetTab(validModuleTab) expect(mockedXMLHttpRequest.open).toHaveBeenCalledTimes(1) expect(mockedXMLHttpRequest.open).toHaveBeenCalledWith('GET', correctUrl, false) expect(mockedXMLHttpRequest.send).toHaveBeenCalledTimes(1) @@ -112,7 +110,8 @@ describe('Testing modules/moduleLoader.ts in a jsdom environment', () => { mockXMLHttpRequest({ responseText: sampleResponse }) const loadedBundle = moduleLoader.loadModuleBundle( 'module', - createEmptyContext(1, Variant.DEFAULT, []) + createEmptyContext(1, Variant.DEFAULT, []), + false ) expect(loadedBundle.make_empty_array()).toEqual([]) }) @@ -123,7 +122,7 @@ describe('Testing modules/moduleLoader.ts in a jsdom environment', () => { const wrongModuleText = `export function es6_function(params) {};` mockXMLHttpRequest({ responseText: wrongModuleText }) expect(() => - moduleLoader.loadModuleBundle('module', createEmptyContext(1, Variant.DEFAULT, [])) + moduleLoader.loadModuleBundle('module', createEmptyContext(1, Variant.DEFAULT, []), false) ).toThrow(ModuleInternalError) }) @@ -183,4 +182,4 @@ describe('Testing modules/moduleLoader.ts in a jsdom environment', () => { expect(mockedXMLHttpRequest3.send).toHaveBeenCalledTimes(1) expect(mockedXMLHttpRequest3.send).toHaveBeenCalledWith(null) }) -}) +}) \ No newline at end of file diff --git a/src/modules/__tests__/moduleLoaderAsync.ts b/src/modules/__tests__/moduleLoaderAsync.ts new file mode 100644 index 000000000..c3f07b4e7 --- /dev/null +++ b/src/modules/__tests__/moduleLoaderAsync.ts @@ -0,0 +1,176 @@ +import type { MockedFunction } from 'jest-mock' + +import { mockContext } from '../../mocks/context' +import { Chapter, Variant } from '../../types' +import { ModuleConnectionError, ModuleInternalError } from '../errors' +import { MODULES_STATIC_URL } from '../moduleLoader' +import type * as moduleLoaderType from '../moduleLoaderAsync' + +const moduleLoader: typeof moduleLoaderType = jest.requireActual('../moduleLoaderAsync') + +const mockedFetch = fetch as MockedFunction +function mockResponse(response: string, status: number = 200) { + mockedFetch.mockResolvedValueOnce({ + text: () => Promise.resolve(response), + json: () => Promise.resolve(JSON.parse(response)), + status + } as any) +} + +async function expectSuccess( + correctUrl: string, + expectedResp: T, + func: () => Promise, + callCount: number = 1 +) { + const response = await func() + + expect(fetch).toHaveBeenCalledTimes(callCount) + + const [calledUrl, callOpts] = mockedFetch.mock.calls[0] + expect(calledUrl).toEqual(correctUrl) + expect(callOpts).toMatchObject({ method: 'GET' }) + if (typeof expectedResp === 'string') { + expect(response).toEqual(expectedResp) + } else { + expect(response).toMatchObject(expectedResp) + } +} + +async function expectFailure(sampleUrl: string, expectedErr: any, func: () => Promise) { + await expect(() => func()).rejects.toBeInstanceOf(expectedErr) + + const [calledUrl, callOpts] = mockedFetch.mock.calls[0] + expect(calledUrl).toEqual(sampleUrl) + expect(callOpts).toMatchObject({ method: 'GET' }) +} + +beforeEach(() => { + jest.clearAllMocks() +}) + +describe('Test httpGetAsync', () => { + test('Http GET function httpGetAsync() works correctly', async () => { + const sampleResponse = `{ "repeat": { "contents": ["Repeat"] } }` + const sampleUrl = 'https://www.example.com' + + mockResponse(sampleResponse) + await expectSuccess(sampleUrl, sampleResponse, () => + moduleLoader.httpGetAsync(sampleUrl, 'text') + ) + }) + + test('Http GET function httpGetAsync() throws ModuleConnectionError', async () => { + const sampleUrl = 'https://www.example.com' + mockResponse('', 404) + + await expectFailure(sampleUrl, ModuleConnectionError, () => + moduleLoader.httpGetAsync(sampleUrl, 'text') + ) + }) + + test('Http GET modules manifest correctly', async () => { + const sampleResponse = `{ "repeat": { "contents": ["Repeat"] } }` + const correctUrl = MODULES_STATIC_URL + `/modules.json` + mockResponse(sampleResponse) + + await expectSuccess(correctUrl, JSON.parse(sampleResponse), () => + moduleLoader.memoizedGetModuleManifestAsync() + ) + }) + + test('Http GET returns objects when "json" is specified', async () => { + const sampleResponse = `{ "repeat": { "contents": ["Repeat"] } }` + const correctUrl = MODULES_STATIC_URL + `/modules.json` + mockResponse(sampleResponse) + const result = await moduleLoader.httpGetAsync(correctUrl, 'json') + expect(result).toMatchObject(JSON.parse(sampleResponse)) + }) +}) + +describe('Test bundle loading', () => { + const sampleModuleName = 'valid_module' + const sampleModuleUrl = MODULES_STATIC_URL + `/bundles/${sampleModuleName}.js` + // const sampleManifest = `{ "${sampleModuleName}": { "tabs": [] } }` + + // beforeEach(() => { + // mockResponse(sampleManifest) + // }) + + test('Http GET module bundle correctly', async () => { + const sampleResponse = `require => ({ foo: () => 'foo' })` + mockResponse(sampleResponse) + + const bundleText = await moduleLoader.memoizedGetModuleBundleAsync(sampleModuleName) + + expect(fetch).toHaveBeenCalledTimes(1) + + const [calledUrl, callOpts] = mockedFetch.mock.calls[0] + expect(calledUrl).toEqual(sampleModuleUrl) + expect(callOpts).toMatchObject({ method: 'GET' }) + expect(bundleText).toEqual(sampleResponse) + }) + + test('Loading a correctly implemented module bundle', async () => { + const context = mockContext(Chapter.SOURCE_4, Variant.DEFAULT) + const sampleResponse = `require => ({ foo: () => 'foo' })` + mockResponse(sampleResponse) + + const loadedModule = await moduleLoader.loadModuleBundleAsync(sampleModuleName, context, false) + + expect(loadedModule.foo()).toEqual('foo') + expect(fetch).toHaveBeenCalledTimes(1) + + const [calledUrl, callOpts] = mockedFetch.mock.calls[0] + expect(calledUrl).toEqual(sampleModuleUrl) + expect(callOpts).toMatchObject({ method: 'GET' }) + }) + + test('Loading a wrongly implemented module bundle throws ModuleInternalError', async () => { + const context = mockContext(Chapter.SOURCE_4, Variant.DEFAULT) + const wrongModuleText = `export function es6_function(params) {};` + mockResponse(wrongModuleText) + await expect(() => + moduleLoader.loadModuleBundleAsync(sampleModuleName, context, true) + ).rejects.toBeInstanceOf(ModuleInternalError) + + expect(fetch).toHaveBeenCalledTimes(1) + }) +}) + +describe('Test tab loading', () => { + const sampleTabUrl = `${MODULES_STATIC_URL}/tabs/Tab1.js` + const sampleManifest = `{ "one_module": { "tabs": ["Tab1", "Tab2"] } }` + + test('Http GET module tab correctly', async () => { + const sampleResponse = `require => ({ foo: () => 'foo' })` + mockResponse(sampleResponse) + + const bundleText = await moduleLoader.memoizedGetModuleTabAsync('Tab1') + + expect(fetch).toHaveBeenCalledTimes(1) + + const [calledUrl, callOpts] = mockedFetch.mock.calls[0] + expect(calledUrl).toEqual(sampleTabUrl) + expect(callOpts).toMatchObject({ method: 'GET' }) + expect(bundleText).toEqual(sampleResponse) + }) + + test('Loading a wrongly implemented tab throws ModuleInternalError', async () => { + mockResponse(sampleManifest) + + const wrongTabText = `export function es6_function(params) {};` + mockResponse(wrongTabText) + mockResponse(wrongTabText) + + await expect(() => moduleLoader.loadModuleTabsAsync('one_module')).rejects.toBeInstanceOf( + ModuleInternalError + ) + expect(fetch).toHaveBeenCalledTimes(3) + + const [[call0Url], [call1Url], [call2Url]] = mockedFetch.mock.calls + expect(call0Url).toEqual(`${MODULES_STATIC_URL}/modules.json`) + expect(call1Url).toEqual(`${MODULES_STATIC_URL}/tabs/Tab1.js`) + expect(call2Url).toEqual(`${MODULES_STATIC_URL}/tabs/Tab2.js`) + }) +}) \ No newline at end of file diff --git a/src/modules/moduleLoaderAsync.ts b/src/modules/moduleLoaderAsync.ts new file mode 100644 index 000000000..deb2c8557 --- /dev/null +++ b/src/modules/moduleLoaderAsync.ts @@ -0,0 +1,102 @@ +import type { Node } from 'estree' +import { memoize } from 'lodash' + +import type { Context } from '..' +import { timeoutPromise } from '../utils' +import { wrapSourceModule } from '../utils/operators' +import { ModuleConnectionError, ModuleInternalError, ModuleNotFoundError } from './errors' +import { MODULES_STATIC_URL } from './moduleLoader' +import type { ModuleBundle, ModuleDocumentation, ModuleManifest } from './moduleTypes' +import { getRequireProvider } from './requireProvider' + +export function httpGetAsync(path: string, type: 'json'): Promise +export function httpGetAsync(path: string, type: 'text'): Promise +export async function httpGetAsync(path: string, type: 'json' | 'text') { + const resp = await fetch(path, { + method: 'GET' + }) + + if (resp.status !== 200 && resp.status !== 304) { + throw new ModuleConnectionError() + } + + const promise = type === 'text' ? resp.text() : resp.json() + if (typeof window === 'undefined') { + return timeoutPromise(promise, 10000) + } + return promise +} + +/** + * Send a HTTP GET request to the modules endpoint to retrieve the manifest + * @return Modules + */ +export const memoizedGetModuleManifestAsync = memoize(getModuleManifestAsync) +function getModuleManifestAsync(): Promise { + return httpGetAsync(`${MODULES_STATIC_URL}/modules.json`, 'json') as Promise +} + +async function checkModuleExists(moduleName: string, node?: Node) { + const modules = await memoizedGetModuleManifestAsync() + // Check if the module exists + if (!(moduleName in modules)) throw new ModuleNotFoundError(moduleName, node) + + return modules[moduleName] +} + +export const memoizedGetModuleBundleAsync = memoize(getModuleBundleAsync) +async function getModuleBundleAsync(moduleName: string): Promise { + return httpGetAsync(`${MODULES_STATIC_URL}/bundles/${moduleName}.js`, 'text') +} + +export const memoizedGetModuleTabAsync = memoize(getModuleTabAsync) +function getModuleTabAsync(tabName: string): Promise { + return httpGetAsync(`${MODULES_STATIC_URL}/tabs/${tabName}.js`, 'text') +} + +export const memoizedGetModuleDocsAsync = memoize(getModuleDocsAsync) +async function getModuleDocsAsync(moduleName: string): Promise { + try { + const result = await httpGetAsync(`${MODULES_STATIC_URL}/jsons/${moduleName}.json`, 'json') + return result as ModuleDocumentation + } catch (error) { + console.warn(`Failed to load documentation for ${moduleName}:`, error) + return null + } +} + +export async function loadModuleTabsAsync(moduleName: string, node?: Node) { + const moduleInfo = await checkModuleExists(moduleName, node) + + // Load the tabs for the current module + return Promise.all( + moduleInfo.tabs.map(async path => { + const rawTabFile = await memoizedGetModuleTabAsync(path) + try { + return eval(rawTabFile) + } catch (error) { + // console.error('tab error:', error); + throw new ModuleInternalError(path, error, node) + } + }) + ) +} + +export async function loadModuleBundleAsync( + moduleName: string, + context: Context, + wrapModule: boolean, + node?: Node +) { + // await checkModuleExists(moduleName, node) + const moduleText = await memoizedGetModuleBundleAsync(moduleName) + try { + const moduleBundle: ModuleBundle = eval(moduleText) + + if (wrapModule) return wrapSourceModule(moduleName, moduleBundle, getRequireProvider(context)) + return moduleBundle(getRequireProvider(context)) + } catch (error) { + // console.error("bundle error: ", error) + throw new ModuleInternalError(moduleName, error, node) + } +} \ No newline at end of file diff --git a/src/modules/moduleTypes.ts b/src/modules/moduleTypes.ts index 974f27de4..baeccc50d 100644 --- a/src/modules/moduleTypes.ts +++ b/src/modules/moduleTypes.ts @@ -1,15 +1,43 @@ import type { RequireProvider } from './requireProvider' -export type Modules = { - [module: string]: { - tabs: string[] - } +export type ModuleManifest = Record +export type ModuleBundle = (require: RequireProvider) => ModuleFunctions +export type ModuleFunctions = Record +export type ModuleDocumentation = Record + +export type ImportTransformOptions = { + /** Set to true to load module tabs */ + loadTabs: boolean + + /** + * Wrapping a Source module involves creating nice toString outputs for + * each of its functions. If this behaviour is desired, set this to true + */ + wrapModules: boolean } -export type ModuleBundle = (require: RequireProvider) => ModuleFunctions +export type ImportResolutionOptions = { + /** + * Set this to true if directories should be resolved + * @example + * ``` + * import { a } from './dir0'; // will resolve to 'dir0/index' + * ``` + */ + resolveDirectories: boolean + + /** + * Pass null to enforce strict file names: `'./dir0/file'` will resolve to exactly that path. + * Otherwise pass an array of file extensions `['js', 'ts']`. For example, if `./dir0/file` is not located, + * it will then search for that file with the given extension, e.g. `./dir0/file.js` + */ + resolveExtensions: string[] | null -export type ModuleFunctions = { - [functionName: string]: Function + /** + * Set this to true to enforce that imports from modules must be of + * defined symbols + */ + allowUndefinedImports: boolean } -export type ModuleDocumentation = Record +export type ImportOptions = ImportResolutionOptions & ImportTransformOptions \ No newline at end of file From f8e33117eedcc52ff8bc73208d35c74143530750 Mon Sep 17 00:00:00 2001 From: Lee Yi Date: Wed, 21 Jun 2023 18:33:41 -0700 Subject: [PATCH 03/50] Add general utils --- src/utils/index.ts | 50 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 src/utils/index.ts diff --git a/src/utils/index.ts b/src/utils/index.ts new file mode 100644 index 000000000..3ff5b77f8 --- /dev/null +++ b/src/utils/index.ts @@ -0,0 +1,50 @@ +/** + * A form of Array.reduce, but using an async reducer + * It doesn't reduce everything asynchronously, but rather waits + * for each problem to resolve sequentially + */ +export async function reduceAsync Promise>( + arr: Iterable, + reducer: Reducer, + init: U +) { + let result: U = init + for (const each of arr) { + result = await reducer(result, each) + } + return result +} + +export async function mapObjectAsync< + T extends Record, + Mapper extends (key: keyof T, value: T[keyof T]) => Promise +>(obj: T, mapper: Mapper) { + const promises = Object.entries(obj).map(async ([key, value]) => [key, await mapper(key, value)]) + const results = await Promise.all(promises) + + return results.reduce( + (res, [k, v]) => ({ + ...res, + [k]: v + }), + {} as Record>> + ) +} + +/** + * Wrap an existing promise that will throw an error after the given timeout + * duration + */ +export const timeoutPromise = (promise: Promise, duration: number) => + new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error('Timeout!')), duration) + promise + .then(result => { + clearTimeout(timeout) + resolve(result) + }) + .catch(err => { + clearTimeout(timeout) + reject(err) + }) + }) \ No newline at end of file From 40b1eb3f2b8757fa4a6beb17621b43e8f215438e Mon Sep 17 00:00:00 2001 From: Lee Yi Date: Wed, 21 Jun 2023 18:33:57 -0700 Subject: [PATCH 04/50] Add helper types --- src/types.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/types.ts b/src/types.ts index 335ff668b..e95d673d7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -473,3 +473,21 @@ export type TypeEnvironment = { declKindMap: Map typeAliasMap: Map }[] + +/** + * Helper type to recursively make properties that are also objects + * partial + * + * By default, `Partial>` is equivalent to `Array`. For this type, `Array` will be + * transformed to Array> instead + */ +export type RecursivePartial = T extends Array + ? Array> + : T extends Record + ? Partial<{ + [K in keyof T]: RecursivePartial + }> + : T + +export type Replace>> = Omit & + Replacer \ No newline at end of file From d6395125035de7a85822f4dafcad824a9d2c9bd6 Mon Sep 17 00:00:00 2001 From: Lee Yi Date: Wed, 21 Jun 2023 18:34:32 -0700 Subject: [PATCH 05/50] Move preprocessor to modules folder --- .../preprocessor}/__tests__/directedGraph.ts | 0 .../preprocessor}/__tests__/errorMessages.ts | 6 +++--- .../preprocessor}/__tests__/preprocessor.ts | 10 +++++----- .../__tests__/transformers/hoistAndMergeImports.ts | 6 +++--- .../__tests__/transformers/removeExports.ts | 6 +++--- .../transformers/removeNonSourceModuleImports.ts | 6 +++--- .../transformProgramToFunctionDeclaration.ts | 8 ++++---- .../preprocessor}/__tests__/utils.ts | 2 +- .../preprocessor}/constructors/baseConstructors.ts | 0 .../constructors/contextSpecificConstructors.ts | 2 +- .../preprocessor}/directedGraph.ts | 0 .../preprocessor}/filePaths.ts | 2 +- .../preprocessor}/preprocessor.ts | 10 +++++----- .../preprocessor}/transformers/hoistAndMergeImports.ts | 0 .../preprocessor}/transformers/removeExports.ts | 2 +- .../transformers/removeNonSourceModuleImports.ts | 2 +- .../transformProgramToFunctionDeclaration.ts | 2 +- .../preprocessor}/typeGuards.ts | 0 18 files changed, 32 insertions(+), 32 deletions(-) rename src/{localImports => modules/preprocessor}/__tests__/directedGraph.ts (100%) rename src/{localImports => modules/preprocessor}/__tests__/errorMessages.ts (97%) rename src/{localImports => modules/preprocessor}/__tests__/preprocessor.ts (98%) rename src/{localImports => modules/preprocessor}/__tests__/transformers/hoistAndMergeImports.ts (92%) rename src/{localImports => modules/preprocessor}/__tests__/transformers/removeExports.ts (96%) rename src/{localImports => modules/preprocessor}/__tests__/transformers/removeNonSourceModuleImports.ts (92%) rename src/{localImports => modules/preprocessor}/__tests__/transformers/transformProgramToFunctionDeclaration.ts (98%) rename src/{localImports => modules/preprocessor}/__tests__/utils.ts (97%) rename src/{localImports => modules/preprocessor}/constructors/baseConstructors.ts (100%) rename src/{localImports => modules/preprocessor}/constructors/contextSpecificConstructors.ts (97%) rename src/{localImports => modules/preprocessor}/directedGraph.ts (100%) rename src/{localImports => modules/preprocessor}/filePaths.ts (98%) rename src/{localImports => modules/preprocessor}/preprocessor.ts (97%) rename src/{localImports => modules/preprocessor}/transformers/hoistAndMergeImports.ts (100%) rename src/{localImports => modules/preprocessor}/transformers/removeExports.ts (98%) rename src/{localImports => modules/preprocessor}/transformers/removeNonSourceModuleImports.ts (98%) rename src/{localImports => modules/preprocessor}/transformers/transformProgramToFunctionDeclaration.ts (99%) rename src/{localImports => modules/preprocessor}/typeGuards.ts (100%) diff --git a/src/localImports/__tests__/directedGraph.ts b/src/modules/preprocessor/__tests__/directedGraph.ts similarity index 100% rename from src/localImports/__tests__/directedGraph.ts rename to src/modules/preprocessor/__tests__/directedGraph.ts diff --git a/src/localImports/__tests__/errorMessages.ts b/src/modules/preprocessor/__tests__/errorMessages.ts similarity index 97% rename from src/localImports/__tests__/errorMessages.ts rename to src/modules/preprocessor/__tests__/errorMessages.ts index 31c912c4b..d636c9ed0 100644 --- a/src/localImports/__tests__/errorMessages.ts +++ b/src/modules/preprocessor/__tests__/errorMessages.ts @@ -1,6 +1,6 @@ -import { parseError, runFilesInContext } from '../../index' -import { mockContext } from '../../mocks/context' -import { Chapter, Variant } from '../../types' +import { parseError, runFilesInContext } from '../../../index' +import { mockContext } from '../../../mocks/context' +import { Chapter, Variant } from '../../../types' describe('syntax errors', () => { let context = mockContext(Chapter.SOURCE_4) diff --git a/src/localImports/__tests__/preprocessor.ts b/src/modules/preprocessor/__tests__/preprocessor.ts similarity index 98% rename from src/localImports/__tests__/preprocessor.ts rename to src/modules/preprocessor/__tests__/preprocessor.ts index 628d32766..df3f6bbf1 100644 --- a/src/localImports/__tests__/preprocessor.ts +++ b/src/modules/preprocessor/__tests__/preprocessor.ts @@ -1,10 +1,10 @@ import es from 'estree' -import { parseError } from '../../index' -import { mockContext } from '../../mocks/context' -import { parse } from '../../parser/parser' -import { accessExportFunctionName, defaultExportLookupName } from '../../stdlib/localImport.prelude' -import { Chapter } from '../../types' +import { parseError } from '../../../index' +import { mockContext } from '../../../mocks/context' +import { parse } from '../../../parser/parser' +import { accessExportFunctionName, defaultExportLookupName } from '../../../stdlib/localImport.prelude' +import { Chapter } from '../../../types' import preprocessFileImports, { getImportedLocalModulePaths } from '../preprocessor' import { parseCodeError, stripLocationInfo } from './utils' diff --git a/src/localImports/__tests__/transformers/hoistAndMergeImports.ts b/src/modules/preprocessor/__tests__/transformers/hoistAndMergeImports.ts similarity index 92% rename from src/localImports/__tests__/transformers/hoistAndMergeImports.ts rename to src/modules/preprocessor/__tests__/transformers/hoistAndMergeImports.ts index 3a492ab24..c06022b51 100644 --- a/src/localImports/__tests__/transformers/hoistAndMergeImports.ts +++ b/src/modules/preprocessor/__tests__/transformers/hoistAndMergeImports.ts @@ -1,6 +1,6 @@ -import { mockContext } from '../../../mocks/context' -import { parse } from '../../../parser/parser' -import { Chapter } from '../../../types' +import { mockContext } from '../../../../mocks/context' +import { parse } from '../../../../parser/parser' +import { Chapter } from '../../../../types' import { hoistAndMergeImports } from '../../transformers/hoistAndMergeImports' import { parseCodeError, stripLocationInfo } from '../utils' diff --git a/src/localImports/__tests__/transformers/removeExports.ts b/src/modules/preprocessor/__tests__/transformers/removeExports.ts similarity index 96% rename from src/localImports/__tests__/transformers/removeExports.ts rename to src/modules/preprocessor/__tests__/transformers/removeExports.ts index 656969a00..9a2c96eb8 100644 --- a/src/localImports/__tests__/transformers/removeExports.ts +++ b/src/modules/preprocessor/__tests__/transformers/removeExports.ts @@ -1,6 +1,6 @@ -import { mockContext } from '../../../mocks/context' -import { parse } from '../../../parser/parser' -import { Chapter } from '../../../types' +import { mockContext } from '../../../../mocks/context' +import { parse } from '../../../../parser/parser' +import { Chapter } from '../../../../types' import { removeExports } from '../../transformers/removeExports' import { parseCodeError, stripLocationInfo } from '../utils' diff --git a/src/localImports/__tests__/transformers/removeNonSourceModuleImports.ts b/src/modules/preprocessor/__tests__/transformers/removeNonSourceModuleImports.ts similarity index 92% rename from src/localImports/__tests__/transformers/removeNonSourceModuleImports.ts rename to src/modules/preprocessor/__tests__/transformers/removeNonSourceModuleImports.ts index 7f6ebd8cb..012d63fd6 100644 --- a/src/localImports/__tests__/transformers/removeNonSourceModuleImports.ts +++ b/src/modules/preprocessor/__tests__/transformers/removeNonSourceModuleImports.ts @@ -1,6 +1,6 @@ -import { mockContext } from '../../../mocks/context' -import { parse } from '../../../parser/parser' -import { Chapter } from '../../../types' +import { mockContext } from '../../../../mocks/context' +import { parse } from '../../../../parser/parser' +import { Chapter } from '../../../../types' import { removeNonSourceModuleImports } from '../../transformers/removeNonSourceModuleImports' import { parseCodeError, stripLocationInfo } from '../utils' diff --git a/src/localImports/__tests__/transformers/transformProgramToFunctionDeclaration.ts b/src/modules/preprocessor/__tests__/transformers/transformProgramToFunctionDeclaration.ts similarity index 98% rename from src/localImports/__tests__/transformers/transformProgramToFunctionDeclaration.ts rename to src/modules/preprocessor/__tests__/transformers/transformProgramToFunctionDeclaration.ts index b3221ec50..50df9c4b0 100644 --- a/src/localImports/__tests__/transformers/transformProgramToFunctionDeclaration.ts +++ b/src/modules/preprocessor/__tests__/transformers/transformProgramToFunctionDeclaration.ts @@ -1,7 +1,7 @@ -import { mockContext } from '../../../mocks/context' -import { parse } from '../../../parser/parser' -import { defaultExportLookupName } from '../../../stdlib/localImport.prelude' -import { Chapter } from '../../../types' +import { mockContext } from '../../../../mocks/context' +import { parse } from '../../../../parser/parser' +import { defaultExportLookupName } from '../../../../stdlib/localImport.prelude' +import { Chapter } from '../../../../types' import { transformProgramToFunctionDeclaration } from '../../transformers/transformProgramToFunctionDeclaration' import { parseCodeError, stripLocationInfo } from '../utils' diff --git a/src/localImports/__tests__/utils.ts b/src/modules/preprocessor/__tests__/utils.ts similarity index 97% rename from src/localImports/__tests__/utils.ts rename to src/modules/preprocessor/__tests__/utils.ts index d2ad6fbc4..25c1861f2 100644 --- a/src/localImports/__tests__/utils.ts +++ b/src/modules/preprocessor/__tests__/utils.ts @@ -1,6 +1,6 @@ import es from 'estree' -import { full, simple } from '../../utils/walkers' +import { full, simple } from '../../../utils/walkers' export const parseCodeError = new Error('Unable to parse code') diff --git a/src/localImports/constructors/baseConstructors.ts b/src/modules/preprocessor/constructors/baseConstructors.ts similarity index 100% rename from src/localImports/constructors/baseConstructors.ts rename to src/modules/preprocessor/constructors/baseConstructors.ts diff --git a/src/localImports/constructors/contextSpecificConstructors.ts b/src/modules/preprocessor/constructors/contextSpecificConstructors.ts similarity index 97% rename from src/localImports/constructors/contextSpecificConstructors.ts rename to src/modules/preprocessor/constructors/contextSpecificConstructors.ts index 4b8c9d8e4..aa16ec4b5 100644 --- a/src/localImports/constructors/contextSpecificConstructors.ts +++ b/src/modules/preprocessor/constructors/contextSpecificConstructors.ts @@ -1,6 +1,6 @@ import es from 'estree' -import { accessExportFunctionName } from '../../stdlib/localImport.prelude' +import { accessExportFunctionName } from '../../../stdlib/localImport.prelude' import { createCallExpression, createIdentifier, diff --git a/src/localImports/directedGraph.ts b/src/modules/preprocessor/directedGraph.ts similarity index 100% rename from src/localImports/directedGraph.ts rename to src/modules/preprocessor/directedGraph.ts diff --git a/src/localImports/filePaths.ts b/src/modules/preprocessor/filePaths.ts similarity index 98% rename from src/localImports/filePaths.ts rename to src/modules/preprocessor/filePaths.ts index 50cd1c4f0..3476cd4fe 100644 --- a/src/localImports/filePaths.ts +++ b/src/modules/preprocessor/filePaths.ts @@ -2,7 +2,7 @@ import { ConsecutiveSlashesInFilePathError, IllegalCharInFilePathError, InvalidFilePathError -} from '../errors/localImportErrors' +} from '../../errors/localImportErrors' /** * Maps non-alphanumeric characters that are legal in file paths diff --git a/src/localImports/preprocessor.ts b/src/modules/preprocessor/preprocessor.ts similarity index 97% rename from src/localImports/preprocessor.ts rename to src/modules/preprocessor/preprocessor.ts index d4f6464c1..c09d7a458 100644 --- a/src/localImports/preprocessor.ts +++ b/src/modules/preprocessor/preprocessor.ts @@ -1,11 +1,11 @@ import es from 'estree' import * as path from 'path' -import { CannotFindModuleError, CircularImportError } from '../errors/localImportErrors' -import { parse } from '../parser/parser' -import { AcornOptions } from '../parser/types' -import { Context } from '../types' -import { isIdentifier } from '../utils/rttc' +import { CannotFindModuleError, CircularImportError } from '../../errors/localImportErrors' +import { parse } from '../../parser/parser' +import { AcornOptions } from '../../parser/types' +import { Context } from '../../types' +import { isIdentifier } from '../../utils/rttc' import { createInvokedFunctionResultVariableDeclaration } from './constructors/contextSpecificConstructors' import { DirectedGraph } from './directedGraph' import { diff --git a/src/localImports/transformers/hoistAndMergeImports.ts b/src/modules/preprocessor/transformers/hoistAndMergeImports.ts similarity index 100% rename from src/localImports/transformers/hoistAndMergeImports.ts rename to src/modules/preprocessor/transformers/hoistAndMergeImports.ts diff --git a/src/localImports/transformers/removeExports.ts b/src/modules/preprocessor/transformers/removeExports.ts similarity index 98% rename from src/localImports/transformers/removeExports.ts rename to src/modules/preprocessor/transformers/removeExports.ts index 7e81e9367..d85355631 100644 --- a/src/localImports/transformers/removeExports.ts +++ b/src/modules/preprocessor/transformers/removeExports.ts @@ -1,6 +1,6 @@ import es from 'estree' -import { ancestor } from '../../utils/walkers' +import { ancestor } from '../../../utils/walkers' import { isDeclaration } from '../typeGuards' /** diff --git a/src/localImports/transformers/removeNonSourceModuleImports.ts b/src/modules/preprocessor/transformers/removeNonSourceModuleImports.ts similarity index 98% rename from src/localImports/transformers/removeNonSourceModuleImports.ts rename to src/modules/preprocessor/transformers/removeNonSourceModuleImports.ts index 0aa3f99e9..43fb8a2d4 100644 --- a/src/localImports/transformers/removeNonSourceModuleImports.ts +++ b/src/modules/preprocessor/transformers/removeNonSourceModuleImports.ts @@ -1,6 +1,6 @@ import es from 'estree' -import { ancestor } from '../../utils/walkers' +import { ancestor } from '../../../utils/walkers' import { isFilePath } from '../filePaths' /** diff --git a/src/localImports/transformers/transformProgramToFunctionDeclaration.ts b/src/modules/preprocessor/transformers/transformProgramToFunctionDeclaration.ts similarity index 99% rename from src/localImports/transformers/transformProgramToFunctionDeclaration.ts rename to src/modules/preprocessor/transformers/transformProgramToFunctionDeclaration.ts index 98a179106..f6fc871d3 100644 --- a/src/localImports/transformers/transformProgramToFunctionDeclaration.ts +++ b/src/modules/preprocessor/transformers/transformProgramToFunctionDeclaration.ts @@ -1,7 +1,7 @@ import es from 'estree' import * as path from 'path' -import { defaultExportLookupName } from '../../stdlib/localImport.prelude' +import { defaultExportLookupName } from '../../../stdlib/localImport.prelude' import { createFunctionDeclaration, createIdentifier, diff --git a/src/localImports/typeGuards.ts b/src/modules/preprocessor/typeGuards.ts similarity index 100% rename from src/localImports/typeGuards.ts rename to src/modules/preprocessor/typeGuards.ts From 6d81ad489183f3638eb27858a33b527efb6c6349 Mon Sep 17 00:00:00 2001 From: Lee Yi Date: Wed, 21 Jun 2023 18:35:00 -0700 Subject: [PATCH 06/50] Relocate ast creator to new utils folder --- src/ec-evaluator/interpreter.ts | 2 +- src/ec-evaluator/utils.ts | 2 +- src/errors/localImportErrors.ts | 2 +- src/gpu/gpu.ts | 2 +- src/gpu/transfomer.ts | 2 +- src/index.ts | 4 +- src/infiniteLoops/instrument.ts | 2 +- src/infiniteLoops/runtime.ts | 2 +- src/infiniteLoops/state.ts | 2 +- src/infiniteLoops/symbolic.ts | 2 +- src/interpreter/closure.ts | 2 +- src/interpreter/interpreter-non-det.ts | 2 +- src/interpreter/interpreter.ts | 4 +- src/lazy/lazy.ts | 2 +- src/modules/errors.ts | 194 ++++++++++++++++++++++++- src/runner/__tests__/runners.ts | 2 +- src/runner/errors.ts | 2 +- src/runner/fullJSRunner.ts | 4 +- src/runner/sourceRunner.ts | 2 +- src/stepper/lib.ts | 2 +- src/stepper/stepper.ts | 2 +- src/transpiler/transpiler.ts | 2 +- src/utils/{ => ast}/astCreator.ts | 2 +- src/utils/ast/types.ts | 73 ++++++++++ src/utils/operators.ts | 4 +- src/validator/__tests__/validator.ts | 2 +- src/validator/validator.ts | 2 +- src/vm/svml-compiler.ts | 2 +- src/vm/svml-machine.ts | 2 +- 29 files changed, 292 insertions(+), 37 deletions(-) rename src/utils/{ => ast}/astCreator.ts (99%) create mode 100644 src/utils/ast/types.ts diff --git a/src/ec-evaluator/interpreter.ts b/src/ec-evaluator/interpreter.ts index 6f18c6798..a60220d65 100644 --- a/src/ec-evaluator/interpreter.ts +++ b/src/ec-evaluator/interpreter.ts @@ -19,7 +19,7 @@ import { loadModuleBundle, loadModuleTabs } from '../modules/moduleLoader' import { ModuleFunctions } from '../modules/moduleTypes' import { checkEditorBreakpoints } from '../stdlib/inspector' import { Context, ContiguousArrayElements, Result, Value } from '../types' -import * as ast from '../utils/astCreator' +import * as ast from '../utils/ast/astCreator' import { evaluateBinaryExpression, evaluateUnaryExpression } from '../utils/operators' import * as rttc from '../utils/rttc' import * as instr from './instrCreator' diff --git a/src/ec-evaluator/utils.ts b/src/ec-evaluator/utils.ts index 23c6bbe56..6760fe9d8 100644 --- a/src/ec-evaluator/utils.ts +++ b/src/ec-evaluator/utils.ts @@ -6,7 +6,7 @@ import * as errors from '../errors/errors' import { RuntimeSourceError } from '../errors/runtimeSourceError' import Closure from '../interpreter/closure' import { Environment, Frame, Value } from '../types' -import * as ast from '../utils/astCreator' +import * as ast from '../utils/ast/astCreator' import * as instr from './instrCreator' import { Agenda } from './interpreter' import { AgendaItem, AppInstr, AssmtInstr, Instr, InstrType } from './types' diff --git a/src/errors/localImportErrors.ts b/src/errors/localImportErrors.ts index 3d4a14b74..c2b627a68 100644 --- a/src/errors/localImportErrors.ts +++ b/src/errors/localImportErrors.ts @@ -1,5 +1,5 @@ import { UNKNOWN_LOCATION } from '../constants' -import { nonAlphanumericCharEncoding } from '../localImports/filePaths' +import { nonAlphanumericCharEncoding } from '../modules/preprocessor/filePaths' import { ErrorSeverity, ErrorType, SourceError } from '../types' export abstract class InvalidFilePathError implements SourceError { diff --git a/src/gpu/gpu.ts b/src/gpu/gpu.ts index 73c4f39e2..663c317d3 100644 --- a/src/gpu/gpu.ts +++ b/src/gpu/gpu.ts @@ -1,6 +1,6 @@ import * as es from 'estree' -import * as create from '../utils/astCreator' +import * as create from '../utils/ast/astCreator' import { getIdentifiersInProgram } from '../utils/uniqueIds' import GPUTransformer from './transfomer' diff --git a/src/gpu/transfomer.ts b/src/gpu/transfomer.ts index 8a9d39d2d..c64f0460a 100644 --- a/src/gpu/transfomer.ts +++ b/src/gpu/transfomer.ts @@ -1,6 +1,6 @@ import * as es from 'estree' -import * as create from '../utils/astCreator' +import * as create from '../utils/ast/astCreator' import { ancestor, make, simple } from '../utils/walkers' import GPUBodyVerifier from './verification/bodyVerifier' import GPULoopVerifier from './verification/loopVerifier' diff --git a/src/index.ts b/src/index.ts index 825f0cd3f..8df2d5264 100644 --- a/src/index.ts +++ b/src/index.ts @@ -29,8 +29,8 @@ import * as es from 'estree' import { ECEResultPromise, resumeEvaluate } from './ec-evaluator/interpreter' import { CannotFindModuleError } from './errors/localImportErrors' -import { validateFilePath } from './localImports/filePaths' -import preprocessFileImports from './localImports/preprocessor' +import { validateFilePath } from './modules/preprocessor/filePaths' +import preprocessFileImports from './modules/preprocessor/preprocessor' import { getKeywords, getProgramNames, NameDeclaration } from './name-extractor' import { parse } from './parser/parser' import { parseWithComments } from './parser/utils' diff --git a/src/infiniteLoops/instrument.ts b/src/infiniteLoops/instrument.ts index c8c98d0e8..b4d1a2656 100644 --- a/src/infiniteLoops/instrument.ts +++ b/src/infiniteLoops/instrument.ts @@ -2,7 +2,7 @@ import { generate } from 'astring' import * as es from 'estree' import { transformImportDeclarations } from '../transpiler/transpiler' -import * as create from '../utils/astCreator' +import * as create from '../utils/ast/astCreator' import { recursive, simple, WalkerCallback } from '../utils/walkers' // transforms AST of program diff --git a/src/infiniteLoops/runtime.ts b/src/infiniteLoops/runtime.ts index a9b81b95f..49443a3a8 100644 --- a/src/infiniteLoops/runtime.ts +++ b/src/infiniteLoops/runtime.ts @@ -6,7 +6,7 @@ import { getRequireProvider } from '../modules/requireProvider' import { parse } from '../parser/parser' import * as stdList from '../stdlib/list' import { Chapter, Variant } from '../types' -import * as create from '../utils/astCreator' +import * as create from '../utils/ast/astCreator' import { checkForInfiniteLoop } from './detect' import { InfiniteLoopError } from './errors' import { diff --git a/src/infiniteLoops/state.ts b/src/infiniteLoops/state.ts index b2ff7fff6..b32918311 100644 --- a/src/infiniteLoops/state.ts +++ b/src/infiniteLoops/state.ts @@ -1,7 +1,7 @@ import { generate } from 'astring' import * as es from 'estree' -import { identifier } from '../utils/astCreator' +import { identifier } from '../utils/ast/astCreator' import * as sym from './symbolic' // Object + functions called during runtime to check for infinite loops diff --git a/src/infiniteLoops/symbolic.ts b/src/infiniteLoops/symbolic.ts index 89e3449cd..07d580b66 100644 --- a/src/infiniteLoops/symbolic.ts +++ b/src/infiniteLoops/symbolic.ts @@ -1,6 +1,6 @@ import * as es from 'estree' -import * as create from '../utils/astCreator' +import * as create from '../utils/ast/astCreator' import { evaluateBinaryExpression, evaluateUnaryExpression } from '../utils/operators' // data structure for symbolic + hybrid values diff --git a/src/interpreter/closure.ts b/src/interpreter/closure.ts index 9519c3140..954a81aab 100644 --- a/src/interpreter/closure.ts +++ b/src/interpreter/closure.ts @@ -8,7 +8,7 @@ import { callExpression, identifier, returnStatement -} from '../utils/astCreator' +} from '../utils/ast/astCreator' import { dummyLocation } from '../utils/dummyAstCreator' import { apply } from './interpreter' diff --git a/src/interpreter/interpreter-non-det.ts b/src/interpreter/interpreter-non-det.ts index 570566d3f..faf61b0f1 100644 --- a/src/interpreter/interpreter-non-det.ts +++ b/src/interpreter/interpreter-non-det.ts @@ -6,7 +6,7 @@ import { CUT, UNKNOWN_LOCATION } from '../constants' import * as errors from '../errors/errors' import { RuntimeSourceError } from '../errors/runtimeSourceError' import { Context, Environment, Frame, Value } from '../types' -import { conditionalExpression, literal, primitive } from '../utils/astCreator' +import { conditionalExpression, literal, primitive } from '../utils/ast/astCreator' import { evaluateBinaryExpression, evaluateUnaryExpression } from '../utils/operators' import * as rttc from '../utils/rttc' import Closure from './closure' diff --git a/src/interpreter/interpreter.ts b/src/interpreter/interpreter.ts index afb66958e..aa968f2e3 100644 --- a/src/interpreter/interpreter.ts +++ b/src/interpreter/interpreter.ts @@ -11,8 +11,8 @@ import { loadModuleBundle, loadModuleTabs } from '../modules/moduleLoader' import { ModuleFunctions } from '../modules/moduleTypes' import { checkEditorBreakpoints } from '../stdlib/inspector' import { Context, ContiguousArrayElements, Environment, Frame, Value, Variant } from '../types' -import * as create from '../utils/astCreator' -import { conditionalExpression, literal, primitive } from '../utils/astCreator' +import * as create from '../utils/ast/astCreator' +import { conditionalExpression, literal, primitive } from '../utils/ast/astCreator' import { evaluateBinaryExpression, evaluateUnaryExpression } from '../utils/operators' import * as rttc from '../utils/rttc' import Closure from './closure' diff --git a/src/lazy/lazy.ts b/src/lazy/lazy.ts index 6dedcf419..8d02437ef 100644 --- a/src/lazy/lazy.ts +++ b/src/lazy/lazy.ts @@ -1,6 +1,6 @@ import * as es from 'estree' -import * as create from '../utils/astCreator' +import * as create from '../utils/ast/astCreator' import { getIdentifiersInProgram } from '../utils/uniqueIds' import { simple } from '../utils/walkers' diff --git a/src/modules/errors.ts b/src/modules/errors.ts index fa3ebaef0..f79a12e44 100644 --- a/src/modules/errors.ts +++ b/src/modules/errors.ts @@ -1,21 +1,203 @@ -import type { ImportDeclaration } from 'estree' - +import { UNKNOWN_LOCATION } from '../constants' import { RuntimeSourceError } from '../errors/runtimeSourceError' +import { ErrorSeverity, ErrorType, SourceError } from '../types' +import type * as es from '../utils/ast/types' +import { nonAlphanumericCharEncoding } from './preprocessor/filePaths' + +export class ModuleConnectionError extends RuntimeSourceError { + private static message: string = `Unable to get modules.` + private static elaboration: string = `You should check your Internet connection, and ensure you have used the correct module path.` + constructor(public readonly error?: any, node?: es.Node) { + super(node) + } + + public explain() { + return ModuleConnectionError.message + } + + public elaborate() { + return ModuleConnectionError.elaboration + } +} + +export class ModuleNotFoundError extends RuntimeSourceError { + constructor(public moduleName: string, node?: es.Node) { + super(node) + } + + public explain() { + return `Module '${this.moduleName}' not found.` + } + + public elaborate() { + return 'You should check your import declarations, and ensure that all are valid modules.' + } +} + +export class ModuleInternalError extends RuntimeSourceError { + constructor(public moduleName: string, public error?: any, node?: es.Node) { + super(node) + } + + public explain() { + return `Error(s) occured when executing the module "${this.moduleName}".` + } + + public elaborate() { + return ` + You may need to contact with the author for this module to fix this error. + ` + } +} -export class UndefinedImportError extends RuntimeSourceError { +export abstract class UndefinedImportErrorBase extends RuntimeSourceError { constructor( - public readonly symbol: string, public readonly moduleName: string, - node?: ImportDeclaration + node?: es.ModuleDeclarationWithSource | es.ExportSpecifier | es.ImportSpecifiers ) { super(node) } + public elaborate(): string { + return "Check your imports and make sure what you're trying to import exists!" + } +} + +export class UndefinedImportError extends UndefinedImportErrorBase { + constructor( + public readonly symbol: string, + moduleName: string, + node?: es.ImportSpecifier | es.ImportDefaultSpecifier | es.ExportSpecifier + ) { + super(moduleName, node) + } + public explain(): string { return `'${this.moduleName}' does not contain a definition for '${this.symbol}'` } +} + +export class UndefinedDefaultImportError extends UndefinedImportErrorBase { + constructor( + moduleName: string, + node?: es.ImportSpecifier | es.ImportDefaultSpecifier | es.ExportSpecifier + ) { + super(moduleName, node) + } + + public explain(): string { + return `'${this.moduleName}' does not contain a default export!` + } +} + +export class UndefinedNamespaceImportError extends UndefinedImportErrorBase { + constructor(moduleName: string, node?: es.ImportNamespaceSpecifier | es.ExportAllDeclaration) { + super(moduleName, node) + } + + public explain(): string { + return `'${this.moduleName}' does not export any symbols!` + } +} + +export abstract class ReexportErrorBase implements SourceError { + public severity = ErrorSeverity.ERROR + public type = ErrorType.RUNTIME + public readonly location: es.SourceLocation + public readonly sourceString: string + + constructor(public readonly modulePath: string, public readonly locations: es.SourceLocation[]) { + this.location = locations[0] ?? UNKNOWN_LOCATION + this.sourceString = locations + .map(({ start: { line, column } }) => `(${line}:${column})`) + .join(', ') + } + + public abstract explain(): string + public abstract elaborate(): string +} + +export class ReexportSymbolError extends ReexportErrorBase { + constructor(modulePath: string, public readonly symbol: string, locations: es.SourceLocation[]) { + super(modulePath, locations) + } + + public explain(): string { + return `Multiple export definitions for the symbol '${this.symbol}' at (${this.sourceString})` + } public elaborate(): string { - return "Check your imports and make sure what you're trying to import exists!" + return 'Check that you are not exporting the same symbol more than once' + } +} + +export class ReexportDefaultError extends ReexportErrorBase { + constructor(modulePath: string, locations: es.SourceLocation[]) { + super(modulePath, locations) + } + + public explain(): string { + return `Multiple default export definitions for the symbol at (${this.sourceString})` + } + + public elaborate(): string { + return 'Check that there is only a single default export' + } +} + +export class CircularImportError implements SourceError { + public type = ErrorType.TYPE + public severity = ErrorSeverity.ERROR + public location = UNKNOWN_LOCATION + + constructor(public filePathsInCycle: string[]) {} + + public explain() { + // We need to reverse the file paths in the cycle so that the + // semantics of "'/a.js' -> '/b.js'" is "'/a.js' imports '/b.js'". + const formattedCycle = this.filePathsInCycle + .map(filePath => `'${filePath}'`) + .reverse() + .join(' -> ') + return `Circular import detected: ${formattedCycle}.` + } + + public elaborate() { + return 'Break the circular import cycle by removing imports from any of the offending files.' } } + +export abstract class InvalidFilePathError implements SourceError { + public type = ErrorType.TYPE + public severity = ErrorSeverity.ERROR + public location = UNKNOWN_LOCATION + + constructor(public filePath: string) {} + + abstract explain(): string + + abstract elaborate(): string +} + +export class IllegalCharInFilePathError extends InvalidFilePathError { + public explain() { + const validNonAlphanumericChars = Object.keys(nonAlphanumericCharEncoding) + .map(char => `'${char}'`) + .join(', ') + return `File path '${this.filePath}' must only contain alphanumeric chars and/or ${validNonAlphanumericChars}.` + } + + public elaborate() { + return 'Rename the offending file path to only use valid chars.' + } +} + +export class ConsecutiveSlashesInFilePathError extends InvalidFilePathError { + public explain() { + return `File path '${this.filePath}' cannot contain consecutive slashes '//'.` + } + + public elaborate() { + return 'Remove consecutive slashes from the offending file path.' + } +} \ No newline at end of file diff --git a/src/runner/__tests__/runners.ts b/src/runner/__tests__/runners.ts index 5e698fb7b..a20b709ae 100644 --- a/src/runner/__tests__/runners.ts +++ b/src/runner/__tests__/runners.ts @@ -3,7 +3,7 @@ import { UndefinedVariable } from '../../errors/errors' import { mockContext } from '../../mocks/context' import { FatalSyntaxError } from '../../parser/errors' import { Chapter, Finished, Variant } from '../../types' -import { locationDummyNode } from '../../utils/astCreator' +import { locationDummyNode } from '../../utils/ast/astCreator' import { CodeSnippetTestCase } from '../../utils/testing' import { htmlErrorHandlingScript } from '../htmlRunner' diff --git a/src/runner/errors.ts b/src/runner/errors.ts index 4f5b2308c..6c1844582 100644 --- a/src/runner/errors.ts +++ b/src/runner/errors.ts @@ -3,7 +3,7 @@ import { NullableMappedPosition, RawSourceMap, SourceMapConsumer } from 'source- import { UNKNOWN_LOCATION } from '../constants' import { ConstAssignment, ExceptionError, UndefinedVariable } from '../errors/errors' import { SourceError } from '../types' -import { locationDummyNode } from '../utils/astCreator' +import { locationDummyNode } from '../utils/ast/astCreator' enum BrowserType { Chrome = 'Chrome', diff --git a/src/runner/fullJSRunner.ts b/src/runner/fullJSRunner.ts index 4924d6de5..abedc11ee 100644 --- a/src/runner/fullJSRunner.ts +++ b/src/runner/fullJSRunner.ts @@ -6,12 +6,12 @@ import { RawSourceMap } from 'source-map' import { IOptions, Result } from '..' import { NATIVE_STORAGE_ID } from '../constants' import { RuntimeSourceError } from '../errors/runtimeSourceError' -import { hoistAndMergeImports } from '../localImports/transformers/hoistAndMergeImports' +import { hoistAndMergeImports } from '../modules/preprocessor/transformers/hoistAndMergeImports' import { getRequireProvider, RequireProvider } from '../modules/requireProvider' import { parse } from '../parser/parser' import { evallerReplacer, getBuiltins, transpile } from '../transpiler/transpiler' import type { Context, NativeStorage } from '../types' -import * as create from '../utils/astCreator' +import * as create from '../utils/ast/astCreator' import { toSourceError } from './errors' import { appendModulesToContext, resolvedErrorPromise } from './utils' diff --git a/src/runner/sourceRunner.ts b/src/runner/sourceRunner.ts index 406f31cb8..701679889 100644 --- a/src/runner/sourceRunner.ts +++ b/src/runner/sourceRunner.ts @@ -15,7 +15,7 @@ import { testForInfiniteLoop } from '../infiniteLoops/runtime' import { evaluateProgram as evaluate } from '../interpreter/interpreter' import { nonDetEvaluate } from '../interpreter/interpreter-non-det' import { transpileToLazy } from '../lazy/lazy' -import preprocessFileImports from '../localImports/preprocessor' +import preprocessFileImports from '../modules/preprocessor/preprocessor' import { getRequireProvider } from '../modules/requireProvider' import { parse } from '../parser/parser' import { AsyncScheduler, NonDetScheduler, PreemptiveScheduler } from '../schedulers' diff --git a/src/stepper/lib.ts b/src/stepper/lib.ts index 741bbac17..7495f9c49 100644 --- a/src/stepper/lib.ts +++ b/src/stepper/lib.ts @@ -2,7 +2,7 @@ import * as es from 'estree' import * as misc from '../stdlib/misc' import { substituterNodes } from '../types' -import * as ast from '../utils/astCreator' +import * as ast from '../utils/ast/astCreator' import { nodeToValue, valueToExpression } from './converter' import { codify } from './stepper' import { isBuiltinFunction, isNumber } from './util' diff --git a/src/stepper/stepper.ts b/src/stepper/stepper.ts index 5fba42e83..8ef1d8303 100644 --- a/src/stepper/stepper.ts +++ b/src/stepper/stepper.ts @@ -11,7 +11,7 @@ import { FunctionDeclarationExpression, substituterNodes } from '../types' -import * as ast from '../utils/astCreator' +import * as ast from '../utils/ast/astCreator' import { dummyBlockExpression, dummyBlockStatement, diff --git a/src/transpiler/transpiler.ts b/src/transpiler/transpiler.ts index df6865089..64404d0b0 100644 --- a/src/transpiler/transpiler.ts +++ b/src/transpiler/transpiler.ts @@ -10,7 +10,7 @@ import { UndefinedImportError } from '../modules/errors' import { memoizedGetModuleFile, memoizedloadModuleDocs } from '../modules/moduleLoader' import { ModuleDocumentation } from '../modules/moduleTypes' import { AllowedDeclarations, Chapter, Context, NativeStorage, Variant } from '../types' -import * as create from '../utils/astCreator' +import * as create from '../utils/ast/astCreator' import { getIdentifiersInNativeStorage, getIdentifiersInProgram, diff --git a/src/utils/astCreator.ts b/src/utils/ast/astCreator.ts similarity index 99% rename from src/utils/astCreator.ts rename to src/utils/ast/astCreator.ts index 78f3e3761..0e21cd776 100644 --- a/src/utils/astCreator.ts +++ b/src/utils/ast/astCreator.ts @@ -1,6 +1,6 @@ import * as es from 'estree' -import { AllowedDeclarations, BlockExpression, FunctionDeclarationExpression } from '../types' +import { AllowedDeclarations, BlockExpression, FunctionDeclarationExpression } from '../../types' export const getVariableDecarationName = (decl: es.VariableDeclaration) => (decl.declarations[0].id as es.Identifier).name diff --git a/src/utils/ast/types.ts b/src/utils/ast/types.ts new file mode 100644 index 000000000..1617df7b3 --- /dev/null +++ b/src/utils/ast/types.ts @@ -0,0 +1,73 @@ +// Reexport the type definitions from estree, but with several helper types as well + +import type * as es from 'estree' + +import type { Replace } from '../../types' + +export type ExportDeclaration = Exclude +export type ImportSpecifiers = + | es.ImportSpecifier + | es.ImportDefaultSpecifier + | es.ImportNamespaceSpecifier +export type BlockArrowFunctionExpression = Replace< + es.ArrowFunctionExpression, + { + expression: false + body: es.BlockStatement + } +> + +export type FunctionNode = + | es.FunctionDeclaration + | es.FunctionExpression + | es.ArrowFunctionExpression + +export type FunctionDeclarationWithId = Replace +export type ClassDeclarationWithId = Replace + +export type ForStatements = es.ForInStatement | es.ForOfStatement | es.ForStatement +export type LoopNode = es.WhileStatement | ForStatements + +export type ExportNamedVariableDeclaration = Replace< + es.ExportNamedDeclaration, + { + declaration: es.VariableDeclaration + source: null + specifiers: never[] + } +> + +export type ExportNamedFunctionDeclaration = Replace< + es.ExportNamedDeclaration, + { + declaration: FunctionDeclarationWithId + source: null + specifiers: never[] + } +> + +export type ExportNamedLocalDeclaration = Replace< + es.ExportNamedDeclaration, + { + source: null + declaration: null + } +> + +/** + * Represents exports of the form `export { a, b } from './a.js';` + */ +export type ExportNamedDeclarationWithSource = Replace< + es.ExportNamedDeclaration, + { + declaration: null + source: es.Literal + } +> + +export type ModuleDeclarationWithSource = + | es.ImportDeclaration + | es.ExportAllDeclaration + | ExportNamedDeclarationWithSource + +export * from 'estree' \ No newline at end of file diff --git a/src/utils/operators.ts b/src/utils/operators.ts index 321ee281e..eaadb8b30 100644 --- a/src/utils/operators.ts +++ b/src/utils/operators.ts @@ -15,8 +15,8 @@ import { import { ModuleBundle, ModuleFunctions } from '../modules/moduleTypes' import { RequireProvider } from '../modules/requireProvider' import { Chapter, NativeStorage, Thunk } from '../types' -import { callExpression, locationDummyNode } from './astCreator' -import * as create from './astCreator' +import { callExpression, locationDummyNode } from './ast/astCreator' +import * as create from './ast/astCreator' import { makeWrapper } from './makeWrapper' import * as rttc from './rttc' diff --git a/src/validator/__tests__/validator.ts b/src/validator/__tests__/validator.ts index ac3ca5092..bc9a8880e 100644 --- a/src/validator/__tests__/validator.ts +++ b/src/validator/__tests__/validator.ts @@ -3,7 +3,7 @@ import * as es from 'estree' import { mockContext } from '../../mocks/context' import { parse } from '../../parser/parser' import { Chapter, NodeWithInferredType } from '../../types' -import { getVariableDecarationName } from '../../utils/astCreator' +import { getVariableDecarationName } from '../../utils/ast/astCreator' import { stripIndent } from '../../utils/formatters' import { expectParsedError } from '../../utils/testing' import { simple } from '../../utils/walkers' diff --git a/src/validator/validator.ts b/src/validator/validator.ts index 7892e75f0..67f1db938 100644 --- a/src/validator/validator.ts +++ b/src/validator/validator.ts @@ -3,7 +3,7 @@ import * as es from 'estree' import { ConstAssignment } from '../errors/errors' import { NoAssignmentToForVariable } from '../errors/validityErrors' import { Context, NodeWithInferredType } from '../types' -import { getVariableDecarationName } from '../utils/astCreator' +import { getVariableDecarationName } from '../utils/ast/astCreator' import { ancestor, base, FullWalkerCallback } from '../utils/walkers' class Declaration { diff --git a/src/vm/svml-compiler.ts b/src/vm/svml-compiler.ts index ed791bf4d..fa0addfd8 100644 --- a/src/vm/svml-compiler.ts +++ b/src/vm/svml-compiler.ts @@ -11,7 +11,7 @@ import { vmPrelude } from '../stdlib/vm.prelude' import { Context, ContiguousArrayElements } from '../types' -import * as create from '../utils/astCreator' +import * as create from '../utils/ast/astCreator' import { recursive, simple } from '../utils/walkers' import OpCodes from './opcodes' diff --git a/src/vm/svml-machine.ts b/src/vm/svml-machine.ts index 064126ff5..370f4eec1 100644 --- a/src/vm/svml-machine.ts +++ b/src/vm/svml-machine.ts @@ -9,7 +9,7 @@ import { VARARGS_NUM_ARGS } from '../stdlib/vm.prelude' import { Context } from '../types' -import { locationDummyNode } from '../utils/astCreator' +import { locationDummyNode } from '../utils/ast/astCreator' import { stringify } from '../utils/stringify' import OpCodes from './opcodes' import { Address, Instruction, Program, SVMFunction } from './svml-compiler' From 8acfd79550b34fb436baca3453afe534b51fc0ad Mon Sep 17 00:00:00 2001 From: Lee Yi Date: Wed, 21 Jun 2023 18:39:13 -0700 Subject: [PATCH 07/50] Add AST utils and relocate walkers --- src/finder.ts | 2 +- src/gpu/transfomer.ts | 2 +- src/gpu/verification/bodyVerifier.ts | 2 +- src/index.ts | 2 +- src/infiniteLoops/detect.ts | 2 +- src/infiniteLoops/instrument.ts | 2 +- src/lazy/lazy.ts | 2 +- src/modules/preprocessor/__tests__/utils.ts | 2 +- src/modules/preprocessor/analyzer.ts | 382 ++++++++++++++++++ .../transformers/removeExports.ts | 2 +- .../removeNonSourceModuleImports.ts | 2 +- src/parser/source/index.ts | 2 +- src/runner/utils.ts | 2 +- src/scope-refactoring.ts | 2 +- src/transpiler/transpiler.ts | 2 +- src/utils/arrayMap.ts | 25 ++ src/utils/assert.ts | 27 ++ src/utils/ast/astUtils.ts | 106 +++++ src/utils/ast/typeGuards.ts | 84 ++++ src/utils/{ => ast}/walkers.ts | 0 src/utils/uniqueIds.ts | 2 +- src/validator/__tests__/validator.ts | 2 +- src/validator/validator.ts | 2 +- src/vm/svml-compiler.ts | 2 +- 24 files changed, 642 insertions(+), 18 deletions(-) create mode 100644 src/modules/preprocessor/analyzer.ts create mode 100644 src/utils/arrayMap.ts create mode 100644 src/utils/assert.ts create mode 100644 src/utils/ast/astUtils.ts create mode 100644 src/utils/ast/typeGuards.ts rename src/utils/{ => ast}/walkers.ts (100%) diff --git a/src/finder.ts b/src/finder.ts index 87b56cab8..cbb000457 100644 --- a/src/finder.ts +++ b/src/finder.ts @@ -18,7 +18,7 @@ import { FullWalkerCallback, recursive, WalkerCallback -} from './utils/walkers' +} from './utils/ast/walkers' // Finds the innermost node that matches the given location export function findIdentifierNode( diff --git a/src/gpu/transfomer.ts b/src/gpu/transfomer.ts index c64f0460a..e9f664d19 100644 --- a/src/gpu/transfomer.ts +++ b/src/gpu/transfomer.ts @@ -1,7 +1,7 @@ import * as es from 'estree' import * as create from '../utils/ast/astCreator' -import { ancestor, make, simple } from '../utils/walkers' +import { ancestor, make, simple } from '../utils/ast/walkers' import GPUBodyVerifier from './verification/bodyVerifier' import GPULoopVerifier from './verification/loopVerifier' diff --git a/src/gpu/verification/bodyVerifier.ts b/src/gpu/verification/bodyVerifier.ts index 706440102..c29abc3c0 100644 --- a/src/gpu/verification/bodyVerifier.ts +++ b/src/gpu/verification/bodyVerifier.ts @@ -1,6 +1,6 @@ import * as es from 'estree' -import { make, simple } from '../../utils/walkers' +import { make, simple } from '../../utils/ast/walkers' /* * GPU Body verifier helps to ensure the body is parallelizable diff --git a/src/index.ts b/src/index.ts index 8df2d5264..25e16e9d0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -21,7 +21,7 @@ import { SVMProgram, Variant } from './types' -import { findNodeAt } from './utils/walkers' +import { findNodeAt } from './utils/ast/walkers' import { assemble } from './vm/svml-assembler' import { compileToIns } from './vm/svml-compiler' export { SourceDocumentation } from './editors/ace/docTooltip' diff --git a/src/infiniteLoops/detect.ts b/src/infiniteLoops/detect.ts index af9b98416..a29de175f 100644 --- a/src/infiniteLoops/detect.ts +++ b/src/infiniteLoops/detect.ts @@ -1,7 +1,7 @@ import { generate } from 'astring' import * as es from 'estree' -import { simple } from '../utils/walkers' +import { simple } from '../utils/ast/walkers' import { InfiniteLoopError, InfiniteLoopErrorType } from './errors' import { getOriginalName } from './instrument' import * as st from './state' diff --git a/src/infiniteLoops/instrument.ts b/src/infiniteLoops/instrument.ts index b4d1a2656..68c72a1fe 100644 --- a/src/infiniteLoops/instrument.ts +++ b/src/infiniteLoops/instrument.ts @@ -3,7 +3,7 @@ import * as es from 'estree' import { transformImportDeclarations } from '../transpiler/transpiler' import * as create from '../utils/ast/astCreator' -import { recursive, simple, WalkerCallback } from '../utils/walkers' +import { recursive, simple, WalkerCallback } from '../utils/ast/walkers' // transforms AST of program const globalIds = { diff --git a/src/lazy/lazy.ts b/src/lazy/lazy.ts index 8d02437ef..6ca59de1a 100644 --- a/src/lazy/lazy.ts +++ b/src/lazy/lazy.ts @@ -2,7 +2,7 @@ import * as es from 'estree' import * as create from '../utils/ast/astCreator' import { getIdentifiersInProgram } from '../utils/uniqueIds' -import { simple } from '../utils/walkers' +import { simple } from '../utils/ast/walkers' const lazyPrimitives = new Set(['makeLazyFunction', 'wrapLazyCallee', 'forceIt', 'delayIt']) diff --git a/src/modules/preprocessor/__tests__/utils.ts b/src/modules/preprocessor/__tests__/utils.ts index 25c1861f2..85b9d798b 100644 --- a/src/modules/preprocessor/__tests__/utils.ts +++ b/src/modules/preprocessor/__tests__/utils.ts @@ -1,6 +1,6 @@ import es from 'estree' -import { full, simple } from '../../../utils/walkers' +import { full, simple } from '../../../utils/ast/walkers' export const parseCodeError = new Error('Unable to parse code') diff --git a/src/modules/preprocessor/analyzer.ts b/src/modules/preprocessor/analyzer.ts new file mode 100644 index 000000000..85fcdc7d8 --- /dev/null +++ b/src/modules/preprocessor/analyzer.ts @@ -0,0 +1,382 @@ + +import { UNKNOWN_LOCATION } from '../../constants' +import { + ModuleInternalError, + ReexportDefaultError, + ReexportSymbolError, + UndefinedDefaultImportError, + UndefinedImportError, + UndefinedNamespaceImportError +} from '../../modules/errors' +import { reduceAsync } from '../../utils' +import ArrayMap from '../../utils/arrayMap' +import assert from '../../utils/assert' +import * as create from '../../utils/ast/astCreator' +import { extractIdsFromPattern, processExportNamedDeclaration } from '../../utils/ast/astUtils' +import { isSourceImport } from '../../utils/ast/typeGuards' +import type * as es from '../../utils/ast/types' +import { memoizedGetModuleDocsAsync } from '../moduleLoaderAsync' + +type ExportRecord = { + /** + * The name of the symbol defined by its source + */ + symbolName: string + + /** + * The actual source in which the symbol is defined + */ + source: string + + loc: es.SourceLocation +} + +/** + * An abstraction of the `Set` type. When `allowUndefinedImports` is true, + * the set is replaced with an object that will never throw an error for any kind + * of imported symbol + */ +type ExportSymbolsRecord = { + has: (symbol: string) => boolean + readonly size: number + [Symbol.iterator]: () => Iterator +} + +/** + * An abstraction of the `Map` type. When `allowUndefinedImports` is true, + * the set is replaced with an object that will ensure no errors are thrown for any kind + * of imported symbol + */ +type ExportSourceMap = { + get: (symbol: string) => ExportRecord | undefined + set: (symbol: string, value: ExportRecord) => void + keys: () => Iterable +} + +const validateDefaultImport = ( + spec: es.ImportDefaultSpecifier | es.ExportSpecifier | es.ImportSpecifier, + sourcePath: string, + modExported: ExportSymbolsRecord +) => { + if (!modExported.has('default')) { + throw new UndefinedDefaultImportError(sourcePath, spec) + } +} + +const validateImport = ( + spec: es.ImportSpecifier | es.ImportDefaultSpecifier | es.ExportSpecifier, + sourcePath: string, + modExported: ExportSymbolsRecord +) => { + let symbol: string + switch (spec.type) { + case 'ExportSpecifier': { + symbol = spec.local.name + break + } + case 'ImportSpecifier': { + symbol = spec.imported.name + break + } + case 'ImportDefaultSpecifier': { + symbol = 'default' + break + } + } + + if (symbol === 'default') { + validateDefaultImport(spec, sourcePath, modExported) + } else if (!modExported.has(symbol)) { + throw new UndefinedImportError(symbol, sourcePath, spec) + } +} + +const validateNamespaceImport = ( + spec: es.ImportNamespaceSpecifier | es.ExportAllDeclaration, + sourcePath: string, + modExported: ExportSymbolsRecord +) => { + if (modExported.size === 0) { + throw new UndefinedNamespaceImportError(sourcePath, spec) + } +} + +/** + * Check for undefined imports, and also for symbols that have multiple export + * definitions, and also resolve export and import directives to their sources + */ +export default async function analyzeImportsAndExports( + programs: Record, + topoOrder: string[], + allowUndefinedImports: boolean +) { + const exportMap: Record = {} + + /** + The idea behind this function is to resolve indirect exports + For example + ``` + // a.js + export const a = "a"; + // b.js + export { a as b } from './a.js' + ``` + + We want to change the following import statement `import { b } from './b.js'` to + `import { a } from './a.js', since the `export` declaration in `b.js` just serves + as a redirection and doesn't affect code behaviour + */ + function resolveSymbol(source: string, desiredSymbol: string): [string, string] { + let symbolName: string + let newSource: string + let loc: es.SourceLocation + + // So for each exported symbol, we return the path to the file where it is actually + // defined and the name it was defined with (since exports can have aliases) + // Kind of like a UFDS, where the roots of each set are symbols that are defined within + // its own file, or imports from Source modules + + // eslint-disable-next-line prefer-const + ;({ source: newSource, symbolName, loc } = exportMap[source].get(desiredSymbol)!) + if (isSourceImport(source) || newSource === source) return [newSource, symbolName] + ;[newSource, symbolName] = resolveSymbol(newSource, symbolName) + exportMap[source].set(desiredSymbol, { source: newSource, symbolName, loc }) + return [newSource, symbolName] + } + + const getDocs = async ( + node: es.ModuleDeclarationWithSource + ): Promise<[ExportSymbolsRecord, string]> => { + const path = node.source!.value as string + + if (allowUndefinedImports) { + exportMap[path] = { + get: (symbol: string) => ({ + source: path, + symbolName: symbol, + loc: UNKNOWN_LOCATION + }), + set: () => {}, + keys: () => [''] + } + + // When undefined imports are allowed, we substitute the list of exported + // symbols for an object that behaves like a set but always returns true when + // `has` is queried + return [ + { + has: () => true, + [Symbol.iterator]: () => ({ next: () => ({ done: true, value: null }) }), + size: 9999 + }, + path + ] + } + + if (!(path in exportMap)) { + // Because modules are loaded in topological order, the exported symbols for a local + // module should be loaded by the time they are needed + // So we can assume that it is the documentation for a Source module that needs to be + // loaded here + assert( + isSourceImport(path), + `Trying to load: ${path}, local modules should already have been loaded in topological order` + ) + + const docs = await memoizedGetModuleDocsAsync(path) + if (!docs) { + throw new ModuleInternalError(path, `Failed to load documentation for ${path}`) + } + exportMap[path] = new Map( + Object.keys(docs).map(symbol => [ + symbol, + { source: path, symbolName: symbol, loc: UNKNOWN_LOCATION } + ]) + ) + } + return [new Set(exportMap[path].keys()), path] + } + + const newImportDeclaration = ( + source: string, + local: es.Identifier, + imported: string + ): es.ImportDeclaration => ({ + type: 'ImportDeclaration', + source: create.literal(source), + specifiers: [ + imported === 'default' + ? { + type: 'ImportDefaultSpecifier', + local + } + : { + type: 'ImportSpecifier', + local, + imported: create.identifier(imported) + } + ] + }) + + const newPrograms: Record = {} + for (const moduleName of topoOrder) { + const program = programs[moduleName] + const exportedSymbols = new ArrayMap() + + const newBody = await reduceAsync( + program.body, + async (body, node) => { + switch (node.type) { + case 'ImportDeclaration': { + const [exports, source] = await getDocs(node) + const newDecls = node.specifiers.map(spec => { + switch (spec.type) { + case 'ImportDefaultSpecifier': + case 'ImportSpecifier': { + if (!allowUndefinedImports) validateImport(spec, source, exports) + + const desiredSymbol = + spec.type === 'ImportSpecifier' ? spec.imported.name : 'default' + const [newSource, symbolName] = resolveSymbol(source, desiredSymbol) + return newImportDeclaration(newSource, spec.local, symbolName) + } + case 'ImportNamespaceSpecifier': { + throw new Error('Namespace imports are not supported!') + // validateNamespaceImport(spec, source, exports) + // return { + // ...node, + // specifiers: [spec] + // } + } + } + }) + return [...body, ...newDecls] + } + case 'ExportDefaultDeclaration': { + exportedSymbols.add('default', { + source: moduleName, + symbolName: 'default', + loc: node.loc! + }) + return [...body, node] + } + case 'ExportNamedDeclaration': { + return await processExportNamedDeclaration(node, { + withVarDecl: async ({ declarations }) => { + for (const { id } of declarations) { + extractIdsFromPattern(id).forEach(({ name }) => { + exportedSymbols.add(name, { + source: moduleName, + symbolName: name, + loc: id.loc! + }) + }) + } + return [...body, node] + }, + withFunction: async ({ id: { name } }) => { + exportedSymbols.add(name, { + source: moduleName, + symbolName: name, + loc: node.loc! + }) + return [...body, node] + }, + withClass: async ({ id: { name } }) => { + exportedSymbols.add(name, { + source: moduleName, + symbolName: name, + loc: node.loc! + }) + return [...body, node] + }, + localExports: async ({ specifiers }) => { + specifiers.forEach(spec => + exportedSymbols.add(spec.exported.name, { + source: moduleName, + symbolName: spec.local.name, + loc: spec.loc! + }) + ) + return [...body, node] + }, + withSource: async node => { + const [exports, source] = await getDocs(node) + const newDecls = node.specifiers.map(spec => { + if (!allowUndefinedImports) validateImport(spec, source, exports) + + const [newSource, symbolName] = resolveSymbol(source, spec.local.name) + exportedSymbols.add(spec.exported.name, { + source: newSource, + symbolName, + loc: spec.loc! + }) + + const newDecl: es.ExportNamedDeclarationWithSource = { + type: 'ExportNamedDeclaration', + declaration: null, + source: create.literal(newSource), + specifiers: [ + { + type: 'ExportSpecifier', + exported: spec.exported, + local: create.identifier(symbolName) + } + ] + } + return newDecl + }) + return [...body, ...newDecls] + } + }) + } + case 'ExportAllDeclaration': { + if (node.exported) { + throw new Error('ExportAllDeclarations with exported name are not supported') + // exportedSymbols.add(node.exported.name, { + // source, + // symbolName: node.exported.name, + // loc: node.loc!, + // }) + } else { + const [exports, source] = await getDocs(node) + if (!allowUndefinedImports) validateNamespaceImport(node, source, exports) + + for (const symbol of exports) { + const [newSource, newSymbol] = resolveSymbol(source, symbol) + exportedSymbols.add(symbol, { + source: newSource, + symbolName: newSymbol, + loc: node.loc! + }) + } + } + } + default: + return [...body, node] + } + }, + [] as es.Program['body'] + ) + + exportMap[moduleName] = new Map( + exportedSymbols.entries().map(([symbol, records]) => { + if (records.length === 1) return [symbol, records[0]] + assert(records.length > 0, 'An exported symbol cannot have zero nodes associated with it') + const locations = records.map(({ loc }) => loc) + if (symbol === 'default') { + throw new ReexportDefaultError(moduleName, locations) + } else { + throw new ReexportSymbolError(moduleName, symbol, locations) + } + }) + ) + + newPrograms[moduleName] = { + ...program, + body: newBody + } + } + + return newPrograms +} \ No newline at end of file diff --git a/src/modules/preprocessor/transformers/removeExports.ts b/src/modules/preprocessor/transformers/removeExports.ts index d85355631..bf8ffba76 100644 --- a/src/modules/preprocessor/transformers/removeExports.ts +++ b/src/modules/preprocessor/transformers/removeExports.ts @@ -1,6 +1,6 @@ import es from 'estree' -import { ancestor } from '../../../utils/walkers' +import { ancestor } from '../../../utils/ast/walkers' import { isDeclaration } from '../typeGuards' /** diff --git a/src/modules/preprocessor/transformers/removeNonSourceModuleImports.ts b/src/modules/preprocessor/transformers/removeNonSourceModuleImports.ts index 43fb8a2d4..dda8b6466 100644 --- a/src/modules/preprocessor/transformers/removeNonSourceModuleImports.ts +++ b/src/modules/preprocessor/transformers/removeNonSourceModuleImports.ts @@ -1,6 +1,6 @@ import es from 'estree' -import { ancestor } from '../../../utils/walkers' +import { ancestor } from '../../../utils/ast/walkers' import { isFilePath } from '../filePaths' /** diff --git a/src/parser/source/index.ts b/src/parser/source/index.ts index 5ccf8c1d8..e478031c4 100644 --- a/src/parser/source/index.ts +++ b/src/parser/source/index.ts @@ -3,7 +3,7 @@ import { Node as ESNode, Program } from 'estree' import { DEFAULT_ECMA_VERSION } from '../../constants' import { Chapter, Context, Rule, SourceError, Variant } from '../../types' -import { ancestor, AncestorWalkerFn } from '../../utils/walkers' +import { ancestor, AncestorWalkerFn } from '../../utils/ast/walkers' import { DisallowedConstructError, FatalSyntaxError } from '../errors' import { AcornOptions, Parser } from '../types' import { createAcornParserOptions, positionToSourceLocation } from '../utils' diff --git a/src/runner/utils.ts b/src/runner/utils.ts index 43b864409..2395e060b 100644 --- a/src/runner/utils.ts +++ b/src/runner/utils.ts @@ -6,7 +6,7 @@ import { loadModuleTabs } from '../modules/moduleLoader' import { parseAt } from '../parser/utils' import { areBreakpointsSet } from '../stdlib/inspector' import { Context, Variant } from '../types' -import { simple } from '../utils/walkers' +import { simple } from '../utils/ast/walkers' // Context Utils diff --git a/src/scope-refactoring.ts b/src/scope-refactoring.ts index a2b542494..a76e2944e 100644 --- a/src/scope-refactoring.ts +++ b/src/scope-refactoring.ts @@ -2,7 +2,7 @@ import * as es from 'estree' import { isInLoc } from './finder' import { BlockFrame, DefinitionNode } from './types' -import { simple } from './utils/walkers' +import { simple } from './utils/ast/walkers' /** * This file parses the original AST Tree into another tree with a similar structure diff --git a/src/transpiler/transpiler.ts b/src/transpiler/transpiler.ts index 64404d0b0..326bf9917 100644 --- a/src/transpiler/transpiler.ts +++ b/src/transpiler/transpiler.ts @@ -16,7 +16,7 @@ import { getIdentifiersInProgram, getUniqueId } from '../utils/uniqueIds' -import { ancestor, simple } from '../utils/walkers' +import { ancestor, simple } from '../utils/ast/walkers' /** * This whole transpiler includes many many many many hacks to get stuff working. diff --git a/src/utils/arrayMap.ts b/src/utils/arrayMap.ts new file mode 100644 index 000000000..3f1ef506c --- /dev/null +++ b/src/utils/arrayMap.ts @@ -0,0 +1,25 @@ +/** + * Convenience class for maps that store an array of values + */ +export default class ArrayMap { + constructor(private readonly map: Map = new Map()) {} + + public get(key: K) { + return this.map.get(key) + } + + public add(key: K, item: V) { + if (!this.map.has(key)) { + this.map.set(key, []) + } + this.map.get(key)!.push(item) + } + + public entries() { + return Array.from(this.map.entries()) + } + + public keys() { + return new Set(this.map.keys()) + } +} \ No newline at end of file diff --git a/src/utils/assert.ts b/src/utils/assert.ts new file mode 100644 index 000000000..8782924f0 --- /dev/null +++ b/src/utils/assert.ts @@ -0,0 +1,27 @@ +/* + * Why not use the nodejs builtin assert? It needs polyfills to work in the browser. + * With this we have a lightweight assert that doesn't need any further packages. + * Plus, we can customize our own assert messages and handling + */ + +import { RuntimeSourceError } from '../errors/runtimeSourceError' + +export class AssertionError extends RuntimeSourceError { + constructor(public readonly message: string) { + super() + } + + public explain(): string { + return this.message + } + + public elaborate(): string { + return 'Please contact the administrators to let them know that this error has occurred' + } +} + +export default function assert(condition: boolean, message: string): asserts condition { + if (!condition) { + throw new AssertionError(message) + } +} \ No newline at end of file diff --git a/src/utils/ast/astUtils.ts b/src/utils/ast/astUtils.ts new file mode 100644 index 000000000..ff5656810 --- /dev/null +++ b/src/utils/ast/astUtils.ts @@ -0,0 +1,106 @@ +import assert from '../assert' +import { isDeclaration } from './typeGuards' +import type * as es from './types' +import { recursive } from './walkers' + +export function extractIdsFromPattern(pattern: es.Pattern): Set { + const ids = new Set() + if (pattern.type === 'MemberExpression') return ids + + recursive(pattern, null, { + ArrayPattern: ({ elements }: es.ArrayPattern, _state, c) => + elements.forEach(elem => { + if (elem) c(elem, null) + }), + AssignmentPattern: (p: es.AssignmentPattern, _state, c) => { + c(p.left, null) + c(p.right, null) + }, + Identifier: (id: es.Identifier) => ids.add(id), + ObjectPattern: ({ properties }: es.ObjectPattern, _state, c) => + properties.forEach(prop => c(prop, null)), + RestElement: ({ argument }: es.RestElement, _state, c) => c(argument, null) + }) + return ids +} + +export function declarationToExpression(node: es.ClassDeclaration): es.ClassExpression +export function declarationToExpression(node: es.FunctionDeclaration): es.FunctionExpression +export function declarationToExpression({ + type, + ...node +}: es.FunctionDeclaration | es.ClassDeclaration) { + return { + ...node, + type: type === 'FunctionDeclaration' ? 'FunctionExpression' : 'ClassExpression' + } as es.FunctionExpression | es.ClassExpression +} + +type ExportDefaultProcessors = { + FunctionDeclaration: (node: es.FunctionDeclarationWithId) => T + ClassDeclaration: (node: es.ClassDeclarationWithId) => T + Expression: (node: es.Expression) => T +} + +export function processExportDefaultDeclaration( + node: es.ExportDefaultDeclaration, + processors: ExportDefaultProcessors +) { + if (isDeclaration(node.declaration)) { + const declaration = node.declaration + assert( + declaration.type !== 'VariableDeclaration', + 'ExportDefaultDeclarations cannot have VariableDeclarations' + ) + + if (declaration.type === 'FunctionDeclaration') { + if (declaration.id) { + return processors.FunctionDeclaration(declaration as es.FunctionDeclarationWithId) + } + + return processors.Expression({ + ...declaration, + type: 'FunctionExpression' + }) + } + + if (declaration.id) { + return processors.ClassDeclaration(declaration as es.ClassDeclarationWithId) + } + + return processors.Expression({ + ...declaration, + type: 'ClassExpression' + }) + } + + return processors.Expression(node.declaration) +} + +type ExportNamedProcessors = { + withVarDecl: (node: es.VariableDeclaration) => T + withClass: (node: es.ClassDeclarationWithId) => T + withFunction: (node: es.FunctionDeclarationWithId) => T + localExports: (node: es.ExportNamedLocalDeclaration) => T + withSource: (node: es.ExportNamedDeclarationWithSource) => T +} + +export function processExportNamedDeclaration( + node: es.ExportNamedDeclaration, + processors: ExportNamedProcessors +) { + if (node.declaration) { + switch (node.declaration.type) { + case 'VariableDeclaration': + return processors.withVarDecl(node.declaration) + case 'FunctionDeclaration': + return processors.withFunction(node.declaration as es.FunctionDeclarationWithId) + case 'ClassDeclaration': + return processors.withClass(node.declaration as es.ClassDeclarationWithId) + } + } else if (node.source) { + return processors.withSource(node as es.ExportNamedDeclarationWithSource) + } else { + return processors.localExports(node as es.ExportNamedLocalDeclaration) + } +} \ No newline at end of file diff --git a/src/utils/ast/typeGuards.ts b/src/utils/ast/typeGuards.ts new file mode 100644 index 000000000..6023e7538 --- /dev/null +++ b/src/utils/ast/typeGuards.ts @@ -0,0 +1,84 @@ +import type * as es from '../../utils/ast/types' + +export function isDeclaration(node: es.Node): node is es.Declaration { + // export type Declaration = + // FunctionDeclaration | VariableDeclaration | ClassDeclaration; + return ( + node.type === 'VariableDeclaration' || + node.type === 'FunctionDeclaration' || + node.type === 'ClassDeclaration' + ) +} + +// It is necessary to write this type guard like this as the 'type' of both +// 'Directive' & 'ExpressionStatement' is 'ExpressionStatement'. +// +// export interface Directive extends BaseNode { +// type: "ExpressionStatement"; +// expression: Literal; +// directive: string; +// } +// +// export interface ExpressionStatement extends BaseStatement { +// type: "ExpressionStatement"; +// expression: Expression; +// } +// +// As such, we check whether the 'directive' property exists on the object +// instead in order to differentiate between the two. +export const isDirective = (node: es.Node): node is es.Directive => { + return 'directive' in node +} + +export const isFunctionNode = (node: es.Node): node is es.FunctionNode => + ['ArrowFunctionExpression', 'FunctionExpression', 'FunctionDeclaration'].includes(node.type) + +export const isIdentifier = (node: es.Node): node is es.Identifier => node.type === 'Identifier' + +export const isImportDeclaration = (node: es.Node): node is es.ImportDeclaration => + node.type === 'ImportDeclaration' + +export const isLoop = (node: es.Node): node is es.LoopNode => + ['WhileStatement', 'ForStatement', 'ForInStatement', 'ForOfStatement'].includes(node.type) + +export const isModuleDeclaration = (node: es.Node): node is es.ModuleDeclaration => { + return [ + 'ImportDeclaration', + 'ExportNamedDeclaration', + 'ExportDefaultDeclaration', + 'ExportAllDeclaration' + ].includes(node.type) +} + +export const isModuleDeclarationWithSource = ( + node: es.Node +): node is es.ModuleDeclarationWithSource => { + switch (node.type) { + case 'ExportNamedDeclaration': + return !!node.source + case 'ExportAllDeclaration': + case 'ImportDeclaration': + return true + default: + return false + } +} + +export const isStatement = ( + node: es.Directive | es.Statement | es.ModuleDeclaration +): node is es.Statement => { + return !isDirective(node) && !isModuleDeclaration(node) +} + +export const isSourceImport = (url: string) => !url.startsWith('.') && !url.startsWith('/') + +export function isPattern(node: es.Node): node is es.Pattern { + return [ + 'ArrayPattern', + 'AssignmentPattern', + 'Identifier', + 'MemberExpression', + 'ObjectPattern', + 'RestElement' + ].includes(node.type) +} \ No newline at end of file diff --git a/src/utils/walkers.ts b/src/utils/ast/walkers.ts similarity index 100% rename from src/utils/walkers.ts rename to src/utils/ast/walkers.ts diff --git a/src/utils/uniqueIds.ts b/src/utils/uniqueIds.ts index 5222be274..f426f8b48 100644 --- a/src/utils/uniqueIds.ts +++ b/src/utils/uniqueIds.ts @@ -1,7 +1,7 @@ import * as es from 'estree' import { NativeStorage } from '../types' -import { simple } from '../utils/walkers' +import { simple } from './ast/walkers' export function getUniqueId(usedIdentifiers: Set, uniqueId = 'unique') { while (usedIdentifiers.has(uniqueId)) { diff --git a/src/validator/__tests__/validator.ts b/src/validator/__tests__/validator.ts index bc9a8880e..e5a85d60c 100644 --- a/src/validator/__tests__/validator.ts +++ b/src/validator/__tests__/validator.ts @@ -6,7 +6,7 @@ import { Chapter, NodeWithInferredType } from '../../types' import { getVariableDecarationName } from '../../utils/ast/astCreator' import { stripIndent } from '../../utils/formatters' import { expectParsedError } from '../../utils/testing' -import { simple } from '../../utils/walkers' +import { simple } from '../../utils/ast/walkers' import { validateAndAnnotate } from '../validator' export function toValidatedAst(code: string) { diff --git a/src/validator/validator.ts b/src/validator/validator.ts index 67f1db938..4178539f6 100644 --- a/src/validator/validator.ts +++ b/src/validator/validator.ts @@ -4,7 +4,7 @@ import { ConstAssignment } from '../errors/errors' import { NoAssignmentToForVariable } from '../errors/validityErrors' import { Context, NodeWithInferredType } from '../types' import { getVariableDecarationName } from '../utils/ast/astCreator' -import { ancestor, base, FullWalkerCallback } from '../utils/walkers' +import { ancestor, base, FullWalkerCallback } from '../utils/ast/walkers' class Declaration { public accessedBeforeDeclaration: boolean = false diff --git a/src/vm/svml-compiler.ts b/src/vm/svml-compiler.ts index fa0addfd8..f6b617982 100644 --- a/src/vm/svml-compiler.ts +++ b/src/vm/svml-compiler.ts @@ -12,7 +12,7 @@ import { } from '../stdlib/vm.prelude' import { Context, ContiguousArrayElements } from '../types' import * as create from '../utils/ast/astCreator' -import { recursive, simple } from '../utils/walkers' +import { recursive, simple } from '../utils/ast/walkers' import OpCodes from './opcodes' const VALID_UNARY_OPERATORS = new Map([ From b00153e1eeb8e0abab06919906864cc268f6da27 Mon Sep 17 00:00:00 2001 From: Lee Yi Date: Wed, 21 Jun 2023 23:53:38 -0700 Subject: [PATCH 08/50] Clean version of async-imports3 --- .babelrc | 2 +- .eslintrc.json | 5 + src/__tests__/environment.ts | 5 +- .../__snapshots__/ec-evaluator-errors.ts.snap | 2 +- .../__tests__/ec-evaluator-errors.ts | 5 +- src/ec-evaluator/interpreter.ts | 98 ++--- src/errors/localImportErrors.ts | 76 ---- src/errors/moduleErrors.ts | 52 --- src/index.ts | 38 +- src/infiniteLoops/__tests__/instrument.ts | 34 +- src/infiniteLoops/__tests__/runtime.ts | 117 +++--- src/infiniteLoops/instrument.ts | 33 +- src/infiniteLoops/runtime.ts | 9 +- .../__snapshots__/interpreter-errors.ts.snap | 13 + .../__tests__/interpreter-errors.ts | 5 +- src/interpreter/interpreter.ts | 40 +- src/lazy/lazy.ts | 2 +- src/modules/__mocks__/moduleLoader.ts | 15 + src/modules/__mocks__/moduleLoaderAsync.ts | 28 ++ src/modules/__tests__/moduleLoader.ts | 2 +- src/modules/__tests__/moduleLoaderAsync.ts | 2 +- src/modules/errors.ts | 2 +- src/modules/moduleLoader.ts | 69 ++-- src/modules/moduleLoaderAsync.ts | 2 +- src/modules/moduleTypes.ts | 2 +- .../__snapshots__/preprocessor.ts.snap | 96 +++++ .../preprocessor/__tests__/preprocessor.ts | 315 ++++++++------ .../hoistAndMergeImports.ts.snap | 21 + .../transformers/hoistAndMergeImports.ts | 65 +-- .../__tests__/transformers/removeExports.ts | 159 ------- .../removeNonSourceModuleImports.ts | 57 --- src/modules/preprocessor/analyzer.ts | 3 +- .../constructors/baseConstructors.ts | 6 +- src/modules/preprocessor/filePaths.ts | 2 +- src/modules/preprocessor/index.ts | 331 +++++++++++++++ src/modules/preprocessor/preprocessor.ts | 290 ------------- src/modules/preprocessor/resolver.ts | 42 ++ .../transformers/hoistAndMergeImports.ts | 168 +++++--- .../transformers/removeImportsAndExports.ts | 25 ++ .../removeNonSourceModuleImports.ts | 113 ----- .../transformProgramToFunctionDeclaration.ts | 239 +++++------ src/modules/utils.ts | 183 ++++++++ src/name-extractor/index.ts | 137 +++--- .../__snapshots__/allowed-syntax.ts.snap | 8 +- src/parser/__tests__/allowed-syntax.ts | 13 +- src/parser/parser.ts | 2 +- src/parser/source/rules/index.ts | 4 +- .../noExportNamedDeclarationWithDefault.ts | 4 +- .../rules/noImportSpecifierWithDefault.ts | 4 +- src/repl/repl.ts | 2 +- src/repl/transpiler.ts | 8 +- src/runner/__tests__/files.ts | 12 +- src/runner/fullJSRunner.ts | 21 +- src/runner/htmlRunner.ts | 6 +- src/runner/sourceRunner.ts | 78 +++- src/runner/utils.ts | 4 +- src/transpiler/__tests__/modules.ts | 100 ++--- src/transpiler/__tests__/transpiled-code.ts | 8 +- src/transpiler/__tests__/variableChecker.ts | 389 ++++++++++++++++++ src/transpiler/transpiler.ts | 277 +++++-------- src/transpiler/variableChecker.ts | 130 ++++++ .../__tests__/source1Typed.test.ts | 14 +- src/typeChecker/typeErrorChecker.ts | 2 +- src/types.ts | 2 +- src/utils/arrayMap.ts | 2 +- src/utils/assert.ts | 2 +- src/utils/ast/astUtils.ts | 2 +- src/utils/ast/typeGuards.ts | 2 +- src/utils/ast/types.ts | 2 +- src/utils/index.ts | 2 +- src/utils/testing.ts | 18 +- src/vm/__tests__/svml-compiler.ts | 4 +- tsconfig.json | 3 +- 73 files changed, 2327 insertions(+), 1708 deletions(-) delete mode 100644 src/errors/localImportErrors.ts delete mode 100644 src/errors/moduleErrors.ts create mode 100644 src/modules/__mocks__/moduleLoader.ts create mode 100644 src/modules/__mocks__/moduleLoaderAsync.ts create mode 100644 src/modules/preprocessor/__tests__/__snapshots__/preprocessor.ts.snap create mode 100644 src/modules/preprocessor/__tests__/transformers/__snapshots__/hoistAndMergeImports.ts.snap delete mode 100644 src/modules/preprocessor/__tests__/transformers/removeExports.ts delete mode 100644 src/modules/preprocessor/__tests__/transformers/removeNonSourceModuleImports.ts create mode 100644 src/modules/preprocessor/index.ts delete mode 100644 src/modules/preprocessor/preprocessor.ts create mode 100644 src/modules/preprocessor/resolver.ts create mode 100644 src/modules/preprocessor/transformers/removeImportsAndExports.ts delete mode 100644 src/modules/preprocessor/transformers/removeNonSourceModuleImports.ts create mode 100644 src/modules/utils.ts create mode 100644 src/transpiler/__tests__/variableChecker.ts create mode 100644 src/transpiler/variableChecker.ts diff --git a/.babelrc b/.babelrc index 05581748b..1320b9a32 100644 --- a/.babelrc +++ b/.babelrc @@ -1,3 +1,3 @@ { - "presets": ["es2015", "stage-2"] + "presets": ["@babel/preset-env"] } diff --git a/.eslintrc.json b/.eslintrc.json index 79e628819..885e082de 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -9,6 +9,11 @@ "plugin:@typescript-eslint/recommended-requiring-type-checking", "prettier" ], + "ignorePatterns": [ + "**/__tests__/**", + "**/__mocks__/**", + "jest.setup.ts" + ], "parser": "@typescript-eslint/parser", "parserOptions": { "project": "tsconfig.json", diff --git a/src/__tests__/environment.ts b/src/__tests__/environment.ts index e8d6cbe57..a86a389d2 100644 --- a/src/__tests__/environment.ts +++ b/src/__tests__/environment.ts @@ -18,7 +18,10 @@ test('Function params and body identifiers are in different environment', () => const context = mockContext(Chapter.SOURCE_4) context.prelude = null // hide the unneeded prelude const parsed = parse(code, context) - const it = evaluate(parsed as any as Program, context, false, false) + const it = evaluate(parsed as any as Program, context, { + loadTabs: false, + wrapModules: false + }) const stepsToComment = 13 // manually counted magic number for (let i = 0; i < stepsToComment; i += 1) { it.next() diff --git a/src/ec-evaluator/__tests__/__snapshots__/ec-evaluator-errors.ts.snap b/src/ec-evaluator/__tests__/__snapshots__/ec-evaluator-errors.ts.snap index 56e49bab0..0d62644a9 100644 --- a/src/ec-evaluator/__tests__/__snapshots__/ec-evaluator-errors.ts.snap +++ b/src/ec-evaluator/__tests__/__snapshots__/ec-evaluator-errors.ts.snap @@ -798,7 +798,7 @@ Object { "code": "import { foo1 } from 'one_module';", "displayResult": Array [], "numErrors": 1, - "parsedErrors": "'one_module' does not contain a definition for 'foo1'", + "parsedErrors": "Line 1: 'one_module' does not contain a definition for 'foo1'", "result": undefined, "resultStatus": "error", "visualiseListResult": Array [], diff --git a/src/ec-evaluator/__tests__/ec-evaluator-errors.ts b/src/ec-evaluator/__tests__/ec-evaluator-errors.ts index b740d9405..746fae77e 100644 --- a/src/ec-evaluator/__tests__/ec-evaluator-errors.ts +++ b/src/ec-evaluator/__tests__/ec-evaluator-errors.ts @@ -1046,5 +1046,8 @@ test('Importing unknown variables throws UndefinedImport error', () => { import { foo1 } from 'one_module'; `, optionEC - ).toMatchInlineSnapshot("\"'one_module' does not contain a definition for 'foo1'\"") + ).toMatchInlineSnapshot( + "\"'one_module' does not contain a definition for 'foo1'\"", + `"Line 1: 'one_module' does not contain a definition for 'foo1'"` + ) }) diff --git a/src/ec-evaluator/interpreter.ts b/src/ec-evaluator/interpreter.ts index a60220d65..58865d290 100644 --- a/src/ec-evaluator/interpreter.ts +++ b/src/ec-evaluator/interpreter.ts @@ -6,7 +6,7 @@ */ /* tslint:disable:max-classes-per-file */ -import * as es from 'estree' +import type * as es from 'estree' import { partition, uniqueId } from 'lodash' import { IOptions } from '..' @@ -14,12 +14,13 @@ import { UNKNOWN_LOCATION } from '../constants' import * as errors from '../errors/errors' import { RuntimeSourceError } from '../errors/runtimeSourceError' import Closure from '../interpreter/closure' -import { UndefinedImportError } from '../modules/errors' -import { loadModuleBundle, loadModuleTabs } from '../modules/moduleLoader' -import { ModuleFunctions } from '../modules/moduleTypes' +import { loadModuleBundleAsync } from '../modules/moduleLoaderAsync' +import { ImportOptions } from '../modules/moduleTypes' +import { transformImportNodesAsync } from '../modules/utils' import { checkEditorBreakpoints } from '../stdlib/inspector' import { Context, ContiguousArrayElements, Result, Value } from '../types' import * as ast from '../utils/ast/astCreator' +import { isImportDeclaration } from '../utils/ast/typeGuards' import { evaluateBinaryExpression, evaluateUnaryExpression } from '../utils/operators' import * as rttc from '../utils/rttc' import * as instr from './instrCreator' @@ -95,11 +96,15 @@ export class Stash extends Stack { * @param context The context to evaluate the program in. * @returns The result of running the ECE machine. */ -export function evaluate(program: es.Program, context: Context, options: IOptions): Value { +export async function evaluate( + program: es.Program, + context: Context, + options: IOptions +): Promise { try { context.runtime.isRunning = true - const nonImportNodes = evaluateImports(program, context, true, true) + const nonImportNodes = await evaluateImports(program, context, options.importOptions) context.runtime.agenda = new Agenda({ ...program, @@ -134,55 +139,43 @@ export function resumeEvaluate(context: Context) { } } -function evaluateImports( +async function evaluateImports( program: es.Program, context: Context, - loadTabs: boolean, - checkImports: boolean + { loadTabs, wrapModules }: ImportOptions ) { - const [importNodes, otherNodes] = partition( - program.body, - ({ type }) => type === 'ImportDeclaration' - ) as [es.ImportDeclaration[], es.Statement[]] + const [importNodes, otherNodes] = partition(program.body, isImportDeclaration) - const moduleFunctions: Record = {} + if (importNodes.length === 0) return otherNodes as es.Statement[] + const environment = currentEnvironment(context) try { - for (const node of importNodes) { - const moduleName = node.source.value - if (typeof moduleName !== 'string') { - throw new Error(`ImportDeclarations should have string sources, got ${moduleName}`) - } - - if (!(moduleName in moduleFunctions)) { - context.moduleContexts[moduleName] = { - state: null, - tabs: loadTabs ? loadModuleTabs(moduleName, node) : null - } - moduleFunctions[moduleName] = loadModuleBundle(moduleName, context, node) - } - - const functions = moduleFunctions[moduleName] - const environment = currentEnvironment(context) - for (const spec of node.specifiers) { - if (spec.type !== 'ImportSpecifier') { - throw new Error(`Only ImportSpecifiers are supported, got: ${spec.type}`) - } - - if (checkImports && !(spec.imported.name in functions)) { - throw new UndefinedImportError(spec.imported.name, moduleName, node) + await transformImportNodesAsync( + importNodes, + context, + loadTabs, + (name, node) => loadModuleBundleAsync(name, context, wrapModules, node), + { + ImportSpecifier: (spec: es.ImportSpecifier, info, node) => { + declareIdentifier(context, spec.local.name, node, environment) + defineVariable(context, spec.local.name, info.content[spec.imported.name], true, node) + }, + ImportDefaultSpecifier: (spec, info, node) => { + declareIdentifier(context, spec.local.name, node, environment) + defineVariable(context, spec.local.name, info.content['default'], true, node) + }, + ImportNamespaceSpecifier: (spec, info, node) => { + declareIdentifier(context, spec.local.name, node, environment) + defineVariable(context, spec.local.name, info.content, true, node) } - - declareIdentifier(context, spec.local.name, node, environment) - defineVariable(context, spec.local.name, functions[spec.imported.name], true, node) } - } + ) } catch (error) { - // console.log(error) + // console.error(error) handleRuntimeError(context, error) } - return otherNodes + return otherNodes as es.Statement[] } /** @@ -192,16 +185,15 @@ function evaluateImports( * @param value The value of ec evaluating the program. * @returns The corresponding promise. */ -export function ECEResultPromise(context: Context, value: Value): Promise { - return new Promise((resolve, reject) => { - if (value instanceof ECEBreak) { - resolve({ status: 'suspended-ec-eval', context }) - } else if (value instanceof ECError) { - resolve({ status: 'error' }) - } else { - resolve({ status: 'finished', context, value }) - } - }) +export async function ECEResultPromise(context: Context, promise: Promise): Promise { + const value = await promise + if (value instanceof ECEBreak) { + return { status: 'suspended-ec-eval', context } + } else if (value instanceof ECError) { + return { status: 'error' } + } else { + return { status: 'finished', context, value } + } } /** diff --git a/src/errors/localImportErrors.ts b/src/errors/localImportErrors.ts deleted file mode 100644 index c2b627a68..000000000 --- a/src/errors/localImportErrors.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { UNKNOWN_LOCATION } from '../constants' -import { nonAlphanumericCharEncoding } from '../modules/preprocessor/filePaths' -import { ErrorSeverity, ErrorType, SourceError } from '../types' - -export abstract class InvalidFilePathError implements SourceError { - public type = ErrorType.TYPE - public severity = ErrorSeverity.ERROR - public location = UNKNOWN_LOCATION - - constructor(public filePath: string) {} - - abstract explain(): string - - abstract elaborate(): string -} - -export class IllegalCharInFilePathError extends InvalidFilePathError { - public explain() { - const validNonAlphanumericChars = Object.keys(nonAlphanumericCharEncoding) - .map(char => `'${char}'`) - .join(', ') - return `File path '${this.filePath}' must only contain alphanumeric chars and/or ${validNonAlphanumericChars}.` - } - - public elaborate() { - return 'Rename the offending file path to only use valid chars.' - } -} - -export class ConsecutiveSlashesInFilePathError extends InvalidFilePathError { - public explain() { - return `File path '${this.filePath}' cannot contain consecutive slashes '//'.` - } - - public elaborate() { - return 'Remove consecutive slashes from the offending file path.' - } -} - -export class CannotFindModuleError implements SourceError { - public type = ErrorType.TYPE - public severity = ErrorSeverity.ERROR - public location = UNKNOWN_LOCATION - - constructor(public moduleFilePath: string) {} - - public explain() { - return `Cannot find module '${this.moduleFilePath}'.` - } - - public elaborate() { - return 'Check that the module file path resolves to an existing file.' - } -} - -export class CircularImportError implements SourceError { - public type = ErrorType.TYPE - public severity = ErrorSeverity.ERROR - public location = UNKNOWN_LOCATION - - constructor(public filePathsInCycle: string[]) {} - - public explain() { - // We need to reverse the file paths in the cycle so that the - // semantics of "'/a.js' -> '/b.js'" is "'/a.js' imports '/b.js'". - const formattedCycle = this.filePathsInCycle - .map(filePath => `'${filePath}'`) - .reverse() - .join(' -> ') - return `Circular import detected: ${formattedCycle}.` - } - - public elaborate() { - return 'Break the circular import cycle by removing imports from any of the offending files.' - } -} diff --git a/src/errors/moduleErrors.ts b/src/errors/moduleErrors.ts deleted file mode 100644 index 683d6089a..000000000 --- a/src/errors/moduleErrors.ts +++ /dev/null @@ -1,52 +0,0 @@ -/* tslint:disable: max-classes-per-file */ -import * as es from 'estree' - -import { RuntimeSourceError } from './runtimeSourceError' - -export class ModuleConnectionError extends RuntimeSourceError { - private static message: string = `Unable to get modules.` - private static elaboration: string = `You should check your Internet connection, and ensure you have used the correct module path.` - constructor(node?: es.Node) { - super(node) - } - - public explain() { - return ModuleConnectionError.message - } - - public elaborate() { - return ModuleConnectionError.elaboration - } -} - -export class ModuleNotFoundError extends RuntimeSourceError { - constructor(public moduleName: string, node?: es.Node) { - super(node) - } - - public explain() { - return `Module "${this.moduleName}" not found.` - } - - public elaborate() { - return ` - You should check your import declarations, and ensure that all are valid modules. - ` - } -} - -export class ModuleInternalError extends RuntimeSourceError { - constructor(public moduleName: string, public error?: any, node?: es.Node) { - super(node) - } - - public explain() { - return `Error(s) occured when executing the module "${this.moduleName}".` - } - - public elaborate() { - return ` - You may need to contact with the author for this module to fix this error. - ` - } -} diff --git a/src/index.ts b/src/index.ts index 25e16e9d0..8cf026c40 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,6 +16,7 @@ import { FuncDeclWithInferredTypeAnnotation, ModuleContext, NodeWithInferredType, + RecursivePartial, Result, SourceError, SVMProgram, @@ -25,14 +26,16 @@ import { findNodeAt } from './utils/ast/walkers' import { assemble } from './vm/svml-assembler' import { compileToIns } from './vm/svml-compiler' export { SourceDocumentation } from './editors/ace/docTooltip' -import * as es from 'estree' +import type * as es from 'estree' import { ECEResultPromise, resumeEvaluate } from './ec-evaluator/interpreter' -import { CannotFindModuleError } from './errors/localImportErrors' +import { ModuleNotFoundError } from './modules/errors' +import type { ImportOptions } from './modules/moduleTypes' +import preprocessFileImports from './modules/preprocessor' import { validateFilePath } from './modules/preprocessor/filePaths' -import preprocessFileImports from './modules/preprocessor/preprocessor' import { getKeywords, getProgramNames, NameDeclaration } from './name-extractor' import { parse } from './parser/parser' +import { decodeError, decodeValue } from './parser/scheme' import { parseWithComments } from './parser/utils' import { fullJSRunner, @@ -43,7 +46,6 @@ import { } from './runner' import { typeCheck } from './typeChecker/typeChecker' import { typeToString } from './utils/stringify' -import { decodeError, decodeValue } from './parser/scheme' export interface IOptions { scheduler: 'preemptive' | 'async' @@ -55,6 +57,14 @@ export interface IOptions { useSubst: boolean isPrelude: boolean throwInfiniteLoops: boolean + + importOptions: ImportOptions + + /** Set to true to console log the transpiler's transpiled code */ + logTranspilerOutput: boolean + + /** Set to true to console log the preprocessor's output */ + logPreprocessorOutput: boolean } // needed to work on browsers @@ -192,7 +202,7 @@ export async function getNames( } const cursorLoc: es.Position = { line, column: col } - const [progNames, displaySuggestions] = getProgramNames(program, comments, cursorLoc) + const [progNames, displaySuggestions] = await getProgramNames(program, comments, cursorLoc) const keywords = getKeywords(program, cursorLoc, context) return [progNames.concat(keywords), displaySuggestions] } @@ -302,7 +312,7 @@ export function getTypeInformation( export async function runInContext( code: string, context: Context, - options: Partial = {} + options: RecursivePartial = {} ): Promise { const defaultFilePath = '/default.js' const files: Partial> = {} @@ -314,7 +324,7 @@ export async function runFilesInContext( files: Partial>, entrypointFilePath: string, context: Context, - options: Partial = {} + options: RecursivePartial = {} ): Promise { for (const filePath in files) { const filePathError = validateFilePath(filePath) @@ -326,12 +336,12 @@ export async function runFilesInContext( const code = files[entrypointFilePath] if (code === undefined) { - context.errors.push(new CannotFindModuleError(entrypointFilePath)) + context.errors.push(new ModuleNotFoundError(entrypointFilePath)) return resolvedErrorPromise } if ( - context.chapter === Chapter.FULL_JS || + // context.chapter === Chapter.FULL_JS || context.chapter === Chapter.FULL_TS || context.chapter === Chapter.PYTHON_1 ) { @@ -398,19 +408,19 @@ export function compile( code: string, context: Context, vmInternalFunctions?: string[] -): SVMProgram | undefined { +): Promise { const defaultFilePath = '/default.js' const files: Partial> = {} files[defaultFilePath] = code return compileFiles(files, defaultFilePath, context, vmInternalFunctions) } -export function compileFiles( +export async function compileFiles( files: Partial>, entrypointFilePath: string, context: Context, vmInternalFunctions?: string[] -): SVMProgram | undefined { +): Promise { for (const filePath in files) { const filePathError = validateFilePath(filePath) if (filePathError !== null) { @@ -421,11 +431,11 @@ export function compileFiles( const entrypointCode = files[entrypointFilePath] if (entrypointCode === undefined) { - context.errors.push(new CannotFindModuleError(entrypointFilePath)) + context.errors.push(new ModuleNotFoundError(entrypointFilePath)) return undefined } - const preprocessedProgram = preprocessFileImports(files, entrypointFilePath, context) + const preprocessedProgram = await preprocessFileImports(files, entrypointFilePath, context) if (!preprocessedProgram) { return undefined } diff --git a/src/infiniteLoops/__tests__/instrument.ts b/src/infiniteLoops/__tests__/instrument.ts index f4f69c924..f2c92a10a 100644 --- a/src/infiniteLoops/__tests__/instrument.ts +++ b/src/infiniteLoops/__tests__/instrument.ts @@ -37,7 +37,11 @@ function mockFunctionsAndState() { * Returns the value saved in the code using the builtin 'output'. * e.g. runWithMock('output(2)') --> 2 */ -function runWithMock(main: string, codeHistory?: string[], builtins: Map = new Map()) { +async function runWithMock( + main: string, + codeHistory?: string[], + builtins: Map = new Map() +) { let output = undefined builtins.set('output', (x: any) => (output = x)) builtins.set('undefined', undefined) @@ -53,7 +57,7 @@ function runWithMock(main: string, codeHistory?: string[], builtins: Map { const main = 'output(2);' - expect(runWithMock(main, [])).toBe(2) + expect(runWithMock(main, [])).resolves.toBe(2) }) test('binary and unary expressions work', () => { - expect(runWithMock('output(1+1);', [])).toBe(2) - expect(runWithMock('output(!true);', [])).toBe(false) + expect(runWithMock('output(1+1);', [])).resolves.toBe(2) + expect(runWithMock('output(!true);', [])).resolves.toBe(false) }) test('assignment works as expected', () => { @@ -75,13 +79,13 @@ test('assignment works as expected', () => { let a = []; a[0] = 3; output(x+a[0]);` - expect(runWithMock(main)).toBe(5) + expect(runWithMock(main)).resolves.toBe(5) }) test('globals from old code accessible', () => { const main = 'output(z+1);' const prev = ['const z = w+1;', 'let w = 10;'] - expect(runWithMock(main, prev)).toBe(12) + expect(runWithMock(main, prev)).resolves.toBe(12) }) test('functions run as expected', () => { @@ -89,7 +93,7 @@ test('functions run as expected', () => { return x===0?x:f(x-1,y)+y; } output(f(5,2));` - expect(runWithMock(main)).toBe(10) + expect(runWithMock(main)).resolves.toBe(10) }) test('nested functions run as expected', () => { @@ -100,7 +104,7 @@ test('nested functions run as expected', () => { return x===0?x:f(x-1,y)+y; } output(f(5,2));` - expect(runWithMock(main)).toBe(2) + expect(runWithMock(main)).resolves.toBe(2) }) test('higher order functions run as expected', () => { @@ -108,14 +112,14 @@ test('higher order functions run as expected', () => { return f(x+1); } output(run(x=>x+1, 1));` - expect(runWithMock(main)).toBe(3) + expect(runWithMock(main)).resolves.toBe(3) }) test('loops run as expected', () => { const main = `let w = 0; for (let i = w; i < 10; i=i+1) {w = i;} output(w);` - expect(runWithMock(main)).toBe(9) + expect(runWithMock(main)).resolves.toBe(9) }) test('nested loops run as expected', () => { @@ -126,13 +130,13 @@ test('nested loops run as expected', () => { } } output(w);` - expect(runWithMock(main)).toBe(100) + expect(runWithMock(main)).resolves.toBe(100) }) test('multidimentional arrays work', () => { const main = `const x = [[1],[2]]; output(x[1] === undefined? undefined: x[1][0]);` - expect(runWithMock(main)).toBe(2) + expect(runWithMock(main)).resolves.toBe(2) }) test('if statements work as expected', () => { @@ -141,7 +145,7 @@ test('if statements work as expected', () => { x = x + 1; } else {} output(x);` - expect(runWithMock(main)).toBe(2) + expect(runWithMock(main)).resolves.toBe(2) }) test('combination of loops and functions run as expected', () => { @@ -158,5 +162,5 @@ test('combination of loops and functions run as expected', () => { w = minus(w,1); } output(z);` - expect(runWithMock(main)).toBe(100) + expect(runWithMock(main)).resolves.toBe(100) }) diff --git a/src/infiniteLoops/__tests__/runtime.ts b/src/infiniteLoops/__tests__/runtime.ts index 4eb48dae8..9b35ea233 100644 --- a/src/infiniteLoops/__tests__/runtime.ts +++ b/src/infiniteLoops/__tests__/runtime.ts @@ -1,43 +1,16 @@ import * as es from 'estree' +import type { MockedFunction } from 'jest-mock' import { runInContext } from '../..' import createContext from '../../createContext' import { mockContext } from '../../mocks/context' -import * as moduleLoader from '../../modules/moduleLoader' +import * as moduleLoader from '../../modules/moduleLoaderAsync' import { parse } from '../../parser/parser' import { Chapter, Variant } from '../../types' -import { stripIndent } from '../../utils/formatters' import { getInfiniteLoopData, InfiniteLoopError, InfiniteLoopErrorType } from '../errors' import { testForInfiniteLoop } from '../runtime' -jest.spyOn(moduleLoader, 'memoizedGetModuleFile').mockImplementationOnce(() => { - return stripIndent` - require => { - 'use strict'; - var exports = {}; - function repeat(func, n) { - return n === 0 ? function (x) { - return x; - } : function (x) { - return func(repeat(func, n - 1)(x)); - }; - } - function twice(func) { - return repeat(func, 2); - } - function thrice(func) { - return repeat(func, 3); - } - exports.repeat = repeat; - exports.thrice = thrice; - exports.twice = twice; - Object.defineProperty(exports, '__esModule', { - value: true - }); - return exports; - } - ` -}) +jest.spyOn(moduleLoader, 'memoizedGetModuleBundleAsync') test('works in runInContext when throwInfiniteLoops is true', async () => { const code = `function fib(x) { @@ -45,6 +18,7 @@ test('works in runInContext when throwInfiniteLoops is true', async () => { } fib(100000);` const context = mockContext(Chapter.SOURCE_4) + await runInContext(code, context, { throwInfiniteLoops: true }) const lastError = context.errors[context.errors.length - 1] expect(lastError instanceof InfiniteLoopError).toBe(true) @@ -77,84 +51,84 @@ const testForInfiniteLoopWithCode = (code: string, previousPrograms: es.Program[ return testForInfiniteLoop(program, previousPrograms) } -test('non-infinite recursion not detected', () => { +test('non-infinite recursion not detected', async () => { const code = `function fib(x) { return x<=1?x:fib(x-1) + fib(x-2); } fib(100000); ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result).toBeUndefined() }) -test('non-infinite loop not detected', () => { +test('non-infinite loop not detected', async () => { const code = `for(let i = 0;i<2000;i=i+1){i+1;} let j = 0; while(j<2000) {j=j+1;} ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result).toBeUndefined() }) -test('no base case function detected', () => { +test('no base case function detected', async () => { const code = `function fib(x) { return fib(x-1) + fib(x-2); } fib(100000); ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.NoBaseCase) expect(result?.streamMode).toBe(false) }) -test('no base case loop detected', () => { +test('no base case loop detected', async () => { const code = `for(let i = 0;true;i=i+1){i+1;} ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.NoBaseCase) expect(result?.streamMode).toBe(false) }) -test('no variables changing function detected', () => { +test('no variables changing function detected', async () => { const code = `let x = 1; function f() { return x===0?x:f(); } f(); ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.Cycle) expect(result?.streamMode).toBe(false) expect(result?.explain()).toContain('None of the variables are being updated.') }) -test('no state change function detected', () => { +test('no state change function detected', async () => { const code = `let x = 1; function f() { return x===0?x:f(); } f(); ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.Cycle) expect(result?.streamMode).toBe(false) expect(result?.explain()).toContain('None of the variables are being updated.') }) -test('infinite cycle detected', () => { +test('infinite cycle detected', async () => { const code = `function f(x) { return x[0] === 1? x : f(x); } f([2,3,4]); ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.Cycle) expect(result?.streamMode).toBe(false) expect(result?.explain()).toContain('cycle') expect(result?.explain()).toContain('[2,3,4]') }) -test('infinite data structures detected', () => { +test('infinite data structures detected', async () => { const code = `function f(x) { return is_null(x)? x : f(tail(x)); } @@ -162,32 +136,32 @@ test('infinite data structures detected', () => { set_tail(tail(tail(circ)), circ); f(circ); ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.Cycle) expect(result?.streamMode).toBe(false) expect(result?.explain()).toContain('cycle') expect(result?.explain()).toContain('(CIRCULAR)') }) -test('functions using SMT work', () => { +test('functions using SMT work', async () => { const code = `function f(x) { return x===0? x: f(x+1); } f(1); ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.FromSmt) expect(result?.streamMode).toBe(false) }) -test('detect forcing infinite streams', () => { +test('detect forcing infinite streams', async () => { const code = `stream_to_list(integers_from(0));` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.NoBaseCase) expect(result?.streamMode).toBe(true) }) -test('detect mutual recursion', () => { +test('detect mutual recursion', async () => { const code = `function e(x){ return x===0?1:1-o(x-1); } @@ -195,23 +169,23 @@ test('detect mutual recursion', () => { return x===1?0:1-e(x-1); } e(9);` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.FromSmt) expect(result?.streamMode).toBe(false) }) -test('functions passed as arguments not checked', () => { +test('functions passed as arguments not checked', async () => { // if they are checked -> this will throw no base case const code = `const twice = f => x => f(f(x)); const thrice = f => x => f(f(f(x))); const add = x => x + 1; (thrice)(twice(twice))(twice(add))(0);` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result).toBeUndefined() }) -test('detect complicated cycle example', () => { +test('detect complicated cycle example', async () => { const code = `function permutations(s) { return is_null(s) ? list(null) @@ -230,12 +204,12 @@ test('detect complicated cycle example', () => { remove_duplicate(list(list(1,2,3), list(1,2,3))); ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.Cycle) expect(result?.streamMode).toBe(false) }) -test('detect complicated cycle example 2', () => { +test('detect complicated cycle example 2', async () => { const code = `function make_big_int_from_number(num){ let output = num; while(output !== 0){ @@ -246,12 +220,12 @@ test('detect complicated cycle example 2', () => { } make_big_int_from_number(1234); ` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.Cycle) expect(result?.streamMode).toBe(false) }) -test('detect complicated fromSMT example 2', () => { +test('detect complicated fromSMT example 2', async () => { const code = `function fast_power(b,n){ if (n % 2 === 0){ return b* fast_power(b, n-2); @@ -261,47 +235,54 @@ test('detect complicated fromSMT example 2', () => { } fast_power(2,3);` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.FromSmt) expect(result?.streamMode).toBe(false) }) -test('detect complicated stream example', () => { +test('detect complicated stream example', async () => { const code = `function up(a, b) { return (a > b) ? up(1, 1 + b) : pair(a, () => stream_reverse(up(a + 1, b))); } eval_stream(up(1,1), 22);` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result).toBeDefined() expect(result?.streamMode).toBe(true) }) -test('math functions are disabled in smt solver', () => { +test('math functions are disabled in smt solver', async () => { const code = ` function f(x) { return x===0? x: f(math_floor(x+1)); } f(1);` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result).toBeUndefined() }) -test('cycle detection ignores non deterministic functions', () => { +test('cycle detection ignores non deterministic functions', async () => { const code = ` function f(x) { return x===0?0:f(math_floor(math_random()/2) + 1); } f(1);` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result).toBeUndefined() }) -test('handle imports properly', () => { +test('handle imports properly', async () => { + const mockedBundleLoader = moduleLoader.memoizedGetModuleBundleAsync as MockedFunction< + typeof moduleLoader.memoizedGetModuleBundleAsync + > + mockedBundleLoader.mockResolvedValueOnce(`require => ({ + thrice: f => x => f(f(f(x))) + })`) + const code = `import {thrice} from "repeat"; function f(x) { return is_number(x) ? f(x) : 42; } display(f(thrice(x=>x+1)(0)));` - const result = testForInfiniteLoopWithCode(code, []) + const result = await testForInfiniteLoopWithCode(code, []) expect(result?.infiniteLoopType).toBe(InfiniteLoopErrorType.Cycle) }) diff --git a/src/infiniteLoops/instrument.ts b/src/infiniteLoops/instrument.ts index 68c72a1fe..fe851e97a 100644 --- a/src/infiniteLoops/instrument.ts +++ b/src/infiniteLoops/instrument.ts @@ -574,15 +574,28 @@ function trackLocations(program: es.Program) { }) } -function handleImports(programs: es.Program[]): [string, string[]] { - const [prefixes, imports] = programs.reduce( - ([prefix, moduleNames], program) => { - const [prefixToAdd, importsToAdd, otherNodes] = transformImportDeclarations( +async function handleImports(programs: es.Program[]): Promise<[string, string[]]> { + const results = await Promise.all( + programs.map(async program => { + const [prefix, declNodes, otherNodes] = await transformImportDeclarations( program, - new Set(), - false + new Set(), + null, + { + loadTabs: false, + wrapModules: false + } ) - program.body = (importsToAdd as es.Program['body']).concat(otherNodes) + + program.body = (declNodes as es.Program['body']).concat(otherNodes) + return [prefix, declNodes, otherNodes] as Awaited< + ReturnType + > + }) + ) + + const [prefixes, imports] = results.reduce( + ([prefix, moduleNames], [prefixToAdd, importsToAdd]) => { prefix.push(prefixToAdd) const importedNames = importsToAdd.flatMap(node => @@ -606,11 +619,11 @@ function handleImports(programs: es.Program[]): [string, string[]] { * @param builtins Names of builtin functions. * @returns code with instrumentations. */ -function instrument( +async function instrument( previous: es.Program[], program: es.Program, builtins: Iterable -): string { +): Promise { const { builtinsId, functionsId, stateId } = globalIds const predefined = {} predefined[builtinsId] = builtinsId @@ -618,7 +631,7 @@ function instrument( predefined[stateId] = stateId const innerProgram = { ...program } - const [prefix, moduleNames] = handleImports([program].concat(previous)) + const [prefix, moduleNames] = await handleImports([program].concat(previous)) for (const name of moduleNames) { predefined[name] = name } diff --git a/src/infiniteLoops/runtime.ts b/src/infiniteLoops/runtime.ts index 49443a3a8..d85ec3406 100644 --- a/src/infiniteLoops/runtime.ts +++ b/src/infiniteLoops/runtime.ts @@ -305,7 +305,10 @@ functions[FunctionNames.evalU] = sym.evaluateHybridUnary * @param previousProgramsStack Any code previously entered in the REPL & parsed into AST. * @returns SourceError if an infinite loop was detected, undefined otherwise. */ -export function testForInfiniteLoop(program: es.Program, previousProgramsStack: es.Program[]) { +export async function testForInfiniteLoop( + program: es.Program, + previousProgramsStack: es.Program[] +) { const context = createContext(Chapter.SOURCE_4, Variant.DEFAULT, undefined, undefined) const prelude = parse(context.prelude as string, context) as es.Program context.prelude = null @@ -313,7 +316,7 @@ export function testForInfiniteLoop(program: es.Program, previousProgramsStack: const newBuiltins = prepareBuiltins(context.nativeStorage.builtins) const { builtinsId, functionsId, stateId } = InfiniteLoopRuntimeObjectNames - const instrumentedCode = instrument(previous, program, newBuiltins.keys()) + const instrumentedCode = await instrument(previous, program, newBuiltins.keys()) const state = new st.State() const sandboxedRun = new Function( @@ -327,7 +330,7 @@ export function testForInfiniteLoop(program: es.Program, previousProgramsStack: ) try { - sandboxedRun(instrumentedCode, functions, state, newBuiltins, getRequireProvider(context)) + await sandboxedRun(instrumentedCode, functions, state, newBuiltins, getRequireProvider(context)) } catch (error) { if (error instanceof InfiniteLoopError) { if (state.lastLocation !== undefined) { diff --git a/src/interpreter/__tests__/__snapshots__/interpreter-errors.ts.snap b/src/interpreter/__tests__/__snapshots__/interpreter-errors.ts.snap index 4f232772a..f172a5554 100644 --- a/src/interpreter/__tests__/__snapshots__/interpreter-errors.ts.snap +++ b/src/interpreter/__tests__/__snapshots__/interpreter-errors.ts.snap @@ -849,6 +849,19 @@ Object { } `; +exports[`Importing unknown variables throws error: expectParsedError 1`] = ` +Object { + "alertResult": Array [], + "code": "import { foo1 } from 'one_module';", + "displayResult": Array [], + "numErrors": 1, + "parsedErrors": "Line 1: 'one_module' does not contain a definition for 'foo1'", + "result": undefined, + "resultStatus": "error", + "visualiseListResult": Array [], +} +`; + exports[`Nice errors when errors occur inside builtins: expectParsedError 1`] = ` Object { "alertResult": Array [], diff --git a/src/interpreter/__tests__/interpreter-errors.ts b/src/interpreter/__tests__/interpreter-errors.ts index 3319dabaf..64d077c2a 100644 --- a/src/interpreter/__tests__/interpreter-errors.ts +++ b/src/interpreter/__tests__/interpreter-errors.ts @@ -1145,5 +1145,8 @@ test('Cascading js errors work properly', () => { test('Importing unknown variables throws error', () => { expectParsedError(stripIndent` import { foo1 } from 'one_module'; - `).toMatchInlineSnapshot("'one_module' does not contain definitions for 'foo1'") + `).toMatchInlineSnapshot( + "'one_module' does not contain definitions for 'foo1'", + `"Line 1: 'one_module' does not contain a definition for 'foo1'"` + ) }) diff --git a/src/interpreter/interpreter.ts b/src/interpreter/interpreter.ts index aa968f2e3..1200f6d44 100644 --- a/src/interpreter/interpreter.ts +++ b/src/interpreter/interpreter.ts @@ -1,18 +1,19 @@ /* tslint:disable:max-classes-per-file */ -import * as es from 'estree' +import type * as es from 'estree' import { isEmpty, uniqueId } from 'lodash' import { UNKNOWN_LOCATION } from '../constants' import { LazyBuiltIn } from '../createContext' import * as errors from '../errors/errors' import { RuntimeSourceError } from '../errors/runtimeSourceError' -import { UndefinedImportError } from '../modules/errors' -import { loadModuleBundle, loadModuleTabs } from '../modules/moduleLoader' -import { ModuleFunctions } from '../modules/moduleTypes' +import { loadModuleBundle } from '../modules/moduleLoader' +import type { ImportTransformOptions, ModuleFunctions } from '../modules/moduleTypes' +import { initModuleContext } from '../modules/utils' import { checkEditorBreakpoints } from '../stdlib/inspector' import { Context, ContiguousArrayElements, Environment, Frame, Value, Variant } from '../types' import * as create from '../utils/ast/astCreator' import { conditionalExpression, literal, primitive } from '../utils/ast/astCreator' +import { simple } from '../utils/ast/walkers' import { evaluateBinaryExpression, evaluateUnaryExpression } from '../utils/operators' import * as rttc from '../utils/rttc' import Closure from './closure' @@ -712,8 +713,7 @@ function getNonEmptyEnv(environment: Environment): Environment { export function* evaluateProgram( program: es.Program, context: Context, - checkImports: boolean, - loadTabs: boolean + { loadTabs, wrapModules }: ImportTransformOptions ) { yield* visit(context, program) @@ -739,26 +739,26 @@ export function* evaluateProgram( } if (!(moduleName in moduleFunctions)) { - context.moduleContexts[moduleName] = { - state: null, - tabs: loadTabs ? loadModuleTabs(moduleName, node) : null - } - moduleFunctions[moduleName] = loadModuleBundle(moduleName, context, node) + initModuleContext(moduleName, context, loadTabs, node) + moduleFunctions[moduleName] = loadModuleBundle(moduleName, context, wrapModules, node) } const functions = moduleFunctions[moduleName] for (const spec of node.specifiers) { - if (spec.type !== 'ImportSpecifier') { - throw new Error(`Only Import Specifiers are supported, got ${spec.type}`) - } - - if (checkImports && !(spec.imported.name in functions)) { - throw new UndefinedImportError(spec.imported.name, moduleName, node) - } - declareIdentifier(context, spec.local.name, node) - defineVariable(context, spec.local.name, functions[spec.imported.name], true) + simple(spec, { + ImportSpecifier: () => + defineVariable( + context, + spec.local.name, + functions[(spec as es.ImportSpecifier).imported.name], + true + ), + ImportDefaultSpecifier: () => + defineVariable(context, spec.local.name, functions['default'], true), + ImportNamespaceSpecifier: () => defineVariable(context, spec.local.name, functions, true) + }) } yield* leave(context) } diff --git a/src/lazy/lazy.ts b/src/lazy/lazy.ts index 6ca59de1a..a53b89aca 100644 --- a/src/lazy/lazy.ts +++ b/src/lazy/lazy.ts @@ -1,8 +1,8 @@ import * as es from 'estree' import * as create from '../utils/ast/astCreator' -import { getIdentifiersInProgram } from '../utils/uniqueIds' import { simple } from '../utils/ast/walkers' +import { getIdentifiersInProgram } from '../utils/uniqueIds' const lazyPrimitives = new Set(['makeLazyFunction', 'wrapLazyCallee', 'forceIt', 'delayIt']) diff --git a/src/modules/__mocks__/moduleLoader.ts b/src/modules/__mocks__/moduleLoader.ts new file mode 100644 index 000000000..a7391b8e0 --- /dev/null +++ b/src/modules/__mocks__/moduleLoader.ts @@ -0,0 +1,15 @@ +export function loadModuleBundle() { + return { + foo: () => 'foo', + bar: () => 'bar' + } +} + +export function loadModuleTabs() { + return [] +} +export const memoizedGetModuleManifest = () => ({ + one_module: { tabs: [] }, + other_module: { tabs: [] }, + another_module: { tabs: [] } +}) diff --git a/src/modules/__mocks__/moduleLoaderAsync.ts b/src/modules/__mocks__/moduleLoaderAsync.ts new file mode 100644 index 000000000..d7b03147c --- /dev/null +++ b/src/modules/__mocks__/moduleLoaderAsync.ts @@ -0,0 +1,28 @@ +export const memoizedGetModuleDocsAsync = jest.fn().mockResolvedValue({ + foo: 'foo', + bar: 'bar' +}) + +export const memoizedGetModuleBundleAsync = jest.fn().mockResolvedValue( + `require => ({ + foo: () => 'foo', + bar: () => 'bar', +})` +) + +export const memoizedGetModuleManifestAsync = jest.fn().mockResolvedValue({ + one_module: { tabs: [] }, + other_module: { tabs: [] }, + another_module: { tabs: [] } +}) + +export function loadModuleBundleAsync() { + return Promise.resolve({ + foo: () => 'foo', + bar: () => 'bar' + }) +} + +export function loadModuleTabsAsync() { + return Promise.resolve([]) +} diff --git a/src/modules/__tests__/moduleLoader.ts b/src/modules/__tests__/moduleLoader.ts index 9d7471ba9..eb3a97b7b 100644 --- a/src/modules/__tests__/moduleLoader.ts +++ b/src/modules/__tests__/moduleLoader.ts @@ -182,4 +182,4 @@ describe('Testing modules/moduleLoader.ts in a jsdom environment', () => { expect(mockedXMLHttpRequest3.send).toHaveBeenCalledTimes(1) expect(mockedXMLHttpRequest3.send).toHaveBeenCalledWith(null) }) -}) \ No newline at end of file +}) diff --git a/src/modules/__tests__/moduleLoaderAsync.ts b/src/modules/__tests__/moduleLoaderAsync.ts index c3f07b4e7..28492169e 100644 --- a/src/modules/__tests__/moduleLoaderAsync.ts +++ b/src/modules/__tests__/moduleLoaderAsync.ts @@ -173,4 +173,4 @@ describe('Test tab loading', () => { expect(call1Url).toEqual(`${MODULES_STATIC_URL}/tabs/Tab1.js`) expect(call2Url).toEqual(`${MODULES_STATIC_URL}/tabs/Tab2.js`) }) -}) \ No newline at end of file +}) diff --git a/src/modules/errors.ts b/src/modules/errors.ts index f79a12e44..a32e8b2f5 100644 --- a/src/modules/errors.ts +++ b/src/modules/errors.ts @@ -200,4 +200,4 @@ export class ConsecutiveSlashesInFilePathError extends InvalidFilePathError { public elaborate() { return 'Remove consecutive slashes from the offending file path.' } -} \ No newline at end of file +} diff --git a/src/modules/moduleLoader.ts b/src/modules/moduleLoader.ts index 2801ce81a..cb73acd48 100644 --- a/src/modules/moduleLoader.ts +++ b/src/modules/moduleLoader.ts @@ -2,14 +2,15 @@ import es from 'estree' import { memoize } from 'lodash' import { XMLHttpRequest as NodeXMLHttpRequest } from 'xmlhttprequest-ts' -import { - ModuleConnectionError, - ModuleInternalError, - ModuleNotFoundError -} from '../errors/moduleErrors' import { Context } from '../types' import { wrapSourceModule } from '../utils/operators' -import { ModuleBundle, ModuleDocumentation, ModuleFunctions, Modules } from './moduleTypes' +import { ModuleConnectionError, ModuleInternalError, ModuleNotFoundError } from './errors' +import type { + ModuleBundle, + ModuleDocumentation, + ModuleFunctions, + ModuleManifest +} from './moduleTypes' import { getRequireProvider } from './requireProvider' // Supports both JSDom (Web Browser) environment and Node environment @@ -46,23 +47,37 @@ export function httpGet(url: string): string { * @return Modules */ export const memoizedGetModuleManifest = memoize(getModuleManifest) -function getModuleManifest(): Modules { - const rawManifest = httpGet(`${MODULES_STATIC_URL}/modules.json`) - return JSON.parse(rawManifest) +function getModuleManifest(): ModuleManifest { + try { + const rawManifest = httpGet(`${MODULES_STATIC_URL}/modules.json`) + return JSON.parse(rawManifest) + } catch (error) { + throw new ModuleConnectionError(error) + } } -/** - * Send a HTTP GET request to the modules endpoint to retrieve the specified file - * @return String of module file contents - */ +export const memoizedGetBundle = memoize(getModuleBundle) +function getModuleBundle(path: string) { + return httpGet(`${MODULES_STATIC_URL}/bundles/${path}.js`) +} -const memoizedGetModuleFileInternal = memoize(getModuleFile) -export const memoizedGetModuleFile = (name: string, type: 'tab' | 'bundle' | 'json') => - memoizedGetModuleFileInternal({ name, type }) -function getModuleFile({ name, type }: { name: string; type: 'tab' | 'bundle' | 'json' }): string { - return httpGet(`${MODULES_STATIC_URL}/${type}s/${name}.js${type === 'json' ? 'on' : ''}`) +export const memoizedGetTab = memoize(getModuleTab) +function getModuleTab(path: string) { + return httpGet(`${MODULES_STATIC_URL}/tabs/${path}.js`) } +// /** +// * Send a HTTP GET request to the modules endpoint to retrieve the specified file +// * @return String of module file contents +// */ + +// const memoizedGetModuleFileInternal = memoize(getModuleFile) +// export const memoizedGetModuleFile = (name: string, type: 'tab' | 'bundle' | 'json') => +// memoizedGetModuleFileInternal({ name, type }) +// function getModuleFile({ name, type }: { name: string; type: 'tab' | 'bundle' | 'json' }): string { +// return httpGet(`${MODULES_STATIC_URL}/${type}s/${name}.js${type === 'json' ? 'on' : ''}`) +// } + /** * Loads the respective module package (functions from the module) * @param path imported module name @@ -70,7 +85,12 @@ function getModuleFile({ name, type }: { name: string; type: 'tab' | 'bundle' | * @param node import declaration node * @returns the module's functions object */ -export function loadModuleBundle(path: string, context: Context, node?: es.Node): ModuleFunctions { +export function loadModuleBundle( + path: string, + context: Context, + wrapModules: boolean, + node?: es.Node +): ModuleFunctions { const modules = memoizedGetModuleManifest() // Check if the module exists @@ -78,9 +98,10 @@ export function loadModuleBundle(path: string, context: Context, node?: es.Node) if (moduleList.includes(path) === false) throw new ModuleNotFoundError(path, node) // Get module file - const moduleText = memoizedGetModuleFile(path, 'bundle') + const moduleText = memoizedGetBundle(path) try { const moduleBundle: ModuleBundle = eval(moduleText) + if (wrapModules) return moduleBundle(getRequireProvider(context)) return wrapSourceModule(path, moduleBundle, getRequireProvider(context)) } catch (error) { // console.error("bundle error: ", error) @@ -105,7 +126,7 @@ export function loadModuleTabs(path: string, node?: es.Node) { const sideContentTabPaths: string[] = modules[path].tabs // Load the tabs for the current module return sideContentTabPaths.map(path => { - const rawTabFile = memoizedGetModuleFile(path, 'tab') + const rawTabFile = memoizedGetTab(path) try { return eval(rawTabFile) } catch (error) { @@ -115,17 +136,17 @@ export function loadModuleTabs(path: string, node?: es.Node) { }) } -export const memoizedloadModuleDocs = memoize(loadModuleDocs) +export const memoizedGetModuleDocs = memoize(loadModuleDocs) export function loadModuleDocs(path: string, node?: es.Node) { try { const modules = memoizedGetModuleManifest() // Check if the module exists const moduleList = Object.keys(modules) if (!moduleList.includes(path)) throw new ModuleNotFoundError(path, node) - const result = getModuleFile({ name: path, type: 'json' }) + const result = httpGet(`${MODULES_STATIC_URL}/jsons/${path}.json`) return JSON.parse(result) as ModuleDocumentation } catch (error) { - console.warn('Failed to load module documentation') + console.warn(`Failed to load documentation for ${path}:`, error) return null } } diff --git a/src/modules/moduleLoaderAsync.ts b/src/modules/moduleLoaderAsync.ts index deb2c8557..1e3ffecb3 100644 --- a/src/modules/moduleLoaderAsync.ts +++ b/src/modules/moduleLoaderAsync.ts @@ -99,4 +99,4 @@ export async function loadModuleBundleAsync( // console.error("bundle error: ", error) throw new ModuleInternalError(moduleName, error, node) } -} \ No newline at end of file +} diff --git a/src/modules/moduleTypes.ts b/src/modules/moduleTypes.ts index baeccc50d..72a170c1f 100644 --- a/src/modules/moduleTypes.ts +++ b/src/modules/moduleTypes.ts @@ -40,4 +40,4 @@ export type ImportResolutionOptions = { allowUndefinedImports: boolean } -export type ImportOptions = ImportResolutionOptions & ImportTransformOptions \ No newline at end of file +export type ImportOptions = ImportResolutionOptions & ImportTransformOptions diff --git a/src/modules/preprocessor/__tests__/__snapshots__/preprocessor.ts.snap b/src/modules/preprocessor/__tests__/__snapshots__/preprocessor.ts.snap new file mode 100644 index 000000000..c44e78257 --- /dev/null +++ b/src/modules/preprocessor/__tests__/__snapshots__/preprocessor.ts.snap @@ -0,0 +1,96 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`preprocessFileImports collates Source module imports at the start of the top-level environment of the preprocessed program 1`] = ` +"import {x, y, z, w} from \\"one_module\\"; +import {h} from \\"another_module\\"; +import {f, g} from \\"other_module\\"; +import {x, y, z, w} from \\"one_module\\"; +import {h} from \\"another_module\\"; +import {f, g} from \\"other_module\\"; +function __$c$$dot$$js__() { + const square = x => x * x; + return pair(null, list(pair(\\"square\\", square))); +} +function __$b$$dot$$js__(___$c$$dot$$js___) { + const square = __access_export__(___$c$$dot$$js___, \\"square\\"); + const b = square(5); + return pair(null, list(pair(\\"b\\", b))); +} +const ___$c$$dot$$js___ = __$c$$dot$$js__(); +const ___$b$$dot$$js___ = __$b$$dot$$js__(___$c$$dot$$js___); +const b = __access_export__(___$b$$dot$$js___, \\"b\\"); +b; +" +`; + +exports[`preprocessFileImports ignores Source module imports & removes all non-Source module import-related AST nodes in the preprocessed program 1`] = ` +"import {a, b, c} from \\"one_module\\"; +import d from \\"one_module\\"; +import {a, b, c} from \\"one_module\\"; +import d from \\"one_module\\"; +function __$not$$dash$$source$$dash$$module$$dot$$js__() { + const x = 1; + const y = 2; + const z = 3; + function square(x) { + return x * x; + } + return pair(square, list(pair(\\"x\\", x), pair(\\"y\\", y), pair(\\"z\\", z))); +} +const ___$not$$dash$$source$$dash$$module$$dot$$js___ = __$not$$dash$$source$$dash$$module$$dot$$js__(); +const w = __access_export__(___$not$$dash$$source$$dash$$module$$dot$$js___, \\"default\\"); +const x = __access_export__(___$not$$dash$$source$$dash$$module$$dot$$js___, \\"x\\"); +const y = __access_export__(___$not$$dash$$source$$dash$$module$$dot$$js___, \\"y\\"); +const z = __access_export__(___$not$$dash$$source$$dash$$module$$dot$$js___, \\"z\\"); +" +`; + +exports[`preprocessFileImports removes all export-related AST nodes 1`] = ` +"const x = 42; +let y = 53; +function square(x) { + return x * x; +} +const id = x => x; +function cube(x) { + return x * x * x; +} +" +`; + +exports[`preprocessFileImports returns a preprocessed program with all imports 1`] = ` +"function __$d$$dot$$js__() { + const addTwo = x => x + 2; + return pair(null, list(pair(\\"mysteryFunction\\", addTwo))); +} +function __$c$$dot$$js__(___$d$$dot$$js___) { + const mysteryFunction = __access_export__(___$d$$dot$$js___, \\"mysteryFunction\\"); + const x = mysteryFunction(5); + function square(x) { + return x * x; + } + return pair(x, list(pair(\\"square\\", square))); +} +function __$b$$dot$$js__(___$c$$dot$$js___) { + const y = __access_export__(___$c$$dot$$js___, \\"default\\"); + const square = __access_export__(___$c$$dot$$js___, \\"square\\"); + const a = square(y); + const b = 3; + return pair(null, list(pair(\\"a\\", a), pair(\\"b\\", b))); +} +const ___$d$$dot$$js___ = __$d$$dot$$js__(); +const ___$c$$dot$$js___ = __$c$$dot$$js__(___$d$$dot$$js___); +const ___$b$$dot$$js___ = __$b$$dot$$js__(___$c$$dot$$js___); +const x = __access_export__(___$b$$dot$$js___, \\"a\\"); +const y = __access_export__(___$b$$dot$$js___, \\"b\\"); +x + y; +" +`; + +exports[`preprocessFileImports returns the same AST if the entrypoint file does not contain import/export statements 1`] = ` +"function square(x) { + return x * x; +} +square(5); +" +`; diff --git a/src/modules/preprocessor/__tests__/preprocessor.ts b/src/modules/preprocessor/__tests__/preprocessor.ts index df3f6bbf1..b1b7ca54c 100644 --- a/src/modules/preprocessor/__tests__/preprocessor.ts +++ b/src/modules/preprocessor/__tests__/preprocessor.ts @@ -1,13 +1,19 @@ -import es from 'estree' +import { generate } from 'astring' +import type { Program } from 'estree' +import type { MockedFunction } from 'jest-mock' -import { parseError } from '../../../index' +import { parseError } from '../../..' import { mockContext } from '../../../mocks/context' -import { parse } from '../../../parser/parser' -import { accessExportFunctionName, defaultExportLookupName } from '../../../stdlib/localImport.prelude' import { Chapter } from '../../../types' -import preprocessFileImports, { getImportedLocalModulePaths } from '../preprocessor' -import { parseCodeError, stripLocationInfo } from './utils' +import { memoizedGetModuleDocsAsync } from '../../moduleLoaderAsync' +import preprocessFileImports from '..' +import hoistAndMergeImports from '../transformers/hoistAndMergeImports' +import { parseCodeError } from './utils' +// The preprocessor now checks for the existence of source modules +// so this is here to solve that issue + +/* describe('getImportedLocalModulePaths', () => { let context = mockContext(Chapter.LIBRARY_PARSER) @@ -78,55 +84,68 @@ describe('getImportedLocalModulePaths', () => { assertCorrectModulePathsAreReturned(code, '/dir/a.js', ['/dir/b.js', '/dir/c.js']) }) }) +*/ describe('preprocessFileImports', () => { let actualContext = mockContext(Chapter.LIBRARY_PARSER) - let expectedContext = mockContext(Chapter.LIBRARY_PARSER) + // let expectedContext = mockContext(Chapter.LIBRARY_PARSER) beforeEach(() => { actualContext = mockContext(Chapter.LIBRARY_PARSER) - expectedContext = mockContext(Chapter.LIBRARY_PARSER) + // expectedContext = mockContext(Chapter.LIBRARY_PARSER) }) - const assertASTsAreEquivalent = ( - actualProgram: es.Program | undefined, - expectedCode: string - ): void => { - if (actualProgram === undefined) { + // const assertASTsAreEquivalent = ( + // actualProgram: Program | undefined, + // expectedCode: string + // ): void => { + // // assert(actualProgram !== undefined, 'Actual program should not be undefined') + // if (!actualProgram) { + // // console.log(actualContext.errors[0], 'occurred at:', actualContext.errors[0].location.start) + // throw new Error('Actual program should not be undefined!') + // } + + // const expectedProgram = parse(expectedCode, expectedContext) + // if (expectedProgram === null) { + // throw parseCodeError + // } + + // expect(stripLocationInfo(actualProgram)).toEqual(stripLocationInfo(expectedProgram)) + // } + + const testAgainstSnapshot = (program: Program | undefined | null) => { + if (!program) { throw parseCodeError } - const expectedProgram = parse(expectedCode, expectedContext) - if (expectedProgram === null) { - throw parseCodeError - } + hoistAndMergeImports(program, { '': program }) - expect(stripLocationInfo(actualProgram)).toEqual(stripLocationInfo(expectedProgram)) + expect(generate(program)).toMatchSnapshot() } - it('returns undefined & adds CannotFindModuleError to context if the entrypoint file does not exist', () => { + it('returns undefined & adds ModuleNotFoundError to context if the entrypoint file does not exist', async () => { const files: Record = { '/a.js': '1 + 2;' } - const actualProgram = preprocessFileImports(files, '/non-existent-file.js', actualContext) + const actualProgram = await preprocessFileImports(files, '/non-existent-file.js', actualContext) expect(actualProgram).toBeUndefined() expect(parseError(actualContext.errors)).toMatchInlineSnapshot( - `"Cannot find module '/non-existent-file.js'."` + `"Module '/non-existent-file.js' not found."` ) }) - it('returns undefined & adds CannotFindModuleError to context if an imported file does not exist', () => { + it('returns undefined & adds ModuleNotFoundError to context if an imported file does not exist', async () => { const files: Record = { '/a.js': `import { x } from './non-existent-file.js';` } - const actualProgram = preprocessFileImports(files, '/a.js', actualContext) + const actualProgram = await preprocessFileImports(files, '/a.js', actualContext) expect(actualProgram).toBeUndefined() expect(parseError(actualContext.errors)).toMatchInlineSnapshot( - `"Cannot find module '/non-existent-file.js'."` + `"Line 1: Module '/non-existent-file.js' not found."` ) }) - it('returns the same AST if the entrypoint file does not contain import/export statements', () => { + it('returns the same AST if the entrypoint file does not contain import/export statements', async () => { const files: Record = { '/a.js': ` function square(x) { @@ -135,12 +154,12 @@ describe('preprocessFileImports', () => { square(5); ` } - const expectedCode = files['/a.js'] - const actualProgram = preprocessFileImports(files, '/a.js', actualContext) - assertASTsAreEquivalent(actualProgram, expectedCode) + // const expectedCode = files['/a.js'] + const actualProgram = await preprocessFileImports(files, '/a.js', actualContext) + testAgainstSnapshot(actualProgram) }) - it('removes all export-related AST nodes', () => { + it('removes all export-related AST nodes', async () => { const files: Record = { '/a.js': ` export const x = 42; @@ -154,25 +173,35 @@ describe('preprocessFileImports', () => { } ` } - const expectedCode = ` - const x = 42; - let y = 53; - function square(x) { - return x * x; - } - const id = x => x; - function cube(x) { - return x * x * x; - } - ` - const actualProgram = preprocessFileImports(files, '/a.js', actualContext) - assertASTsAreEquivalent(actualProgram, expectedCode) + // const expectedCode = ` + // const x = 42; + // let y = 53; + // function square(x) { + // return x * x; + // } + // const id = x => x; + // function cube(x) { + // return x * x * x; + // } + // ` + const actualProgram = await preprocessFileImports(files, '/a.js', actualContext) + testAgainstSnapshot(actualProgram) }) - it('ignores Source module imports & removes all non-Source module import-related AST nodes in the preprocessed program', () => { + it('ignores Source module imports & removes all non-Source module import-related AST nodes in the preprocessed program', async () => { + const docsMocked = memoizedGetModuleDocsAsync as MockedFunction< + typeof memoizedGetModuleDocsAsync + > + docsMocked.mockResolvedValueOnce({ + default: '', + a: '', + b: '', + c: '' + }) + const files: Record = { '/a.js': ` - import d, { a, b, c } from "source-module"; + import d, { a, b, c } from "one_module"; import w, { x, y, z } from "./not-source-module.js"; `, '/not-source-module.js': ` @@ -184,84 +213,102 @@ describe('preprocessFileImports', () => { } ` } - const expectedCode = ` - import { a, b, c } from "source-module"; - - function __$not$$dash$$source$$dash$$module$$dot$$js__() { - const x = 1; - const y = 2; - const z = 3; - function square(x) { - return x * x; - } - - return pair(square, list(pair("x", x), pair("y", y), pair("z", z))); - } - - const ___$not$$dash$$source$$dash$$module$$dot$$js___ = __$not$$dash$$source$$dash$$module$$dot$$js__(); - - const w = ${accessExportFunctionName}(___$not$$dash$$source$$dash$$module$$dot$$js___, "${defaultExportLookupName}"); - const x = ${accessExportFunctionName}(___$not$$dash$$source$$dash$$module$$dot$$js___, "x"); - const y = ${accessExportFunctionName}(___$not$$dash$$source$$dash$$module$$dot$$js___, "y"); - const z = ${accessExportFunctionName}(___$not$$dash$$source$$dash$$module$$dot$$js___, "z"); - ` - const actualProgram = preprocessFileImports(files, '/a.js', actualContext) - assertASTsAreEquivalent(actualProgram, expectedCode) + // const expectedCode = ` + // import { a, b, c } from "one_module"; + + // function __$not$$dash$$source$$dash$$module$$dot$$js__() { + // const x = 1; + // const y = 2; + // const z = 3; + // function square(x) { + // return x * x; + // } + + // return pair(square, list(pair("x", x), pair("y", y), pair("z", z))); + // } + + // const ___$not$$dash$$source$$dash$$module$$dot$$js___ = __$not$$dash$$source$$dash$$module$$dot$$js__(); + + // const w = ${accessExportFunctionName}(___$not$$dash$$source$$dash$$module$$dot$$js___, "${defaultExportLookupName}"); + // const x = ${accessExportFunctionName}(___$not$$dash$$source$$dash$$module$$dot$$js___, "x"); + // const y = ${accessExportFunctionName}(___$not$$dash$$source$$dash$$module$$dot$$js___, "y"); + // const z = ${accessExportFunctionName}(___$not$$dash$$source$$dash$$module$$dot$$js___, "z"); + // ` + const actualProgram = await preprocessFileImports(files, '/a.js', actualContext, { + allowUndefinedImports: true + }) + testAgainstSnapshot(actualProgram) + // assertASTsAreEquivalent(actualProgram, expectedCode) }) - it('collates Source module imports at the start of the top-level environment of the preprocessed program', () => { + it('collates Source module imports at the start of the top-level environment of the preprocessed program', async () => { + const docsMocked = memoizedGetModuleDocsAsync as MockedFunction< + typeof memoizedGetModuleDocsAsync + > + docsMocked.mockResolvedValue({ + f: '', + g: '', + h: '', + w: '', + x: '', + y: '', + z: '' + }) const files: Record = { '/a.js': ` import { b } from "./b.js"; - import { w, x } from "source-module"; - import { f, g } from "other-source-module"; + import { w, x } from "one_module"; + import { f, g } from "other_module"; b; `, '/b.js': ` import { square } from "./c.js"; - import { x, y } from "source-module"; - import { h } from "another-source-module"; + import { x, y } from "one_module"; + import { h } from "another_module"; export const b = square(5); `, '/c.js': ` - import { x, y, z } from "source-module"; + import { x, y, z } from "one_module"; export const square = x => x * x; ` } - const expectedCode = ` - import { w, x, y, z } from "source-module"; - import { f, g } from "other-source-module"; - import { h } from "another-source-module"; + // const expectedCode = ` + // import { w, x, y, z } from "one_module"; + // import { f, g } from "other_module"; + // import { h } from "another_module"; - function __$b$$dot$$js__(___$c$$dot$$js___) { - const square = ${accessExportFunctionName}(___$c$$dot$$js___, "square"); + // function __$b$$dot$$js__(___$c$$dot$$js___) { + // const square = ${accessExportFunctionName}(___$c$$dot$$js___, "square"); - const b = square(5); + // const b = square(5); - return pair(null, list(pair("b", b))); - } + // return pair(null, list(pair("b", b))); + // } - function __$c$$dot$$js__() { - const square = x => x * x; + // function __$c$$dot$$js__() { + // const square = x => x * x; - return pair(null, list(pair("square", square))); - } + // return pair(null, list(pair("square", square))); + // } - const ___$c$$dot$$js___ = __$c$$dot$$js__(); - const ___$b$$dot$$js___ = __$b$$dot$$js__(___$c$$dot$$js___); + // const ___$c$$dot$$js___ = __$c$$dot$$js__(); + // const ___$b$$dot$$js___ = __$b$$dot$$js__(___$c$$dot$$js___); - const b = ${accessExportFunctionName}(___$b$$dot$$js___, "b"); + // const b = ${accessExportFunctionName}(___$b$$dot$$js___, "b"); - b; - ` - const actualProgram = preprocessFileImports(files, '/a.js', actualContext) - assertASTsAreEquivalent(actualProgram, expectedCode) + // b; + // ` + const actualProgram = await preprocessFileImports(files, '/a.js', actualContext, { + allowUndefinedImports: true + }) + testAgainstSnapshot(actualProgram) + // assertASTsAreEquivalent(actualProgram, expectedCode) }) - it('returns CircularImportError if there are circular imports', () => { + it('returns CircularImportError if there are circular imports', async () => { const files: Record = { '/a.js': ` import { b } from "./b.js"; @@ -279,13 +326,13 @@ describe('preprocessFileImports', () => { export const c = 3; ` } - preprocessFileImports(files, '/a.js', actualContext) + await preprocessFileImports(files, '/a.js', actualContext) expect(parseError(actualContext.errors)).toMatchInlineSnapshot( - `"Circular import detected: '/a.js' -> '/b.js' -> '/c.js' -> '/a.js'."` + `"Circular import detected: '/c.js' -> '/a.js' -> '/b.js' -> '/c.js'."` ) }) - it('returns CircularImportError if there are circular imports - verbose', () => { + it('returns CircularImportError if there are circular imports - verbose', async () => { const files: Record = { '/a.js': ` import { b } from "./b.js"; @@ -303,15 +350,15 @@ describe('preprocessFileImports', () => { export const c = 3; ` } - preprocessFileImports(files, '/a.js', actualContext) + await preprocessFileImports(files, '/a.js', actualContext) expect(parseError(actualContext.errors, true)).toMatchInlineSnapshot(` - "Circular import detected: '/a.js' -> '/b.js' -> '/c.js' -> '/a.js'. + "Circular import detected: '/c.js' -> '/a.js' -> '/b.js' -> '/c.js'. Break the circular import cycle by removing imports from any of the offending files. " `) }) - it('returns CircularImportError if there are self-imports', () => { + it('returns CircularImportError if there are self-imports', async () => { const files: Record = { '/a.js': ` import { y } from "./a.js"; @@ -319,13 +366,13 @@ describe('preprocessFileImports', () => { export { x as y }; ` } - preprocessFileImports(files, '/a.js', actualContext) + await preprocessFileImports(files, '/a.js', actualContext) expect(parseError(actualContext.errors)).toMatchInlineSnapshot( `"Circular import detected: '/a.js' -> '/a.js'."` ) }) - it('returns CircularImportError if there are self-imports - verbose', () => { + it('returns CircularImportError if there are self-imports - verbose', async () => { const files: Record = { '/a.js': ` import { y } from "./a.js"; @@ -333,7 +380,7 @@ describe('preprocessFileImports', () => { export { x as y }; ` } - preprocessFileImports(files, '/a.js', actualContext) + await preprocessFileImports(files, '/a.js', actualContext) expect(parseError(actualContext.errors, true)).toMatchInlineSnapshot(` "Circular import detected: '/a.js' -> '/a.js'. Break the circular import cycle by removing imports from any of the offending files. @@ -341,7 +388,7 @@ describe('preprocessFileImports', () => { `) }) - it('returns a preprocessed program with all imports', () => { + it('returns a preprocessed program with all imports', async () => { const files: Record = { '/a.js': ` import { a as x, b as y } from "./b.js"; @@ -369,44 +416,46 @@ describe('preprocessFileImports', () => { export { addTwo as mysteryFunction }; ` } - const expectedCode = ` - function __$b$$dot$$js__(___$c$$dot$$js___) { - const y = ${accessExportFunctionName}(___$c$$dot$$js___, "${defaultExportLookupName}"); - const square = ${accessExportFunctionName}(___$c$$dot$$js___, "square"); + // const expectedCode = ` + // function __$b$$dot$$js__(___$c$$dot$$js___) { + // const y = ${accessExportFunctionName}(___$c$$dot$$js___, "${defaultExportLookupName}"); + // const square = ${accessExportFunctionName}(___$c$$dot$$js___, "square"); - const a = square(y); - const b = 3; + // const a = square(y); + // const b = 3; - return pair(null, list(pair("a", a), pair("b", b))); - } + // return pair(null, list(pair("a", a), pair("b", b))); + // } - function __$c$$dot$$js__(___$d$$dot$$js___) { - const mysteryFunction = ${accessExportFunctionName}(___$d$$dot$$js___, "mysteryFunction"); + // function __$c$$dot$$js__(___$d$$dot$$js___) { + // const mysteryFunction = ${accessExportFunctionName}(___$d$$dot$$js___, "mysteryFunction"); - const x = mysteryFunction(5); - function square(x) { - return x * x; - } + // const x = mysteryFunction(5); + // function square(x) { + // return x * x; + // } - return pair(x, list(pair("square", square))); - } + // return pair(x, list(pair("square", square))); + // } - function __$d$$dot$$js__() { - const addTwo = x => x + 2; + // function __$d$$dot$$js__() { + // const addTwo = x => x + 2; - return pair(null, list(pair("mysteryFunction", addTwo))); - } + // return pair(null, list(pair("mysteryFunction", addTwo))); + // } - const ___$d$$dot$$js___ = __$d$$dot$$js__(); - const ___$c$$dot$$js___ = __$c$$dot$$js__(___$d$$dot$$js___); - const ___$b$$dot$$js___ = __$b$$dot$$js__(___$c$$dot$$js___); + // const ___$d$$dot$$js___ = __$d$$dot$$js__(); + // const ___$c$$dot$$js___ = __$c$$dot$$js__(___$d$$dot$$js___); + // const ___$b$$dot$$js___ = __$b$$dot$$js__(___$c$$dot$$js___); - const x = ${accessExportFunctionName}(___$b$$dot$$js___, "a"); - const y = ${accessExportFunctionName}(___$b$$dot$$js___, "b"); + // const x = ${accessExportFunctionName}(___$b$$dot$$js___, "a"); + // const y = ${accessExportFunctionName}(___$b$$dot$$js___, "b"); - x + y; - ` - const actualProgram = preprocessFileImports(files, '/a.js', actualContext) - assertASTsAreEquivalent(actualProgram, expectedCode) + // x + y; + // ` + const actualProgram = await preprocessFileImports(files, '/a.js', actualContext, { + allowUndefinedImports: true + }) + testAgainstSnapshot(actualProgram) }) }) diff --git a/src/modules/preprocessor/__tests__/transformers/__snapshots__/hoistAndMergeImports.ts.snap b/src/modules/preprocessor/__tests__/transformers/__snapshots__/hoistAndMergeImports.ts.snap new file mode 100644 index 000000000..0e387b4ca --- /dev/null +++ b/src/modules/preprocessor/__tests__/transformers/__snapshots__/hoistAndMergeImports.ts.snap @@ -0,0 +1,21 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`hoistAndMergeImports hoists import declarations to the top of the program 1`] = ` +"import x from \\"source-module\\"; +function square(x) { + return x * x; +} +import {a, b, c} from \\"./a.js\\"; +export {square}; +import x from \\"source-module\\"; +square(3); +" +`; + +exports[`hoistAndMergeImports merges import declarations from the same module 1`] = ` +"import {a, b, c} from \\"./a.js\\"; +import {d} from \\"./a.js\\"; +import {x} from \\"./b.js\\"; +import {e, f} from \\"./a.js\\"; +" +`; diff --git a/src/modules/preprocessor/__tests__/transformers/hoistAndMergeImports.ts b/src/modules/preprocessor/__tests__/transformers/hoistAndMergeImports.ts index c06022b51..4e92660db 100644 --- a/src/modules/preprocessor/__tests__/transformers/hoistAndMergeImports.ts +++ b/src/modules/preprocessor/__tests__/transformers/hoistAndMergeImports.ts @@ -1,27 +1,39 @@ +import { generate } from 'astring' + import { mockContext } from '../../../../mocks/context' import { parse } from '../../../../parser/parser' import { Chapter } from '../../../../types' -import { hoistAndMergeImports } from '../../transformers/hoistAndMergeImports' -import { parseCodeError, stripLocationInfo } from '../utils' +import hoistAndMergeImports from '../../transformers/hoistAndMergeImports' +import { parseCodeError } from '../utils' describe('hoistAndMergeImports', () => { let actualContext = mockContext(Chapter.LIBRARY_PARSER) - let expectedContext = mockContext(Chapter.LIBRARY_PARSER) + // let expectedContext = mockContext(Chapter.LIBRARY_PARSER) beforeEach(() => { actualContext = mockContext(Chapter.LIBRARY_PARSER) - expectedContext = mockContext(Chapter.LIBRARY_PARSER) + // expectedContext = mockContext(Chapter.LIBRARY_PARSER) }) - const assertASTsAreEquivalent = (actualCode: string, expectedCode: string): void => { - const actualProgram = parse(actualCode, actualContext) - const expectedProgram = parse(expectedCode, expectedContext) - if (actualProgram === null || expectedProgram === null) { + // const assertASTsAreEquivalent = (actualCode: string, expectedCode: string): void => { + // const actualProgram = parse(actualCode, actualContext) + // const expectedProgram = parse(expectedCode, expectedContext) + // if (actualProgram === null || expectedProgram === null) { + // throw parseCodeError + // } + + // hoistAndMergeImports(actualProgram, [actualProgram]) + // expect(stripLocationInfo(actualProgram)).toEqual(stripLocationInfo(expectedProgram)) + // } + + const testAgainstSnapshot = (code: string) => { + const program = parse(code, actualContext) + if (program === null) { throw parseCodeError } - hoistAndMergeImports(actualProgram) - expect(stripLocationInfo(actualProgram)).toEqual(stripLocationInfo(expectedProgram)) + hoistAndMergeImports(program, { '': program }) + expect(generate(program)).toMatchSnapshot() } test('hoists import declarations to the top of the program', () => { @@ -38,19 +50,20 @@ describe('hoistAndMergeImports', () => { square(3); ` - const expectedCode = ` - import { a, b, c } from "./a.js"; - import x from "source-module"; + testAgainstSnapshot(actualCode) + // const expectedCode = ` + // import { a, b, c } from "./a.js"; + // import x from "source-module"; - function square(x) { - return x * x; - } + // function square(x) { + // return x * x; + // } - export { square }; + // export { square }; - square(3); - ` - assertASTsAreEquivalent(actualCode, expectedCode) + // square(3); + // ` + // assertASTsAreEquivalent(actualCode, expectedCode) }) test('merges import declarations from the same module', () => { @@ -60,10 +73,12 @@ describe('hoistAndMergeImports', () => { import { x } from "./b.js"; import { e, f } from "./a.js"; ` - const expectedCode = ` - import { a, b, c, d, e, f } from "./a.js"; - import { x } from "./b.js"; - ` - assertASTsAreEquivalent(actualCode, expectedCode) + + testAgainstSnapshot(actualCode) + // const expectedCode = ` + // import { a, b, c, d, e, f } from "./a.js"; + // import { x } from "./b.js"; + // ` + // assertASTsAreEquivalent(actualCode, expectedCode) }) }) diff --git a/src/modules/preprocessor/__tests__/transformers/removeExports.ts b/src/modules/preprocessor/__tests__/transformers/removeExports.ts deleted file mode 100644 index 9a2c96eb8..000000000 --- a/src/modules/preprocessor/__tests__/transformers/removeExports.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { mockContext } from '../../../../mocks/context' -import { parse } from '../../../../parser/parser' -import { Chapter } from '../../../../types' -import { removeExports } from '../../transformers/removeExports' -import { parseCodeError, stripLocationInfo } from '../utils' - -describe('removeExports', () => { - let actualContext = mockContext(Chapter.LIBRARY_PARSER) - let expectedContext = mockContext(Chapter.LIBRARY_PARSER) - - beforeEach(() => { - actualContext = mockContext(Chapter.LIBRARY_PARSER) - expectedContext = mockContext(Chapter.LIBRARY_PARSER) - }) - - const assertASTsAreEquivalent = (actualCode: string, expectedCode: string): void => { - const actualProgram = parse(actualCode, actualContext) - const expectedProgram = parse(expectedCode, expectedContext) - if (actualProgram === null || expectedProgram === null) { - throw parseCodeError - } - - removeExports(actualProgram) - expect(stripLocationInfo(actualProgram)).toEqual(stripLocationInfo(expectedProgram)) - } - - describe('removes ExportNamedDeclaration nodes', () => { - test('when exporting variable declarations', () => { - const actualCode = ` - export const x = 42; - export let y = 53; - ` - const expectedCode = ` - const x = 42; - let y = 53; - ` - assertASTsAreEquivalent(actualCode, expectedCode) - }) - - test('when exporting function declarations', () => { - const actualCode = ` - export function square(x) { - return x * x; - } - ` - const expectedCode = ` - function square(x) { - return x * x; - } - ` - assertASTsAreEquivalent(actualCode, expectedCode) - }) - - test('when exporting arrow function declarations', () => { - const actualCode = ` - export const square = x => x * x; - ` - const expectedCode = ` - const square = x => x * x; - ` - assertASTsAreEquivalent(actualCode, expectedCode) - }) - - test('when exporting (renamed) identifiers', () => { - const actualCode = ` - const x = 42; - let y = 53; - function square(x) { - return x * x; - } - const id = x => x; - export { x, y, square as sq, id as default }; - ` - const expectedCode = ` - const x = 42; - let y = 53; - function square(x) { - return x * x; - } - const id = x => x; - ` - assertASTsAreEquivalent(actualCode, expectedCode) - }) - }) - - describe('removes ExportDefaultDeclaration nodes', () => { - // Default exports of variable declarations and arrow function declarations - // is not allowed in ES6, and will be caught by the Acorn parser. - test('when exporting function declarations', () => { - const actualCode = ` - export default function square(x) { - return x * x; - } - ` - const expectedCode = ` - function square(x) { - return x * x; - } - ` - assertASTsAreEquivalent(actualCode, expectedCode) - }) - - test('when exporting constants', () => { - const actualCode = ` - const x = 42; - export default x; - ` - const expectedCode = ` - const x = 42; - ` - assertASTsAreEquivalent(actualCode, expectedCode) - }) - - test('when exporting variables', () => { - const actualCode = ` - let y = 53; - export default y; - ` - const expectedCode = ` - let y = 53; - ` - assertASTsAreEquivalent(actualCode, expectedCode) - }) - - test('when exporting functions', () => { - const actualCode = ` - function square(x) { - return x * x; - } - export default square; - ` - const expectedCode = ` - function square(x) { - return x * x; - } - ` - assertASTsAreEquivalent(actualCode, expectedCode) - }) - - test('when exporting arrow functions', () => { - const actualCode = ` - const id = x => x; - export default id; - ` - const expectedCode = ` - const id = x => x; - ` - assertASTsAreEquivalent(actualCode, expectedCode) - }) - - test('when exporting expressions', () => { - const actualCode = ` - export default 123 + 456; - ` - const expectedCode = '' - assertASTsAreEquivalent(actualCode, expectedCode) - }) - }) -}) diff --git a/src/modules/preprocessor/__tests__/transformers/removeNonSourceModuleImports.ts b/src/modules/preprocessor/__tests__/transformers/removeNonSourceModuleImports.ts deleted file mode 100644 index 012d63fd6..000000000 --- a/src/modules/preprocessor/__tests__/transformers/removeNonSourceModuleImports.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { mockContext } from '../../../../mocks/context' -import { parse } from '../../../../parser/parser' -import { Chapter } from '../../../../types' -import { removeNonSourceModuleImports } from '../../transformers/removeNonSourceModuleImports' -import { parseCodeError, stripLocationInfo } from '../utils' - -describe('removeNonSourceModuleImports', () => { - let actualContext = mockContext(Chapter.LIBRARY_PARSER) - let expectedContext = mockContext(Chapter.LIBRARY_PARSER) - - beforeEach(() => { - actualContext = mockContext(Chapter.LIBRARY_PARSER) - expectedContext = mockContext(Chapter.LIBRARY_PARSER) - }) - - const assertASTsAreEquivalent = (actualCode: string, expectedCode: string): void => { - const actualProgram = parse(actualCode, actualContext) - const expectedProgram = parse(expectedCode, expectedContext) - if (actualProgram === null || expectedProgram === null) { - throw parseCodeError - } - - removeNonSourceModuleImports(actualProgram) - expect(stripLocationInfo(actualProgram)).toEqual(stripLocationInfo(expectedProgram)) - } - - test('removes ImportDefaultSpecifier nodes', () => { - const actualCode = ` - import a from "./a.js"; - import x from "source-module"; - ` - const expectedCode = '' - assertASTsAreEquivalent(actualCode, expectedCode) - }) - - // While 'removeNonSourceModuleImports' will remove ImportNamespaceSpecifier nodes, we - // cannot actually test it because ImportNamespaceSpecifier nodes are banned in the parser. - // test('removes ImportNamespaceSpecifier nodes', () => { - // const actualCode = ` - // import * as a from "./a.js"; - // import * as x from "source-module"; - // ` - // const expectedCode = '' - // assertASTsAreEquivalent(actualCode, expectedCode) - // }) - - test('removes only non-Source module ImportSpecifier nodes', () => { - const actualCode = ` - import { a, b, c } from "./a.js"; - import { x, y, z } from "source-module"; - ` - const expectedCode = ` - import { x, y, z } from "source-module"; - ` - assertASTsAreEquivalent(actualCode, expectedCode) - }) -}) diff --git a/src/modules/preprocessor/analyzer.ts b/src/modules/preprocessor/analyzer.ts index 85fcdc7d8..50460487d 100644 --- a/src/modules/preprocessor/analyzer.ts +++ b/src/modules/preprocessor/analyzer.ts @@ -1,4 +1,3 @@ - import { UNKNOWN_LOCATION } from '../../constants' import { ModuleInternalError, @@ -379,4 +378,4 @@ export default async function analyzeImportsAndExports( } return newPrograms -} \ No newline at end of file +} diff --git a/src/modules/preprocessor/constructors/baseConstructors.ts b/src/modules/preprocessor/constructors/baseConstructors.ts index f67fb5c6c..eea13ac83 100644 --- a/src/modules/preprocessor/constructors/baseConstructors.ts +++ b/src/modules/preprocessor/constructors/baseConstructors.ts @@ -1,4 +1,4 @@ -import es from 'estree' +import type * as es from '../../../utils/ast/types' // Note that typecasting is done on some of the constructed AST nodes because // the ESTree AST node types are not fully aligned with the actual AST that @@ -74,7 +74,7 @@ export const createFunctionDeclaration = ( name: string, params: es.Pattern[], body: es.Statement[] -): es.FunctionDeclaration => { +): es.FunctionDeclarationWithId => { return { type: 'FunctionDeclaration', expression: false, @@ -90,7 +90,7 @@ export const createFunctionDeclaration = ( } // The 'expression' property is not typed in ESTree, but it exists // on FunctionDeclaration nodes in the AST generated by acorn parser. - } as es.FunctionDeclaration + } as es.FunctionDeclarationWithId } export const createImportDeclaration = ( diff --git a/src/modules/preprocessor/filePaths.ts b/src/modules/preprocessor/filePaths.ts index 3476cd4fe..8c12af17a 100644 --- a/src/modules/preprocessor/filePaths.ts +++ b/src/modules/preprocessor/filePaths.ts @@ -2,7 +2,7 @@ import { ConsecutiveSlashesInFilePathError, IllegalCharInFilePathError, InvalidFilePathError -} from '../../errors/localImportErrors' +} from '../../modules/errors' /** * Maps non-alphanumeric characters that are legal in file paths diff --git a/src/modules/preprocessor/index.ts b/src/modules/preprocessor/index.ts new file mode 100644 index 000000000..cbc6e6f02 --- /dev/null +++ b/src/modules/preprocessor/index.ts @@ -0,0 +1,331 @@ +import * as pathlib from 'path' + +import { parse } from '../../parser/parser' +import type { AcornOptions } from '../../parser/types' +import type { Context } from '../../types' +import assert from '../../utils/assert' +import { + isIdentifier, + isModuleDeclaration, + isModuleDeclarationWithSource, + isSourceImport +} from '../../utils/ast/typeGuards' +import type * as es from '../../utils/ast/types' +import { CircularImportError, ModuleNotFoundError } from '../errors' +import type { ImportResolutionOptions } from '../moduleTypes' +import analyzeImportsAndExports from './analyzer' +import { createInvokedFunctionResultVariableDeclaration } from './constructors/contextSpecificConstructors' +import { DirectedGraph } from './directedGraph' +import { transformFunctionNameToInvokedFunctionResultVariableName } from './filePaths' +import resolveModule from './resolver' +import hoistAndMergeImports from './transformers/hoistAndMergeImports' +import removeImportsAndExports from './transformers/removeImportsAndExports' +import { + createAccessImportStatements, + getInvokedFunctionResultVariableNameToImportSpecifiersMap, + transformProgramToFunctionDeclaration +} from './transformers/transformProgramToFunctionDeclaration' + +/** + * Error type to indicate that preprocessing has failed but that the context + * contains the underlying errors + */ +class PreprocessError extends Error {} + +const defaultResolutionOptions: Required = { + allowUndefinedImports: false, + resolveDirectories: false, + resolveExtensions: null +} + +/** + * Parse all of the provided files and figure out which modules + * are dependent on which, returning that result in the form + * of a DAG + */ +export const parseProgramsAndConstructImportGraph = async ( + files: Partial>, + entrypointFilePath: string, + context: Context, + rawResolutionOptions: Partial = {} +): Promise<{ + programs: Record + importGraph: DirectedGraph +}> => { + const resolutionOptions = { + ...defaultResolutionOptions, + ...rawResolutionOptions + } + const programs: Record = {} + const importGraph = new DirectedGraph() + + // If there is more than one file, tag AST nodes with the source file path. + const numOfFiles = Object.keys(files).length + const shouldAddSourceFileToAST = numOfFiles > 1 + + async function resolve(path: string, node?: es.ModuleDeclarationWithSource) { + let source: string + if (node) { + assert( + typeof node.source.value === 'string', + `${node.type} should have a source of type string, got ${node.source}` + ) + source = node.source.value + } else { + source = path + } + + const [resolved, modAbsPath] = await resolveModule( + node ? path : '.', + source, + p => files[p] !== undefined, + resolutionOptions + ) + if (!resolved) throw new ModuleNotFoundError(modAbsPath, node) + return modAbsPath + } + + /** + * Process each file (as a module) and determine which other (local and source) + * modules are required. This function should always be called with absolute + * paths + * + * @param currentFilePath Current absolute file path of the module + */ + async function parseFile(currentFilePath: string): Promise { + if (currentFilePath in programs) { + return + } + + const code = files[currentFilePath] + assert( + code !== undefined, + "Module resolver should've thrown an error if the file path did not resolve" + ) + + // Tag AST nodes with the source file path for use in error messages. + const parserOptions: Partial = shouldAddSourceFileToAST + ? { + sourceFile: currentFilePath + } + : {} + const program = parse(code, context, parserOptions, false) + if (!program) { + // Due to a bug in the typed parser where throwOnError isn't respected, + // we need to throw a quick exit error here instead + throw new PreprocessError() + } + + const dependencies = new Set() + programs[currentFilePath] = program + + for (const node of program.body) { + if (!isModuleDeclarationWithSource(node)) continue + + const modAbsPath = await resolve(currentFilePath, node) + if (modAbsPath === currentFilePath) { + throw new CircularImportError([modAbsPath, currentFilePath]) + } + + dependencies.add(modAbsPath) + + // Replace the source of the node with the resolved path + node.source.value = modAbsPath + } + + await Promise.all( + Array.from(dependencies).map(async dependency => { + // There is no need to track Source modules as dependencies, as it can be assumed + // that they will always come first in the topological order + if (!isSourceImport(dependency)) { + await parseFile(dependency) + // If the edge has already been traversed before, the import graph + // must contain a cycle. Then we can exit early and proceed to find the cycle + if (importGraph.hasEdge(dependency, currentFilePath)) { + throw new PreprocessError() + } + + importGraph.addEdge(dependency, currentFilePath) + } + }) + ) + } + + try { + // Remember to resolve the entrypoint file too! + const entrypointAbsPath = await resolve(entrypointFilePath) + await parseFile(entrypointAbsPath) + } catch (error) { + if (!(error instanceof PreprocessError)) { + context.errors.push(error) + } + } + + return { + programs, + importGraph + } +} + +export type PreprocessOptions = { + allowUndefinedImports?: boolean +} & ImportResolutionOptions + +const defaultOptions: Required = { + ...defaultResolutionOptions, + allowUndefinedImports: false +} + +/** + * Preprocesses file imports and returns a transformed Abstract Syntax Tree (AST). + * If an error is encountered at any point, returns `undefined` to signify that an + * error occurred. Details of the error can be found inside `context.errors`. + * + * The preprocessing works by transforming each imported file into a function whose + * parameters are other files (results of transformed functions) and return value + * is a pair where the head is the default export or null, and the tail is a list + * of pairs that map from exported names to identifiers. + * + * See https://github.com/source-academy/js-slang/wiki/Local-Module-Import-&-Export + * for more information. + * + * @param files An object mapping absolute file paths to file content. + * @param entrypointFilePath The absolute path of the entrypoint file. + * @param context The information associated with the program evaluation. + */ +const preprocessFileImports = async ( + files: Partial>, + entrypointFilePath: string, + context: Context, + rawOptions: Partial = {} +): Promise => { + const { allowUndefinedImports, ...resolutionOptions } = { + ...defaultOptions, + ...rawOptions + } + + // Parse all files into ASTs and build the import graph. + const { programs, importGraph } = await parseProgramsAndConstructImportGraph( + files, + entrypointFilePath, + context, + resolutionOptions + ) + + // Return 'undefined' if there are errors while parsing. + if (context.errors.length !== 0) { + return undefined + } + + // Check for circular imports. + const topologicalOrderResult = importGraph.getTopologicalOrder() + if (!topologicalOrderResult.isValidTopologicalOrderFound) { + context.errors.push(new CircularImportError(topologicalOrderResult.firstCycleFound)) + return undefined + } + + let newPrograms: Record = {} + try { + // Based on how the import graph is constructed, it could be the case that the entrypoint + // file is never included in the topo order. This is only an issue for the import export + // analyzer, hence the following code + const fullTopoOrder = topologicalOrderResult.topologicalOrder + if (!fullTopoOrder.includes(entrypointFilePath)) { + // Since it's the entrypoint, it must be loaded last + fullTopoOrder.push(entrypointFilePath) + } + + // This check is performed after cycle detection because if we tried to resolve export symbols + // and there is a cycle in the import graph the constructImportGraph function may end up in an + // infinite loop + // For example, a.js: export * from './b.js' and b.js: export * from './a.js' + // Then trying to discover what symbols are exported by a.js will require determining what symbols + // b.js exports, which would in turn require the symbols exported by a.js + // If the topological order exists, then this is guaranteed not to occur + newPrograms = await analyzeImportsAndExports(programs, fullTopoOrder, allowUndefinedImports) + } catch (error) { + context.errors.push(error) + return undefined + } + + // We want to operate on the entrypoint program to get the eventual + // preprocessed program. + const entrypointProgram = newPrograms[entrypointFilePath] + const entrypointDirPath = pathlib.resolve(entrypointFilePath, '..') + + // Create variables to hold the imported statements. + const entrypointProgramModuleDeclarations = entrypointProgram.body.filter(isModuleDeclaration) + const entrypointProgramInvokedFunctionResultVariableNameToImportSpecifiersMap = + getInvokedFunctionResultVariableNameToImportSpecifiersMap( + entrypointProgramModuleDeclarations, + entrypointDirPath + ) + const entrypointProgramAccessImportStatements = createAccessImportStatements( + entrypointProgramInvokedFunctionResultVariableNameToImportSpecifiersMap + ) + + // Transform all programs into their equivalent function declaration + // except for the entrypoint program. + const [functionDeclarations, invokedFunctionResultVariableDeclarations] = + topologicalOrderResult.topologicalOrder + // The entrypoint program does not need to be transformed into its + // function declaration equivalent as its enclosing environment is + // simply the overall program's (constructed program's) environment. + .filter(path => path !== entrypointFilePath) + .reduce( + ([funcDecls, invokeDecls], filePath) => { + const program = newPrograms[filePath] + const functionDeclaration = transformProgramToFunctionDeclaration(program, filePath) + + const functionName = functionDeclaration.id.name + const invokedFunctionResultVariableName = + transformFunctionNameToInvokedFunctionResultVariableName(functionName) + + const functionParams = functionDeclaration.params.filter(isIdentifier) + assert( + functionParams.length === functionDeclaration.params.length, + 'Function declaration contains non-Identifier AST nodes as params. This should never happen.' + ) + + // Invoke each of the transformed functions and store the result in a variable. + const invokedFunctionResultVariableDeclaration = + createInvokedFunctionResultVariableDeclaration( + functionName, + invokedFunctionResultVariableName, + functionParams + ) + + return [ + [...funcDecls, functionDeclaration], + [...invokeDecls, invokedFunctionResultVariableDeclaration] + ] + }, + [[], []] as [es.FunctionDeclaration[], es.VariableDeclaration[]] + ) + + // Re-assemble the program. + const preprocessedProgram: es.Program = { + ...entrypointProgram, + body: [ + ...functionDeclarations, + ...invokedFunctionResultVariableDeclarations, + ...entrypointProgramAccessImportStatements, + ...entrypointProgram.body + ] + } + + // console.log(generate(preprocessedProgram)) + + // Import and Export related nodes are no longer necessary, so we can remove them from the program entirely + removeImportsAndExports(preprocessedProgram) + + // Finally, we need to hoist all remaining imports to the top of the + // program. These imports should be source module imports since + // non-Source module imports would have already been removed. As part + // of this step, we also merge imports from the same module so as to + // import each unique name per module only once. + hoistAndMergeImports(preprocessedProgram, newPrograms) + return preprocessedProgram +} + +export default preprocessFileImports diff --git a/src/modules/preprocessor/preprocessor.ts b/src/modules/preprocessor/preprocessor.ts deleted file mode 100644 index c09d7a458..000000000 --- a/src/modules/preprocessor/preprocessor.ts +++ /dev/null @@ -1,290 +0,0 @@ -import es from 'estree' -import * as path from 'path' - -import { CannotFindModuleError, CircularImportError } from '../../errors/localImportErrors' -import { parse } from '../../parser/parser' -import { AcornOptions } from '../../parser/types' -import { Context } from '../../types' -import { isIdentifier } from '../../utils/rttc' -import { createInvokedFunctionResultVariableDeclaration } from './constructors/contextSpecificConstructors' -import { DirectedGraph } from './directedGraph' -import { - transformFilePathToValidFunctionName, - transformFunctionNameToInvokedFunctionResultVariableName -} from './filePaths' -import { hoistAndMergeImports } from './transformers/hoistAndMergeImports' -import { removeExports } from './transformers/removeExports' -import { - isSourceModule, - removeNonSourceModuleImports -} from './transformers/removeNonSourceModuleImports' -import { - createAccessImportStatements, - getInvokedFunctionResultVariableNameToImportSpecifiersMap, - transformProgramToFunctionDeclaration -} from './transformers/transformProgramToFunctionDeclaration' -import { isImportDeclaration, isModuleDeclaration } from './typeGuards' - -/** - * Returns all absolute local module paths which should be imported. - * This function makes use of the file path of the current file to - * determine the absolute local module paths. - * - * Note that the current file path must be absolute. - * - * @param program The program to be operated on. - * @param currentFilePath The file path of the current file. - */ -export const getImportedLocalModulePaths = ( - program: es.Program, - currentFilePath: string -): Set => { - if (!path.isAbsolute(currentFilePath)) { - throw new Error(`Current file path '${currentFilePath}' is not absolute.`) - } - - const baseFilePath = path.resolve(currentFilePath, '..') - const importedLocalModuleNames: Set = new Set() - const importDeclarations = program.body.filter(isImportDeclaration) - importDeclarations.forEach((importDeclaration: es.ImportDeclaration): void => { - const modulePath = importDeclaration.source.value - if (typeof modulePath !== 'string') { - throw new Error('Module names must be strings.') - } - if (!isSourceModule(modulePath)) { - const absoluteModulePath = path.resolve(baseFilePath, modulePath) - importedLocalModuleNames.add(absoluteModulePath) - } - }) - return importedLocalModuleNames -} - -const parseProgramsAndConstructImportGraph = ( - files: Partial>, - entrypointFilePath: string, - context: Context -): { - programs: Record - importGraph: DirectedGraph -} => { - const programs: Record = {} - const importGraph = new DirectedGraph() - - // If there is more than one file, tag AST nodes with the source file path. - const numOfFiles = Object.keys(files).length - const shouldAddSourceFileToAST = numOfFiles > 1 - - const parseFile = (currentFilePath: string): void => { - const code = files[currentFilePath] - if (code === undefined) { - context.errors.push(new CannotFindModuleError(currentFilePath)) - return - } - - // Tag AST nodes with the source file path for use in error messages. - const parserOptions: Partial = shouldAddSourceFileToAST - ? { - sourceFile: currentFilePath - } - : {} - const program = parse(code, context, parserOptions) - if (program === null) { - return - } - - programs[currentFilePath] = program - - const importedLocalModulePaths = getImportedLocalModulePaths(program, currentFilePath) - for (const importedLocalModulePath of importedLocalModulePaths) { - // If the source & destination nodes in the import graph are the - // same, then the file is trying to import from itself. This is a - // special case of circular imports. - if (importedLocalModulePath === currentFilePath) { - context.errors.push(new CircularImportError([importedLocalModulePath, currentFilePath])) - return - } - // If we traverse the same edge in the import graph twice, it means - // that there is a cycle in the graph. We terminate early so as not - // to get into an infinite loop (and also because there is no point - // in traversing cycles when our goal is to build up the import - // graph). - if (importGraph.hasEdge(importedLocalModulePath, currentFilePath)) { - continue - } - // Since the file at 'currentFilePath' contains the import statement - // from the file at 'importedLocalModulePath', we treat the former - // as the destination node and the latter as the source node in our - // import graph. This is because when we insert the transformed - // function declarations into the resulting program, we need to start - // with the function declarations that do not depend on other - // function declarations. - importGraph.addEdge(importedLocalModulePath, currentFilePath) - // Recursively parse imported files. - parseFile(importedLocalModulePath) - } - } - - parseFile(entrypointFilePath) - - return { - programs, - importGraph - } -} - -const getSourceModuleImports = (programs: Record): es.ImportDeclaration[] => { - const sourceModuleImports: es.ImportDeclaration[] = [] - Object.values(programs).forEach((program: es.Program): void => { - const importDeclarations = program.body.filter(isImportDeclaration) - importDeclarations.forEach((importDeclaration: es.ImportDeclaration): void => { - const importSource = importDeclaration.source.value - if (typeof importSource !== 'string') { - throw new Error('Module names must be strings.') - } - if (isSourceModule(importSource)) { - sourceModuleImports.push(importDeclaration) - } - }) - }) - return sourceModuleImports -} - -/** - * Preprocesses file imports and returns a transformed Abstract Syntax Tree (AST). - * If an error is encountered at any point, returns `undefined` to signify that an - * error occurred. Details of the error can be found inside `context.errors`. - * - * The preprocessing works by transforming each imported file into a function whose - * parameters are other files (results of transformed functions) and return value - * is a pair where the head is the default export or null, and the tail is a list - * of pairs that map from exported names to identifiers. - * - * See https://github.com/source-academy/js-slang/wiki/Local-Module-Import-&-Export - * for more information. - * - * @param files An object mapping absolute file paths to file content. - * @param entrypointFilePath The absolute path of the entrypoint file. - * @param context The information associated with the program evaluation. - */ -const preprocessFileImports = ( - files: Partial>, - entrypointFilePath: string, - context: Context -): es.Program | undefined => { - // Parse all files into ASTs and build the import graph. - const { programs, importGraph } = parseProgramsAndConstructImportGraph( - files, - entrypointFilePath, - context - ) - // Return 'undefined' if there are errors while parsing. - if (context.errors.length !== 0) { - return undefined - } - - // Check for circular imports. - const topologicalOrderResult = importGraph.getTopologicalOrder() - if (!topologicalOrderResult.isValidTopologicalOrderFound) { - context.errors.push(new CircularImportError(topologicalOrderResult.firstCycleFound)) - return undefined - } - - // We want to operate on the entrypoint program to get the eventual - // preprocessed program. - const entrypointProgram = programs[entrypointFilePath] - const entrypointDirPath = path.resolve(entrypointFilePath, '..') - - // Create variables to hold the imported statements. - const entrypointProgramModuleDeclarations = entrypointProgram.body.filter(isModuleDeclaration) - const entrypointProgramInvokedFunctionResultVariableNameToImportSpecifiersMap = - getInvokedFunctionResultVariableNameToImportSpecifiersMap( - entrypointProgramModuleDeclarations, - entrypointDirPath - ) - const entrypointProgramAccessImportStatements = createAccessImportStatements( - entrypointProgramInvokedFunctionResultVariableNameToImportSpecifiersMap - ) - - // Transform all programs into their equivalent function declaration - // except for the entrypoint program. - const functionDeclarations: Record = {} - for (const [filePath, program] of Object.entries(programs)) { - // The entrypoint program does not need to be transformed into its - // function declaration equivalent as its enclosing environment is - // simply the overall program's (constructed program's) environment. - if (filePath === entrypointFilePath) { - continue - } - - const functionDeclaration = transformProgramToFunctionDeclaration(program, filePath) - const functionName = functionDeclaration.id?.name - if (functionName === undefined) { - throw new Error( - 'A transformed function declaration is missing its name. This should never happen.' - ) - } - - functionDeclarations[functionName] = functionDeclaration - } - - // Invoke each of the transformed functions and store the result in a variable. - const invokedFunctionResultVariableDeclarations: es.VariableDeclaration[] = [] - topologicalOrderResult.topologicalOrder.forEach((filePath: string): void => { - // As mentioned above, the entrypoint program does not have a function - // declaration equivalent, so there is no need to process it. - if (filePath === entrypointFilePath) { - return - } - - const functionName = transformFilePathToValidFunctionName(filePath) - const invokedFunctionResultVariableName = - transformFunctionNameToInvokedFunctionResultVariableName(functionName) - - const functionDeclaration = functionDeclarations[functionName] - const functionParams = functionDeclaration.params.filter(isIdentifier) - if (functionParams.length !== functionDeclaration.params.length) { - throw new Error( - 'Function declaration contains non-Identifier AST nodes as params. This should never happen.' - ) - } - - const invokedFunctionResultVariableDeclaration = createInvokedFunctionResultVariableDeclaration( - functionName, - invokedFunctionResultVariableName, - functionParams - ) - invokedFunctionResultVariableDeclarations.push(invokedFunctionResultVariableDeclaration) - }) - - // Get all Source module imports across the entrypoint program & all imported programs. - const sourceModuleImports = getSourceModuleImports(programs) - - // Re-assemble the program. - const preprocessedProgram: es.Program = { - ...entrypointProgram, - body: [ - ...sourceModuleImports, - ...Object.values(functionDeclarations), - ...invokedFunctionResultVariableDeclarations, - ...entrypointProgramAccessImportStatements, - ...entrypointProgram.body - ] - } - - // After this pre-processing step, all export-related nodes in the AST - // are no longer needed and are thus removed. - removeExports(preprocessedProgram) - // Likewise, all import-related nodes in the AST which are not Source - // module imports are no longer needed and are also removed. - removeNonSourceModuleImports(preprocessedProgram) - // Finally, we need to hoist all remaining imports to the top of the - // program. These imports should be source module imports since - // non-Source module imports would have already been removed. As part - // of this step, we also merge imports from the same module so as to - // import each unique name per module only once. - hoistAndMergeImports(preprocessedProgram) - - return preprocessedProgram -} - -export default preprocessFileImports diff --git a/src/modules/preprocessor/resolver.ts b/src/modules/preprocessor/resolver.ts new file mode 100644 index 000000000..08a42c737 --- /dev/null +++ b/src/modules/preprocessor/resolver.ts @@ -0,0 +1,42 @@ +import * as pathlib from 'path' + +import { isSourceImport } from '../../utils/ast/typeGuards' +import { memoizedGetModuleManifestAsync } from '../moduleLoaderAsync' +import type { ImportResolutionOptions } from '../moduleTypes' + +/** + * Function that returns the full, absolute path to the module being imported + * @param ourPath Path of the current module + * @param source Path to the module being imported + * @param modulePredicate Predicate for checking if the given module exists + * @param options Import resolution options + */ +export default async function resolveModule( + ourPath: string, + source: string, + modulePredicate: (p: string) => boolean, + options: Omit +): Promise<[resolved: boolean, modAbsPath: string]> { + if (isSourceImport(source)) { + const moduleManifest = await memoizedGetModuleManifestAsync() + return [source in moduleManifest, source] + } else { + const modAbsPath = pathlib.resolve(ourPath, '..', source) + if (modulePredicate(modAbsPath)) return [true, modAbsPath] + + if (options.resolveDirectories && modulePredicate(`${modAbsPath}/index`)) { + return [true, `${modAbsPath}/index`] + } + + if (options.resolveExtensions) { + for (const ext of options.resolveExtensions) { + if (modulePredicate(`${modAbsPath}.${ext}`)) return [true, `${modAbsPath}.${ext}`] + + if (options.resolveDirectories && modulePredicate(`${modAbsPath}/index.${ext}`)) { + return [true, `${modAbsPath}/index.${ext}`] + } + } + } + return [false, modAbsPath] + } +} diff --git a/src/modules/preprocessor/transformers/hoistAndMergeImports.ts b/src/modules/preprocessor/transformers/hoistAndMergeImports.ts index 96699587c..5f0968fa3 100644 --- a/src/modules/preprocessor/transformers/hoistAndMergeImports.ts +++ b/src/modules/preprocessor/transformers/hoistAndMergeImports.ts @@ -1,9 +1,13 @@ -import es from 'estree' -import * as _ from 'lodash' - -import { createImportDeclaration, createLiteral } from '../constructors/baseConstructors' -import { cloneAndStripImportSpecifier } from '../constructors/contextSpecificConstructors' -import { isImportDeclaration } from '../typeGuards' +import { isImportDeclaration, isSourceImport } from '../../../utils/ast/typeGuards' +import type * as es from '../../../utils/ast/types' +import { + createIdentifier, + createImportDeclaration, + createImportDefaultSpecifier, + createImportNamespaceSpecifier, + createImportSpecifier, + createLiteral +} from '../constructors/baseConstructors' /** * Hoists import declarations to the top of the program & merges duplicate @@ -14,68 +18,106 @@ import { isImportDeclaration } from '../typeGuards' * directory. If such a functionality is required, this function will * need to be modified. * - * @param program The AST which should have its ImportDeclaration nodes - * hoisted & duplicate imports merged. + * @param outputProgram The AST which should have its ImportDeclaration nodes + * hoisted & duplicate imports merged. */ -export const hoistAndMergeImports = (program: es.Program): void => { - // Separate import declarations from non-import declarations. - const importDeclarations = program.body.filter(isImportDeclaration) - const nonImportDeclarations = program.body.filter( - (node: es.Directive | es.Statement | es.ModuleDeclaration): boolean => - !isImportDeclaration(node) - ) - - // Merge import sources & specifiers. - const importSourceToSpecifiersMap: Map< +export default function hoistAndMergeImports( + outputProgram: es.Program, + programs: Record +) { + const importsToSpecifiers = new Map< string, - Array - > = new Map() - for (const importDeclaration of importDeclarations) { - const importSource = importDeclaration.source.value - if (typeof importSource !== 'string') { - throw new Error('Module names must be strings.') - } - const specifiers = importSourceToSpecifiersMap.get(importSource) ?? [] - for (const specifier of importDeclaration.specifiers) { - // The Acorn parser adds extra information to AST nodes that are not - // part of the ESTree types. As such, we need to clone and strip - // the import specifier AST nodes to get a canonical representation - // that we can use to keep track of whether the import specifier - // is a duplicate or not. - const strippedSpecifier = cloneAndStripImportSpecifier(specifier) - // Note that we cannot make use of JavaScript's built-in Set class - // as it compares references for objects. - const isSpecifierDuplicate = - specifiers.filter( - ( - specifier: es.ImportSpecifier | es.ImportDefaultSpecifier | es.ImportNamespaceSpecifier - ): boolean => { - return _.isEqual(strippedSpecifier, specifier) + { namespaceSymbols: Set; imports: Map> } + >() + + // Now we go over the programs again + Object.values(programs).forEach(program => { + program.body.forEach(node => { + if (!isImportDeclaration(node)) return + + const source = node.source!.value as string + // We no longer need imports from non-source modules, so we can just ignore them + if (!isSourceImport(source)) return + + if (!importsToSpecifiers.has(source)) { + importsToSpecifiers.set(source, { + namespaceSymbols: new Set(), + imports: new Map() + }) + } + const specifierMap = importsToSpecifiers.get(source)! + node.specifiers.forEach(spec => { + let importingName: string + switch (spec.type) { + case 'ImportSpecifier': { + importingName = spec.imported.name + break + } + case 'ImportDefaultSpecifier': { + importingName = 'default' + break } - ).length !== 0 - if (isSpecifierDuplicate) { - continue + case 'ImportNamespaceSpecifier': { + specifierMap.namespaceSymbols.add(spec.local.name) + return + } + } + + if (!specifierMap.imports.has(importingName)) { + specifierMap.imports.set(importingName, new Set()) + } + specifierMap.imports.get(importingName)!.add(spec.local.name) + }) + }) + }) + + // Every distinct source module being imported is given its own ImportDeclaration node + const importDeclarations = Array.from(importsToSpecifiers.entries()).flatMap( + ([moduleName, { imports, namespaceSymbols }]) => { + // Across different modules, the user may choose to alias some of the declarations, so we keep track, + // of all the different aliases used for each unique imported symbol + const specifiers = Array.from(imports.entries()).flatMap(([importedName, aliases]) => { + if (importedName !== 'default') { + return Array.from(aliases).map(alias => + createImportSpecifier(createIdentifier(alias), createIdentifier(importedName)) + ) + } else { + return [] + } + }) + + let output = + specifiers.length > 0 + ? [createImportDeclaration(specifiers, createLiteral(moduleName))] + : [] + if (imports.has('default')) { + // You can't have multiple default specifiers per node, so we need to create + // a new node for each + output = output.concat( + Array.from(imports.get('default')!.values()).map(alias => + createImportDeclaration( + [createImportDefaultSpecifier(createIdentifier(alias))], + createLiteral(moduleName) + ) + ) + ) + } + + if (namespaceSymbols.size > 0) { + // You can't have multiple namespace specifiers per node, so we need to create + // a new node for each + output = output.concat( + Array.from(namespaceSymbols).map(alias => + createImportDeclaration( + [createImportNamespaceSpecifier(createIdentifier(alias))], + createLiteral(moduleName) + ) + ) + ) } - specifiers.push(strippedSpecifier) - } - importSourceToSpecifiersMap.set(importSource, specifiers) - } - // Convert the merged import sources & specifiers back into import declarations. - const mergedImportDeclarations: es.ImportDeclaration[] = [] - importSourceToSpecifiersMap.forEach( - ( - specifiers: Array< - es.ImportSpecifier | es.ImportDefaultSpecifier | es.ImportNamespaceSpecifier - >, - importSource: string - ): void => { - mergedImportDeclarations.push( - createImportDeclaration(specifiers, createLiteral(importSource)) - ) + return output } ) - - // Hoist the merged import declarations to the top of the program body. - program.body = [...mergedImportDeclarations, ...nonImportDeclarations] + outputProgram.body = [...importDeclarations, ...outputProgram.body] } diff --git a/src/modules/preprocessor/transformers/removeImportsAndExports.ts b/src/modules/preprocessor/transformers/removeImportsAndExports.ts new file mode 100644 index 000000000..072304926 --- /dev/null +++ b/src/modules/preprocessor/transformers/removeImportsAndExports.ts @@ -0,0 +1,25 @@ +import type { Program, Statement } from 'estree' + +import { processExportDefaultDeclaration } from '../../../utils/ast/astUtils' + +export default function removeImportsAndExports(program: Program) { + const newBody = program.body.reduce((res, node) => { + switch (node.type) { + case 'ExportDefaultDeclaration': + return processExportDefaultDeclaration(node, { + ClassDeclaration: decl => [...res, decl], + FunctionDeclaration: decl => [...res, decl], + Expression: () => res + }) + case 'ExportNamedDeclaration': + return node.declaration ? [...res, node.declaration] : res + case 'ImportDeclaration': + case 'ExportAllDeclaration': + return res + default: + return [...res, node] + } + }, [] as Statement[]) + + program.body = newBody +} diff --git a/src/modules/preprocessor/transformers/removeNonSourceModuleImports.ts b/src/modules/preprocessor/transformers/removeNonSourceModuleImports.ts deleted file mode 100644 index dda8b6466..000000000 --- a/src/modules/preprocessor/transformers/removeNonSourceModuleImports.ts +++ /dev/null @@ -1,113 +0,0 @@ -import es from 'estree' - -import { ancestor } from '../../../utils/ast/walkers' -import { isFilePath } from '../filePaths' - -/** - * Returns whether a module name refers to a Source module. - * We define a Source module name to be any string that is not - * a file path. - * - * Source module import: `import { x } from "module";` - * Local (relative) module import: `import { x } from "./module";` - * Local (absolute) module import: `import { x } from "/dir/dir2/module";` - * - * @param moduleName The name of the module. - */ -export const isSourceModule = (moduleName: string): boolean => { - return !isFilePath(moduleName) -} - -/** - * Removes all non-Source module import-related nodes from the AST. - * - * All import-related nodes which are not removed in the pre-processing - * step will be treated by the Source modules loader as a Source module. - * If a Source module by the same name does not exist, the program - * evaluation will error out. As such, this function removes all - * import-related AST nodes which the Source module loader does not - * support, as well as ImportDeclaration nodes for local module imports. - * - * The definition of whether a module is a local module or a Source - * module depends on the implementation of the `isSourceModule` function. - * - * @param program The AST which should be stripped of non-Source module - * import-related nodes. - */ -export const removeNonSourceModuleImports = (program: es.Program): void => { - // First pass: remove all import AST nodes which are unused by Source modules. - ancestor(program, { - ImportSpecifier(_node: es.ImportSpecifier, _state: es.Node[], _ancestors: es.Node[]): void { - // Nothing to do here since ImportSpecifier nodes are used by Source modules. - }, - ImportDefaultSpecifier( - node: es.ImportDefaultSpecifier, - _state: es.Node[], - ancestors: es.Node[] - ): void { - // The ancestors array contains the current node, meaning that the - // parent node is the second last node of the array. - const parent = ancestors[ancestors.length - 2] - // The parent node of an ImportDefaultSpecifier node must be an ImportDeclaration node. - if (parent.type !== 'ImportDeclaration') { - return - } - const nodeIndex = parent.specifiers.findIndex(n => n === node) - // Remove the ImportDefaultSpecifier node in its parent node's array of specifiers. - // This is because Source modules do not support default imports. - parent.specifiers.splice(nodeIndex, 1) - }, - ImportNamespaceSpecifier( - node: es.ImportNamespaceSpecifier, - _state: es.Node[], - ancestors: es.Node[] - ): void { - // The ancestors array contains the current node, meaning that the - // parent node is the second last node of the array. - const parent = ancestors[ancestors.length - 2] - // The parent node of an ImportNamespaceSpecifier node must be an ImportDeclaration node. - if (parent.type !== 'ImportDeclaration') { - return - } - const nodeIndex = parent.specifiers.findIndex(n => n === node) - // Remove the ImportNamespaceSpecifier node in its parent node's array of specifiers. - // This is because Source modules do not support namespace imports. - parent.specifiers.splice(nodeIndex, 1) - } - }) - - // Operate on a copy of the Program node's body to prevent the walk from missing ImportDeclaration nodes. - const programBody = [...program.body] - const removeImportDeclaration = (node: es.ImportDeclaration, ancestors: es.Node[]): void => { - // The ancestors array contains the current node, meaning that the - // parent node is the second last node of the array. - const parent = ancestors[ancestors.length - 2] - // The parent node of an ImportDeclaration node must be a Program node. - if (parent.type !== 'Program') { - return - } - const nodeIndex = programBody.findIndex(n => n === node) - // Remove the ImportDeclaration node in its parent node's body. - programBody.splice(nodeIndex, 1) - } - // Second pass: remove all ImportDeclaration nodes for non-Source modules, or that do not - // have any specifiers (thus being functionally useless). - ancestor(program, { - ImportDeclaration(node: es.ImportDeclaration, _state: es.Node[], ancestors: es.Node[]): void { - if (typeof node.source.value !== 'string') { - throw new Error('Module names must be strings.') - } - // ImportDeclaration nodes without any specifiers are functionally useless and are thus removed. - if (node.specifiers.length === 0) { - removeImportDeclaration(node, ancestors) - return - } - // Non-Source modules should already have been handled in the pre-processing step and are no - // longer needed. They must be removed to avoid being treated as Source modules. - if (!isSourceModule(node.source.value)) { - removeImportDeclaration(node, ancestors) - } - } - }) - program.body = programBody -} diff --git a/src/modules/preprocessor/transformers/transformProgramToFunctionDeclaration.ts b/src/modules/preprocessor/transformers/transformProgramToFunctionDeclaration.ts index f6fc871d3..4e542d81e 100644 --- a/src/modules/preprocessor/transformers/transformProgramToFunctionDeclaration.ts +++ b/src/modules/preprocessor/transformers/transformProgramToFunctionDeclaration.ts @@ -1,8 +1,21 @@ -import es from 'estree' -import * as path from 'path' +import * as pathlib from 'path' -import { defaultExportLookupName } from '../../../stdlib/localImport.prelude' import { + accessExportFunctionName, + defaultExportLookupName +} from '../../../stdlib/localImport.prelude' +import assert from '../../../utils/assert' +import { processExportDefaultDeclaration } from '../../../utils/ast/astUtils' +import { + isDeclaration, + isDirective, + isModuleDeclaration, + isSourceImport, + isStatement +} from '../../../utils/ast/typeGuards' +import type * as es from '../../../utils/ast/types' +import { + createCallExpression, createFunctionDeclaration, createIdentifier, createLiteral, @@ -17,32 +30,38 @@ import { transformFilePathToValidFunctionName, transformFunctionNameToInvokedFunctionResultVariableName } from '../filePaths' -import { isDeclaration, isDirective, isModuleDeclaration, isStatement } from '../typeGuards' -import { isSourceModule } from './removeNonSourceModuleImports' - -type ImportSpecifier = es.ImportSpecifier | es.ImportDefaultSpecifier | es.ImportNamespaceSpecifier export const getInvokedFunctionResultVariableNameToImportSpecifiersMap = ( nodes: es.ModuleDeclaration[], currentDirPath: string -): Record => { - const invokedFunctionResultVariableNameToImportSpecifierMap: Record = - {} +): Record => { + const invokedFunctionResultVariableNameToImportSpecifierMap: Record< + string, + (es.ImportSpecifiers | es.ExportSpecifier)[] + > = {} nodes.forEach((node: es.ModuleDeclaration): void => { - // Only ImportDeclaration nodes specify imported names. - if (node.type !== 'ImportDeclaration') { - return - } - const importSource = node.source.value - if (typeof importSource !== 'string') { - throw new Error( - 'Encountered an ImportDeclaration node with a non-string source. This should never occur.' - ) + switch (node.type) { + case 'ExportNamedDeclaration': { + if (!node.source) return + break + } + case 'ImportDeclaration': + break + default: + return } + + const importSource = node.source!.value + assert( + typeof importSource === 'string', + `Encountered an ${node.type} node with a non-string source. This should never occur.` + ) + // Only handle import declarations for non-Source modules. - if (isSourceModule(importSource)) { + if (isSourceImport(importSource)) { return } + // Different import sources can refer to the same file. For example, // both './b.js' & '../dir/b.js' can refer to the same file if the // current file path is '/dir/a.js'. To ensure that every file is @@ -50,13 +69,15 @@ export const getInvokedFunctionResultVariableNameToImportSpecifiersMap = ( // current file path to get the absolute file path of the file to // be imported. Since the absolute file path is guaranteed to be // unique, it is also the canonical file path. - const importFilePath = path.resolve(currentDirPath, importSource) + const importFilePath = pathlib.resolve(currentDirPath, importSource) + // Even though we limit the chars that can appear in Source file // paths, some chars in file paths (such as '/') cannot be used // in function names. As such, we substitute illegal chars with // legal ones in a manner that gives us a bijective mapping from // file paths to function names. const importFunctionName = transformFilePathToValidFunctionName(importFilePath) + // In the top-level environment of the resulting program, for every // imported file, we will end up with two different names; one for // the function declaration, and another for the variable holding @@ -71,6 +92,7 @@ export const getInvokedFunctionResultVariableNameToImportSpecifiersMap = ( // Having the two different names helps us to achieve this objective. const invokedFunctionResultVariableName = transformFunctionNameToInvokedFunctionResultVariableName(importFunctionName) + // If this is the file ImportDeclaration node for the canonical // file path, instantiate the entry in the map. if ( @@ -83,116 +105,82 @@ export const getInvokedFunctionResultVariableNameToImportSpecifiersMap = ( ...node.specifiers ) }) + return invokedFunctionResultVariableNameToImportSpecifierMap } -const getIdentifier = (node: es.Declaration): es.Identifier | null => { - switch (node.type) { - case 'FunctionDeclaration': - if (node.id === null) { - throw new Error( - 'Encountered a FunctionDeclaration node without an identifier. This should have been caught when parsing.' - ) - } - return node.id - case 'VariableDeclaration': - const id = node.declarations[0].id - // In Source, variable names are Identifiers. - if (id.type !== 'Identifier') { - throw new Error(`Expected variable name to be an Identifier, but was ${id.type} instead.`) - } - return id - case 'ClassDeclaration': - throw new Error('Exporting of class is not supported.') - } -} +const getExportExpressions = ( + nodes: es.ModuleDeclaration[], + invokedFunctionResultVariableNameToImportSpecifierMap: Record< + string, + (es.ImportSpecifiers | es.ExportSpecifier)[] + > +) => { + const exportExpressions: Record = {} -const getExportedNameToIdentifierMap = ( - nodes: es.ModuleDeclaration[] -): Record => { - const exportedNameToIdentifierMap: Record = {} - nodes.forEach((node: es.ModuleDeclaration): void => { - // Only ExportNamedDeclaration nodes specify exported names. - if (node.type !== 'ExportNamedDeclaration') { - return - } - if (node.declaration) { - const identifier = getIdentifier(node.declaration) - if (identifier === null) { - return + for (const node of nodes) { + switch (node.type) { + case 'ExportNamedDeclaration': { + if (node.declaration) { + let identifier: es.Identifier + if (node.declaration.type === 'VariableDeclaration') { + const { + declarations: [{ id }] + } = node.declaration + identifier = id as es.Identifier + } else { + identifier = node.declaration.id! + } + exportExpressions[identifier.name] = identifier + } else if (!node.source) { + node.specifiers.forEach(({ exported: { name }, local }) => { + exportExpressions[name] = local + }) + } + break + } + case 'ExportDefaultDeclaration': { + exportExpressions[defaultExportLookupName] = processExportDefaultDeclaration(node, { + ClassDeclaration: ({ id }) => id, + FunctionDeclaration: ({ id }) => id, + Expression: expr => expr + }) + break } - // When an ExportNamedDeclaration node has a declaration, the - // identifier is the same as the exported name (i.e., no renaming). - const exportedName = identifier.name - exportedNameToIdentifierMap[exportedName] = identifier - } else { - // When an ExportNamedDeclaration node does not have a declaration, - // it contains a list of names to export, i.e., export { a, b as c, d };. - // Exported names can be renamed using the 'as' keyword. As such, the - // exported names and their corresponding identifiers might be different. - node.specifiers.forEach((node: es.ExportSpecifier): void => { - const exportedName = node.exported.name - const identifier = node.local - exportedNameToIdentifierMap[exportedName] = identifier - }) } - }) - return exportedNameToIdentifierMap -} + } -const getDefaultExportExpression = ( - nodes: es.ModuleDeclaration[], - exportedNameToIdentifierMap: Partial> -): es.Expression | null => { - let defaultExport: es.Expression | null = null + for (const [source, nodes] of Object.entries( + invokedFunctionResultVariableNameToImportSpecifierMap + )) { + for (const node of nodes) { + if (node.type !== 'ExportSpecifier') continue - // Handle default exports which are parsed as ExportNamedDeclaration AST nodes. - // 'export { name as default };' is equivalent to 'export default name;' but - // is represented by an ExportNamedDeclaration node instead of an - // ExportedDefaultDeclaration node. - // - // NOTE: If there is a named export representing the default export, its entry - // in the map must be removed to prevent it from being treated as a named export. - if (exportedNameToIdentifierMap['default'] !== undefined) { - defaultExport = exportedNameToIdentifierMap['default'] - delete exportedNameToIdentifierMap['default'] + const { + exported: { name: exportName }, + local: { name: localName } + } = node + exportExpressions[exportName] = createCallExpression(accessExportFunctionName, [ + createIdentifier(source), + createLiteral(localName) + ]) + } } - nodes.forEach((node: es.ModuleDeclaration): void => { - // Only ExportDefaultDeclaration nodes specify the default export. - if (node.type !== 'ExportDefaultDeclaration') { - return - } - if (defaultExport !== null) { - // This should never occur because multiple default exports should have - // been caught by the Acorn parser when parsing into an AST. - throw new Error('Encountered multiple default exports!') - } - if (isDeclaration(node.declaration)) { - const identifier = getIdentifier(node.declaration) - if (identifier === null) { - return - } - // When an ExportDefaultDeclaration node has a declaration, the - // identifier is the same as the exported name (i.e., no renaming). - defaultExport = identifier - } else { - // When an ExportDefaultDeclaration node does not have a declaration, - // it has an expression. - defaultExport = node.declaration - } - }) - return defaultExport + return exportExpressions } export const createAccessImportStatements = ( - invokedFunctionResultVariableNameToImportSpecifiersMap: Record + invokedFunctionResultVariableNameToImportSpecifiersMap: Record< + string, + (es.ImportSpecifiers | es.ExportSpecifier)[] + > ): es.VariableDeclaration[] => { const importDeclarations: es.VariableDeclaration[] = [] for (const [invokedFunctionResultVariableName, importSpecifiers] of Object.entries( invokedFunctionResultVariableNameToImportSpecifiersMap )) { - importSpecifiers.forEach((importSpecifier: ImportSpecifier): void => { + importSpecifiers.forEach(importSpecifier => { let importDeclaration switch (importSpecifier.type) { case 'ImportSpecifier': @@ -212,6 +200,8 @@ export const createAccessImportStatements = ( case 'ImportNamespaceSpecifier': // In order to support namespace imports, Source would need to first support objects. throw new Error('Namespace imports are not supported.') + case 'ExportSpecifier': + return } importDeclarations.push(importDeclaration) }) @@ -220,12 +210,12 @@ export const createAccessImportStatements = ( } const createReturnListArguments = ( - exportedNameToIdentifierMap: Record + exportedNameToIdentifierMap: Record ): Array => { return Object.entries(exportedNameToIdentifierMap).map( - ([exportedName, identifier]: [string, es.Identifier]): es.SimpleCallExpression => { + ([exportedName, expr]: [string, es.Identifier]): es.SimpleCallExpression => { const head = createLiteral(exportedName) - const tail = identifier + const tail = expr return createPairCallExpression(head, tail) } ) @@ -285,29 +275,26 @@ const removeModuleDeclarations = ( export const transformProgramToFunctionDeclaration = ( program: es.Program, currentFilePath: string -): es.FunctionDeclaration => { +): es.FunctionDeclarationWithId => { const moduleDeclarations = program.body.filter(isModuleDeclaration) - const currentDirPath = path.resolve(currentFilePath, '..') + const currentDirPath = pathlib.resolve(currentFilePath, '..') // Create variables to hold the imported statements. const invokedFunctionResultVariableNameToImportSpecifiersMap = getInvokedFunctionResultVariableNameToImportSpecifiersMap(moduleDeclarations, currentDirPath) + const accessImportStatements = createAccessImportStatements( invokedFunctionResultVariableNameToImportSpecifiersMap ) // Create the return value of all exports for the function. - const exportedNameToIdentifierMap = getExportedNameToIdentifierMap(moduleDeclarations) - const defaultExportExpression = getDefaultExportExpression( + const { [defaultExportLookupName]: defaultExport, ...exportExpressions } = getExportExpressions( moduleDeclarations, - exportedNameToIdentifierMap - ) - const defaultExport = defaultExportExpression ?? createLiteral(null) - const namedExports = createListCallExpression( - createReturnListArguments(exportedNameToIdentifierMap) + invokedFunctionResultVariableNameToImportSpecifiersMap ) + const namedExports = createListCallExpression(createReturnListArguments(exportExpressions)) const returnStatement = createReturnStatement( - createPairCallExpression(defaultExport, namedExports) + createPairCallExpression(defaultExport ?? createLiteral(null), namedExports) ) // Assemble the function body. diff --git a/src/modules/utils.ts b/src/modules/utils.ts new file mode 100644 index 000000000..0da46ebf9 --- /dev/null +++ b/src/modules/utils.ts @@ -0,0 +1,183 @@ +import type { ImportDeclaration, Node } from 'estree' + +import type { Context } from '..' +import assert from '../utils/assert' +import { getUniqueId } from '../utils/uniqueIds' +import { loadModuleTabs } from './moduleLoader' +import { loadModuleTabsAsync } from './moduleLoaderAsync' + +/** + * Create the module's context and load its tabs (if `loadTabs` is true) + */ +export async function initModuleContext( + moduleName: string, + context: Context, + loadTabs: boolean, + node?: Node +) { + if (!(moduleName in context.moduleContexts)) { + context.moduleContexts[moduleName] = { + state: null, + tabs: loadTabs ? loadModuleTabs(moduleName, node) : null + } + } else if (context.moduleContexts[moduleName].tabs === null && loadTabs) { + context.moduleContexts[moduleName].tabs = loadModuleTabs(moduleName, node) + } +} + +/** + * Create the module's context and load its tabs (if `loadTabs` is true) + */ +export async function initModuleContextAsync( + moduleName: string, + context: Context, + loadTabs: boolean, + node?: Node +) { + if (!(moduleName in context.moduleContexts)) { + context.moduleContexts[moduleName] = { + state: null, + tabs: loadTabs ? await loadModuleTabsAsync(moduleName, node) : null + } + } else if (context.moduleContexts[moduleName].tabs === null && loadTabs) { + context.moduleContexts[moduleName].tabs = await loadModuleTabsAsync(moduleName, node) + } +} + +/** + * Represents a loaded Source module + */ +export type ModuleInfo = { + /** + * `ImportDeclarations` that import from this module. + */ + nodes: ImportDeclaration[] + + /** + * Represents the loaded module. It can be the module's functions itself (see the ec-evaluator), + * or just the module text (see the transpiler), or any other type. + * + * This field should not be null when the function returns. + */ + content: T + + /** + * The unique name given to this module. If `usedIdentifiers` is not provided, this field will be `null`. + */ + namespaced: string | null +} + +/** + * Function that converts an `ImportSpecifier` into the given Transformed type. + * It can be used as a `void` returning function as well, in case the specifiers + * don't need to be transformed, just acted upon. + * @example + * ImportSpecifier(specifier, node, info) => { + * return create.constantDeclaration( + * spec.local.name, + * create.memberExpression( + * create.identifier(info.namespaced), + * spec.imported.name + * ), + * ) + * } + */ +export type SpecifierProcessor = ( + spec: ImportDeclaration['specifiers'][0], + moduleInfo: ModuleInfo, + node: ImportDeclaration +) => Transformed + +export type ImportSpecifierType = + | 'ImportSpecifier' + | 'ImportDefaultSpecifier' + | 'ImportNamespaceSpecifier' + +/** + * This function is intended to unify how each of the different Source runners load imports. It handles + * namespacing (if `usedIdentifiers` is provided), loading the module's context (if `context` is not `null`), + * loading the module's tabs (if `loadTabs` is given as `true`) and the conversion + * of import specifiers to the relevant type used by the runner. + * @param nodes Nodes to transform + * @param context Context to transform with, or `null`. Setting this to null prevents module contexts and tabs from being loaded. + * @param loadTabs Set this to false to prevent tabs from being loaded even if a context is provided. + * @param moduleLoader Function that takes the name of the module and returns its loaded representation. + * @param processors Functions for working with each type of import specifier. + * @param usedIdentifiers Set containing identifiers already used in code. If null, namespacing is not conducted. + * @returns The loaded modules, along with the transformed versions of the given nodes + */ +export async function transformImportNodesAsync( + nodes: ImportDeclaration[], + context: Context | null, + loadTabs: boolean, + moduleLoader: (name: string, node?: Node) => Promise, + processors: Record>, + usedIdentifiers?: Set +) { + const internalLoader = async (name: string, node?: Node) => { + // Make sure that module contexts are initialized before + // loading the bundles + if (context) { + await initModuleContextAsync(name, context, loadTabs, node) + } + + return moduleLoader(name, node) + } + + const promises: Promise[] = [] + const moduleInfos = nodes.reduce((res, node) => { + const moduleName = node.source.value + assert( + typeof moduleName === 'string', + `Expected ImportDeclaration to have a source of type string, got ${moduleName}` + ) + + if (!(moduleName in res)) { + // First time we are loading this module + res[moduleName] = { + nodes: [], + content: null as any, + namespaced: null + } + const loadPromise = internalLoader(moduleName, node).then(content => { + res[moduleName].content = content + }) + + promises.push(loadPromise) + } + + res[moduleName].nodes.push(node) + + // Collate all the identifiers introduced by specifiers to prevent collisions when + // the import declaration has aliases, e.g import { show as __MODULE__ } from 'rune'; + if (usedIdentifiers) { + node.specifiers.forEach(spec => usedIdentifiers.add(spec.local.name)) + } + return res + }, {} as Record>) + + // Wait for all module and symbol loading to finish + await Promise.all(promises) + + return Object.entries(moduleInfos).reduce((res, [moduleName, info]) => { + // Now for each module, we give it a unique namespaced id + const namespaced = usedIdentifiers ? getUniqueId(usedIdentifiers, '__MODULE__') : null + info.namespaced = namespaced + + assert(info.content !== null, `${moduleName} was not loaded properly. This should never happen`) + + return { + ...res, + [moduleName]: { + content: info.nodes.flatMap(node => + node.specifiers.flatMap(spec => { + // Finally, transform that specifier into the form needed + // by the runner + return processors[spec.type](spec, info, node) + }) + ), + info + } + } + }, {} as Record; content: Transformed[] }>) +} diff --git a/src/name-extractor/index.ts b/src/name-extractor/index.ts index 14656b8ae..f48338907 100644 --- a/src/name-extractor/index.ts +++ b/src/name-extractor/index.ts @@ -1,11 +1,18 @@ -import * as es from 'estree' +import type * as es from 'estree' -import { Context } from '../' +import type { Context } from '../' import { UNKNOWN_LOCATION } from '../constants' -import { ModuleConnectionError, ModuleNotFoundError } from '../errors/moduleErrors' import { findAncestors, findIdentifierNode } from '../finder' -import { memoizedloadModuleDocs } from '../modules/moduleLoader' +import { ModuleConnectionError, ModuleNotFoundError } from '../modules/errors' +import { memoizedGetModuleDocsAsync } from '../modules/moduleLoaderAsync' import syntaxBlacklist from '../parser/source/syntax' +import { + isDeclaration, + isFunctionNode, + isImportDeclaration, + isLoop, + isSourceImport +} from '../utils/ast/typeGuards' export interface NameDeclaration { name: string @@ -19,26 +26,6 @@ const KIND_FUNCTION = 'func' const KIND_PARAM = 'param' const KIND_CONST = 'const' -function isImportDeclaration(node: es.Node): boolean { - return node.type === 'ImportDeclaration' -} - -function isDeclaration(node: es.Node): boolean { - return node.type === 'VariableDeclaration' || node.type === 'FunctionDeclaration' -} - -function isFunction(node: es.Node): boolean { - return ( - node.type === 'FunctionDeclaration' || - node.type === 'FunctionExpression' || - node.type === 'ArrowFunctionExpression' - ) -} - -function isLoop(node: es.Node): boolean { - return node.type === 'WhileStatement' || node.type === 'ForStatement' -} - // Update this to use exported check from "acorn-loose" package when it is released function isDummyName(name: string): boolean { return name === '✖' @@ -116,7 +103,7 @@ export function getKeywords( ) { addAllowedKeywords(keywordsInBlock) // Keywords only allowed in functions - if (ancestors.some(node => isFunction(node))) { + if (ancestors.some(node => isFunctionNode(node))) { addAllowedKeywords(keywordsInFunction) } @@ -139,11 +126,11 @@ export function getKeywords( * @returns Tuple consisting of the list of suggestions, and a boolean value indicating if * suggestions should be displayed, i.e. `[suggestions, shouldPrompt]` */ -export function getProgramNames( +export async function getProgramNames( prog: es.Node, comments: acorn.Comment[], cursorLoc: es.Position -): [NameDeclaration[], boolean] { +): Promise<[NameDeclaration[], boolean]> { function before(first: es.Position, second: es.Position) { return first.line < second.line || (first.line === second.line && first.column <= second.column) } @@ -170,7 +157,7 @@ export function getProgramNames( // Workaround due to minification problem // tslint:disable-next-line const node = queue.shift()! - if (isFunction(node)) { + if (isFunctionNode(node)) { // This is the only time we want raw identifiers nameQueue.push(...(node as any).params) } @@ -198,13 +185,22 @@ export function getProgramNames( } } - const res: any = {} - nameQueue - .map(node => getNames(node, n => cursorInLoc(n.loc))) - .reduce((prev, cur) => prev.concat(cur), []) // no flatmap feelsbad - .forEach((decl, idx) => { - res[decl.name] = { ...decl, score: idx } - }) // Deduplicate, ensure deeper declarations overwrite + const names = await Promise.all(nameQueue.map(node => getNames(node, n => cursorInLoc(n.loc)))) + const res = names.flat().reduce( + (prev, each, idx) => ({ + ...prev, + [each.name]: { ...each, score: idx } // Deduplicate, ensure deeper declarations overwrite + }), + {} as Record + ) + + // const res: any = {} + // nameQueue + // .map(node => getNames(node, n => cursorInLoc(n.loc))) + // .reduce((prev, cur) => prev.concat(cur), []) // no flatmap feelsbad + // .forEach((decl, idx) => { + // res[decl.name] = { ...decl, score: idx } + // }) return [Object.values(res), true] } @@ -305,44 +301,79 @@ function cursorInIdentifier(node: es.Node, locTest: (node: es.Node) => boolean): * is located within the node, false otherwise * @returns List of found names */ -function getNames(node: es.Node, locTest: (node: es.Node) => boolean): NameDeclaration[] { +async function getNames( + node: es.Node, + locTest: (node: es.Node) => boolean +): Promise { + const createDocHtml = (header: string, desc: string) => + `

${header}

${desc}
` + switch (node.type) { case 'ImportDeclaration': + const source = node.source.value as string + if (!isSourceImport(source)) { + return node.specifiers.map(({ local: { name } }) => ({ + name, + meta: KIND_IMPORT, + docHTML: createDocHtml( + `Imported symbol ${name} from ${source}`, + `No documentation available for ${name} from ${source}` + ) + })) + } + const specs = node.specifiers.filter(x => !isDummyName(x.local.name)) try { - const docs = memoizedloadModuleDocs(node.source.value as string, node) + const docs = await memoizedGetModuleDocsAsync(source) if (!docs) { - return specs.map(spec => ({ - name: spec.local.name, + return specs.map(({ local: { name } }) => ({ + name, meta: KIND_IMPORT, - docHTML: `Unable to retrieve documentation for ${spec.local.name} from ${node.source.value} module` + docHTML: `Unable to retrieve documentation for ${name} from ${source} module` })) } return specs.map(spec => { - if (spec.type !== 'ImportSpecifier' || docs[spec.local.name] === undefined) { - return { - name: spec.local.name, - meta: KIND_IMPORT, - docHTML: `No documentation available for ${spec.local.name} from ${node.source.value} module` - } - } else { + const localName = spec.local.name + + if (docs[spec.local.name] === undefined) { return { name: spec.local.name, meta: KIND_IMPORT, - docHTML: docs[spec.local.name] + docHTML: `No documentation available for ${localName} from ${source} module` } } + + switch (spec.type) { + case 'ImportSpecifier': + return { + name: localName, + meta: KIND_IMPORT, + docHTML: docs[spec.imported.name] + } + case 'ImportDefaultSpecifier': + return { + name: localName, + meta: KIND_IMPORT, + docHTML: docs['default'] + } + case 'ImportNamespaceSpecifier': + return { + name: localName, + meta: KIND_IMPORT, + docHTML: `${source} module namespace import` + } + } }) } catch (err) { if (!(err instanceof ModuleNotFoundError || err instanceof ModuleConnectionError)) throw err - return specs.map(spec => ({ - name: spec.local.name, + return specs.map(({ local: { name } }) => ({ + name, meta: KIND_IMPORT, - docHTML: `Unable to retrieve documentation for ${spec.local.name} from ${node.source.value} module` + docHTML: `Unable to retrieve documentation for ${name} from ${source} module` })) } case 'VariableDeclaration': @@ -353,12 +384,12 @@ function getNames(node: es.Node, locTest: (node: es.Node) => boolean): NameDecla if ( !name || isDummyName(name) || - (decl.init && !isFunction(decl.init) && locTest(decl.init)) // Avoid suggesting `let foo = foo`, but suggest recursion with arrow functions + (decl.init && !isFunctionNode(decl.init) && locTest(decl.init)) // Avoid suggesting `let foo = foo`, but suggest recursion with arrow functions ) { continue } - if (node.kind === KIND_CONST && decl.init && isFunction(decl.init)) { + if (node.kind === KIND_CONST && decl.init && isFunctionNode(decl.init)) { // constant initialized with arrow function will always be a function declarations.push({ name, meta: KIND_FUNCTION }) } else { diff --git a/src/parser/__tests__/__snapshots__/allowed-syntax.ts.snap b/src/parser/__tests__/__snapshots__/allowed-syntax.ts.snap index bfc5a3d5e..85eed8940 100644 --- a/src/parser/__tests__/__snapshots__/allowed-syntax.ts.snap +++ b/src/parser/__tests__/__snapshots__/allowed-syntax.ts.snap @@ -4290,7 +4290,7 @@ x[key] = 3;", exports[`Syntaxes are allowed in the chapter they are introduced 35: fails a chapter below 1`] = ` Object { "alertResult": Array [], - "code": "import defaultExport from \\"module-name\\";", + "code": "import defaultExport from \\"one_module\\";", "displayResult": Array [], "numErrors": 1, "parsedErrors": "Line 1: Import default specifiers are not allowed", @@ -4303,7 +4303,7 @@ Object { exports[`Syntaxes are allowed in the chapter they are introduced 35: parse passes 1`] = ` Object { "alertResult": Array [], - "code": "parse(\\"import defaultExport from \\\\\\"module-name\\\\\\";\\");", + "code": "parse(\\"import defaultExport from \\\\\\"one_module\\\\\\";\\");", "displayResult": Array [], "numErrors": 0, "parsedErrors": "", @@ -4318,7 +4318,7 @@ Object { null, ], Array [ - "module-name", + "one_module", null, ], ], @@ -4331,7 +4331,7 @@ Object { exports[`Syntaxes are allowed in the chapter they are introduced 35: passes 1`] = ` Object { "alertResult": Array [], - "code": "import defaultExport from \\"module-name\\";", + "code": "import defaultExport from \\"one_module\\";", "displayResult": Array [], "numErrors": 0, "parsedErrors": "", diff --git a/src/parser/__tests__/allowed-syntax.ts b/src/parser/__tests__/allowed-syntax.ts index b94cc193d..379843185 100644 --- a/src/parser/__tests__/allowed-syntax.ts +++ b/src/parser/__tests__/allowed-syntax.ts @@ -294,7 +294,7 @@ test.each([ [ Chapter.LIBRARY_PARSER, ` - import defaultExport from "module-name"; + import defaultExport from "one_module"; ` ], @@ -340,15 +340,20 @@ test.each([ (chapter: Chapter, snippet: string, skipSuccessTests: boolean = false) => { snippet = stripIndent(snippet) const parseSnippet = `parse(${JSON.stringify(snippet)});` - const tests = [] + + const tests: Promise[] = [] if (!skipSuccessTests) { tests.push( - snapshotSuccess(snippet, { chapter, native: chapter !== Chapter.LIBRARY_PARSER }, 'passes') + snapshotSuccess( + snippet, + { chapter, native: chapter !== Chapter.LIBRARY_PARSER, allowUndefinedImports: true }, + 'passes' + ) ) tests.push( snapshotSuccess( parseSnippet, - { chapter: Math.max(4, chapter), native: true }, + { chapter: Math.max(4, chapter), native: true, allowUndefinedImports: true }, 'parse passes' ) ) diff --git a/src/parser/parser.ts b/src/parser/parser.ts index 220950114..7d7602297 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -4,9 +4,9 @@ import { Context } from '..' import { Chapter, Variant } from '../types' import { FullJSParser } from './fullJS' import { FullTSParser } from './fullTS' +import { PythonParser } from './python' import { SchemeParser } from './scheme' import { SourceParser } from './source' -import { PythonParser } from './python' import { SourceTypedParser } from './source/typed' import { AcornOptions, Parser } from './types' diff --git a/src/parser/source/rules/index.ts b/src/parser/source/rules/index.ts index 1dff0c7c3..ab04cd6f1 100644 --- a/src/parser/source/rules/index.ts +++ b/src/parser/source/rules/index.ts @@ -8,11 +8,13 @@ import forStatementMustHaveAllParts from './forStatementMustHaveAllParts' import noDeclareMutable from './noDeclareMutable' import noDotAbbreviation from './noDotAbbreviation' import noEval from './noEval' +import noExportNamedDeclarationWithDefault from './noExportNamedDeclarationWithDefault' import noFunctionDeclarationWithoutIdentifier from './noFunctionDeclarationWithoutIdentifier' import noHolesInArrays from './noHolesInArrays' import noIfWithoutElse from './noIfWithoutElse' import noImplicitDeclareUndefined from './noImplicitDeclareUndefined' import noImplicitReturnUndefined from './noImplicitReturnUndefined' +import noImportSpecifierWithDefault from './noImportSpecifierWithDefault' import noNull from './noNull' import noSpreadInArray from './noSpreadInArray' import noTemplateExpression from './noTemplateExpression' @@ -22,8 +24,6 @@ import noUnspecifiedOperator from './noUnspecifiedOperator' import noUpdateAssignment from './noUpdateAssignment' import noVar from './noVar' import singleVariableDeclaration from './singleVariableDeclaration' -import noExportNamedDeclarationWithDefault from './noExportNamedDeclarationWithDefault' -import noImportSpecifierWithDefault from './noImportSpecifierWithDefault' const rules: Rule[] = [ bracesAroundFor, diff --git a/src/parser/source/rules/noExportNamedDeclarationWithDefault.ts b/src/parser/source/rules/noExportNamedDeclarationWithDefault.ts index ebbef0a8f..4d4d3b7b5 100644 --- a/src/parser/source/rules/noExportNamedDeclarationWithDefault.ts +++ b/src/parser/source/rules/noExportNamedDeclarationWithDefault.ts @@ -1,9 +1,9 @@ import * as es from 'estree' +import { UNKNOWN_LOCATION } from '../../../constants' +import { defaultExportLookupName } from '../../../stdlib/localImport.prelude' import { ErrorSeverity, ErrorType, Rule, SourceError } from '../../../types' import syntaxBlacklist from '../syntax' -import { defaultExportLookupName } from '../../../stdlib/localImport.prelude' -import { UNKNOWN_LOCATION } from '../../../constants' export class NoExportNamedDeclarationWithDefaultError implements SourceError { public type = ErrorType.SYNTAX diff --git a/src/parser/source/rules/noImportSpecifierWithDefault.ts b/src/parser/source/rules/noImportSpecifierWithDefault.ts index 3241e37e2..e776441bd 100644 --- a/src/parser/source/rules/noImportSpecifierWithDefault.ts +++ b/src/parser/source/rules/noImportSpecifierWithDefault.ts @@ -1,9 +1,9 @@ import * as es from 'estree' +import { UNKNOWN_LOCATION } from '../../../constants' +import { defaultExportLookupName } from '../../../stdlib/localImport.prelude' import { ErrorSeverity, ErrorType, Rule, SourceError } from '../../../types' import syntaxBlacklist from '../syntax' -import { defaultExportLookupName } from '../../../stdlib/localImport.prelude' -import { UNKNOWN_LOCATION } from '../../../constants' export class NoImportSpecifierWithDefaultError implements SourceError { public type = ErrorType.SYNTAX diff --git a/src/repl/repl.ts b/src/repl/repl.ts index a448326d5..55bba9d6e 100644 --- a/src/repl/repl.ts +++ b/src/repl/repl.ts @@ -2,7 +2,7 @@ import { start } from 'repl' // 'repl' here refers to the module named 'repl' in index.d.ts import { inspect } from 'util' -import { scmLanguages, sourceLanguages, pyLanguages } from '../constants' +import { pyLanguages,scmLanguages, sourceLanguages } from '../constants' import { createContext, IOptions, parseError, runInContext } from '../index' import Closure from '../interpreter/closure' import { ExecutionMethod, Variant } from '../types' diff --git a/src/repl/transpiler.ts b/src/repl/transpiler.ts index db7d3906b..bd441a00e 100644 --- a/src/repl/transpiler.ts +++ b/src/repl/transpiler.ts @@ -11,7 +11,7 @@ import { transpile } from '../transpiler/transpiler' import { Chapter, Variant } from '../types' import { validateAndAnnotate } from '../validator/validator' -function transpileCode( +async function transpileCode( chapter: Chapter = Chapter.SOURCE_1, variant: Variant = Variant.DEFAULT, code = '', @@ -35,7 +35,7 @@ function transpileCode( if (pretranspile) { return generate(program) } else { - return transpile(program as Program, context).transpiled + return (await transpile(program as Program, context)).transpiled } } @@ -90,8 +90,8 @@ function main() { }) process.stdin.on('end', () => { const code = Buffer.concat(chunks).toString('utf-8') - const transpiled = transpileCode(chapter, variant, code, pretranspile) - process.stdout.write(transpiled) + transpileCode(chapter, variant, code, pretranspile).then(data => process.stdout.write(data)) + // process.stdout.write(transpiled) }) } diff --git a/src/runner/__tests__/files.ts b/src/runner/__tests__/files.ts index df213f7d0..9a64d7b86 100644 --- a/src/runner/__tests__/files.ts +++ b/src/runner/__tests__/files.ts @@ -60,15 +60,15 @@ describe('runFilesInContext', () => { it('returns CannotFindModuleError if entrypoint file does not exist', () => { const files: Record = {} runFilesInContext(files, '/a.js', context) - expect(parseError(context.errors)).toMatchInlineSnapshot(`"Cannot find module '/a.js'."`) + expect(parseError(context.errors)).toMatchInlineSnapshot(`"Module '/a.js' not found."`) }) it('returns CannotFindModuleError if entrypoint file does not exist - verbose', () => { const files: Record = {} runFilesInContext(files, '/a.js', context) expect(parseError(context.errors, true)).toMatchInlineSnapshot(` - "Cannot find module '/a.js'. - Check that the module file path resolves to an existing file. + "Module '/a.js' not found. + You should check your import declarations, and ensure that all are valid modules. " `) }) @@ -132,15 +132,15 @@ describe('compileFiles', () => { it('returns CannotFindModuleError if entrypoint file does not exist', () => { const files: Record = {} compileFiles(files, '/a.js', context) - expect(parseError(context.errors)).toMatchInlineSnapshot(`"Cannot find module '/a.js'."`) + expect(parseError(context.errors)).toMatchInlineSnapshot(`"Module '/a.js' not found."`) }) it('returns CannotFindModuleError if entrypoint file does not exist - verbose', () => { const files: Record = {} compileFiles(files, '/a.js', context) expect(parseError(context.errors, true)).toMatchInlineSnapshot(` - "Cannot find module '/a.js'. - Check that the module file path resolves to an existing file. + "Module '/a.js' not found. + You should check your import declarations, and ensure that all are valid modules. " `) }) diff --git a/src/runner/fullJSRunner.ts b/src/runner/fullJSRunner.ts index abedc11ee..143775a8c 100644 --- a/src/runner/fullJSRunner.ts +++ b/src/runner/fullJSRunner.ts @@ -1,19 +1,18 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import { generate } from 'astring' -import * as es from 'estree' +import type * as es from 'estree' import { RawSourceMap } from 'source-map' import { IOptions, Result } from '..' import { NATIVE_STORAGE_ID } from '../constants' import { RuntimeSourceError } from '../errors/runtimeSourceError' -import { hoistAndMergeImports } from '../modules/preprocessor/transformers/hoistAndMergeImports' import { getRequireProvider, RequireProvider } from '../modules/requireProvider' import { parse } from '../parser/parser' import { evallerReplacer, getBuiltins, transpile } from '../transpiler/transpiler' -import type { Context, NativeStorage } from '../types' +import type { Context, NativeStorage, RecursivePartial } from '../types' import * as create from '../utils/ast/astCreator' import { toSourceError } from './errors' -import { appendModulesToContext, resolvedErrorPromise } from './utils' +import { resolvedErrorPromise } from './utils' function fullJSEval( code: string, @@ -48,7 +47,7 @@ function containsPrevEval(context: Context): boolean { export async function fullJSRunner( program: es.Program, context: Context, - options: Partial = {} + options: RecursivePartial = {} ): Promise { // prelude & builtins // only process builtins and preludes if it is a fresh eval context @@ -60,10 +59,6 @@ export async function fullJSRunner( ? [] : [...getBuiltins(context.nativeStorage), ...prelude] - // modules - hoistAndMergeImports(program) - appendModulesToContext(program, context) - // evaluate and create a separate block for preludes and builtins const preEvalProgram: es.Program = create.program([ ...preludeAndBuiltins, @@ -76,13 +71,15 @@ export async function fullJSRunner( let transpiled let sourceMapJson: RawSourceMap | undefined try { - ;({ transpiled, sourceMapJson } = transpile(program, context)) - return Promise.resolve({ + ;({ transpiled, sourceMapJson } = await transpile(program, context, options.importOptions)) + if (options.logTranspilerOutput) console.log(transpiled) + return { status: 'finished', context, value: await fullJSEval(transpiled, requireProvider, context.nativeStorage) - }) + } } catch (error) { + // console.log(error) context.errors.push( error instanceof RuntimeSourceError ? error : await toSourceError(error, sourceMapJson) ) diff --git a/src/runner/htmlRunner.ts b/src/runner/htmlRunner.ts index f52c688f8..76d00cbde 100644 --- a/src/runner/htmlRunner.ts +++ b/src/runner/htmlRunner.ts @@ -1,5 +1,5 @@ -import { IOptions, Result } from '..' -import { Context } from '../types' +import type { IOptions, Result } from '..' +import type { Context, RecursivePartial } from '../types' const HTML_ERROR_HANDLING_SCRIPT_TEMPLATE = `