From 572589cbce247b322925cd3c1274ba56d72d3741 Mon Sep 17 00:00:00 2001 From: Christopher Hiller Date: Tue, 26 Mar 2024 15:08:58 -0700 Subject: [PATCH] feat(compartment-mapper): support for dynamic requires This change adds support for _dynamic requires_, where a Node.js script does something like this: ```js const someModuleSpecifier = getSpecifier(); const idk = require(someModuleSpecifier); ``` This behavior is not enabled by default; specific options and Powers are required. See the signature of `loadFromMap()` in `import-lite.js` for more information. Dynamic loading of exit modules (e.g., if `someModuleSpecifier` was `node:fs`) is handled by a user-provided "exit module import hook". Policy applies. Dynamic requires work as you'd probably expected _except_: - A compartment may dynamically require from its *dependee*, since this is a common pattern in the ecosystem (see [node-gyp-build](https://npm.im/node-gyp-build)). - The special "attenuators" compartment may not be dynamically required. Horsefeathers! Some relevant bits, if you're mining history: - All internal parsers are now _synchronous_, which was made possible by the introduction of `evadeCensorSync` in `@endo/evasive-transform`. - Async parsers are still supported, but they would only be user-defined. Async parsers are incompatible with dynamic requires. - Added property `{Set} compartments` to `CompartmentDescriptor`. This is essentially a view into the compartment names within `CompartmentDescriptor.scopes`. - The `mapParsers()` function has moved into `map-parser.js`. - `@endo/trampoline` is leveraged in a couple places (`import-hook.js`, `map-parser.js`) to avoid code duplication. - Introduced `FsInterface`, `UrlInterface`, `CryptoInterface` and `PathInterface` (and their ilk) for use with `makeReadPowers()` and the new `makeReadNowPowers()`. --- packages/compartment-mapper/NEWS.md | 7 + packages/compartment-mapper/node-powers.js | 6 +- packages/compartment-mapper/package.json | 1 + .../compartment-mapper/src/import-hook.js | 637 ++++++++++++++---- .../compartment-mapper/src/import-lite.js | 152 ++++- packages/compartment-mapper/src/import.js | 58 +- packages/compartment-mapper/src/link.js | 230 +++---- packages/compartment-mapper/src/map-parser.js | 339 ++++++++++ .../compartment-mapper/src/node-modules.js | 17 +- .../compartment-mapper/src/node-powers.js | 127 +++- .../src/parse-archive-cjs.js | 3 +- .../src/parse-archive-mjs.js | 3 +- .../compartment-mapper/src/parse-bytes.js | 8 +- .../src/parse-cjs-shared-export-wrapper.js | 29 +- packages/compartment-mapper/src/parse-cjs.js | 5 +- packages/compartment-mapper/src/parse-json.js | 18 +- packages/compartment-mapper/src/parse-mjs.js | 3 +- .../compartment-mapper/src/parse-pre-cjs.js | 5 +- .../compartment-mapper/src/parse-pre-mjs.js | 3 +- packages/compartment-mapper/src/parse-text.js | 8 +- packages/compartment-mapper/src/policy.js | 96 +-- packages/compartment-mapper/src/powers.js | 70 +- packages/compartment-mapper/src/search.js | 3 +- packages/compartment-mapper/src/types.js | 391 ++++++++++- .../test/dynamic-require.test.js | 417 ++++++++++++ .../node_modules/absolute-app/README.md | 1 + .../node_modules/absolute-app/index.js | 1 + .../node_modules/absolute-app/package.json | 12 + .../node_modules/app/README.md | 1 + .../node_modules/app/index.js | 1 + .../node_modules/app/package.json | 12 + .../node_modules/badsprunt/README.md | 1 + .../node_modules/badsprunt/index.js | 5 + .../node_modules/badsprunt/package.json | 12 + .../node_modules/broken-app/README.md | 1 + .../node_modules/broken-app/index.js | 1 + .../node_modules/broken-app/package.json | 12 + .../node_modules/dynamic/README.md | 1 + .../node_modules/dynamic/index.js | 3 + .../node_modules/dynamic/package.json | 10 + .../node_modules/hooked-app/README.md | 1 + .../node_modules/hooked-app/index.js | 5 + .../node_modules/hooked-app/package.json | 12 + .../node_modules/hooked/README.md | 1 + .../node_modules/hooked/index.js | 3 + .../node_modules/hooked/package.json | 9 + .../node_modules/is-ok/README.md | 1 + .../node_modules/is-ok/index.js | 1 + .../node_modules/is-ok/package.json | 7 + .../node_modules/node-tammy-build/README.md | 1 + .../node_modules/node-tammy-build/index.js | 3 + .../node-tammy-build/package.json | 7 + .../node_modules/sprunt/README.md | 1 + .../node_modules/sprunt/index.js | 4 + .../node_modules/sprunt/package.json | 11 + .../node_modules/sprunt/sprunt.js | 1 + yarn.lock | 3 +- 57 files changed, 2318 insertions(+), 463 deletions(-) create mode 100644 packages/compartment-mapper/src/map-parser.js create mode 100644 packages/compartment-mapper/test/dynamic-require.test.js create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/absolute-app/README.md create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/absolute-app/index.js create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/absolute-app/package.json create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/app/README.md create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/app/index.js create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/app/package.json create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/badsprunt/README.md create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/badsprunt/index.js create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/badsprunt/package.json create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/broken-app/README.md create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/broken-app/index.js create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/broken-app/package.json create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/dynamic/README.md create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/dynamic/index.js create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/dynamic/package.json create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked-app/README.md create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked-app/index.js create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked-app/package.json create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked/README.md create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked/index.js create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked/package.json create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/is-ok/README.md create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/is-ok/index.js create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/is-ok/package.json create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/node-tammy-build/README.md create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/node-tammy-build/index.js create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/node-tammy-build/package.json create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/sprunt/README.md create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/sprunt/index.js create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/sprunt/package.json create mode 100644 packages/compartment-mapper/test/fixtures-dynamic/node_modules/sprunt/sprunt.js diff --git a/packages/compartment-mapper/NEWS.md b/packages/compartment-mapper/NEWS.md index eadaaa3155..e8ab9a7708 100644 --- a/packages/compartment-mapper/NEWS.md +++ b/packages/compartment-mapper/NEWS.md @@ -1,5 +1,12 @@ User-visible changes to `@endo/compartment-mapper`: +# Next release + +- Adds support for dynamic requires in CommonJS modules. This requires specific + configuration to be passed in (including new read powers), and is _not_ + enabled by default. See the signature of `loadFromMap()` in `import-lite.js` + for details. + # v1.2.0 (2024-07-30) - Fixes incompatible behavior with Node.js package conditional exports #2276. diff --git a/packages/compartment-mapper/node-powers.js b/packages/compartment-mapper/node-powers.js index 9a4c8e9eb4..0ef8fed881 100644 --- a/packages/compartment-mapper/node-powers.js +++ b/packages/compartment-mapper/node-powers.js @@ -1,3 +1,7 @@ -export { makeReadPowers, makeWritePowers } from './src/node-powers.js'; +export { + makeReadPowers, + makeWritePowers, + makeReadNowPowers, +} from './src/node-powers.js'; // Deprecated: export { makeNodeReadPowers, makeNodeWritePowers } from './src/node-powers.js'; diff --git a/packages/compartment-mapper/package.json b/packages/compartment-mapper/package.json index c6c1d36d2f..32ac137911 100644 --- a/packages/compartment-mapper/package.json +++ b/packages/compartment-mapper/package.json @@ -57,6 +57,7 @@ "dependencies": { "@endo/cjs-module-analyzer": "workspace:^", "@endo/module-source": "workspace:^", + "@endo/trampoline": "workspace:^", "@endo/zip": "workspace:^", "ses": "workspace:^" }, diff --git a/packages/compartment-mapper/src/import-hook.js b/packages/compartment-mapper/src/import-hook.js index aad21e4ad1..b62c5ec634 100644 --- a/packages/compartment-mapper/src/import-hook.js +++ b/packages/compartment-mapper/src/import-hook.js @@ -7,20 +7,41 @@ */ // @ts-check +/** + * @import { + * ImportHook, + * ImportNowHook, + * RedirectStaticModuleInterface, + * StaticModuleType + * } from 'ses' + * @import { + * CompartmentDescriptor, + * ChooseModuleDescriptorOperators, + * ChooseModuleDescriptorOptions, + * ChooseModuleDescriptorYieldables, + * ExitModuleImportHook, + * FindRedirectParams, + * HashFn, + * ImportHookMaker, + * ImportNowHookMaker, + * MakeImportNowHookMakerOptions, + * ModuleDescriptor, + * ParseResult, + * ReadFn, + * ReadPowers, + * SourceMapHook, + * Sources, + * ReadNowPowers + * } from './types.js' + */ -/** @import {ImportHook} from 'ses' */ -/** @import {StaticModuleType} from 'ses' */ -/** @import {RedirectStaticModuleInterface} from 'ses' */ -/** @import {ReadFn} from './types.js' */ -/** @import {ReadPowers} from './types.js' */ -/** @import {HashFn} from './types.js' */ -/** @import {Sources} from './types.js' */ -/** @import {CompartmentDescriptor} from './types.js' */ -/** @import {ImportHookMaker} from './types.js' */ -/** @import {ExitModuleImportHook} from './types.js' */ - -import { attenuateModuleHook, enforceModulePolicy } from './policy.js'; +import { asyncTrampoline, syncTrampoline } from '@endo/trampoline'; import { resolve } from './node-module-specifier.js'; +import { + attenuateModuleHook, + ATTENUATORS_COMPARTMENT, + enforceModulePolicy, +} from './policy.js'; import { unpackReadPowers } from './powers.js'; // q, as in quote, for quoting strings in error messages. @@ -36,6 +57,8 @@ const { apply } = Reflect; */ const freeze = Object.freeze; +const { entries, keys, assign, create } = Object; + const { hasOwnProperty } = Object.prototype; /** * @param {Record} haystack @@ -68,10 +91,97 @@ const nodejsConventionSearchSuffixes = [ '/index.node', ]; +/** + * Given a module specifier which is an absolute path, attempt to match it with + * an existing compartment; return a {@link RedirectStaticModuleInterface} if found. + * + * @throws If we determine `absoluteModuleSpecifier` is unknown + * @param {FindRedirectParams} params Parameters + * @returns {RedirectStaticModuleInterface|undefined} A redirect or nothing + */ +const findRedirect = ({ + compartmentDescriptor, + compartmentDescriptors, + compartments, + absoluteModuleSpecifier, + packageLocation, +}) => { + const moduleSpecifierLocation = new URL( + absoluteModuleSpecifier, + packageLocation, + ).href; + + // a file:// URL string + let someLocation = new URL('./', moduleSpecifierLocation).href; + + // we are guaranteed an absolute path, so we can search "up" for the compartment + // due to the structure of `node_modules` + + // n === count of path components to the fs root + for (;;) { + if ( + someLocation !== ATTENUATORS_COMPARTMENT && + someLocation in compartments + ) { + const location = someLocation; + const someCompartmentDescriptor = compartmentDescriptors[location]; + if (compartmentDescriptor === someCompartmentDescriptor) { + // this compartmentDescriptor wants to dynamically load its own module + // using an absolute path + return undefined; + } + + // this tests the compartment referred to by the absolute path + // is a dependency of the compartment descriptor + if (compartmentDescriptor.compartments.has(location)) { + return { + specifier: absoluteModuleSpecifier, + compartment: compartments[location], + }; + } + + // this tests if the compartment descriptor is a dependency of the + // compartment referred to by the absolute path. + // it may be in scope, but disallowed by policy. + if ( + someCompartmentDescriptor.compartments.has( + compartmentDescriptor.location, + ) + ) { + enforceModulePolicy( + compartmentDescriptor.name, + someCompartmentDescriptor, + { + errorHint: `Blocked in import hook. ${q(absoluteModuleSpecifier)} is part of the compartment map and resolves to ${location}`, + }, + ); + return { + specifier: absoluteModuleSpecifier, + compartment: compartments[location], + }; + } + + throw new Error(`Could not import module: ${q(absoluteModuleSpecifier)}`); + } else { + // go up a directory + const parentLocation = new URL('../', someLocation).href; + + // afaict this behavior is consistent across both windows and posix + if (parentLocation === someLocation) { + throw new Error( + `Could not import unknown module: ${q(absoluteModuleSpecifier)}`, + ); + } + + someLocation = parentLocation; + } + } +}; + /** * @param {object} params * @param {Record=} params.modules - * @param {ExitModuleImportHook=} params.exitModuleImportHook + * @param {ExitModuleImportHook} [params.exitModuleImportHook] * @returns {ExitModuleImportHook|undefined} */ export const exitModuleImportHookMaker = ({ @@ -84,12 +194,12 @@ export const exitModuleImportHookMaker = ({ return async specifier => { if (modules && has(modules, specifier)) { const ns = modules[specifier]; - return Object.freeze({ + return freeze({ imports: [], - exports: ns ? Object.keys(ns) : [], + exports: ns ? keys(ns) : [], execute: moduleExports => { moduleExports.default = ns; - Object.assign(moduleExports, ns); + assign(moduleExports, ns); }, }); } @@ -100,6 +210,185 @@ export const exitModuleImportHookMaker = ({ }; }; +/** + * Expands a module specifier into a list of potential candidates based on + * `searchSuffixes`. + * + * @param {string} moduleSpecifier Module specifier + * @param {string[]} searchSuffixes Suffixes to search if the unmodified + * specifier is not found + * @returns {string[]} A list of potential candidates (including + * `moduleSpecifier` itself) + */ +const nominateCandidates = (moduleSpecifier, searchSuffixes) => { + // Collate candidate locations for the moduleSpecifier, + // to support Node.js conventions and similar. + const candidates = [moduleSpecifier]; + for (const candidateSuffix of searchSuffixes) { + candidates.push(`${moduleSpecifier}${candidateSuffix}`); + } + return candidates; +}; + +/** + * Returns a generator which applies {@link ChooseModuleDescriptorOperators} in + * `operators` using the options in options to ultimately result in a + * {@link StaticModuleType} for a particular {@link CompartmentDescriptor} (or + * `undefined`). + * + * Supports both {@link SyncChooseModuleDescriptorOperators sync} and + * {@link AsyncChooseModuleDescriptorOperators async} operators. + * + * Used by both {@link makeImportNowHookMaker} and {@link makeImportHookMaker}. + * + * @template {ChooseModuleDescriptorOperators} Operators Type of operators (sync + * or async) + * @param {ChooseModuleDescriptorOptions} options Options/context + * @param {Operators} operators Operators + * @returns {Generator>} + * Generator + */ +function* chooseModuleDescriptor( + { + candidates, + compartmentDescriptor, + compartmentDescriptors, + compartments, + computeSha512, + moduleDescriptors, + moduleSpecifier, + packageLocation, + packageSources, + readPowers, + sourceMapHook, + strictlyRequiredForCompartment, + }, + { maybeRead, parse, shouldDeferError = () => false }, +) { + for (const candidateSpecifier of candidates) { + const candidateModuleDescriptor = moduleDescriptors[candidateSpecifier]; + if (candidateModuleDescriptor !== undefined) { + const { compartment: candidateCompartmentName = packageLocation } = + candidateModuleDescriptor; + const candidateCompartment = compartments[candidateCompartmentName]; + if (candidateCompartment === undefined) { + throw Error( + `compartment missing for candidate ${candidateSpecifier} in ${candidateCompartmentName}`, + ); + } + // modify compartmentMap to include this redirect + const candidateCompartmentDescriptor = + compartmentDescriptors[candidateCompartmentName]; + if (candidateCompartmentDescriptor === undefined) { + throw Error( + `compartmentDescriptor missing for candidate ${candidateSpecifier} in ${candidateCompartmentName}`, + ); + } + candidateCompartmentDescriptor.modules[moduleSpecifier] = + candidateModuleDescriptor; + // return a redirect + /** @type {RedirectStaticModuleInterface} */ + const record = { + specifier: candidateSpecifier, + compartment: candidateCompartment, + }; + return record; + } + + // Using a specifier as a location. + // This is not always valid. + // But, for Node.js, when the specifier is relative and not a directory + // name, they are usable as URL's. + const moduleLocation = resolveLocation(candidateSpecifier, packageLocation); + + // "next" values must have type assertions for narrowing because we have + // multiple yielded types + const moduleBytes = /** @type {Uint8Array|undefined} */ ( + yield maybeRead(moduleLocation) + ); + + if (moduleBytes !== undefined) { + /** @type {string | undefined} */ + let sourceMap; + // must be narrowed + const envelope = /** @type {ParseResult} */ ( + yield parse( + moduleBytes, + candidateSpecifier, + moduleLocation, + packageLocation, + { + readPowers, + sourceMapHook: + sourceMapHook && + (nextSourceMapObject => { + sourceMap = JSON.stringify(nextSourceMapObject); + }), + compartmentDescriptor, + }, + ) + ); + const { + parser, + bytes: transformedBytes, + record: concreteRecord, + } = envelope; + + // Facilitate a redirect if the returned record has a different + // module specifier than the requested one. + if (candidateSpecifier !== moduleSpecifier) { + moduleDescriptors[moduleSpecifier] = { + module: candidateSpecifier, + compartment: packageLocation, + }; + } + /** @type {StaticModuleType} */ + const record = { + record: concreteRecord, + specifier: candidateSpecifier, + importMeta: { url: moduleLocation }, + }; + + let sha512; + if (computeSha512 !== undefined) { + sha512 = computeSha512(transformedBytes); + + if (sourceMapHook !== undefined && sourceMap !== undefined) { + sourceMapHook(sourceMap, { + compartment: packageLocation, + module: candidateSpecifier, + location: moduleLocation, + sha512, + }); + } + } + + const packageRelativeLocation = moduleLocation.slice( + packageLocation.length, + ); + packageSources[candidateSpecifier] = { + location: packageRelativeLocation, + sourceLocation: moduleLocation, + parser, + bytes: transformedBytes, + record: concreteRecord, + sha512, + }; + if (!shouldDeferError(parser)) { + for (const importSpecifier of getImportsFromRecord(record)) { + strictlyRequiredForCompartment(packageLocation).add( + resolve(importSpecifier, moduleSpecifier), + ); + } + } + + return record; + } + } + return undefined; +} + /** * @param {ReadFn|ReadPowers} readPowers * @param {string} baseLocation @@ -110,23 +399,23 @@ export const exitModuleImportHookMaker = ({ * @param {HashFn} [options.computeSha512] * @param {Array} [options.searchSuffixes] - Suffixes to search if the * unmodified specifier is not found. - * Pass [] to emulate Node.js’s strict behavior. - * The default handles Node.js’s CommonJS behavior. + * Pass [] to emulate Node.js' strict behavior. + * The default handles Node.js' CommonJS behavior. * Unlike Node.js, the Compartment Mapper lifts CommonJS up, more like a * bundler, and does not attempt to vary the behavior of resolution depending * on the language of the importing module. * @param {string} options.entryCompartmentName * @param {string} options.entryModuleSpecifier * @param {ExitModuleImportHook} [options.exitModuleImportHook] - * @param {import('./types.js').SourceMapHook} [options.sourceMapHook] + * @param {SourceMapHook} [options.sourceMapHook] * @returns {ImportHookMaker} */ export const makeImportHookMaker = ( readPowers, baseLocation, { - sources = Object.create(null), - compartmentDescriptors = Object.create(null), + sources = create(null), + compartmentDescriptors = create(null), archiveOnly = false, computeSha512 = undefined, searchSuffixes = nodejsConventionSearchSuffixes, @@ -168,11 +457,10 @@ export const makeImportHookMaker = ( }) => { // per-compartment: packageLocation = resolveLocation(packageLocation, baseLocation); - const packageSources = sources[packageLocation] || Object.create(null); + const packageSources = sources[packageLocation] || create(null); sources[packageLocation] = packageSources; const compartmentDescriptor = compartmentDescriptors[packageLocation] || {}; - const { modules: moduleDescriptors = Object.create(null) } = - compartmentDescriptor; + const { modules: moduleDescriptors = create(null) } = compartmentDescriptor; compartmentDescriptor.modules = moduleDescriptors; /** @@ -209,9 +497,11 @@ export const makeImportHookMaker = ( /** @type {ImportHook} */ const importHook = async moduleSpecifier => { - await null; compartmentDescriptor.retained = true; + // for lint rule + await null; + // per-module: // In Node.js, an absolute specifier always indicates a built-in or @@ -257,130 +547,31 @@ export const makeImportHookMaker = ( ); } - // Collate candidate locations for the moduleSpecifier, - // to support Node.js conventions and similar. - const candidates = [moduleSpecifier]; - for (const candidateSuffix of searchSuffixes) { - candidates.push(`${moduleSpecifier}${candidateSuffix}`); - } - const { maybeRead } = unpackReadPowers(readPowers); - for (const candidateSpecifier of candidates) { - const candidateModuleDescriptor = moduleDescriptors[candidateSpecifier]; - if (candidateModuleDescriptor !== undefined) { - const { compartment: candidateCompartmentName = packageLocation } = - candidateModuleDescriptor; - const candidateCompartment = compartments[candidateCompartmentName]; - if (candidateCompartment === undefined) { - throw Error( - `compartment missing for candidate ${candidateSpecifier} in ${candidateCompartmentName}`, - ); - } - // modify compartmentMap to include this redirect - const candidateCompartmentDescriptor = - compartmentDescriptors[candidateCompartmentName]; - if (candidateCompartmentDescriptor === undefined) { - throw Error( - `compartmentDescriptor missing for candidate ${candidateSpecifier} in ${candidateCompartmentName}`, - ); - } - candidateCompartmentDescriptor.modules[moduleSpecifier] = - candidateModuleDescriptor; - // return a redirect - /** @type {RedirectStaticModuleInterface} */ - const record = { - specifier: candidateSpecifier, - compartment: candidateCompartment, - }; - return record; - } + const candidates = nominateCandidates(moduleSpecifier, searchSuffixes); - // Using a specifier as a location. - // This is not always valid. - // But, for Node.js, when the specifier is relative and not a directory - // name, they are usable as URL's. - const moduleLocation = resolveLocation( - candidateSpecifier, + const record = await asyncTrampoline( + chooseModuleDescriptor, + { + candidates, + compartmentDescriptor, + compartmentDescriptors, + compartments, + computeSha512, + moduleDescriptors, + moduleSpecifier, packageLocation, - ); - // eslint-disable-next-line no-await-in-loop - const moduleBytes = await maybeRead(moduleLocation); - if (moduleBytes !== undefined) { - /** @type {string | undefined} */ - let sourceMap; - // eslint-disable-next-line no-await-in-loop - const envelope = await parse( - moduleBytes, - candidateSpecifier, - moduleLocation, - packageLocation, - { - readPowers, - sourceMapHook: - sourceMapHook && - (nextSourceMapObject => { - sourceMap = JSON.stringify(nextSourceMapObject); - }), - compartmentDescriptor, - }, - ); - const { - parser, - bytes: transformedBytes, - record: concreteRecord, - } = envelope; - - // Facilitate a redirect if the returned record has a different - // module specifier than the requested one. - if (candidateSpecifier !== moduleSpecifier) { - moduleDescriptors[moduleSpecifier] = { - module: candidateSpecifier, - compartment: packageLocation, - }; - } - /** @type {StaticModuleType} */ - const record = { - record: concreteRecord, - specifier: candidateSpecifier, - importMeta: { url: moduleLocation }, - }; - - let sha512; - if (computeSha512 !== undefined) { - sha512 = computeSha512(transformedBytes); - - if (sourceMapHook !== undefined && sourceMap !== undefined) { - sourceMapHook(sourceMap, { - compartment: packageLocation, - module: candidateSpecifier, - location: moduleLocation, - sha512, - }); - } - } - - const packageRelativeLocation = moduleLocation.slice( - packageLocation.length, - ); - packageSources[candidateSpecifier] = { - location: packageRelativeLocation, - sourceLocation: moduleLocation, - parser, - bytes: transformedBytes, - record: concreteRecord, - sha512, - }; - if (!shouldDeferError(parser)) { - for (const importSpecifier of getImportsFromRecord(record)) { - strictlyRequiredForCompartment(packageLocation).add( - resolve(importSpecifier, moduleSpecifier), - ); - } - } + packageSources, + readPowers, + sourceMapHook, + strictlyRequiredForCompartment, + }, + { maybeRead, parse, shouldDeferError }, + ); - return record; - } + if (record) { + return record; } return deferError( @@ -399,3 +590,161 @@ export const makeImportHookMaker = ( }; return makeImportHook; }; + +/** + * Synchronous import for dynamic requires. + * + * @param {ReadNowPowers} readPowers + * @param {string} baseLocation + * @param {MakeImportNowHookMakerOptions} options + * @returns {ImportNowHookMaker} + */ +export function makeImportNowHookMaker( + readPowers, + baseLocation, + { + sources = create(null), + compartmentDescriptors = create(null), + computeSha512 = undefined, + searchSuffixes = nodejsConventionSearchSuffixes, + sourceMapHook = undefined, + exitModuleImportNowHook, + }, +) { + // Set of specifiers for modules (scoped to compartment) whose parser is not + // using heuristics to determine imports. + /** @type {Map>} compartment name ->* module specifier */ + const strictlyRequired = new Map(); + + /** + * @param {string} compartmentName + */ + const strictlyRequiredForCompartment = compartmentName => { + let compartmentStrictlyRequired = strictlyRequired.get(compartmentName); + if (compartmentStrictlyRequired !== undefined) { + return compartmentStrictlyRequired; + } + compartmentStrictlyRequired = new Set(); + strictlyRequired.set(compartmentName, compartmentStrictlyRequired); + return compartmentStrictlyRequired; + }; + + /** + * @type {ImportNowHookMaker} + */ + const makeImportNowHook = ({ + packageLocation, + packageName: _packageName, + parse, + compartments, + }) => { + if (!('isSyncParser' in parse)) { + return function impossibleTransformImportNowHook() { + throw new Error( + 'Dynamic requires are only possible with synchronous parsers and no asynchronous module transforms in options', + ); + }; + } + + const compartmentDescriptor = compartmentDescriptors[packageLocation] || {}; + + packageLocation = resolveLocation(packageLocation, baseLocation); + const packageSources = sources[packageLocation] || create(null); + sources[packageLocation] = packageSources; + const { + modules: + moduleDescriptors = /** @type {Record} */ ( + create(null) + ), + } = compartmentDescriptor; + compartmentDescriptor.modules = moduleDescriptors; + + let { policy } = compartmentDescriptor; + policy = policy || create(null); + + // Associates modules with compartment descriptors based on policy + // in cases where the association was not made when building the + // compartment map but is indicated by the policy. + if ('packages' in policy && typeof policy.packages === 'object') { + for (const [packageName, packagePolicyItem] of entries(policy.packages)) { + if ( + !(packageName in compartmentDescriptor.modules) && + packageName in compartmentDescriptor.scopes && + packagePolicyItem + ) { + compartmentDescriptor.modules[packageName] = + compartmentDescriptor.scopes[packageName]; + } + } + } + + const { maybeReadNow, isAbsolute } = readPowers; + + /** @type {ImportNowHook} */ + const importNowHook = moduleSpecifier => { + if (isAbsolute(moduleSpecifier)) { + const record = findRedirect({ + compartmentDescriptor, + compartmentDescriptors, + compartments, + absoluteModuleSpecifier: moduleSpecifier, + packageLocation, + }); + if (record) { + return record; + } + } + + const candidates = nominateCandidates(moduleSpecifier, searchSuffixes); + + const record = syncTrampoline( + chooseModuleDescriptor, + { + candidates, + compartmentDescriptor, + compartmentDescriptors, + compartments, + computeSha512, + moduleDescriptors, + moduleSpecifier, + packageLocation, + packageSources, + readPowers, + sourceMapHook, + strictlyRequiredForCompartment, + }, + { + maybeRead: maybeReadNow, + parse, + }, + ); + + if (record) { + return record; + } + + if (exitModuleImportNowHook) { + // this hook is responsible for ensuring that the moduleSpecifier actually refers to an exit module + const exitRecord = exitModuleImportNowHook( + moduleSpecifier, + packageLocation, + ); + + if (!exitRecord) { + throw new Error(`Could not import module: ${q(moduleSpecifier)}`); + } + + return exitRecord; + } + + throw new Error( + `Could not import module: ${q( + moduleSpecifier, + )}; try providing an importNowHook`, + ); + }; + + return importNowHook; + }; + return makeImportNowHook; +} diff --git a/packages/compartment-mapper/src/import-lite.js b/packages/compartment-mapper/src/import-lite.js index 68c80143a3..408ac58a23 100644 --- a/packages/compartment-mapper/src/import-lite.js +++ b/packages/compartment-mapper/src/import-lite.js @@ -16,11 +16,13 @@ // @ts-check /* eslint no-shadow: "off" */ - /** @import {CompartmentMapDescriptor} from './types.js' */ +/** @import {SyncImportLocationOptions} from './types.js' */ +/** @import {ImportNowHookMaker} from './types.js' */ +/** @import {ModuleTransforms} from './types.js' */ +/** @import {ReadNowPowers} from './types.js' */ /** @import {Application} from './types.js' */ /** @import {ImportLocationOptions} from './types.js' */ -/** @import {LoadLocationOptions} from './types.js' */ /** @import {ExecuteFn} from './types.js' */ /** @import {ReadFn} from './types.js' */ /** @import {ReadPowers} from './types.js' */ @@ -30,19 +32,63 @@ import { link } from './link.js'; import { exitModuleImportHookMaker, makeImportHookMaker, + makeImportNowHookMaker, } from './import-hook.js'; +import { isReadNowPowers } from './powers.js'; + +const { assign, create, freeze, entries } = Object; + +/** + * Returns `true` if `value` is a {@link SyncImportLocationOptions}. + * + * The requirements here are: + * - `moduleTransforms` _is not_ present in `value` + * - `parserForLanguage` - if set, contains synchronous parsers only + * + * @param {ImportLocationOptions|SyncImportLocationOptions} value + * @returns {value is SyncImportLocationOptions} + */ +const isSyncOptions = value => { + if (!value || (typeof value === 'object' && !('moduleTransforms' in value))) { + if (value.parserForLanguage) { + for (const [_language, { synchronous }] of entries( + value.parserForLanguage, + )) { + if (!synchronous) { + return false; + } + } + } + return true; + } + return false; +}; -const { assign, create, freeze } = Object; +/** + * @overload + * @param {ReadNowPowers} readPowers + * @param {CompartmentMapDescriptor} compartmentMap + * @param {SyncImportLocationOptions} [opts] + * @returns {Promise} + */ /** + * @overload * @param {ReadFn | ReadPowers} readPowers * @param {CompartmentMapDescriptor} compartmentMap - * @param {LoadLocationOptions} [options] + * @param {ImportLocationOptions} [opts] * @returns {Promise} */ + +/** + * @param {ReadFn|ReadPowers|ReadNowPowers} readPowers + * @param {CompartmentMapDescriptor} compartmentMap + * @param {ImportLocationOptions} [options] + * @returns {Promise} + */ + export const loadFromMap = async (readPowers, compartmentMap, options = {}) => { const { - moduleTransforms = {}, searchSuffixes = undefined, parserForLanguage: parserForLanguageOption = {}, languageForExtension: languageForExtensionOption = {}, @@ -55,6 +101,44 @@ export const loadFromMap = async (readPowers, compartmentMap, options = {}) => { assign(create(null), languageForExtensionOption), ); + /** + * Object containing options and read powers that fulfills all requirements + * for creation of a {@link ImportNowHookMaker}, thus enabling dynamic import. + * + * @typedef SyncBehavior + * @property {ReadNowPowers} readPowers + * @property {SyncImportLocationOptions} options + * @property {'SYNC'} type + */ + + /** + * Object containing options and read powers which is incompatible with + * creation of an {@link ImportNowHookMaker}, thus disabling dynamic import. + * + * @typedef AsyncBehavior + * @property {ReadFn|ReadPowers} readPowers + * @property {ImportLocationOptions} options + * @property {'ASYNC'} type + */ + + /** + * When we must control flow based on _n_ type guards consdering _n_ discrete + * values, grouping the values into an object, then leveraging a discriminated + * union (the `type` property) is one way to approach the problem. + */ + const behavior = + isReadNowPowers(readPowers) && isSyncOptions(options) + ? /** @type {SyncBehavior} */ ({ + readPowers, + options: options || {}, + type: 'SYNC', + }) + : /** @type {AsyncBehavior} */ ({ + readPowers, + options: options || {}, + type: 'ASYNC', + }); + const { entry: { compartment: entryCompartmentName, module: entryModuleSpecifier }, } = compartmentMap; @@ -85,16 +169,54 @@ export const loadFromMap = async (readPowers, compartmentMap, options = {}) => { exitModuleImportHook: compartmentExitModuleImportHook, }, ); - const { compartment, pendingJobsPromise } = link(compartmentMap, { - makeImportHook, - parserForLanguage, - languageForExtension, - globals, - transforms, - moduleTransforms, - __shimTransforms__, - Compartment, - }); + + /** @type {ImportNowHookMaker | undefined} */ + let makeImportNowHook; + + /** @type {Compartment} */ + let compartment; + /** @type {Promise} */ + let pendingJobsPromise; + + if (behavior.type === 'SYNC') { + const { importNowHook: exitModuleImportNowHook, syncModuleTransforms } = + behavior.options; + makeImportNowHook = makeImportNowHookMaker( + /** @type {ReadNowPowers} */ (readPowers), + entryCompartmentName, + { + compartmentDescriptors: compartmentMap.compartments, + searchSuffixes, + exitModuleImportNowHook, + }, + ); + ({ compartment, pendingJobsPromise } = link(compartmentMap, { + makeImportHook, + makeImportNowHook, + parserForLanguage, + languageForExtension, + globals, + transforms, + syncModuleTransforms, + __shimTransforms__, + Compartment, + })); + } else { + // sync module transforms are allowed, because they are "compatible" + // with async module transforms (not vice-versa) + const { moduleTransforms, syncModuleTransforms } = behavior.options; + ({ compartment, pendingJobsPromise } = link(compartmentMap, { + makeImportHook, + parserForLanguage, + languageForExtension, + globals, + transforms, + moduleTransforms, + syncModuleTransforms, + __shimTransforms__, + Compartment, + })); + } await pendingJobsPromise; diff --git a/packages/compartment-mapper/src/import.js b/packages/compartment-mapper/src/import.js index 151b4c3985..9c7561afca 100644 --- a/packages/compartment-mapper/src/import.js +++ b/packages/compartment-mapper/src/import.js @@ -19,8 +19,13 @@ import { loadFromMap } from './import-lite.js'; const { assign, create, freeze } = Object; /** @import {Application} from './types.js' */ +/** @import {ImportLocationOptions} from './types.js' */ +/** @import {SyncArchiveOptions} from './types.js' */ +/** @import {LoadLocationOptions} from './types.js' */ +/** @import {SyncImportLocationOptions} from './types.js' */ +/** @import {SomeObject} from './types.js' */ +/** @import {ReadNowPowers} from './types.js' */ /** @import {ArchiveOptions} from './types.js' */ -/** @import {ExecuteOptions} from './types.js' */ /** @import {ReadFn} from './types.js' */ /** @import {ReadPowers} from './types.js' */ @@ -38,9 +43,25 @@ const assignParserForLanguage = (options = {}) => { }; /** + * @overload + * @param {ReadNowPowers} readPowers + * @param {string} moduleLocation + * @param {SyncArchiveOptions} options + * @returns {Promise} + */ + +/** + * @overload * @param {ReadFn | ReadPowers} readPowers * @param {string} moduleLocation - * @param {ArchiveOptions} [options] + * @param {LoadLocationOptions} [options] + * @returns {Promise} + */ + +/** + * @param {ReadFn|ReadPowers|ReadNowPowers} readPowers + * @param {string} moduleLocation + * @param {LoadLocationOptions} [options] * @returns {Promise} */ export const loadLocation = async ( @@ -48,7 +69,10 @@ export const loadLocation = async ( moduleLocation, options = {}, ) => { - const { dev, tags, conditions = tags, commonDependencies, policy } = options; + const { dev, tags, commonDependencies, policy } = options; + // conditions are not present in SyncArchiveOptions + const conditions = + 'conditions' in options ? options.conditions || tags : tags; const compartmentMap = await mapNodeModules(readPowers, moduleLocation, { dev, conditions, @@ -63,10 +87,32 @@ export const loadLocation = async ( }; /** - * @param {ReadFn | ReadPowers} readPowers + * Allows dynamic requires + * + * @overload + * @param {ReadNowPowers} readPowers + * @param {string} moduleLocation + * @param {SyncImportLocationOptions} options + * @returns {Promise} the object of the imported modules exported + * names. + */ + +/** + * Disallows dynamic requires + * + * @overload + * @param {ReadPowers|ReadFn} readPowers + * @param {string} moduleLocation + * @param {ImportLocationOptions} [options] + * @returns {Promise} the object of the imported modules exported + * names. + */ + +/** + * @param {ReadPowers|ReadFn|ReadNowPowers} readPowers * @param {string} moduleLocation - * @param {ExecuteOptions & ArchiveOptions} [options] - * @returns {Promise} the object of the imported modules exported + * @param {ImportLocationOptions} [options] + * @returns {Promise} the object of the imported modules exported * names. */ export const importLocation = async (readPowers, moduleLocation, options) => { diff --git a/packages/compartment-mapper/src/link.js b/packages/compartment-mapper/src/link.js index f080687565..d099f9adbb 100644 --- a/packages/compartment-mapper/src/link.js +++ b/packages/compartment-mapper/src/link.js @@ -9,27 +9,34 @@ // @ts-check /** @import {ModuleMapHook} from 'ses' */ -/** @import {ParseFn, ParserForLanguage} from './types.js' */ -/** @import {ParserImplementation} from './types.js' */ -/** @import {ShouldDeferError} from './types.js' */ -/** @import {ModuleTransforms} from './types.js' */ -/** @import {LanguageForExtension} from './types.js' */ -/** @import {ModuleDescriptor} from './types.js' */ -/** @import {CompartmentDescriptor} from './types.js' */ -/** @import {CompartmentMapDescriptor} from './types.js' */ -/** @import {LinkOptions} from './types.js' */ +/** + * @import { + * CompartmentDescriptor, + * CompartmentMapDescriptor, + * ImportNowHookMaker, + * LanguageForExtension, + * LinkOptions, + * LinkResult, + * ModuleDescriptor, + * ParseFn, + * ParseFnAsync, + * ParserForLanguage, + * ParserImplementation, + * ShouldDeferError, + * } from './types.js' + */ /** @import {ERef} from '@endo/eventual-send' */ +import { makeMapParsers } from './map-parser.js'; import { resolve as resolveFallback } from './node-module-specifier.js'; -import { parseExtension } from './extension.js'; import { - enforceModulePolicy, ATTENUATORS_COMPARTMENT, attenuateGlobals, + enforceModulePolicy, makeDeferredAttenuatorsProvider, } from './policy.js'; -const { assign, create, entries, freeze, fromEntries } = Object; +const { assign, create, entries, freeze } = Object; const { hasOwnProperty } = Object.prototype; const { apply } = Reflect; const { allSettled } = Promise; @@ -52,130 +59,6 @@ const q = JSON.stringify; */ const has = (object, key) => apply(hasOwnProperty, object, [key]); -/** - * Decide if extension is clearly indicating a parser/language for a file - * - * @param {string} extension - * @returns {boolean} - */ -const extensionImpliesLanguage = extension => extension !== 'js'; - -/** - * `makeExtensionParser` produces a `parser` that parses the content of a - * module according to the corresponding module language, given the extension - * of the module specifier and the configuration of the containing compartment. - * We do not yet support import assertions and we do not have a mechanism - * for validating the MIME type of the module content against the - * language implied by the extension or file name. - * - * @param {Record} languageForExtension - maps a file extension - * to the corresponding language. - * @param {Record} languageForModuleSpecifier - In a rare case, - * the type of a module is implied by package.json and should not be inferred - * from its extension. - * @param {ParserForLanguage} parserForLanguage - * @param {ModuleTransforms} moduleTransforms - * @returns {ParseFn} - */ -const makeExtensionParser = ( - languageForExtension, - languageForModuleSpecifier, - parserForLanguage, - moduleTransforms, -) => { - return async (bytes, specifier, location, packageLocation, options) => { - await null; - let language; - const extension = parseExtension(location); - - if ( - !extensionImpliesLanguage(extension) && - has(languageForModuleSpecifier, specifier) - ) { - language = languageForModuleSpecifier[specifier]; - } else { - language = languageForExtension[extension] || extension; - } - - let sourceMap; - - if (has(moduleTransforms, language)) { - try { - ({ - bytes, - parser: language, - sourceMap, - } = await moduleTransforms[language]( - bytes, - specifier, - location, - packageLocation, - { - // At time of writing, sourceMap is always undefined, but keeping - // it here is more resilient if the surrounding if block becomes a - // loop for multi-step transforms. - sourceMap, - }, - )); - } catch (err) { - throw Error( - `Error transforming ${q(language)} source in ${q(location)}: ${ - err.message - }`, - { cause: err }, - ); - } - } - - if (!has(parserForLanguage, language)) { - throw Error( - `Cannot parse module ${specifier} at ${location}, no parser configured for the language ${language}`, - ); - } - const { parse } = /** @type {ParserImplementation} */ ( - parserForLanguage[language] - ); - return parse(bytes, specifier, location, packageLocation, { - sourceMap, - ...options, - }); - }; -}; - -/** - * @param {LanguageForExtension} languageForExtension - * @param {Record} languageForModuleSpecifier - In a rare case, the type of a module - * is implied by package.json and should not be inferred from its extension. - * @param {ParserForLanguage} parserForLanguage - * @param {ModuleTransforms} moduleTransforms - * @returns {ParseFn} - */ -export const mapParsers = ( - languageForExtension, - languageForModuleSpecifier, - parserForLanguage, - moduleTransforms = {}, -) => { - const languageForExtensionEntries = []; - const problems = []; - for (const [extension, language] of entries(languageForExtension)) { - if (has(parserForLanguage, language)) { - languageForExtensionEntries.push([extension, language]); - } else { - problems.push(`${q(language)} for extension ${q(extension)}`); - } - } - if (problems.length > 0) { - throw Error(`No parser available for language: ${problems.join(', ')}`); - } - return makeExtensionParser( - fromEntries(languageForExtensionEntries), - languageForModuleSpecifier, - parserForLanguage, - moduleTransforms, - ); -}; - /** * For a full, absolute module specifier like "dependency", * produce the module specifier in the dependency, like ".". @@ -328,36 +211,57 @@ const makeModuleMapHook = ( return moduleMapHook; }; +/** + * @type {ImportNowHookMaker} + */ +const impossibleImportNowHookMaker = () => { + return function impossibleImportNowHook() { + throw new Error('Provided read powers do not support dynamic requires'); + }; +}; + /** * Assemble a DAG of compartments as declared in a compartment map starting at * the named compartment and building all compartments that it depends upon, * recursively threading the modules exported by one compartment into the * compartment that imports them. - * Returns the root of the compartment DAG. - * Does not load or execute any modules. - * Uses makeImportHook with the given "location" string of each compartment in - * the DAG. - * Passes the given globals and external modules into the root compartment - * only. * + * - Returns the root of the compartment DAG. + * - Does not load or execute any modules. + * - Uses `makeImportHook` with the given "location" string of each compartment + * in the DAG. + * - Passes the given globals and external modules into the root compartment + * only. + * + * @param {CompartmentMapDescriptor} compartmentMap + * @param {LinkOptions} options + * @returns {LinkResult} the root compartment of the compartment DAG + */ + +/** * @param {CompartmentMapDescriptor} compartmentMap * @param {LinkOptions} options + * @returns {LinkResult} */ export const link = ( { entry, compartments: compartmentDescriptors }, - { + options, +) => { + const { resolve = resolveFallback, makeImportHook, + makeImportNowHook = impossibleImportNowHookMaker, parserForLanguage: parserForLanguageOption = {}, languageForExtension: languageForExtensionOption = {}, globals = {}, transforms = [], - moduleTransforms = {}, + moduleTransforms, + syncModuleTransforms, __shimTransforms__ = [], archiveOnly = false, Compartment = defaultCompartment, - }, -) => { + } = options; + const { compartment: entryCompartmentName } = entry; /** @type {Record} */ @@ -382,19 +286,28 @@ export const link = ( assign(create(null), parserForLanguageOption), ); + const mapParsers = makeMapParsers({ + parserForLanguage, + moduleTransforms, + syncModuleTransforms, + }); + for (const [compartmentName, compartmentDescriptor] of entries( compartmentDescriptors, )) { - // TODO: The default assignments seem to break type inference const { location, name, - modules = create(null), parsers: languageForExtensionOverrides = {}, types: languageForModuleSpecifierOverrides = {}, - scopes = create(null), } = compartmentDescriptor; + // this is for retaining the correct type inference about these values + // without use of `let` + const { scopes: _scopes, modules: _modules } = compartmentDescriptor; + const modules = _modules || create(null); + const scopes = _scopes || create(null); + // Capture the default. // The `moduleMapHook` writes back to the compartment map. compartmentDescriptor.modules = modules; @@ -412,12 +325,12 @@ export const link = ( ), ); - const parse = mapParsers( - languageForExtension, - languageForModuleSpecifier, - parserForLanguage, - moduleTransforms, + // TS is kind of dumb about this, so we can use a type assertion to avoid a + // pointless ternary. + const parse = /** @type {ParseFn|ParseFnAsync} */ ( + mapParsers(languageForExtension, languageForModuleSpecifier) ); + /** @type {ShouldDeferError} */ const shouldDeferError = language => { if (language && has(parserForLanguage, language)) { @@ -441,6 +354,14 @@ export const link = ( shouldDeferError, compartments, }); + + const importNowHook = makeImportNowHook({ + packageLocation: location, + packageName: name, + parse, + compartments, + }); + const moduleMapHook = makeModuleMapHook( compartmentDescriptor, compartments, @@ -453,6 +374,7 @@ export const link = ( name: location, resolveHook, importHook, + importNowHook, moduleMapHook, transforms, __shimTransforms__, diff --git a/packages/compartment-mapper/src/map-parser.js b/packages/compartment-mapper/src/map-parser.js new file mode 100644 index 0000000000..0283660996 --- /dev/null +++ b/packages/compartment-mapper/src/map-parser.js @@ -0,0 +1,339 @@ +/** + * Exports {@link makeMapParsers}, which creates a function which matches a + * module to a parser based on reasons. + * + * @module + */ + +// @ts-check + +import { syncTrampoline, asyncTrampoline } from '@endo/trampoline'; +import { parseExtension } from './extension.js'; + +/** + * @import { + * LanguageForExtension, + * LanguageForModuleSpecifier, + * MakeMapParsersOptions, + * MapParsersFn, + * ModuleTransform, + * ModuleTransforms, + * ParseFn, + * ParseFnAsync, + * ParseResult, + * ParserForLanguage, + * SyncModuleTransform, + * SyncModuleTransforms + * } from './types.js'; + */ + +const { entries, fromEntries, keys, hasOwnProperty, values } = Object; +const { apply } = Reflect; +// q, as in quote, for strings in error messages. +const q = JSON.stringify; + +/** + * @param {Record} object + * @param {string} key + * @returns {boolean} + */ +const has = (object, key) => apply(hasOwnProperty, object, [key]); + +/** + * Decide if extension is clearly indicating a parser/language for a file + * + * @param {string} extension + * @returns {boolean} + */ +const extensionImpliesLanguage = extension => extension !== 'js'; + +/** + * Produces a `parser` that parses the content of a module according to the + * corresponding module language, given the extension of the module specifier + * and the configuration of the containing compartment. We do not yet support + * import assertions and we do not have a mechanism for validating the MIME type + * of the module content against the language implied by the extension or file + * name. + * + * @param {boolean} preferSynchronous + * @param {Record} languageForExtension - maps a file extension + * to the corresponding language. + * @param {Record} languageForModuleSpecifier - In a rare case, + * the type of a module is implied by package.json and should not be inferred + * from its extension. + * @param {ParserForLanguage} parserForLanguage + * @param {ModuleTransforms} moduleTransforms + * @param {SyncModuleTransforms} syncModuleTransforms + * @returns {ParseFnAsync|ParseFn} + */ +const makeExtensionParser = ( + preferSynchronous, + languageForExtension, + languageForModuleSpecifier, + parserForLanguage, + moduleTransforms, + syncModuleTransforms, +) => { + /** @type {Record} */ + let transforms; + + /** + * Function returning a generator which executes a parser for a module in either sync or async context. + * + * @param {Uint8Array} bytes + * @param {string} specifier + * @param {string} location + * @param {string} packageLocation + * @param {*} options + * @returns {Generator|ReturnType, ParseResult, Awaited|ReturnType>>} + */ + function* getParserGenerator( + bytes, + specifier, + location, + packageLocation, + options, + ) { + /** @type {string} */ + let language; + const extension = parseExtension(location); + + if ( + !extensionImpliesLanguage(extension) && + has(languageForModuleSpecifier, specifier) + ) { + language = languageForModuleSpecifier[specifier]; + } else { + language = languageForExtension[extension] || extension; + } + + /** @type {string|undefined} */ + let sourceMap; + + if (has(transforms, language)) { + try { + ({ + bytes, + parser: language, + sourceMap, + } = yield transforms[language]( + bytes, + specifier, + location, + packageLocation, + { + // At time of writing, sourceMap is always undefined, but keeping + // it here is more resilient if the surrounding if block becomes a + // loop for multi-step transforms. + sourceMap, + }, + )); + } catch (err) { + throw Error( + `Error transforming ${q(language)} source in ${q(location)}: ${err.message}`, + { cause: err }, + ); + } + } + if (!has(parserForLanguage, language)) { + throw Error( + `Cannot parse module ${specifier} at ${location}, no parser configured for the language ${language}`, + ); + } + const { parse } = parserForLanguage[language]; + return parse(bytes, specifier, location, packageLocation, { + sourceMap, + ...options, + }); + } + + /** + * @type {ParseFn} + */ + const syncParser = (bytes, specifier, location, packageLocation, options) => { + const result = syncTrampoline( + getParserGenerator, + bytes, + specifier, + location, + packageLocation, + options, + ); + if ('then' in result && typeof result.then === 'function') { + throw new TypeError( + 'Sync parser cannot return a Thenable; ensure parser is actually synchronous', + ); + } + return result; + }; + syncParser.isSyncParser = true; + + /** + * @type {ParseFnAsync} + */ + const asyncParser = async ( + bytes, + specifier, + location, + packageLocation, + options, + ) => { + return asyncTrampoline( + getParserGenerator, + bytes, + specifier, + location, + packageLocation, + options, + ); + }; + + // Unfortunately, typescript was not smart enough to figure out the return + // type depending on a boolean in arguments, so it has to be + // ParseFnAsync|ParseFn + if (preferSynchronous) { + transforms = syncModuleTransforms; + return syncParser; + } else { + // we can fold syncModuleTransforms into moduleTransforms because + // async supports sync, but not vice-versa + transforms = { + ...syncModuleTransforms, + ...moduleTransforms, + }; + + return asyncParser; + } +}; + +/** + * Creates a synchronous parser + * + * @overload + * @param {LanguageForExtension} languageForExtension + * @param {LanguageForModuleSpecifier} languageForModuleSpecifier - In a rare case, + * the type of a module is implied by package.json and should not be inferred + * from its extension. + * @param {ParserForLanguage} parserForLanguage + * @param {ModuleTransforms} [moduleTransforms] + * @param {SyncModuleTransforms} [syncModuleTransforms] + * @param {true} preferSynchronous If `true`, will create a `ParseFn` + * @returns {ParseFn} + */ + +/** + * Creates an asynchronous parser + * + * @overload + * @param {LanguageForExtension} languageForExtension + * @param {LanguageForModuleSpecifier} languageForModuleSpecifier - In a rare case, + * the type of a module is implied by package.json and should not be inferred + * from its extension. + * @param {ParserForLanguage} parserForLanguage + * @param {ModuleTransforms} [moduleTransforms] + * @param {SyncModuleTransforms} [syncModuleTransforms] + * @param {false} [preferSynchronous] + * @returns {ParseFnAsync} + */ + +/** + * @param {LanguageForExtension} languageForExtension + * @param {LanguageForModuleSpecifier} languageForModuleSpecifier - In a rare case, + * the type of a module is implied by package.json and should not be inferred + * from its extension. + * @param {ParserForLanguage} parserForLanguage + * @param {ModuleTransforms} [moduleTransforms] + * @param {SyncModuleTransforms} [syncModuleTransforms] + * @param {boolean} [preferSynchronous] If `true`, will create a `ParseFn` + * @returns {ParseFnAsync|ParseFn} + */ +function mapParsers( + languageForExtension, + languageForModuleSpecifier, + parserForLanguage, + moduleTransforms = {}, + syncModuleTransforms = {}, + preferSynchronous = false, +) { + const languageForExtensionEntries = []; + const problems = []; + for (const [extension, language] of entries(languageForExtension)) { + if (has(parserForLanguage, language)) { + languageForExtensionEntries.push([extension, language]); + } else { + problems.push(`${q(language)} for extension ${q(extension)}`); + } + } + if (problems.length > 0) { + throw Error(`No parser available for language: ${problems.join(', ')}`); + } + return makeExtensionParser( + preferSynchronous, + fromEntries(languageForExtensionEntries), + languageForModuleSpecifier, + parserForLanguage, + moduleTransforms, + syncModuleTransforms, + ); +} + +/** + * Prepares a function to map parsers after verifying whether synchronous + * behavior is preferred. Synchronous behavior is selected if all parsers are + * synchronous and no async transforms are provided. + * + * _Note:_ The type argument for {@link MapParsersFn} _could_ be inferred from + * the function arguments _if_ {@link ParserForLanguage} contained only values + * of type `ParserImplementation & {synchronous: true}` (and also considering + * the emptiness of `moduleTransforms`); may only be worth the effort if it was + * causing issues in practice. + * + * @param {MakeMapParsersOptions} options + * @returns {MapParsersFn} + */ +export const makeMapParsers = ({ + parserForLanguage, + moduleTransforms, + syncModuleTransforms, +}) => { + /** + * Async `mapParsers()` function; returned when a non-synchronous parser is + * present _or_ when `moduleTransforms` is non-empty. + * + * @type {MapParsersFn} + */ + const asyncParseFn = (languageForExtension, languageForModuleSpecifier) => + mapParsers( + languageForExtension, + languageForModuleSpecifier, + parserForLanguage, + moduleTransforms, + syncModuleTransforms, + ); + + if (moduleTransforms && keys(moduleTransforms).length > 0) { + return asyncParseFn; + } + + // all parsers must explicitly be flagged `synchronous` to return a + // `MapParsersFn` + if (values(parserForLanguage).some(({ synchronous }) => !synchronous)) { + return asyncParseFn; + } + + /** + * Synchronous `mapParsers()` function; returned when all parsers are + * synchronous and `moduleTransforms` is empty. + * + * @type {MapParsersFn} + */ + return (languageForExtension, languageForModuleSpecifier) => + mapParsers( + languageForExtension, + languageForModuleSpecifier, + parserForLanguage, + moduleTransforms, + syncModuleTransforms, + true, + ); +}; diff --git a/packages/compartment-mapper/src/node-modules.js b/packages/compartment-mapper/src/node-modules.js index 609dddb7f6..fde2c0fce8 100644 --- a/packages/compartment-mapper/src/node-modules.js +++ b/packages/compartment-mapper/src/node-modules.js @@ -12,6 +12,8 @@ /* eslint no-shadow: 0 */ /** @import {CanonicalFn} from './types.js' */ +/** @import {CompartmentMapForNodeModulesOptions} from './types.js' */ +/** @import {SomePolicy} from './types.js' */ /** @import {CompartmentDescriptor} from './types.js' */ /** @import {CompartmentMapDescriptor} from './types.js' */ /** @import {Language} from './types.js' */ @@ -605,7 +607,7 @@ const graphPackages = async ( * @param {Graph} graph * @param {Set} conditions - build conditions about the target environment * for selecting relevant exports, e.g., "browser" or "node". - * @param {import('./types.js').Policy} [policy] + * @param {SomePolicy} [policy] * @returns {CompartmentMapDescriptor} */ const translateGraph = ( @@ -642,6 +644,12 @@ const translateGraph = ( /** @type {Record} */ const scopes = Object.create(null); + /** + * List of all the compartments (by name) that this compartment can import from. + * + * @type {Set} + */ + const compartmentNames = new Set(); const packagePolicy = getPolicyForPackage( { isEntry: dependeeLocation === entryPackageLocation, @@ -699,6 +707,7 @@ const translateGraph = ( for (const dependencyName of keys(dependencyLocations).sort()) { const dependencyLocation = dependencyLocations[dependencyName]; digestExternalAliases(dependencyName, dependencyLocation); + compartmentNames.add(dependencyLocation); } // digest own internal aliases for (const modulePath of keys(internalAliases).sort()) { @@ -724,6 +733,7 @@ const translateGraph = ( parsers, types, policy: /** @type {SomePackagePolicy} */ (packagePolicy), + compartments: compartmentNames, }; } @@ -745,10 +755,7 @@ const translateGraph = ( * @param {Set} conditions * @param {object} packageDescriptor * @param {string} moduleSpecifier - * @param {object} [options] - * @param {boolean} [options.dev] - * @param {object} [options.commonDependencies] - * @param {object} [options.policy] + * @param {CompartmentMapForNodeModulesOptions} [options] * @returns {Promise} */ export const compartmentMapForNodeModules = async ( diff --git a/packages/compartment-mapper/src/node-powers.js b/packages/compartment-mapper/src/node-powers.js index 0f315fe9c3..a4c084eaf0 100644 --- a/packages/compartment-mapper/src/node-powers.js +++ b/packages/compartment-mapper/src/node-powers.js @@ -8,14 +8,28 @@ // @ts-check -/** @import {ReadPowers} from './types.js' */ +/** @import {CanonicalFn} from './types.js' */ +/** @import {CryptoInterface} from './types.js' */ +/** @import {FileURLToPathFn} from './types.js' */ +/** @import {FsInterface} from './types.js' */ /** @import {HashFn} from './types.js' */ +/** @import {IsAbsoluteFn} from './types.js' */ +/** @import {MaybeReadFn} from './types.js' */ +/** @import {MaybeReadPowers} from './types.js' */ +/** @import {PathInterface} from './types.js' */ +/** @import {PathToFileURLFn} from './types.js' */ +/** @import {ReadFn} from './types.js' */ +/** @import {ReadPowers} from './types.js' */ +/** @import {RequireResolveFn} from './types.js' */ +/** @import {ReadNowPowers} from './types.js' */ +/** @import {UrlInterface} from './types.js' */ /** @import {WritePowers} from './types.js' */ +/** @import {MaybeReadNowFn} from './types.js' */ import { createRequire } from 'module'; /** - * @param {string} location + * @type {FileURLToPathFn} */ const fakeFileURLToPath = location => { const url = new URL(location); @@ -26,32 +40,45 @@ const fakeFileURLToPath = location => { }; /** - * @param {string} path + * @type {PathToFileURLFn} path */ const fakePathToFileURL = path => { return new URL(path, 'file://').toString(); }; +/** + * @type {IsAbsoluteFn} + */ +const fakeIsAbsolute = () => false; + /** * The implementation of `makeReadPowers` and the deprecated * `makeNodeReadPowers` handles the case when the `url` power is not provided, * but `makeReadPowers` presents a type that requires `url`. * * @param {object} args - * @param {typeof import('fs')} args.fs - * @param {typeof import('url')} [args.url] - * @param {typeof import('crypto')} [args.crypto] + * @param {FsInterface} args.fs + * @param {UrlInterface} [args.url] + * @param {CryptoInterface} [args.crypto] + * @param {PathInterface} [args.path] + * @returns {MaybeReadPowers} */ -const makeReadPowersSloppy = ({ fs, url = undefined, crypto = undefined }) => { +const makeReadPowersSloppy = ({ + fs, + url = undefined, + crypto = undefined, + path = undefined, +}) => { const fileURLToPath = url === undefined ? fakeFileURLToPath : url.fileURLToPath; const pathToFileURL = url === undefined ? fakePathToFileURL : url.pathToFileURL; + const isAbsolute = path === undefined ? fakeIsAbsolute : path.isAbsolute; let readMutex = Promise.resolve(undefined); /** - * @param {string} location + * @type {ReadFn} */ const read = async location => { const promise = readMutex; @@ -61,18 +88,18 @@ const makeReadPowersSloppy = ({ fs, url = undefined, crypto = undefined }) => { }); await promise; - const path = fileURLToPath(location); + const filepath = fileURLToPath(location); try { // We await here to ensure that we release the mutex only after // completing the read. - return await fs.promises.readFile(path); + return await fs.promises.readFile(filepath); } finally { release(undefined); } }; /** - * @param {string} location + * @type {MaybeReadFn} */ const maybeRead = location => read(location).catch(error => { @@ -85,6 +112,7 @@ const makeReadPowersSloppy = ({ fs, url = undefined, crypto = undefined }) => { throw error; }); + /** @type {RequireResolveFn} */ const requireResolve = (from, specifier, options) => createRequire(from).resolve(specifier, options); @@ -101,7 +129,7 @@ const makeReadPowersSloppy = ({ fs, url = undefined, crypto = undefined }) => { * non-existent directory on the next step after canonicalizing the package * location. * - * @param {string} location + * @type {CanonicalFn} */ const canonical = async location => { await null; @@ -120,7 +148,7 @@ const makeReadPowersSloppy = ({ fs, url = undefined, crypto = undefined }) => { } }; - /** @type {HashFn=} */ + /** @type {HashFn | undefined} */ const computeSha512 = crypto ? bytes => { const hash = crypto.createHash('sha512'); @@ -137,6 +165,53 @@ const makeReadPowersSloppy = ({ fs, url = undefined, crypto = undefined }) => { canonical, computeSha512, requireResolve, + isAbsolute, + }; +}; + +/** + * Creates {@link ReadPowers} for dynamic module support + * + * @param {object} args + * @param {FsInterface} args.fs + * @param {UrlInterface} [args.url] + * @param {CryptoInterface} [args.crypto] + * @param {PathInterface} [args.path] + * @returns {MaybeReadPowers & ReadNowPowers} + */ +export const makeReadNowPowers = ({ + fs, + url = undefined, + crypto = undefined, + path = undefined, +}) => { + const powers = makeReadPowersSloppy({ fs, url, crypto, path }); + const fileURLToPath = powers.fileURLToPath || fakeFileURLToPath; + const isAbsolute = powers.isAbsolute || fakeIsAbsolute; + + /** + * @type {MaybeReadNowFn} + */ + const maybeReadNow = location => { + const filePath = fileURLToPath(location); + try { + return fs.readFileSync(filePath); + } catch (error) { + if ( + 'code' in error && + (error.code === 'ENOENT' || error.code === 'EISDIR') + ) { + return undefined; + } + throw error; + } + }; + + return { + ...powers, + maybeReadNow, + fileURLToPath, + isAbsolute, }; }; @@ -146,8 +221,8 @@ const makeReadPowersSloppy = ({ fs, url = undefined, crypto = undefined }) => { * but `makeWritePowers` presents a type that requires `url`. * * @param {object} args - * @param {typeof import('fs')} args.fs - * @param {typeof import('url')} [args.url] + * @param {FsInterface} args.fs + * @param {UrlInterface} [args.url] */ const makeWritePowersSloppy = ({ fs, url = undefined }) => { const fileURLToPath = @@ -171,39 +246,41 @@ const makeWritePowersSloppy = ({ fs, url = undefined }) => { /** * @param {object} args - * @param {typeof import('fs')} args.fs - * @param {typeof import('url')} args.url - * @param {typeof import('crypto')} [args.crypto] + * @param {FsInterface} args.fs + * @param {UrlInterface} args.url + * @param {CryptoInterface} [args.crypto] */ export const makeReadPowers = makeReadPowersSloppy; /** * @param {object} args - * @param {typeof import('fs')} args.fs - * @param {typeof import('url')} args.url + * @param {FsInterface} args.fs + * @param {UrlInterface} args.url */ export const makeWritePowers = makeWritePowersSloppy; /** - * @deprecated in favor of makeReadPowers. + * Deprecated in favor of {@link makeReadPowers}. * It transpires that positional arguments needed to become an arguments bag to * reasonably expand to multiple optional dependencies. * - * @param {typeof import('fs')} fs - * @param {typeof import('crypto')} [crypto] + * @param {FsInterface} fs + * @param {CryptoInterface} [crypto] * @returns {ReadPowers} + * @deprecated */ export const makeNodeReadPowers = (fs, crypto = undefined) => { return makeReadPowersSloppy({ fs, crypto }); }; /** - * @deprecated in favor of makeWritePowers. + * Deprecated in favor of {@link makeWritePowers}. * It transpires that positional arguments needed to become an arguments bag to * reasonably expand to multiple optional dependencies. * - * @param {typeof import('fs')} fs + * @param {FsInterface} fs * @returns {WritePowers} + * @deprecated */ export const makeNodeWritePowers = fs => { return makeWritePowersSloppy({ fs }); diff --git a/packages/compartment-mapper/src/parse-archive-cjs.js b/packages/compartment-mapper/src/parse-archive-cjs.js index b0881cb02c..87c906b45a 100644 --- a/packages/compartment-mapper/src/parse-archive-cjs.js +++ b/packages/compartment-mapper/src/parse-archive-cjs.js @@ -16,7 +16,7 @@ const noopExecute = () => {}; freeze(noopExecute); /** @type {import('./types.js').ParseFn} */ -export const parseArchiveCjs = async ( +export const parseArchiveCjs = ( bytes, specifier, location, @@ -66,4 +66,5 @@ export const parseArchiveCjs = async ( export default { parse: parseArchiveCjs, heuristicImports: true, + synchronous: true, }; diff --git a/packages/compartment-mapper/src/parse-archive-mjs.js b/packages/compartment-mapper/src/parse-archive-mjs.js index 1894021386..0442c4d69f 100644 --- a/packages/compartment-mapper/src/parse-archive-mjs.js +++ b/packages/compartment-mapper/src/parse-archive-mjs.js @@ -9,7 +9,7 @@ const textEncoder = new TextEncoder(); const textDecoder = new TextDecoder(); /** @type {import('./types.js').ParseFn} */ -export const parseArchiveMjs = async ( +export const parseArchiveMjs = ( bytes, specifier, sourceUrl, @@ -35,4 +35,5 @@ export const parseArchiveMjs = async ( export default { parse: parseArchiveMjs, heuristicImports: false, + synchronous: true, }; diff --git a/packages/compartment-mapper/src/parse-bytes.js b/packages/compartment-mapper/src/parse-bytes.js index d2de12e733..66a9f53def 100644 --- a/packages/compartment-mapper/src/parse-bytes.js +++ b/packages/compartment-mapper/src/parse-bytes.js @@ -13,12 +13,7 @@ const freeze = Object.freeze; /** @type {import('./types.js').ParseFn} */ -export const parseBytes = async ( - bytes, - _specifier, - _location, - _packageLocation, -) => { +export const parseBytes = (bytes, _specifier, _location, _packageLocation) => { // Snapshot ArrayBuffer const buffer = new ArrayBuffer(bytes.length); const bytesView = new Uint8Array(buffer); @@ -49,4 +44,5 @@ export const parseBytes = async ( export default { parse: parseBytes, heuristicImports: false, + synchronous: true, }; diff --git a/packages/compartment-mapper/src/parse-cjs-shared-export-wrapper.js b/packages/compartment-mapper/src/parse-cjs-shared-export-wrapper.js index 1029ddbd29..ee9cab48c5 100644 --- a/packages/compartment-mapper/src/parse-cjs-shared-export-wrapper.js +++ b/packages/compartment-mapper/src/parse-cjs-shared-export-wrapper.js @@ -3,6 +3,8 @@ * module source. */ +import { findInvalidReadNowPowersProps, isReadNowPowers } from './powers.js'; + // @ts-check /** @import {ReadFn} from './types.js' */ @@ -24,7 +26,9 @@ const noTrailingSlash = path => { }; /** - * Generates values for __filename and __dirname from location + * Generates values for __filename and __dirname from location _if and only if_ + * `readPowers` is of type {@link ReadPowers} containing a + * {@link ReadPowers.fileURLToPath} method. * * @param {ReadPowers | ReadFn | undefined} readPowers * @param {string} location @@ -141,13 +145,26 @@ export const wrap = ({ }, }); - const require = (/** @type {string} */ importSpecifier) => { + /** @param {string} importSpecifier */ + const require = importSpecifier => { + // if this fails, tell user + + /** @type {import('ses').ModuleExportsNamespace} */ + let namespace; + if (!has(resolvedImports, importSpecifier)) { - throw new Error( - `Cannot find module "${importSpecifier}" in "${location}"`, - ); + if (isReadNowPowers(readPowers)) { + namespace = compartment.importNow(importSpecifier); + } else { + const invalidProps = findInvalidReadNowPowersProps(readPowers).sort(); + throw new Error( + `Synchronous readPowers required for dynamic import of ${assert.quote(importSpecifier)}; missing or invalid prop(s): ${invalidProps.join(', ')}`, + ); + } + } else { + namespace = compartment.importNow(resolvedImports[importSpecifier]); } - const namespace = compartment.importNow(resolvedImports[importSpecifier]); + // If you read this file carefully, you'll see it's not possible for a cjs module to not have the default anymore. // It's currently possible to require modules that were not created by this file though. if (has(namespace, 'default')) { diff --git a/packages/compartment-mapper/src/parse-cjs.js b/packages/compartment-mapper/src/parse-cjs.js index dccd7d0cea..b440479634 100644 --- a/packages/compartment-mapper/src/parse-cjs.js +++ b/packages/compartment-mapper/src/parse-cjs.js @@ -12,7 +12,7 @@ const textDecoder = new TextDecoder(); const { freeze } = Object; /** @type {import('./types.js').ParseFn} */ -export const parseCjs = async ( +export const parseCjs = ( bytes, _specifier, location, @@ -31,7 +31,7 @@ export const parseCjs = async ( exports.push('default'); } - const { filename, dirname } = await getModulePaths(readPowers, location); + const { filename, dirname } = getModulePaths(readPowers, location); /** * @param {object} moduleEnvironmentRecord @@ -67,4 +67,5 @@ export const parseCjs = async ( export default { parse: parseCjs, heuristicImports: true, + synchronous: true, }; diff --git a/packages/compartment-mapper/src/parse-json.js b/packages/compartment-mapper/src/parse-json.js index 1a315a3fcd..a981076f96 100644 --- a/packages/compartment-mapper/src/parse-json.js +++ b/packages/compartment-mapper/src/parse-json.js @@ -2,25 +2,24 @@ // @ts-check +/** @import {Harden} from 'ses' */ +/** @import {ParseFn} from './types.js' */ +/** @import {ParserImplementation} from './types.js' */ + import { parseLocatedJson } from './json.js'; /** * TypeScript cannot be relied upon to deal with the nuances of Readonly, so we * borrow the pass-through type definition of harden here. * - * @type {import('ses').Harden} + * @type {Harden} */ const freeze = Object.freeze; const textDecoder = new TextDecoder(); -/** @type {import('./types.js').ParseFn} */ -export const parseJson = async ( - bytes, - _specifier, - location, - _packageLocation, -) => { +/** @type {ParseFn} */ +export const parseJson = (bytes, _specifier, location, _packageLocation) => { const source = textDecoder.decode(bytes); const imports = freeze([]); @@ -41,8 +40,9 @@ export const parseJson = async ( }; }; -/** @type {import('./types.js').ParserImplementation} */ +/** @type {ParserImplementation} */ export default { parse: parseJson, heuristicImports: false, + synchronous: true, }; diff --git a/packages/compartment-mapper/src/parse-mjs.js b/packages/compartment-mapper/src/parse-mjs.js index 684a494d4e..21823741af 100644 --- a/packages/compartment-mapper/src/parse-mjs.js +++ b/packages/compartment-mapper/src/parse-mjs.js @@ -7,7 +7,7 @@ import { ModuleSource } from '@endo/module-source'; const textDecoder = new TextDecoder(); /** @type {import('./types.js').ParseFn} */ -export const parseMjs = async ( +export const parseMjs = ( bytes, _specifier, sourceUrl, @@ -33,4 +33,5 @@ export const parseMjs = async ( export default { parse: parseMjs, heuristicImports: false, + synchronous: true, }; diff --git a/packages/compartment-mapper/src/parse-pre-cjs.js b/packages/compartment-mapper/src/parse-pre-cjs.js index d6e02f3068..b94d47df4b 100644 --- a/packages/compartment-mapper/src/parse-pre-cjs.js +++ b/packages/compartment-mapper/src/parse-pre-cjs.js @@ -12,7 +12,7 @@ import { wrap, getModulePaths } from './parse-cjs-shared-export-wrapper.js'; const textDecoder = new TextDecoder(); /** @type {import('./types.js').ParseFn} */ -export const parsePreCjs = async ( +export const parsePreCjs = ( bytes, _specifier, location, @@ -25,7 +25,7 @@ export const parsePreCjs = async ( location, ); - const { filename, dirname } = await getModulePaths(readPowers, location); + const { filename, dirname } = getModulePaths(readPowers, location); /** * @param {object} moduleEnvironmentRecord @@ -64,4 +64,5 @@ export const parsePreCjs = async ( export default { parse: parsePreCjs, heuristicImports: true, + synchronous: true, }; diff --git a/packages/compartment-mapper/src/parse-pre-mjs.js b/packages/compartment-mapper/src/parse-pre-mjs.js index 331927b83c..1acf7e44ce 100644 --- a/packages/compartment-mapper/src/parse-pre-mjs.js +++ b/packages/compartment-mapper/src/parse-pre-mjs.js @@ -11,7 +11,7 @@ import { parseLocatedJson } from './json.js'; const textDecoder = new TextDecoder(); /** @type {import('./types.js').ParseFn} */ -export const parsePreMjs = async ( +export const parsePreMjs = ( bytes, _specifier, location, @@ -38,4 +38,5 @@ export const parsePreMjs = async ( export default { parse: parsePreMjs, heuristicImports: false, + synchronous: true, }; diff --git a/packages/compartment-mapper/src/parse-text.js b/packages/compartment-mapper/src/parse-text.js index 301b7f6042..2f4008bb6c 100644 --- a/packages/compartment-mapper/src/parse-text.js +++ b/packages/compartment-mapper/src/parse-text.js @@ -16,12 +16,7 @@ const freeze = Object.freeze; const textDecoder = new TextDecoder(); /** @type {import('./types.js').ParseFn} */ -export const parseText = async ( - bytes, - _specifier, - _location, - _packageLocation, -) => { +export const parseText = (bytes, _specifier, _location, _packageLocation) => { const text = textDecoder.decode(bytes); /** @type {Array} */ @@ -49,4 +44,5 @@ export const parseText = async ( export default { parse: parseText, heuristicImports: false, + synchronous: true, }; diff --git a/packages/compartment-mapper/src/policy.js b/packages/compartment-mapper/src/policy.js index 7306ae7496..978c1f5d38 100644 --- a/packages/compartment-mapper/src/policy.js +++ b/packages/compartment-mapper/src/policy.js @@ -3,6 +3,16 @@ // @ts-check +/** @import {Policy} from './types.js' */ +/** @import {PackagePolicy} from './types.js' */ +/** @import {AttenuationDefinition} from './types.js' */ +/** @import {PackageNamingKit} from './types.js' */ +/** @import {DeferredAttenuatorsProvider} from './types.js' */ +/** @import {CompartmentDescriptor} from './types.js' */ +/** @import {Attenuator} from './types.js' */ +/** @import {SomePolicy} from './types.js' */ +/** @import {SomePackagePolicy} from './types.js' */ + import { getAttenuatorFromDefinition, isAllowingEverything, @@ -21,26 +31,26 @@ const q = JSON.stringify; export const ATTENUATORS_COMPARTMENT = ''; /** - * Copies properties (optionally limited to a specific list) from one object - * to another. By default, copies all enumerable, own, string- and - * symbol-named properties. - * - * @param {object} from - * @param {object} to - * @param {Array} [list] - * @returns {object} + * Copies properties (optionally limited to a specific list) from one object to another. + * @template {Record} T + * @template {Record} U + * @template {Array>} [K=Array] + * @param {T} from + * @param {U} to + * @param {K} [list] + * @returns {Omit & Pick} */ const selectiveCopy = (from, to, list) => { + /** @type {Array>} */ + let props; if (!list) { const descs = getOwnPropertyDescriptors(from); - list = ownKeys(from).filter( - key => - // @ts-expect-error TypeScript still confused about a symbol as index - descs[key].enumerable, - ); + props = ownKeys(from).filter(key => descs[key].enumerable); + } else { + props = list; } - for (let index = 0; index < list.length; index += 1) { - const key = list[index]; + for (let index = 0; index < props.length; index += 1) { + const key = props[index]; // If an endowment is missing, global value is undefined. // This is an expected behavior if globals are used for platform feature detection to[key] = from[key]; @@ -53,7 +63,7 @@ const selectiveCopy = (from, to, list) => { * * Note: this function is recursive * @param {string[]} attenuators - List of attenuator names; may be mutated - * @param {import('./types.js').AttenuationDefinition|import('./types.js').Policy} policyFragment + * @param {AttenuationDefinition|Policy} policyFragment */ const collectAttenuators = (attenuators, policyFragment) => { if ('attenuate' in policyFragment) { @@ -72,7 +82,7 @@ const attenuatorsCache = new WeakMap(); * Goes through policy and lists all attenuator specifiers used. * Memoization keyed on policy object reference * - * @param {import('./types.js').Policy} [policy] + * @param {Policy} [policy] * @returns {Array} attenuators */ export const detectAttenuators = policy => { @@ -93,7 +103,7 @@ export const detectAttenuators = policy => { /** * Generates a string identifying a package for policy lookup purposes. * - * @param {import('./types.js').PackageNamingKit} namingKit + * @param {PackageNamingKit} namingKit * @returns {string} */ const generateCanonicalName = ({ isEntry = false, name, path }) => { @@ -110,8 +120,8 @@ const generateCanonicalName = ({ isEntry = false, name, path }) => { * Verifies if a module identified by `namingKit` can be a dependency of a package per `packagePolicy`. * `packagePolicy` is required, when policy is not set, skipping needs to be handled by the caller. * - * @param {import('./types.js').PackageNamingKit} namingKit - * @param {import('./types.js').PackagePolicy} packagePolicy + * @param {PackageNamingKit} namingKit + * @param {PackagePolicy} packagePolicy * @returns {boolean} */ export const dependencyAllowedByPolicy = (namingKit, packagePolicy) => { @@ -127,25 +137,25 @@ export const dependencyAllowedByPolicy = (namingKit, packagePolicy) => { * Returns the policy applicable to the canonicalName of the package * * @overload - * @param {import('./types.js').PackageNamingKit} namingKit - a key in the policy resources spec is derived from these - * @param {import('./types.js').Policy} policy - user supplied policy - * @returns {import('./types.js').PackagePolicy} packagePolicy if policy was specified + * @param {PackageNamingKit} namingKit - a key in the policy resources spec is derived from these + * @param {SomePolicy} policy - user supplied policy + * @returns {SomePackagePolicy} packagePolicy if policy was specified */ /** * Returns `undefined` * * @overload - * @param {import('./types.js').PackageNamingKit} namingKit - a key in the policy resources spec is derived from these - * @param {import('./types.js').Policy} [policy] - user supplied policy - * @returns {import('./types.js').PackagePolicy|undefined} packagePolicy if policy was specified + * @param {PackageNamingKit} namingKit - a key in the policy resources spec is derived from these + * @param {SomePolicy} [policy] - user supplied policy + * @returns {SomePackagePolicy|undefined} packagePolicy if policy was specified */ /** * Returns the policy applicable to the canonicalName of the package * - * @param {import('./types.js').PackageNamingKit} namingKit - a key in the policy resources spec is derived from these - * @param {import('./types.js').Policy} [policy] - user supplied policy + * @param {PackageNamingKit} namingKit - a key in the policy resources spec is derived from these + * @param {SomePolicy} [policy] - user supplied policy */ export const getPolicyForPackage = (namingKit, policy) => { if (!policy) { @@ -171,7 +181,7 @@ export const getPolicyForPackage = (namingKit, policy) => { /** * Get list of globals from package policy - * @param {import('./types.js').PackagePolicy} [packagePolicy] + * @param {PackagePolicy} [packagePolicy] * @returns {Array} */ const getGlobalsList = packagePolicy => { @@ -188,8 +198,8 @@ const MODULE_ATTENUATOR = 'attenuateModule'; /** * Imports attenuator per its definition and provider - * @param {import('./types.js').AttenuationDefinition} attenuationDefinition - * @param {import('./types.js').DeferredAttenuatorsProvider} attenuatorsProvider + * @param {AttenuationDefinition} attenuationDefinition + * @param {DeferredAttenuatorsProvider} attenuatorsProvider * @param {string} attenuatorExportName * @returns {Promise} */ @@ -218,8 +228,8 @@ const importAttenuatorForDefinition = async ( /** * Makes an async provider for attenuators * @param {Record} compartments - * @param {Record} compartmentDescriptors - * @returns {import('./types.js').DeferredAttenuatorsProvider} + * @param {Record} compartmentDescriptors + * @returns {DeferredAttenuatorsProvider} */ export const makeDeferredAttenuatorsProvider = ( compartments, @@ -243,7 +253,7 @@ export const makeDeferredAttenuatorsProvider = ( /** * * @param {string} attenuatorSpecifier - * @returns {Promise} + * @returns {Promise} */ importAttenuator = async attenuatorSpecifier => { if (!attenuatorSpecifier) { @@ -267,8 +277,8 @@ export const makeDeferredAttenuatorsProvider = ( * Attenuates the `globalThis` object * * @param {object} options - * @param {import('./types.js').DeferredAttenuatorsProvider} options.attenuators - * @param {import('./types.js').AttenuationDefinition} options.attenuationDefinition + * @param {DeferredAttenuatorsProvider} options.attenuators + * @param {AttenuationDefinition} options.attenuationDefinition * @param {object} options.globalThis * @param {object} options.globals */ @@ -307,8 +317,8 @@ async function attenuateGlobalThis({ * * @param {object} globalThis * @param {object} globals - * @param {import('./types.js').PackagePolicy} packagePolicy - * @param {import('./types.js').DeferredAttenuatorsProvider} attenuators + * @param {PackagePolicy} packagePolicy + * @param {DeferredAttenuatorsProvider} attenuators * @param {Array} pendingJobs * @param {string} name * @returns {void} @@ -383,7 +393,7 @@ const diagnoseModulePolicy = errorHint => { * Throws if importing of the specifier is not allowed by the policy * * @param {string} specifier - * @param {import('./types.js').CompartmentDescriptor} compartmentDescriptor + * @param {CompartmentDescriptor} compartmentDescriptor * @param {EnforceModulePolicyOptions} [options] */ export const enforceModulePolicy = ( @@ -421,8 +431,8 @@ export const enforceModulePolicy = ( /** * Attenuates a module * @param {object} options - * @param {import('./types.js').DeferredAttenuatorsProvider} options.attenuators - * @param {import('./types.js').AttenuationDefinition} options.attenuationDefinition + * @param {DeferredAttenuatorsProvider} options.attenuators + * @param {AttenuationDefinition} options.attenuationDefinition * @param {import('ses').ThirdPartyStaticModuleInterface} options.originalModuleRecord * @returns {Promise} */ @@ -460,8 +470,8 @@ async function attenuateModule({ * * @param {string} specifier - exit module name * @param {import('ses').ThirdPartyStaticModuleInterface} originalModuleRecord - reference to the exit module - * @param {import('./types.js').PackagePolicy} policy - local compartment policy - * @param {import('./types.js').DeferredAttenuatorsProvider} attenuators - a key-value where attenuations can be found + * @param {PackagePolicy} policy - local compartment policy + * @param {DeferredAttenuatorsProvider} attenuators - a key-value where attenuations can be found * @returns {Promise} - the attenuated module */ export const attenuateModuleHook = async ( diff --git a/packages/compartment-mapper/src/powers.js b/packages/compartment-mapper/src/powers.js index be37b6c235..15ab5b299d 100644 --- a/packages/compartment-mapper/src/powers.js +++ b/packages/compartment-mapper/src/powers.js @@ -7,25 +7,35 @@ // @ts-check -/** @type {import('./types.js').CanonicalFn} */ +/** @import {CanonicalFn} from './types.js' */ +/** @import {ReadNowPowers} from './types.js' */ +/** @import {ReadNowPowersProp} from './types.js' */ +/** @import {ReadFn} from './types.js' */ +/** @import {ReadPowers} from './types.js' */ +/** @import {MaybeReadPowers} from './types.js' */ +/** @import {MaybeReadFn} from './types.js' */ + +const { freeze } = Object; + +/** @type {CanonicalFn} */ const canonicalShim = async path => path; /** - * @param {import('./types.js').ReadFn | import('./types.js').ReadPowers | import('./types.js').MaybeReadPowers} powers - * @returns {import('./types.js').MaybeReadPowers} + * @param {ReadFn | ReadPowers | MaybeReadPowers} powers + * @returns {MaybeReadPowers} */ export const unpackReadPowers = powers => { - /** @type {import('./types.js').ReadFn | undefined} */ + /** @type {ReadFn | undefined} */ let read; - /** @type {import('./types.js').MaybeReadFn | undefined} */ + /** @type {MaybeReadFn | undefined} */ let maybeRead; - /** @type {import('./types.js').CanonicalFn | undefined} */ + /** @type {CanonicalFn | undefined} */ let canonical; if (typeof powers === 'function') { read = powers; } else { - ({ read, maybeRead, canonical } = powers); + ({ read, maybeRead, canonical } = /** @type {MaybeReadPowers} */ (powers)); } if (canonical === undefined) { @@ -35,9 +45,7 @@ export const unpackReadPowers = powers => { if (maybeRead === undefined) { /** @param {string} path */ maybeRead = path => - /** @type {import('./types.js').ReadFn} */ (read)(path).catch( - _error => undefined, - ); + /** @type {ReadFn} */ (read)(path).catch(_error => undefined); } return { @@ -47,3 +55,45 @@ export const unpackReadPowers = powers => { canonical, }; }; + +/** + * Ordered array of every property in {@link ReadNowPowers} which is _required_. + * + * @satisfies {Readonly<{[K in ReadNowPowersProp]-?: {} extends Pick ? never : K}[ReadNowPowersProp][]>} + */ +const requiredReadNowPowersProps = freeze( + /** @type {const} */ (['fileURLToPath', 'isAbsolute', 'maybeReadNow']), +); + +/** + * Returns `true` if `value` is a {@link ReadNowPowers} + * + * @param {ReadPowers|ReadFn|undefined} value + * @returns {value is ReadNowPowers} + */ +export const isReadNowPowers = value => + Boolean( + value && + typeof value === 'object' && + requiredReadNowPowersProps.every( + prop => prop in value && typeof value[prop] === 'function', + ), + ); + +/** + * Returns a list of the properties missing from (or invalid within) `value` that are required for + * `value` to be a {@link ReadNowPowers}. + * + * Used for human-friendly error messages + * + * @param {ReadPowers | ReadFn} [value] The value to check for missing properties. + * @returns {ReadNowPowersProp[]} + */ +export const findInvalidReadNowPowersProps = value => { + if (!value || typeof value === 'function') { + return [...requiredReadNowPowersProps]; + } + return requiredReadNowPowersProps.filter( + prop => !(prop in value) || typeof value[prop] !== 'function', + ); +}; diff --git a/packages/compartment-mapper/src/search.js b/packages/compartment-mapper/src/search.js index a8ec3b04d3..9a76e75b65 100644 --- a/packages/compartment-mapper/src/search.js +++ b/packages/compartment-mapper/src/search.js @@ -96,10 +96,9 @@ const maybeReadDescriptorDefault = async ( * }>} */ export const search = async (readPowers, moduleLocation) => { - const { maybeRead } = unpackReadPowers(readPowers); const { data, directory, location, packageDescriptorLocation } = await searchDescriptor(moduleLocation, loc => - maybeReadDescriptorDefault(maybeRead, loc), + maybeReadDescriptorDefault(readPowers, loc), ); if (!data) { diff --git a/packages/compartment-mapper/src/types.js b/packages/compartment-mapper/src/types.js index a22702c3c9..f022c9f032 100644 --- a/packages/compartment-mapper/src/types.js +++ b/packages/compartment-mapper/src/types.js @@ -4,6 +4,7 @@ export {}; /** @import {FinalStaticModuleType} from 'ses' */ /** @import {ImportHook} from 'ses' */ +/** @import {ImportNowHook} from 'ses' */ /** @import {StaticModuleType} from 'ses' */ /** @import {ThirdPartyStaticModuleInterface} from 'ses' */ /** @import {Transform} from 'ses' */ @@ -34,7 +35,7 @@ export {}; /** * A compartment descriptor corresponds to a single Compartment * of an assembled Application and describes how to construct - * one for a given library or application package.json. + * one for a given library or application `package.json`. * * @typedef {object} CompartmentDescriptor * @property {string} label @@ -52,11 +53,12 @@ export {}; * @property {LanguageForExtension} parsers - language for extension * @property {LanguageForModuleSpecifier} types - language for module specifier * @property {SomePackagePolicy} policy - policy specific to compartment + * @property {Set} compartments - List of compartment names this Compartment depends upon */ /** * For every module explicitly mentioned in an `exports` field of a - * package.json, there is a corresponding module descriptor. + * `package.json`, there is a corresponding module descriptor. * * @typedef {object} ModuleDescriptor * @property {string=} [compartment] @@ -72,7 +74,7 @@ export {}; * Scope descriptors link all names under a prefix to modules in another * compartment, like a wildcard. * These are employed to link any module not explicitly mentioned - * in a package.json file, when that package.json file does not have + * in a `package.json` file, when that `package.json` file does not have * an explicit `exports` map. * * @typedef {object} ScopeDescriptor @@ -120,6 +122,12 @@ export {}; * @returns {Promise} bytes */ +/** + * @callback ReadNowFn + * @param {string} location + * @returns {Uint8Array} bytes + */ + /** * A resolution of `undefined` indicates `ENOENT` or the equivalent. * @@ -128,6 +136,14 @@ export {}; * @returns {Promise} bytes */ +/** + * A resolution of `undefined` indicates `ENOENT` or the equivalent. + * + * @callback MaybeReadNowFn + * @param {string} location + * @returns {Uint8Array | undefined} bytes + */ + /** * Returns a canonical URL for a given URL, following redirects or symbolic * links if any exist along the path. @@ -161,19 +177,80 @@ export {}; * @returns {Promise} */ +/** + * @callback FileURLToPathFn + * @param {string|URL} location + * @returns {string} + */ + +/** + * @callback IsAbsoluteFn + * @param {string} location + * @returns {boolean} + */ + +/** + * Node.js' `url.pathToFileURL` only returns a {@link URL}. + * @callback PathToFileURLFn + * @param {string} location + * @returns {URL|string} + */ + +/** + * @callback RequireResolveFn + * @param {string} fromLocation + * @param {string} specifier + * @param {{paths?: string[]}} [options] + */ + /** * @typedef {object} ReadPowers * @property {ReadFn} read * @property {CanonicalFn} canonical + * @property {MaybeReadNowFn} [maybeReadNow] + * @property {HashFn} [computeSha512] + * @property {FileURLToPathFn} [fileURLToPath] + * @property {PathToFileURLFn} [pathToFileURL] + * @property {RequireResolveFn} [requireResolve] + * @property {IsAbsoluteFn} [isAbsolute] + */ + +/** + * These properties are necessary for dynamic require support + * + * @typedef {'fileURLToPath' | 'isAbsolute' | 'maybeReadNow'} ReadNowPowersProp + * @see {@link ReadNowPowers} + */ + +/** + * The extension of {@link ReadPowers} necessary for dynamic require support + * + * For a `ReadPowers` to be a `ReadNowPowers`: + * + * 1. It must be an object (not a {@link ReadFn}) + * 2. Prop `maybeReadNow` is a function + * 3. Prop `fileURLToPath` is a function + * 4. Prop `isAbsolute` is a function + * + * @typedef {Omit & Required>} ReadNowPowers + */ + +/** + * @typedef MakeImportNowHookMakerOptions + * @property {Sources} [sources] + * @property {Record} [compartmentDescriptors] * @property {HashFn} [computeSha512] - * @property {Function} [fileURLToPath] - * @property {Function} [pathToFileURL] - * @property {Function} [requireResolve] + * @property {string[]} [searchSuffixes] Suffixes to search if the unmodified + * specifier is not found. Pass `[]` to emulate Node.js' strict behavior. The + * default handles Node.js' CommonJS behavior. Unlike Node.js, the Compartment + * Mapper lifts CommonJS up, more like a bundler, and does not attempt to vary + * the behavior of resolution depending on the language of the importing module. + * @property {SourceMapHook} [sourceMapHook] + * @property {ExitModuleImportNowHook} [exitModuleImportNowHook] */ /** - * @typedef {ReadPowers | object} MaybeReadPowers - * @property {MaybeReadFn} maybeRead + * @typedef {ReadPowers & {maybeRead: MaybeReadFn}} MaybeReadPowers */ /** @@ -210,7 +287,7 @@ export {}; * @property {string} packageLocation * @property {string} packageName * @property {DeferredAttenuatorsProvider} attenuators - * @property {ParseFn} parse + * @property {ParseFn|ParseFnAsync} parse * @property {ShouldDeferError} shouldDeferError * @property {Record} compartments */ @@ -221,6 +298,20 @@ export {}; * @returns {ImportHook} */ +/** + * @typedef {object} ImportNowHookMakerParams + * @property {string} packageLocation + * @property {string} packageName + * @property {ParseFn|ParseFnAsync} parse + * @property {Record} compartments + */ + +/** + * @callback ImportNowHookMaker + * @param {ImportNowHookMakerParams} params + * @returns {ImportNowHook} + */ + /** * @typedef {object} SourceMapHookDetails * @property {string} compartment @@ -250,7 +341,17 @@ export {}; */ /** - * @callback ParseFn + * Result of a {@link ParseFn} + * + * @typedef ParseResult + * @property {Uint8Array} bytes + * @property {Language} parser + * @property {FinalStaticModuleType} record + * @property {string} [sourceMap] + */ + +/** + * @callback ParseFn_ * @param {Uint8Array} bytes * @param {string} specifier * @param {string} location @@ -261,12 +362,25 @@ export {}; * @param {string} [options.sourceMapUrl] * @param {ReadFn | ReadPowers} [options.readPowers] * @param {CompartmentDescriptor} [options.compartmentDescriptor] - * @returns {Promise<{ - * bytes: Uint8Array, - * parser: Language, - * record: FinalStaticModuleType, - * sourceMap?: string, - * }>} + * @returns {ParseResult} + */ + +/** + * @typedef {ParseFn_ & {isSyncParser?: true}} ParseFn + */ + +/** + * @callback ParseFnAsync + * @param {Uint8Array} bytes + * @param {string} specifier + * @param {string} location + * @param {string} packageLocation + * @param {object} [options] + * @param {string} [options.sourceMap] + * @param {SourceMapHook} [options.sourceMapHook] + * @param {string} [options.sourceMapUrl] + * @param {ReadFn | ReadPowers} [options.readPowers] + * @returns {Promise} */ /** @@ -275,6 +389,7 @@ export {}; * * @typedef {object} ParserImplementation * @property {boolean} heuristicImports + * @property {boolean} [synchronous] * @property {ParseFn} parse */ @@ -291,6 +406,13 @@ export {}; * @returns {Promise} module namespace */ +/** + * @callback ExitModuleImportNowHook + * @param {string} specifier + * @param {string} referrer + * @returns {ThirdPartyStaticModuleInterface|undefined} module namespace + */ + /** * @see {@link LoadArchiveOptions} * @typedef {object} ExtraLoadArchiveOptions @@ -345,7 +467,7 @@ export {}; /** * Options for `loadLocation()` * - * @typedef {ArchiveOptions} LoadLocationOptions + * @typedef {ArchiveOptions|SyncArchiveOptions} LoadLocationOptions */ /** @@ -353,12 +475,22 @@ export {}; * @typedef {object} ExtraLinkOptions * @property {ResolveHook} [resolve] * @property {ImportHookMaker} makeImportHook + * @property {ImportNowHookMaker} [makeImportNowHook] * @property {ParserForLanguage} [parserForLanguage] * @property {LanguageForExtension} [languageForExtension] * @property {ModuleTransforms} [moduleTransforms] + * @property {SyncModuleTransforms} [syncModuleTransforms] * @property {boolean} [archiveOnly] */ +/** + * @typedef LinkResult + * @property {Compartment} compartment, + * @property {Record} compartments + * @property {Compartment} attenuatorsCompartment + * @property {Promise} pendingJobsPromise + */ + /** * Options for `link()` * @@ -369,17 +501,32 @@ export {}; * @typedef {Record} ModuleTransforms */ +/** + * @typedef {Record} SyncModuleTransforms + */ + /** * @callback ModuleTransform * @param {Uint8Array} bytes * @param {string} specifier * @param {string} location * @param {string} packageLocation - * @param {object} [options] - * @param {string} [options.sourceMap] + * @param {object} [params] + * @param {string} [params.sourceMap] * @returns {Promise<{bytes: Uint8Array, parser: Language, sourceMap?: string}>} */ +/** + * @callback SyncModuleTransform + * @param {Uint8Array} bytes + * @param {string} specifier + * @param {string} location + * @param {string} packageLocation + * @param {object} [params] + * @param {string} [params.sourceMap] + * @returns {{bytes: Uint8Array, parser: Language, sourceMap?: string}} + */ + // ///////////////////////////////////////////////////////////////////////////// // Communicating source files from an archive snapshot, from archive.js to @@ -421,6 +568,7 @@ export {}; /** * @typedef {object} ArchiveOptions * @property {ModuleTransforms} [moduleTransforms] + * @property {SyncModuleTransforms} [syncModuleTransforms] * @property {Record} [modules] * @property {boolean} [dev] * @property {SomePolicy} [policy] @@ -435,6 +583,23 @@ export {}; * @property {LanguageForExtension} [languageForExtension] */ +/** + * @typedef SyncArchiveOptions + * @property {SyncModuleTransforms} [syncModuleTransforms] + * @property {Record} [modules] + * @property {boolean} [dev] + * @property {object} [policy] + * @property {Set} [tags] + * @property {CaptureSourceLocationHook} [captureSourceLocation] + * @property {ExitModuleImportHook} [importHook] + * @property {Array} [searchSuffixes] + * @property {Record} [commonDependencies] + * @property {SourceMapHook} [sourceMapHook] + * @property {ExitModuleImportNowHook} [importNowHook] + * @property {Record} [parserForLanguage] + * @property {LanguageForExtension} [languageForExtension] + */ + // ///////////////////////////////////////////////////////////////////////////// // Policy enforcement infrastructure @@ -490,7 +655,7 @@ export {}; /** * @template {[any, ...any[]]} [Params=[any, ...any[]]] - * @template [T=unknown] + * @template [T=SomeObject] * @template [U=T] * @callback ModuleAttenuatorFn * @param {Params} params @@ -509,7 +674,7 @@ export {}; */ /** - * A type representing a property policy, which is a record of string keys and boolean values. + * A type representing a property policy, which is a record of string keys and boolean values * @typedef {Record} PropertyPolicy */ @@ -537,6 +702,7 @@ export {}; * @property {PolicyItem|AttenuationDefinition} [globals] - The policy item or full attenuation definition for globals. * @property {PolicyItem|NestedAttenuationDefinition} [builtins] - The policy item or nested attenuation definition for builtins. * @property {boolean} [noGlobalFreeze] - Whether to disable global freeze. + * @property {boolean} [dynamic] - Whether to allow dynamic imports * @property {ExtraOptions} [options] - Any additional user-defined options can be added to the policy here */ @@ -558,10 +724,82 @@ export {}; * @typedef {Record} SomeObject */ +/** + * Function in {@link CryptoInterface} + * + * @callback CreateHashFn + * @param {'sha512'} algorithm + * @returns {Hash} + */ + +/** + * Object returned by function in {@link CryptoInterface} + * + * @typedef Hash + * @property {(data: Uint8Array|string) => Hash} update + * @property {() => Buffer} digest + * @see {@link https://nodejs.org/api/crypto.html#class-hash} + */ + +/** + * Function in {@link FsPromisesInterface} + * + * @callback RealpathFn + * @param {string} filepath + * @returns {Promise} + */ + +/** + * Object within {@link FsPromisesInterface} + * + * @typedef FsPromisesInterface + * @property {RealpathFn} realpath + * @property {WriteFn} writeFile + * @property {ReadFn} readFile + * @see {@link https://nodejs.org/api/fs.html#promises-api} + */ + +/** + * For creating {@link ReadPowers} + * + * @typedef FsInterface + * @property {FsPromisesInterface} promises + * @property {ReadNowFn} readFileSync + * @see {@link https://nodejs.org/api/fs.html} + */ + +/** + * For creating {@link ReadPowers} + * + * @typedef UrlInterface + * @property {FileURLToPathFn} fileURLToPath + * @property {PathToFileURLFn} pathToFileURL + * @see {@link https://nodejs.org/api/url.html} + */ + +/** + * For creating {@link ReadPowers} + * @typedef CryptoInterface + * @property {CreateHashFn} createHash + * @see {@link https://nodejs.org/api/crypto.html} + */ + +/** + * @typedef PathInterface + * @property {IsAbsoluteFn} isAbsolute + * @see {@link https://nodejs.org/api/path.html} + */ + +/** + * Options for `compartmentMapForNodeModules` + * + * @typedef {Pick} CompartmentMapForNodeModulesOptions + */ + /** * Any {@link PackagePolicy} * - * @typedef {PackagePolicy} SomePackagePolicy + * @typedef {PackagePolicy} SomePackagePolicy */ /** @@ -620,11 +858,19 @@ export {}; * @typedef {ExecuteOptions & ArchiveOptions} ImportLocationOptions */ +/** + * Options for `importLocation()` necessary (but not sufficient--see + * {@link ReadNowPowers}) for dynamic require support + * + * @typedef {ExecuteOptions & SyncArchiveOptions} SyncImportLocationOptions + */ + /** * Options for `captureFromMap()` * * @typedef CaptureOptions * @property {ModuleTransforms} [moduleTransforms] + * @property {SyncModuleTransforms} [syncModuleTransforms] * @property {Record} [modules] * @property {boolean} [dev] * @property {SomePolicy} [policy] @@ -636,6 +882,7 @@ export {}; * @property {SourceMapHook} [sourceMapHook] * @property {Record} [parserForLanguage] * @property {LanguageForExtension} [languageForExtension] + * @property {ExitModuleImportNowHook} [importNowHook] */ /** @@ -646,3 +893,103 @@ export {}; * @property {Sources} captureSources * @property {Record} compartmentRenames */ + +/** + * Options object for `chooseModuleDescriptor`. + * + * @typedef ChooseModuleDescriptorOptions + * @property {string[]} candidates List of `moduleSpecifier` with search + * suffixes appended + * @property {CompartmentDescriptor} compartmentDescriptor Compartment + * descriptor + * @property {Record} compartmentDescriptors All + * compartment descriptors + * @property {Record} compartments All compartments + * @property {HashFn} [computeSha512] Function to compute SHA-512 hash + * @property {Record} moduleDescriptors All module + * descriptors + * @property {string} moduleSpecifier Module specifier + * @property {string} packageLocation Package location + * @property {CompartmentSources} packageSources Sources + * @property {ReadPowers|ReadFn} readPowers Powers + * @property {SourceMapHook} [sourceMapHook] Source map hook + * @property {(compartmentName: string) => Set} strictlyRequiredForCompartment Function + * returning a set of module names (scoped to the compartment) whose parser is not using + * heuristics to determine imports. + */ + +/** + * Operators for `chooseModuleDescriptor` representing synchronous operation. + * + * @typedef SyncChooseModuleDescriptorOperators + * @property {MaybeReadNowFn} maybeRead A function that reads a file, returning + * its binary contents _or_ `undefined` if the file is not found + * @property {ParseFn} parse A function which parses the (defined) binary + * contents from `maybeRead` into a `ParseResult` + * @property {never} [shouldDeferError] Should be omitted. + */ + +/** + * Operators for `chooseModuleDescriptor` representing asynchronous operation. + * + * @typedef AsyncChooseModuleDescriptorOperators + * @property {MaybeReadFn} maybeRead A function that reads a file, resolving w/ + * its binary contents _or_ `undefined` if the file is not found + * @property {ParseFnAsync|ParseFn} parse A function which parses the (defined) + * binary contents from `maybeRead` into a `ParseResult` + * @property {(language: Language) => boolean} shouldDeferError A function that + * returns `true` if the language returned by `parse` should defer errors. + */ + +/** + * Either synchronous or asynchronous operators for `chooseModuleDescriptor`. + * + * @typedef {AsyncChooseModuleDescriptorOperators | SyncChooseModuleDescriptorOperators} ChooseModuleDescriptorOperators + */ + +/** + * The agglomeration of things that the `chooseModuleDescriptor` generator can + * yield. + * + * The generator does not necessarily yield _all_ of these; it depends on + * whether the operators are {@link AsyncChooseModuleDescriptorOperators} or + * {@link SyncChooseModuleDescriptorOperators}. + * + * @typedef {ReturnType | + * ReturnType} ChooseModuleDescriptorYieldables + */ + +/** + * Parameters for `findRedirect()`. + * + * @typedef FindRedirectParams + * @property {CompartmentDescriptor} compartmentDescriptor + * @property {Record} compartmentDescriptors + * @property {Record} compartments + * @property {string} absoluteModuleSpecifier A module specifier which is an absolute path. NOT a file:// URL. + * @property {string} packageLocation Location of the compartment descriptor's package + */ + +/** + * Options for `makeMapParsers()` + * + * @typedef MakeMapParsersOptions + * @property {ParserForLanguage} parserForLanguage Mapping of language to + * {@link ParserImplementation} + * @property {ModuleTransforms} [moduleTransforms] Async or sync module + * transforms. If non-empty, dynamic requires are unsupported. + * @property {SyncModuleTransforms} [syncModuleTransforms] Sync module + * transforms + */ + +/** + * The value returned by `makeMapParsers()` + * + * @template {ParseFn|ParseFnAsync} [T=ParseFn|ParseFnAsync] + * @callback MapParsersFn + * @param {LanguageForExtension} languageForExtension Mapping of file extension + * to {@link Language} + * @param {LanguageForModuleSpecifier} languageForModuleSpecifier Mapping of + * module specifier to {@link Language} + * @returns {T} Parser function + */ diff --git a/packages/compartment-mapper/test/dynamic-require.test.js b/packages/compartment-mapper/test/dynamic-require.test.js new file mode 100644 index 0000000000..e2e2694e05 --- /dev/null +++ b/packages/compartment-mapper/test/dynamic-require.test.js @@ -0,0 +1,417 @@ +/* eslint-disable no-shadow */ +// @ts-check +/* eslint-disable import/no-dynamic-require */ + +/** @import {ExitModuleImportNowHook, Policy} from '../src/types.js' */ +/** @import {SyncModuleTransforms} from '../src/types.js' */ + +import 'ses'; +import test from 'ava'; +import fs from 'node:fs'; +import { Module } from 'node:module'; +import path from 'node:path'; +import url from 'node:url'; +import { importLocation } from '../src/import.js'; +import { makeReadNowPowers } from '../src/node-powers.js'; + +const readPowers = makeReadNowPowers({ fs, url, path }); +const { freeze, keys, assign } = Object; + +const importNowHook = (specifier, packageLocation) => { + const require = Module.createRequire( + readPowers.fileURLToPath(packageLocation), + ); + /** @type {object} */ + const ns = require(specifier); + return freeze( + /** @type {import('ses').ThirdPartyStaticModuleInterface} */ ({ + imports: [], + exports: keys(ns), + execute: moduleExports => { + moduleExports.default = ns; + assign(moduleExports, ns); + }, + }), + ); +}; + +test('intra-package dynamic require works without invoking the exitModuleImportNowHook', async t => { + t.plan(2); + const fixture = new URL( + 'fixtures-dynamic/node_modules/app/index.js', + import.meta.url, + ).toString(); + let importNowHookCallCount = 0; + const importNowHook = (specifier, packageLocation) => { + importNowHookCallCount += 1; + const require = Module.createRequire( + readPowers.fileURLToPath(packageLocation), + ); + /** @type {object} */ + const ns = require(specifier); + return freeze( + /** @type {import('ses').ThirdPartyStaticModuleInterface} */ ({ + imports: [], + exports: keys(ns), + execute: moduleExports => { + moduleExports.default = ns; + assign(moduleExports, ns); + }, + }), + ); + }; + + /** @type {Policy} */ + const policy = { + entry: { + packages: 'any', + }, + resources: { + dynamic: { + packages: { + 'is-ok': true, + }, + }, + }, + }; + const { namespace } = await importLocation(readPowers, fixture, { + policy, + importNowHook, + }); + + t.deepEqual( + { + default: { + isOk: 1, + }, + isOk: 1, + }, + { ...namespace }, + ); + t.is(importNowHookCallCount, 0); +}); + +// this test mimics how node-gyp-require works; you pass it a directory and it +// figures out what file to require within that directory. there is no +// reciprocal dependency on wherever that directory lives (usually it's +// somewhere in the dependent package) +test('intra-package dynamic require with inter-package absolute path works without invoking the exitModuleImportNowHook', async t => { + t.plan(2); + const fixture = new URL( + 'fixtures-dynamic/node_modules/absolute-app/index.js', + import.meta.url, + ).toString(); + let importNowHookCallCount = 0; + const importNowHook = (specifier, packageLocation) => { + importNowHookCallCount += 1; + const require = Module.createRequire( + readPowers.fileURLToPath(packageLocation), + ); + /** @type {object} */ + const ns = require(specifier); + return freeze( + /** @type {import('ses').ThirdPartyStaticModuleInterface} */ ({ + imports: [], + exports: keys(ns), + execute: moduleExports => { + moduleExports.default = ns; + assign(moduleExports, ns); + }, + }), + ); + }; + /** @type {Policy} */ + const policy = { + entry: { + packages: 'any', + }, + resources: { + sprunt: { + packages: { + 'node-tammy-build': true, + }, + }, + }, + }; + + const { namespace } = await importLocation(readPowers, fixture, { + policy, + importNowHook, + }); + + t.deepEqual( + { + default: { + isOk: 1, + }, + isOk: 1, + }, + { ...namespace }, + ); + t.is(importNowHookCallCount, 0); +}); + +test('intra-package dynamic require using known-but-restricted absolute path fails', async t => { + const fixture = new URL( + 'fixtures-dynamic/node_modules/broken-app/index.js', + import.meta.url, + ).toString(); + /** @type {ExitModuleImportNowHook} */ + const importNowHook = (specifier, packageLocation) => { + const require = Module.createRequire( + readPowers.fileURLToPath(packageLocation), + ); + /** @type {object} */ + const ns = require(specifier); + return freeze( + /** @type {import('ses').ThirdPartyStaticModuleInterface} */ ({ + imports: [], + exports: keys(ns), + execute: moduleExports => { + moduleExports.default = ns; + assign(moduleExports, ns); + }, + }), + ); + }; + /** @type {Policy} */ + const policy = { + entry: { + packages: 'any', + }, + resources: { + badsprunt: { + packages: { + 'node-tammy-build': true, + }, + }, + 'badsprunt>node-tammy-build': { + packages: { sprunt: false }, + }, + }, + }; + + await t.throwsAsync( + importLocation(readPowers, fixture, { + policy, + importNowHook, + }), + { + message: /Blocked in import hook/, + }, + ); +}); + +test('dynamic require fails without maybeReadNow in read powers', async t => { + const fixture = new URL( + 'fixtures-dynamic/node_modules/app/index.js', + import.meta.url, + ).toString(); + + const { maybeReadNow: _, ...lessPower } = readPowers; + await t.throwsAsync( + // @ts-expect-error bad type + importLocation(lessPower, fixture, { + importNowHook, + policy: { + entry: { + packages: 'any', + }, + resources: { + dynamic: { + packages: { + 'is-ok': true, + }, + }, + }, + }, + }), + { + message: + /Synchronous readPowers required for dynamic import of "is-ok"; missing or invalid prop\(s\): maybeReadNow/, + }, + ); +}); + +test('dynamic require fails without isAbsolute & fileURLToPath in read powers', async t => { + const fixture = new URL( + 'fixtures-dynamic/node_modules/app/index.js', + import.meta.url, + ).toString(); + const { isAbsolute: _, fileURLToPath: ___, ...lessPower } = readPowers; + await t.throwsAsync( + // @ts-expect-error bad types + importLocation(lessPower, fixture, { + importNowHook, + policy: { + entry: { + packages: 'any', + }, + resources: { + dynamic: { + packages: { + 'is-ok': true, + }, + }, + }, + }, + }), + { + message: + /Synchronous readPowers required for dynamic import of "is-ok"; missing or invalid prop\(s\): fileURLToPath, isAbsolute/, + }, + ); +}); + +test('inter-package and exit module dynamic require works', async t => { + t.plan(3); + + const fixture = new URL( + 'fixtures-dynamic/node_modules/hooked-app/index.js', + import.meta.url, + ).toString(); + + // number of times the `importNowHook` got called + let importNowHookCallCount = 0; + /** @type {string[]} */ + const importNowHookSpecifiers = []; + + /** @type {ExitModuleImportNowHook} */ + const importNowHook = (specifier, packageLocation) => { + importNowHookCallCount += 1; + importNowHookSpecifiers.push(specifier); + const require = Module.createRequire( + readPowers.fileURLToPath(packageLocation), + ); + /** @type {object} */ + const ns = require(specifier); + return freeze( + /** @type {import('ses').ThirdPartyStaticModuleInterface} */ ({ + imports: [], + exports: keys(ns), + execute: moduleExports => { + moduleExports.default = ns; + assign(moduleExports, ns); + }, + }), + ); + }; + + const { namespace } = await importLocation(readPowers, fixture, { + importNowHook, + policy: { + entry: { + packages: 'any', + }, + resources: { + hooked: { + packages: { + dynamic: true, + }, + builtins: { + cluster: true, + }, + }, + 'hooked>dynamic': { + packages: { + 'is-ok': true, + }, + }, + }, + }, + }); + + t.deepEqual( + { + default: { + isOk: 1, + }, + isOk: 1, + }, + { ...namespace }, + ); + + t.is(importNowHookCallCount, 1); + t.deepEqual(importNowHookSpecifiers, ['cluster']); +}); + +test('sync module transforms work with dynamic require support', async t => { + const fixture = new URL( + 'fixtures-dynamic/node_modules/app/index.js', + import.meta.url, + ).toString(); + + t.plan(2); + + let transformCount = 0; + + /** @type {SyncModuleTransforms} */ + const syncModuleTransforms = { + cjs: sourceBytes => { + transformCount += 1; + return { + bytes: sourceBytes, + parser: 'cjs', + }; + }, + }; + + /** @type {Policy} */ + const policy = { + entry: { + packages: 'any', + }, + resources: { + dynamic: { + packages: { + 'is-ok': true, + }, + }, + }, + }; + + const { namespace } = await importLocation(readPowers, fixture, { + syncModuleTransforms, + importNowHook, + policy, + }); + + t.deepEqual( + { + default: { + isOk: 1, + }, + isOk: 1, + }, + { ...namespace }, + ); + + t.true(transformCount > 0); +}); + +test('sync module transforms work without dynamic require support', async t => { + const fixture = new URL( + 'fixtures-cjs-compat/node_modules/app/index.js', + import.meta.url, + ).toString(); + + let transformCount = 0; + + /** @type {SyncModuleTransforms} */ + const syncModuleTransforms = { + cjs: sourceBytes => { + transformCount += 1; + return { + bytes: sourceBytes, + parser: 'cjs', + }; + }, + }; + + const { read } = readPowers; + await importLocation(read, fixture, { + syncModuleTransforms, + }); + + t.true(transformCount === 29); +}); diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/absolute-app/README.md b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/absolute-app/README.md new file mode 100644 index 0000000000..dd078ca01a --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/absolute-app/README.md @@ -0,0 +1 @@ +An app requiring a package which dynamically requires using an absolute path as a module specifier. \ No newline at end of file diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/absolute-app/index.js b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/absolute-app/index.js new file mode 100644 index 0000000000..3b28e3b6f3 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/absolute-app/index.js @@ -0,0 +1 @@ +exports.isOk = require('sprunt').isOk; \ No newline at end of file diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/absolute-app/package.json b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/absolute-app/package.json new file mode 100644 index 0000000000..23f3bda899 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/absolute-app/package.json @@ -0,0 +1,12 @@ +{ + "name": "absolute-app", + "version": "1.0.0", + "main": "./index.js", + "type": "commonjs", + "dependencies": { + "sprunt": "^1.0.0" + }, + "scripts": { + "preinstall": "echo DO NOT INSTALL TEST FIXTURES; exit -1" + } +} diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/app/README.md b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/app/README.md new file mode 100644 index 0000000000..4e523dedbd --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/app/README.md @@ -0,0 +1 @@ +App which requires a package which dynamically loads its own module. \ No newline at end of file diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/app/index.js b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/app/index.js new file mode 100644 index 0000000000..0e019702e6 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/app/index.js @@ -0,0 +1 @@ +exports.isOk = require('dynamic').isOk; \ No newline at end of file diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/app/package.json b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/app/package.json new file mode 100644 index 0000000000..3c3f0c73d4 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/app/package.json @@ -0,0 +1,12 @@ +{ + "name": "app", + "version": "1.0.0", + "main": "./index.js", + "type": "commonjs", + "dependencies": { + "dynamic": "^1.0.0" + }, + "scripts": { + "preinstall": "echo DO NOT INSTALL TEST FIXTURES; exit -1" + } +} diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/badsprunt/README.md b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/badsprunt/README.md new file mode 100644 index 0000000000..7c312aa751 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/badsprunt/README.md @@ -0,0 +1 @@ +This package calls the function returned by `node-tammy-build` with an absolute path to a file contained in a compartment which it `node-tammy-build` explicitly cannot access. \ No newline at end of file diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/badsprunt/index.js b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/badsprunt/index.js new file mode 100644 index 0000000000..4ea6411342 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/badsprunt/index.js @@ -0,0 +1,5 @@ +const tammy = require('node-tammy-build'); + +const sprunt = __dirname + '/../sprunt/sprunt.js'; + +module.exports = tammy(sprunt); diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/badsprunt/package.json b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/badsprunt/package.json new file mode 100644 index 0000000000..cab79816cf --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/badsprunt/package.json @@ -0,0 +1,12 @@ +{ + "name": "badsprunt", + "version": "1.0.0", + "scripts": { + "preinstall": "echo DO NOT INSTALL TEST FIXTURES; exit -1" + }, + "dependencies": { + "node-tammy-build": "^1.0.0", + "sprunt": "^1.0.0" + }, + "main": "index.js" +} diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/broken-app/README.md b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/broken-app/README.md new file mode 100644 index 0000000000..8320940da4 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/broken-app/README.md @@ -0,0 +1 @@ +This package loads package `badsprunt`, which is bad. See `badsprunt`'s `README` for the dirt \ No newline at end of file diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/broken-app/index.js b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/broken-app/index.js new file mode 100644 index 0000000000..45c1f737b7 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/broken-app/index.js @@ -0,0 +1 @@ +exports.isOk = require('badsprunt').isOk; \ No newline at end of file diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/broken-app/package.json b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/broken-app/package.json new file mode 100644 index 0000000000..795233b9d6 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/broken-app/package.json @@ -0,0 +1,12 @@ +{ + "name": "app", + "version": "1.0.0", + "main": "./index.js", + "type": "commonjs", + "dependencies": { + "badsprunt": "^1.0.0" + }, + "scripts": { + "preinstall": "echo DO NOT INSTALL TEST FIXTURES; exit -1" + } +} diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/dynamic/README.md b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/dynamic/README.md new file mode 100644 index 0000000000..cd88399922 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/dynamic/README.md @@ -0,0 +1 @@ +This package dynamically requires another package by name. diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/dynamic/index.js b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/dynamic/index.js new file mode 100644 index 0000000000..a5e016d9cd --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/dynamic/index.js @@ -0,0 +1,3 @@ +const pkg = 'is-ok'; + +exports.isOk = require(pkg).isOk; diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/dynamic/package.json b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/dynamic/package.json new file mode 100644 index 0000000000..10f27e1811 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/dynamic/package.json @@ -0,0 +1,10 @@ +{ + "name": "dynamic", + "version": "1.0.0", + "scripts": { + "preinstall": "echo DO NOT INSTALL TEST FIXTURES; exit -1" + }, + "dependencies": { + "is-ok": "^1.0.0" + } +} diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked-app/README.md b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked-app/README.md new file mode 100644 index 0000000000..b4ab3eb0d4 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked-app/README.md @@ -0,0 +1 @@ +App which requires a package which dynamically requires other packages. \ No newline at end of file diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked-app/index.js b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked-app/index.js new file mode 100644 index 0000000000..f64a83d77a --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked-app/index.js @@ -0,0 +1,5 @@ +exports.isOk = require('hooked').isOk; + +const builtin = 'cluster'; + +require(builtin); \ No newline at end of file diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked-app/package.json b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked-app/package.json new file mode 100644 index 0000000000..26550f869b --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked-app/package.json @@ -0,0 +1,12 @@ +{ + "name": "app", + "version": "1.0.0", + "main": "./index.js", + "type": "commonjs", + "dependencies": { + "hooked": "^1.0.0" + }, + "scripts": { + "preinstall": "echo DO NOT INSTALL TEST FIXTURES; exit -1" + } +} diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked/README.md b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked/README.md new file mode 100644 index 0000000000..c1f8965a0b --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked/README.md @@ -0,0 +1 @@ +This package dynamically requires another package and dynamically requires a builtin. \ No newline at end of file diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked/index.js b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked/index.js new file mode 100644 index 0000000000..0e814097e6 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked/index.js @@ -0,0 +1,3 @@ +const dynamic = require.resolve('dynamic'); + +exports.isOk = require(dynamic).isOk; diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked/package.json b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked/package.json new file mode 100644 index 0000000000..7de2fbff21 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/hooked/package.json @@ -0,0 +1,9 @@ +{ + "name": "hooked", + "scripts": { + "preinstall": "echo DO NOT INSTALL TEST FIXTURES; exit -1" + }, + "dependencies": { + "dynamic": "^1.0.0" + } +} diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/is-ok/README.md b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/is-ok/README.md new file mode 100644 index 0000000000..4c2628e835 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/is-ok/README.md @@ -0,0 +1 @@ +This package does nothing interesting. \ No newline at end of file diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/is-ok/index.js b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/is-ok/index.js new file mode 100644 index 0000000000..4198dd8caf --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/is-ok/index.js @@ -0,0 +1 @@ +exports.isOk = 1 \ No newline at end of file diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/is-ok/package.json b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/is-ok/package.json new file mode 100644 index 0000000000..e8b1b84d8f --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/is-ok/package.json @@ -0,0 +1,7 @@ +{ + "name": "is-ok", + "version": "1.0.0", + "scripts": { + "preinstall": "echo DO NOT INSTALL TEST FIXTURES; exit -1" + } +} diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/node-tammy-build/README.md b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/node-tammy-build/README.md new file mode 100644 index 0000000000..6fb4a336d8 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/node-tammy-build/README.md @@ -0,0 +1 @@ +This package exports a function which accepts a path and returns the result of `require`-ing aforementioned path. It mimics the behavior of `node-gyp-build`. \ No newline at end of file diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/node-tammy-build/index.js b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/node-tammy-build/index.js new file mode 100644 index 0000000000..bcd910f190 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/node-tammy-build/index.js @@ -0,0 +1,3 @@ +module.exports = (path) => { + return require(path); +} \ No newline at end of file diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/node-tammy-build/package.json b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/node-tammy-build/package.json new file mode 100644 index 0000000000..17af880ed5 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/node-tammy-build/package.json @@ -0,0 +1,7 @@ +{ + "name": "node-tammy-build", + "version": "1.0.0", + "scripts": { + "preinstall": "echo DO NOT INSTALL TEST FIXTURES; exit -1" + } +} diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/sprunt/README.md b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/sprunt/README.md new file mode 100644 index 0000000000..0f5d064200 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/sprunt/README.md @@ -0,0 +1 @@ +This package mimics a consumer of `node-gyp-build`, where a package is required and a path is provided to that package, which is then dynamically required \ No newline at end of file diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/sprunt/index.js b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/sprunt/index.js new file mode 100644 index 0000000000..9770e24b16 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/sprunt/index.js @@ -0,0 +1,4 @@ +const dynamo = require('node-tammy-build'); + +const sprunt = __dirname + '/sprunt.js'; +module.exports = dynamo(sprunt); diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/sprunt/package.json b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/sprunt/package.json new file mode 100644 index 0000000000..26601c14b9 --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/sprunt/package.json @@ -0,0 +1,11 @@ +{ + "name": "sprunt", + "version": "1.0.0", + "scripts": { + "preinstall": "echo DO NOT INSTALL TEST FIXTURES; exit -1" + }, + "dependencies": { + "node-tammy-build": "^1.0.0" + }, + "main": "index.js" +} diff --git a/packages/compartment-mapper/test/fixtures-dynamic/node_modules/sprunt/sprunt.js b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/sprunt/sprunt.js new file mode 100644 index 0000000000..009118fb8c --- /dev/null +++ b/packages/compartment-mapper/test/fixtures-dynamic/node_modules/sprunt/sprunt.js @@ -0,0 +1 @@ +exports.isOk = 1; \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 47f382d4b7..c3ee35612b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -348,6 +348,7 @@ __metadata: dependencies: "@endo/cjs-module-analyzer": "workspace:^" "@endo/module-source": "workspace:^" + "@endo/trampoline": "workspace:^" "@endo/zip": "workspace:^" ava: "npm:^6.1.3" babel-eslint: "npm:^10.1.0" @@ -889,7 +890,7 @@ __metadata: languageName: unknown linkType: soft -"@endo/trampoline@workspace:packages/trampoline": +"@endo/trampoline@workspace:^, @endo/trampoline@workspace:packages/trampoline": version: 0.0.0-use.local resolution: "@endo/trampoline@workspace:packages/trampoline" dependencies: