From 9c1c17a84168c5742084c34ac3395ca38bc182eb Mon Sep 17 00:00:00 2001 From: Gus Caplan Date: Tue, 9 Jun 2020 09:12:48 -0500 Subject: [PATCH] esm: loader chaining This patch adds the ability to chain loaders together. Hooks still need auditing on the best way to behave in the context of chaining, and that will be addressed in future patches. --- doc/api/esm.md | 200 ++++++++++-------- lib/internal/main/repl.js | 7 +- lib/internal/modules/cjs/loader.js | 38 +--- lib/internal/modules/esm/get_format.js | 2 +- lib/internal/modules/esm/loader.js | 170 +++++++++------ lib/internal/modules/esm/resolve.js | 3 +- lib/internal/modules/esm/transform_source.js | 7 - lib/internal/modules/esm/translators.js | 112 ++++------ lib/internal/modules/run_main.js | 13 +- lib/internal/process/esm_loader.js | 136 ++++++++---- lib/internal/process/execution.js | 16 +- lib/internal/validators.js | 6 + lib/repl.js | 3 +- node.gyp | 1 - src/node_options.cc | 2 +- src/node_options.h | 2 +- test/es-module/test-loader-chaining.mjs | 8 + .../builtin-named-exports-loader.mjs | 15 +- .../es-module-loaders/example-loader.mjs | 6 +- .../fixtures/es-module-loaders/get-source.mjs | 5 +- test/fixtures/es-module-loaders/js-loader.mjs | 4 +- .../es-module-loaders/loader-chain-a.mjs | 8 + .../es-module-loaders/loader-chain-b.mjs | 8 + .../es-module-loaders/loader-get-format.mjs | 4 +- .../loader-invalid-format.mjs | 8 +- .../es-module-loaders/loader-invalid-url.mjs | 4 +- .../es-module-loaders/loader-shared-dep.mjs | 4 +- .../loader-unknown-builtin-module.mjs | 8 +- .../loader-with-custom-condition.mjs | 6 +- .../es-module-loaders/loader-with-dep.mjs | 7 +- .../not-found-assert-loader.mjs | 20 +- .../es-module-loaders/string-sources.mjs | 12 +- .../es-module-loaders/transform-source.mjs | 7 +- .../esm_display_syntax_error_import.out | 1 - ...esm_display_syntax_error_import_module.out | 1 - test/message/esm_loader_not_found.out | 20 +- .../esm_loader_not_found_cjs_hint_bare.out | 6 +- ...esm_loader_not_found_cjs_hint_relative.out | 20 +- test/parallel/test-bootstrap-modules.js | 1 - 39 files changed, 485 insertions(+), 416 deletions(-) delete mode 100644 lib/internal/modules/esm/transform_source.js create mode 100644 test/es-module/test-loader-chaining.mjs create mode 100644 test/fixtures/es-module-loaders/loader-chain-a.mjs create mode 100644 test/fixtures/es-module-loaders/loader-chain-b.mjs diff --git a/doc/api/esm.md b/doc/api/esm.md index c9bb7473a8e689..da3e981f151060 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -1177,18 +1177,40 @@ node --experimental-top-level-await b.mjs # works To customize the default module resolution, loader hooks can optionally be -provided via a `--experimental-loader ./loader-name.mjs` argument to Node.js. - +provided via `--experimental-loader ./loader-name.mjs` arguments to Node.js. When hooks are used they only apply to ES module loading and not to any CommonJS modules loaded. +Loaders are modules which export functions and data which can be used by +Node.js to modify the behavior of loading ES Modules. Loaders may be “chained” +together by passing multiple `--experimental-loader` arguments to Node.js. A +loader module will be imported once as a singleton. + +Loaders follow an authority model, whereby each loader passed to node, from +left to right, takes precedence over the next. For the `resolve`, `getFormat`, +and `getSource` hooks, this means that the leftmost hook is called, and then +the next, and so on, until one gives an answer. For `transformSource`, this +means the rightmost hook is called first, and the result is passed to the next, +and so on, until it reaches the leftmost hook. The `getGlobalPreloadCode` hooks +run from left to right, so that the loaders to the left have more control over +the environment than loaders to the right. + ### Hooks -#### resolve hook +#### resolve(specifier, context, nextResolve) > Note: The loaders API is being redesigned. This hook may disappear or its > signature may change. Do not rely on the API described below. +* `specifier` {string} +* `context` {Object} + * `conditions` {string[]} + * `parentURL` {string} +* `nextResolve` {Function} +* Returns: {Object} If `null` is returned, Node.js will defer to the next + loader. + * `url` {string} + The `resolve` hook returns the resolved file URL for a given module specifier and parent URL. The module specifier is the string in an `import` statement or `import()` expression, and the parent URL is the URL of the module that imported @@ -1199,51 +1221,47 @@ The `conditions` property on the `context` is an array of conditions for for looking up conditional mappings elsewhere or to modify the list when calling the default resolution logic. -The current [package exports conditions][Conditional Exports] will always be in -the `context.conditions` array passed into the hook. To guarantee _default -Node.js module specifier resolution behavior_ when calling `defaultResolve`, the -`context.conditions` array passed to it _must_ include _all_ elements of the -`context.conditions` array originally passed into the `resolve` hook. +The [current set of Node.js default conditions][Conditional exports] will always +be in the `context.conditions` list passed to the hook. If the hook wants to +ensure Node.js-compatible resolution logic, all items from this default +condition list **must** be passed through to the next loader function. ```js -/** - * @param {string} specifier - * @param {{ - * parentURL: !(string | undefined), - * conditions: !(Array), - * }} context - * @param {Function} defaultResolve - * @returns {!(Promise<{ url: string }>)} - */ -export async function resolve(specifier, context, defaultResolve) { - const { parentURL = null } = context; +export async function resolve(specifier, context, nextResolve) { if (Math.random() > 0.5) { // Some condition. // For some or all specifiers, do some custom logic for resolving. // Always return an object of the form {url: }. return { - url: parentURL ? - new URL(specifier, parentURL).href : + url: context.parentURL ? + new URL(specifier, context.parentURL).href : new URL(specifier).href, }; } if (Math.random() < 0.5) { // Another condition. - // When calling `defaultResolve`, the arguments can be modified. In this - // case it's adding another value for matching conditional exports. - return defaultResolve(specifier, { + // When calling to the next loader, the arguments can be modified. In + // this case it's adding another value for matching conditional exports. + return nextResolve(specifier, { ...context, conditions: [...context.conditions, 'another-condition'], }); } - // Defer to Node.js for all other specifiers. - return defaultResolve(specifier, context, defaultResolve); + // Defer to the next loader for all other specifiers. + return null; } ``` -#### getFormat hook +#### getFormat(url, context, nextGetFormat) > Note: The loaders API is being redesigned. This hook may disappear or its > signature may change. Do not rely on the API described below. +* `url` {string} +* `context` {Object} +* `nextGetFormat` {Function} +* Returns: {Object} If `null` is returned, Node.js will defer to the next + loader. + * `format` {string} + The `getFormat` hook provides a way to define a custom method of determining how a URL should be interpreted. The `format` returned also affects what the acceptable forms of source values are for a module when parsing. This can be one @@ -1255,7 +1273,7 @@ of the following: | `'commonjs'` | Load a Node.js CommonJS module | Not applicable | | `'json'` | Load a JSON file | { [ArrayBuffer][], [string][], [TypedArray][] } | | `'module'` | Load an ES module | { [ArrayBuffer][], [string][], [TypedArray][] } | -| `'wasm'` | Load a WebAssembly module | { [ArrayBuffer][], [string][], [TypedArray][] } | +| `'wasm'` | Load a WebAssembly module | { [ArrayBuffer][], [TypedArray][] } | Note: These types all correspond to classes defined in ECMAScript. @@ -1267,13 +1285,7 @@ Note: If the source value of a text-based format (i.e., `'json'`, `'module'`) is not a string, it will be converted to a string using [`util.TextDecoder`][]. ```js -/** - * @param {string} url - * @param {Object} context (currently empty) - * @param {Function} defaultGetFormat - * @returns {Promise<{ format: string }>} - */ -export async function getFormat(url, context, defaultGetFormat) { +export async function getFormat(url, context, nextGetFormat) { if (Math.random() > 0.5) { // Some condition. // For some or all URLs, do some custom logic for determining format. // Always return an object of the form {format: }, where the @@ -1282,28 +1294,30 @@ export async function getFormat(url, context, defaultGetFormat) { format: 'module', }; } - // Defer to Node.js for all other URLs. - return defaultGetFormat(url, context, defaultGetFormat); + // Defer to the next loader for all other URLs. + return null; } ``` -#### getSource hook +#### getSource(url, context, nextGetSource) > Note: The loaders API is being redesigned. This hook may disappear or its > signature may change. Do not rely on the API described below. +* `url` {string} +* `context` {Object} + * `format` {string} +* `nextgetSource` {Object} +* Returns: {Object} If `null` is returned, Node.js will defer to the next + loader. + * `source` {string|TypedArray|ArrayBuffer|Buffer} + The `getSource` hook provides a way to define a custom method for retrieving the source code of an ES module specifier. This would allow a loader to potentially avoid reading files from disk. ```js -/** - * @param {string} url - * @param {{ format: string }} context - * @param {Function} defaultGetSource - * @returns {Promise<{ source: !(SharedArrayBuffer | string | Uint8Array) }>} - */ -export async function getSource(url, context, defaultGetSource) { +export async function getSource(url, context, nexGetSource) { const { format } = context; if (Math.random() > 0.5) { // Some condition. // For some or all URLs, do some custom logic for retrieving the source. @@ -1312,16 +1326,24 @@ export async function getSource(url, context, defaultGetSource) { source: '...', }; } - // Defer to Node.js for all other URLs. - return defaultGetSource(url, context, defaultGetSource); + // Defer to the next loader for all other URLs. + return null; } ``` -#### transformSource hook +#### transformSource(source, context) > Note: The loaders API is being redesigned. This hook may disappear or its > signature may change. Do not rely on the API described below. +* `source` {string|Buffer} +* `context` {Object} + * `format` {string} + * `originalSource` {string|TypedArray|ArrayBuffer|Buffer} + * `url` {string} +* Returns: {Object} + * `source` {string|TypedArray|ArrayBuffer|Buffer} + The `transformSource` hook provides a way to modify the source code of a loaded ES module file after the source string has been loaded but before Node.js has done anything with it. @@ -1331,17 +1353,7 @@ JavaScript, a resolve hook is also necessary in order to register any unknown-to-Node.js file extensions. See the [transpiler loader example][] below. ```js -/** - * @param {!(SharedArrayBuffer | string | Uint8Array)} source - * @param {{ - * url: string, - * format: string, - * }} context - * @param {Function} defaultTransformSource - * @returns {Promise<{ source: !(SharedArrayBuffer | string | Uint8Array) }>} - */ -export async function transformSource(source, context, defaultTransformSource) { - const { url, format } = context; +export async function transformSource(source, { url, format }) { if (Math.random() > 0.5) { // Some condition. // For some or all URLs, do some custom logic for modifying the source. // Always return an object of the form {source: }. @@ -1349,16 +1361,17 @@ export async function transformSource(source, context, defaultTransformSource) { source: '...', }; } - // Defer to Node.js for all other sources. - return defaultTransformSource(source, context, defaultTransformSource); + return { source }; } ``` -#### getGlobalPreloadCode hook +#### getGlobalPreloadCode() > Note: The loaders API is being redesigned. This hook may disappear or its > signature may change. Do not rely on the API described below. +* Returns: {string} + Sometimes it can be necessary to run some code inside of the same global scope that the application will run in. This hook allows to return a string that will be ran as sloppy-mode script on startup. @@ -1371,9 +1384,6 @@ If the code needs more advanced `require` features, it will have to construct its own `require` using `module.createRequire()`. ```js -/** - * @returns {string} Code to run before application startup - */ export function getGlobalPreloadCode() { return `\ globalThis.someInjectedProperty = 42; @@ -1405,9 +1415,7 @@ and there is no security. // https-loader.mjs import { get } from 'https'; -export function resolve(specifier, context, defaultResolve) { - const { parentURL = null } = context; - +export function resolve(specifier, context, nextResolve) { // Normally Node.js would error on specifiers starting with 'https://', so // this hook intercepts them and converts them into absolute URLs to be // passed along to the later hooks below. @@ -1415,17 +1423,17 @@ export function resolve(specifier, context, defaultResolve) { return { url: specifier }; - } else if (parentURL && parentURL.startsWith('https://')) { + } else if (context.parentURL?.startsWith('https://')) { return { url: new URL(specifier, parentURL).href }; } // Let Node.js handle all other specifiers. - return defaultResolve(specifier, context, defaultResolve); + return null; } -export function getFormat(url, context, defaultGetFormat) { +export function getFormat(url, context, nextGetFormat) { // This loader assumes all network-provided JavaScript is ES module code. if (url.startsWith('https://')) { return { @@ -1434,10 +1442,10 @@ export function getFormat(url, context, defaultGetFormat) { } // Let Node.js handle all other URLs. - return defaultGetFormat(url, context, defaultGetFormat); + return null; } -export function getSource(url, context, defaultGetSource) { +export function getSource(url, context, nextGetSource) { // For JavaScript to be loaded over the network, we need to fetch and // return it. if (url.startsWith('https://')) { @@ -1451,7 +1459,7 @@ export function getSource(url, context, defaultGetSource) { } // Let Node.js handle all other URLs. - return defaultGetSource(url, context, defaultGetSource); + return null; } ``` @@ -1492,9 +1500,7 @@ const baseURL = pathToFileURL(`${process.cwd()}/`).href; // CoffeeScript files end in .coffee, .litcoffee or .coffee.md. const extensionsRegex = /\.coffee$|\.litcoffee$|\.coffee\.md$/; -export function resolve(specifier, context, defaultResolve) { - const { parentURL = baseURL } = context; - +export function resolve(specifier, { parentURL = baseURL }) { // Node.js normally errors on unknown file extensions, so return a URL for // specifiers ending in the CoffeeScript file extensions. if (extensionsRegex.test(specifier)) { @@ -1504,10 +1510,10 @@ export function resolve(specifier, context, defaultResolve) { } // Let Node.js handle all other specifiers. - return defaultResolve(specifier, context, defaultResolve); + return null; } -export function getFormat(url, context, defaultGetFormat) { +export function getFormat(url, context, nextGetFormat) { // Now that we patched resolve to let CoffeeScript URLs through, we need to // tell Node.js what format such URLs should be interpreted as. For the // purposes of this loader, all CoffeeScript URLs are ES modules. @@ -1518,20 +1524,16 @@ export function getFormat(url, context, defaultGetFormat) { } // Let Node.js handle all other URLs. - return defaultGetFormat(url, context, defaultGetFormat); + return null; } -export function transformSource(source, context, defaultTransformSource) { - const { url, format } = context; - +export function transformSource(source, { url }) { if (extensionsRegex.test(url)) { return { - source: CoffeeScript.compile(source, { bare: true }) + source: CoffeeScript.compile(source.toString(), { bare: true }) }; } - - // Let Node.js handle all other sources. - return defaultTransformSource(source, context, defaultTransformSource); + return { source }; } ``` @@ -1560,6 +1562,18 @@ loaded from disk but before Node.js executes it; and so on for any `.coffee`, `.litcoffee` or `.coffee.md` files referenced via `import` statements of any loaded file. +You could also chain this loader together with the HTTPS loader from above: + +```coffee +# remote.coffee +import foo from 'https://example.org/resource.coffee' +console.log foo +``` + +```console +node --experimental-loader ./https-loader.mjs --experimental-loader ./coffeescript-loader.mjs remote.coffee +``` + ## Resolution algorithm ### Features @@ -1909,17 +1923,17 @@ success! [`import`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import [`module.createRequire()`]: modules.html#modules_module_createrequire_filename [`module.syncBuiltinESMExports()`]: modules.html#modules_module_syncbuiltinesmexports -[`transformSource` hook]: #esm_code_transformsource_code_hook -[ArrayBuffer]: https://www.ecma-international.org/ecma-262/6.0/#sec-arraybuffer-constructor +[`transformSource` hook]: #esm_code_transformsource_source_context_code +[ArrayBuffer]: https://tc39.es/ecma262/#sec-arraybuffer-constructor [SharedArrayBuffer]: https://tc39.es/ecma262/#sec-sharedarraybuffer-constructor -[string]: https://www.ecma-international.org/ecma-262/6.0/#sec-string-constructor -[TypedArray]: https://www.ecma-international.org/ecma-262/6.0/#sec-typedarray-objects -[Uint8Array]: https://www.ecma-international.org/ecma-262/6.0/#sec-uint8array +[string]: https://tc39.es/ecma262/#sec-string-constructor +[TypedArray]: https://tc39.es/ecma262/#sec-typedarray-objects +[Uint8Array]: https://tc39.es/ecma262/#sec-uint8array [`util.TextDecoder`]: util.html#util_class_util_textdecoder [import an ES or CommonJS module for its side effects only]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#Import_a_module_for_its_side_effects_only [special scheme]: https://url.spec.whatwg.org/#special-scheme [the full specifier path]: #esm_mandatory_file_extensions -[the official standard format]: https://tc39.github.io/ecma262/#sec-modules +[the official standard format]: https://tc39.es/ecma262/#sec-modules [the dual CommonJS/ES module packages section]: #esm_dual_commonjs_es_module_packages [transpiler loader example]: #esm_transpiler_loader [6.1.7 Array Index]: https://tc39.es/ecma262/#integer-index diff --git a/lib/internal/main/repl.js b/lib/internal/main/repl.js index a8356687ccedf5..7c38ad6abfa2c2 100644 --- a/lib/internal/main/repl.js +++ b/lib/internal/main/repl.js @@ -9,7 +9,8 @@ const { const esmLoader = require('internal/process/esm_loader'); const { - evalScript + evalScript, + uncaughtException, } = require('internal/process/execution'); const console = require('internal/console/global'); @@ -33,7 +34,7 @@ if (process.env.NODE_REPL_EXTERNAL_MODULE) { process.exit(1); } - esmLoader.loadESM(() => { + esmLoader.getLoader().then(() => { console.log(`Welcome to Node.js ${process.version}.\n` + 'Type ".help" for more information.'); @@ -61,5 +62,7 @@ if (process.env.NODE_REPL_EXTERNAL_MODULE) { getOptionValue('--inspect-brk'), getOptionValue('--print')); } + }).catch((e) => { + uncaughtException(e, true /* fromPromise */); }); } diff --git a/lib/internal/modules/cjs/loader.js b/lib/internal/modules/cjs/loader.js index de24f6c409c498..ebadaca824fbe1 100644 --- a/lib/internal/modules/cjs/loader.js +++ b/lib/internal/modules/cjs/loader.js @@ -102,6 +102,13 @@ const { const { validateString } = require('internal/validators'); const pendingDeprecation = getOptionValue('--pending-deprecation'); +const originalModuleExports = new SafeWeakMap(); +module.exports = { + wrapSafe, Module, toRealPath, readPackageScope, + originalModuleExports, + get hasLoadedAnyUserCJSModule() { return hasLoadedAnyUserCJSModule; } +}; + const { CHAR_FORWARD_SLASH, CHAR_BACKWARD_SLASH, @@ -113,8 +120,6 @@ const { } = require('internal/util/types'); const asyncESM = require('internal/process/esm_loader'); -const ModuleJob = require('internal/modules/esm/module_job'); -const { ModuleWrap, kInstantiated } = internalBinding('module_wrap'); const { encodedSepRegEx, packageInternalResolve @@ -1119,30 +1124,7 @@ Module.prototype.load = function(filename) { Module._extensions[extension](this, filename); this.loaded = true; - const ESMLoader = asyncESM.ESMLoader; - const url = `${pathToFileURL(filename)}`; - const module = ESMLoader.moduleMap.get(url); - // Create module entry at load time to snapshot exports correctly - const exports = this.exports; - // Called from cjs translator - if (module !== undefined && module.module !== undefined) { - if (module.module.getStatus() >= kInstantiated) - module.module.setExport('default', exports); - } else { - // Preemptively cache - // We use a function to defer promise creation for async hooks. - ESMLoader.moduleMap.set( - url, - // Module job creation will start promises. - // We make it a function to lazily trigger those promises - // for async hooks compatibility. - () => new ModuleJob(ESMLoader, url, () => - new ModuleWrap(url, undefined, ['default'], function() { - this.setExport('default', exports); - }) - , false /* isMain */, false /* inspectBrk */) - ); - } + originalModuleExports.set(this, this.exports); }; @@ -1176,7 +1158,7 @@ function wrapSafe(filename, content, cjsModuleInstance) { lineOffset: 0, displayErrors: true, importModuleDynamically: async (specifier) => { - const loader = asyncESM.ESMLoader; + const loader = await asyncESM.getLoader(); return loader.import(specifier, normalizeReferrerURL(filename)); }, }); @@ -1209,7 +1191,7 @@ function wrapSafe(filename, content, cjsModuleInstance) { const { callbackMap } = internalBinding('module_wrap'); callbackMap.set(compiled.cacheKey, { importModuleDynamically: async (specifier) => { - const loader = asyncESM.ESMLoader; + const loader = await asyncESM.getLoader(); return loader.import(specifier, normalizeReferrerURL(filename)); } }); diff --git a/lib/internal/modules/esm/get_format.js b/lib/internal/modules/esm/get_format.js index 616b2cf52309ea..697e9c5ab7c9e3 100644 --- a/lib/internal/modules/esm/get_format.js +++ b/lib/internal/modules/esm/get_format.js @@ -33,7 +33,7 @@ if (experimentalWasmModules) if (experimentalJsonModules) extensionFormatMap['.json'] = legacyExtensionFormatMap['.json'] = 'json'; -function defaultGetFormat(url, context, defaultGetFormatUnused) { +function defaultGetFormat(url, context, nextGetFormat) { if (StringPrototypeStartsWith(url, 'nodejs:')) { return { format: 'builtin' }; } diff --git a/lib/internal/modules/esm/loader.js b/lib/internal/modules/esm/loader.js index 37191f65bf0b7e..4ce30b47baa772 100644 --- a/lib/internal/modules/esm/loader.js +++ b/lib/internal/modules/esm/loader.js @@ -6,7 +6,6 @@ require('internal/modules/cjs/loader'); const { FunctionPrototypeBind, ObjectSetPrototypeOf, - SafeMap, } = primordials; const { @@ -14,7 +13,7 @@ const { ERR_INVALID_RETURN_PROPERTY, ERR_INVALID_RETURN_PROPERTY_VALUE, ERR_INVALID_RETURN_VALUE, - ERR_UNKNOWN_MODULE_FORMAT + ERR_UNKNOWN_MODULE_FORMAT, } = require('internal/errors').codes; const { URL, pathToFileURL } = require('internal/url'); const { validateString } = require('internal/validators'); @@ -28,11 +27,30 @@ const { const { defaultGetFormat } = require('internal/modules/esm/get_format'); const { defaultGetSource } = require( 'internal/modules/esm/get_source'); -const { defaultTransformSource } = require( - 'internal/modules/esm/transform_source'); const { translators } = require( 'internal/modules/esm/translators'); const { getOptionValue } = require('internal/options'); +const { + isArrayBufferView, + isAnyArrayBuffer, +} = require('internal/util/types'); + +let cwd; // Initialized in importLoader + +function validateSource(source, hookName, allowString) { + if (allowString && typeof source === 'string') { + return; + } + if (isArrayBufferView(source) || isAnyArrayBuffer(source)) { + return; + } + throw new ERR_INVALID_RETURN_PROPERTY_VALUE( + `${allowString ? 'string, ' : ''}array buffer, or typed array`, + hookName, + 'source', + source, + ); +} /* A Loader instance is used as the main entry point for loading ES modules. * Currently, this is a singleton -- there is only one used for loading @@ -46,33 +64,16 @@ class Loader { // Registry of loaded modules, akin to `require.cache` this.moduleMap = new ModuleMap(); - // Map of already-loaded CJS modules to use - this.cjsCache = new SafeMap(); - - // This hook is called before the first root module is imported. It's a - // function that returns a piece of code that runs as a sloppy-mode script. - // The script may evaluate to a function that can be called with a - // `getBuiltin` helper that can be used to retrieve builtins. - // If the hook returns `null` instead of a source string, it opts out of - // running any preload code. - // The preload code runs as soon as the hook module has finished evaluating. - this._getGlobalPreloadCode = null; - // The resolver has the signature - // (specifier : string, parentURL : string, defaultResolve) - // -> Promise<{ url : string }> - // where defaultResolve is ModuleRequest.resolve (having the same - // signature itself). + // Preload code is provided by loaders to be run after hook initialization. + this.globalPreloadCode = []; + // Loader resolve hook. this._resolve = defaultResolve; - // This hook is called after the module is resolved but before a translator - // is chosen to load it; the format returned by this function is the name - // of a translator. + // Loader getFormat hook. this._getFormat = defaultGetFormat; - // This hook is called just before the source code of an ES module file - // is loaded. + // Loader getSource hook. this._getSource = defaultGetSource; - // This hook is called just after the source code of an ES module file - // is loaded, but before anything is done with the string. - this._transformSource = defaultTransformSource; + // Transform source hooks. + this.transformSourceHooks = []; // The index for assigning unique URLs to anonymous module evaluation this.evalIndex = 0; } @@ -83,7 +84,7 @@ class Loader { validateString(parentURL, 'parentURL'); const resolveResponse = await this._resolve( - specifier, { parentURL, conditions: DEFAULT_CONDITIONS }, defaultResolve); + specifier, { parentURL, conditions: DEFAULT_CONDITIONS }); if (typeof resolveResponse !== 'object') { throw new ERR_INVALID_RETURN_VALUE( 'object', 'loader resolve', resolveResponse); @@ -98,8 +99,7 @@ class Loader { } async getFormat(url) { - const getFormatResponse = await this._getFormat( - url, {}, defaultGetFormat); + const getFormatResponse = await this._getFormat(url, {}); if (typeof getFormatResponse !== 'object') { throw new ERR_INVALID_RETURN_VALUE( 'object', 'loader getFormat', getFormatResponse); @@ -137,6 +137,22 @@ class Loader { return format; } + async getSource(url, format) { + const { source: originalSource } = await this._getSource(url, { format }); + + const allowString = format !== 'wasm'; + validateSource(originalSource, 'getSource', allowString); + + let source = originalSource; + for (let i = 0; i < this.transformSourceHooks.length; i += 1) { + const hook = this.transformSourceHooks[i]; + ({ source } = await hook(source, { url, format, originalSource })); + validateSource(source, 'transformSource', allowString); + } + + return source; + } + async eval( source, url = pathToFileURL(`${process.cwd()}/[eval${++this.evalIndex}]`).href @@ -166,72 +182,84 @@ class Loader { return module.getNamespace(); } + async importLoader(specifier) { + if (cwd === undefined) { + try { + // `process.cwd()` can fail. + cwd = process.cwd() + '/'; + } catch { + cwd = 'file:///'; + } + cwd = pathToFileURL(cwd).href; + } + + const { url } = await defaultResolve(specifier, cwd, + { conditions: DEFAULT_CONDITIONS }); + const { format } = await defaultGetFormat(url, {}); + + // !!! CRITICAL SECTION !!! + // NO AWAIT OPS BETWEEN HERE AND SETTING JOB IN MODULE MAP! + // YIELDING CONTROL COULD RESULT IN MAP BEING OVERRIDDEN! + let job = this.moduleMap.get(url); + if (job === undefined) { + if (!translators.has(format)) + throw new ERR_UNKNOWN_MODULE_FORMAT(format); + + const loaderInstance = translators.get(format); + + job = new ModuleJob(this, url, loaderInstance, false, false); + this.moduleMap.set(url, job); + // !!! END CRITICAL SECTION !!! + } + + const { module } = await job.run(); + return module.getNamespace(); + } + hook(hooks) { const { resolve, - dynamicInstantiate, getFormat, getSource, - transformSource, - getGlobalPreloadCode, } = hooks; // Use .bind() to avoid giving access to the Loader instance when called. if (resolve !== undefined) this._resolve = FunctionPrototypeBind(resolve, null); - if (dynamicInstantiate !== undefined) { - process.emitWarning( - 'The dynamicInstantiate loader hook has been removed.'); - } if (getFormat !== undefined) { this._getFormat = FunctionPrototypeBind(getFormat, null); } if (getSource !== undefined) { this._getSource = FunctionPrototypeBind(getSource, null); } - if (transformSource !== undefined) { - this._transformSource = FunctionPrototypeBind(transformSource, null); - } - if (getGlobalPreloadCode !== undefined) { - this._getGlobalPreloadCode = - FunctionPrototypeBind(getGlobalPreloadCode, null); - } } runGlobalPreloadCode() { - if (!this._getGlobalPreloadCode) { - return; - } - const preloadCode = this._getGlobalPreloadCode(); - if (preloadCode === null) { - return; - } + for (let i = 0; i < this.globalPreloadCode.length; i += 1) { + const preloadCode = this.globalPreloadCode[i]; - if (typeof preloadCode !== 'string') { - throw new ERR_INVALID_RETURN_VALUE( - 'string', 'loader getGlobalPreloadCode', preloadCode); + const { compileFunction } = require('vm'); + const preloadInit = compileFunction(preloadCode, ['getBuiltin'], { + filename: '', + }); + const { NativeModule } = require('internal/bootstrap/loaders'); + + preloadInit.call(globalThis, (builtinName) => { + if (NativeModule.canBeRequiredByUsers(builtinName)) { + return require(builtinName); + } + throw new ERR_INVALID_ARG_VALUE('builtinName', builtinName); + }); } - const { compileFunction } = require('vm'); - const preloadInit = compileFunction(preloadCode, ['getBuiltin'], { - filename: '', - }); - const { NativeModule } = require('internal/bootstrap/loaders'); - - preloadInit.call(globalThis, (builtinName) => { - if (NativeModule.canBeRequiredByUsers(builtinName)) { - return require(builtinName); - } - throw new ERR_INVALID_ARG_VALUE('builtinName', builtinName); - }); } async getModuleJob(specifier, parentURL) { const url = await this.resolve(specifier, parentURL); const format = await this.getFormat(url); + + // !!! CRITICAL SECTION !!! + // NO AWAIT OPS BETWEEN HERE AND SETTING JOB IN MODULE MAP let job = this.moduleMap.get(url); - // CommonJS will set functions for lazy job evaluation. - if (typeof job === 'function') - this.moduleMap.set(url, job = job()); if (job !== undefined) return job; @@ -245,6 +273,8 @@ class Loader { job = new ModuleJob(this, url, loaderInstance, parentURL === undefined, inspectBrk); this.moduleMap.set(url, job); + // !!! END CRITICAL SECTION !!! + return job; } } diff --git a/lib/internal/modules/esm/resolve.js b/lib/internal/modules/esm/resolve.js index 7ea59f30c6894e..b424370f67dfb3 100644 --- a/lib/internal/modules/esm/resolve.js +++ b/lib/internal/modules/esm/resolve.js @@ -764,8 +764,7 @@ function resolveAsCommonJS(specifier, parentURL) { } } -function defaultResolve(specifier, context = {}, defaultResolveUnused) { - let { parentURL, conditions } = context; +function defaultResolve(specifier, { parentURL, conditions } = {}) { let parsed; try { parsed = new URL(specifier); diff --git a/lib/internal/modules/esm/transform_source.js b/lib/internal/modules/esm/transform_source.js deleted file mode 100644 index 2d07dd3607fb66..00000000000000 --- a/lib/internal/modules/esm/transform_source.js +++ /dev/null @@ -1,7 +0,0 @@ -'use strict'; - -function defaultTransformSource(source, { url, format } = {}, - defaultTransformSource) { - return { source }; -} -exports.defaultTransformSource = defaultTransformSource; diff --git a/lib/internal/modules/esm/translators.js b/lib/internal/modules/esm/translators.js index bb3528eddde964..a7085402a17356 100644 --- a/lib/internal/modules/esm/translators.js +++ b/lib/internal/modules/esm/translators.js @@ -11,22 +11,15 @@ const { StringPrototypeReplace, } = primordials; -let _TYPES = null; -function lazyTypes() { - if (_TYPES !== null) return _TYPES; - return _TYPES = require('internal/util/types'); -} - const { stripBOM, loadNativeModule } = require('internal/modules/cjs/helpers'); -const CJSModule = require('internal/modules/cjs/loader').Module; +const { + Module: CJSModule, + originalModuleExports: originalCJSModuleExports, +} = require('internal/modules/cjs/loader'); const internalURLModule = require('internal/url'); -const { defaultGetSource } = require( - 'internal/modules/esm/get_source'); -const { defaultTransformSource } = require( - 'internal/modules/esm/transform_source'); const createDynamicModule = require( 'internal/modules/esm/create_dynamic_module'); const { fileURLToPath, URL } = require('url'); @@ -36,7 +29,6 @@ let debug = require('internal/util/debuglog').debuglog('esm', (fn) => { const { emitExperimentalWarning } = require('internal/util'); const { ERR_UNKNOWN_BUILTIN_MODULE, - ERR_INVALID_RETURN_PROPERTY_VALUE } = require('internal/errors').codes; const { maybeCacheSourceMap } = require('internal/source_map/source_map_cache'); const moduleWrap = internalBinding('module_wrap'); @@ -48,30 +40,6 @@ const experimentalImportMetaResolve = const translators = new SafeMap(); exports.translators = translators; -let DECODER = null; -function assertBufferSource(body, allowString, hookName) { - if (allowString && typeof body === 'string') { - return; - } - const { isArrayBufferView, isAnyArrayBuffer } = lazyTypes(); - if (isArrayBufferView(body) || isAnyArrayBuffer(body)) { - return; - } - throw new ERR_INVALID_RETURN_PROPERTY_VALUE( - `${allowString ? 'string, ' : ''}array buffer, or typed array`, - hookName, - 'source', - body - ); -} - -function stringify(body) { - if (typeof body === 'string') return body; - assertBufferSource(body, false, 'transformSource'); - DECODER = DECODER === null ? new TextDecoder() : DECODER; - return DECODER.decode(body); -} - function errPath(url) { const parsed = new URL(url); if (parsed.protocol === 'file:') { @@ -80,10 +48,21 @@ function errPath(url) { return url; } +let DECODER; +function stringify(source) { + if (typeof source === 'string') { + return source; + } + if (DECODER === undefined) { + DECODER = new TextDecoder(); + } + return DECODER.decode(source); +} + let esmLoader; async function importModuleDynamically(specifier, { url }) { if (!esmLoader) { - esmLoader = require('internal/process/esm_loader').ESMLoader; + esmLoader = await require('internal/process/esm_loader').getLoader(); } return esmLoader.import(specifier, url); } @@ -91,7 +70,7 @@ async function importModuleDynamically(specifier, { url }) { function createImportMetaResolve(defaultParentUrl) { return async function resolve(specifier, parentUrl = defaultParentUrl) { if (!esmLoader) { - esmLoader = require('internal/process/esm_loader').ESMLoader; + esmLoader = await require('internal/process/esm_loader').getLoader(); } return PromisePrototypeCatch( esmLoader.resolve(specifier, parentUrl), @@ -109,14 +88,20 @@ function initializeImportMeta(meta, { url }) { meta.url = url; } +function requireOriginal(filename) { + const resolved = isWindows ? + StringPrototypeReplace(filename, winSepRegEx, '\\') : filename; + if (!CJSModule._cache[resolved]) { + CJSModule._load(filename, undefined, false); + } + const module = CJSModule._cache[resolved]; + return originalCJSModuleExports.get(module); +} + // Strategy for loading a standard JavaScript module translators.set('module', async function moduleStrategy(url) { - let { source } = await this._getSource( - url, { format: 'module' }, defaultGetSource); - assertBufferSource(source, true, 'getSource'); - ({ source } = await this._transformSource( - source, { url, format: 'module' }, defaultTransformSource)); - source = stringify(source); + const source = stringify(await this.getSource(url, 'module')); + maybeCacheSourceMap(url, source); debug(`Translating StandardModule ${url}`); const module = new ModuleWrap(url, undefined, source, 0, 0); @@ -133,26 +118,10 @@ const winSepRegEx = /\//g; translators.set('commonjs', function commonjsStrategy(url, isMain) { debug(`Translating CJSModule ${url}`); const pathname = internalURLModule.fileURLToPath(new URL(url)); - const cached = this.cjsCache.get(url); - if (cached) { - this.cjsCache.delete(url); - return cached; - } - const module = CJSModule._cache[ - isWindows ? StringPrototypeReplace(pathname, winSepRegEx, '\\') : pathname - ]; - if (module && module.loaded) { - const exports = module.exports; - return new ModuleWrap(url, undefined, ['default'], function() { - this.setExport('default', exports); - }); - } return new ModuleWrap(url, undefined, ['default'], function() { debug(`Loading CJSModule ${url}`); - // We don't care about the return val of _load here because Module#load - // will handle it for us by checking the loader registry and filling the - // exports like above - CJSModule._load(pathname, undefined, isMain); + const exports = requireOriginal(pathname); + this.setExport('default', exports); }); }); @@ -189,12 +158,9 @@ translators.set('json', async function jsonStrategy(url) { }); } } - let { source } = await this._getSource( - url, { format: 'json' }, defaultGetSource); - assertBufferSource(source, true, 'getSource'); - ({ source } = await this._transformSource( - source, { url, format: 'json' }, defaultTransformSource)); - source = stringify(source); + + const source = stringify(await this.getSource(url, 'json')); + if (pathname) { // A require call could have been called on the same file during loading and // that resolves synchronously. To make sure we always return the identical @@ -207,6 +173,7 @@ translators.set('json', async function jsonStrategy(url) { }); } } + try { const exports = JSONParse(stripBOM(source)); module = { @@ -221,9 +188,11 @@ translators.set('json', async function jsonStrategy(url) { err.message = errPath(url) + ': ' + err.message; throw err; } + if (pathname) { CJSModule._cache[modulePath] = module; } + return new ModuleWrap(url, undefined, ['default'], function() { debug(`Parsing JSONModule ${url}`); this.setExport('default', module.exports); @@ -233,12 +202,7 @@ translators.set('json', async function jsonStrategy(url) { // Strategy for loading a wasm module translators.set('wasm', async function(url) { emitExperimentalWarning('Importing Web Assembly modules'); - let { source } = await this._getSource( - url, { format: 'wasm' }, defaultGetSource); - assertBufferSource(source, false, 'getSource'); - ({ source } = await this._transformSource( - source, { url, format: 'wasm' }, defaultTransformSource)); - assertBufferSource(source, false, 'transformSource'); + const source = await this.getSource(url, 'wasm'); debug(`Translating WASMModule ${url}`); let compiled; try { diff --git a/lib/internal/modules/run_main.js b/lib/internal/modules/run_main.js index 0967ef539ca20e..cc494fea523fb3 100644 --- a/lib/internal/modules/run_main.js +++ b/lib/internal/modules/run_main.js @@ -21,8 +21,8 @@ function resolveMainPath(main) { } function shouldUseESMLoader(mainPath) { - const userLoader = getOptionValue('--experimental-loader'); - if (userLoader) + const userLoaders = getOptionValue('--experimental-loader'); + if (userLoaders.length > 0) return true; const esModuleSpecifierResolution = getOptionValue('--es-module-specifier-resolution'); @@ -38,12 +38,15 @@ function shouldUseESMLoader(mainPath) { } function runMainESM(mainPath) { - const esmLoader = require('internal/process/esm_loader'); + const { getLoader } = require('internal/process/esm_loader'); const { pathToFileURL } = require('internal/url'); - esmLoader.loadESM((ESMLoader) => { + const { uncaughtException } = require('internal/process/execution'); + return getLoader().then((loader) => { const main = path.isAbsolute(mainPath) ? pathToFileURL(mainPath).href : mainPath; - return ESMLoader.import(main); + return loader.import(main); + }).catch((e) => { + uncaughtException(e, true /* fromPromise */); }); } diff --git a/lib/internal/process/esm_loader.js b/lib/internal/process/esm_loader.js index 8f076b3ef32efd..d839dab98729be 100644 --- a/lib/internal/process/esm_loader.js +++ b/lib/internal/process/esm_loader.js @@ -1,16 +1,21 @@ 'use strict'; +const { + ArrayPrototypePush, + ReflectApply, +} = primordials; + const { ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING, } = require('internal/errors').codes; const { Loader } = require('internal/modules/esm/loader'); -const { - hasUncaughtExceptionCaptureCallback, -} = require('internal/process/execution'); -const { pathToFileURL } = require('internal/url'); const { getModuleFromWrap, } = require('internal/vm/module'); +const { + validateString, + validateFunction, +} = require('internal/validators'); exports.initializeImportMetaObject = function(wrap, meta) { const { callbackMap } = internalBinding('module_wrap'); @@ -34,46 +39,93 @@ exports.importModuleDynamicallyCallback = async function(wrap, specifier) { throw new ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING(); }; -let ESMLoader = new Loader(); -exports.ESMLoader = ESMLoader; +const createLoader = async () => { + const loader = new Loader(); -async function initializeLoader() { const { getOptionValue } = require('internal/options'); - const userLoader = getOptionValue('--experimental-loader'); - if (!userLoader) - return; - let cwd; - try { - cwd = process.cwd() + '/'; - } catch { - cwd = 'file:///'; - } - // If --experimental-loader is specified, create a loader with user hooks. - // Otherwise create the default loader. - const { emitExperimentalWarning } = require('internal/util'); - emitExperimentalWarning('--experimental-loader'); - return (async () => { - const hooks = - await ESMLoader.import(userLoader, pathToFileURL(cwd).href); - ESMLoader = new Loader(); - ESMLoader.hook(hooks); - ESMLoader.runGlobalPreloadCode(); - return exports.ESMLoader = ESMLoader; - })(); -} - -exports.loadESM = async function loadESM(callback) { - try { - await initializeLoader(); - await callback(ESMLoader); - } catch (err) { - if (hasUncaughtExceptionCaptureCallback()) { - process._fatalException(err); - return; + const userLoaders = getOptionValue('--experimental-loader'); + + if (userLoaders.length > 0) { + const { emitExperimentalWarning } = require('internal/util'); + emitExperimentalWarning('--experimental-loader'); + + const importedLoaders = []; + for (let i = 0; i < userLoaders.length; i += 1) { + const ns = await loader.importLoader(userLoaders[i]); + + if (ns.resolve !== undefined) { + validateFunction(ns.resolve, 'resolve'); + } + if (ns.getFormat !== undefined) { + validateFunction(ns.getFormat, 'getFormat'); + } + if (ns.getSource !== undefined) { + validateFunction(ns.getSource, 'getSource'); + } + if (ns.transformSource !== undefined) { + validateFunction(ns.transformSource, 'transformSource'); + ArrayPrototypePush(loader.transformSourceHooks, ns.transformSource); + } + + if (ns.getGlobalPreloadCode !== undefined) { + validateFunction(ns.getGlobalPreloadCode, 'getGlobalPreloadCode'); + const code = ReflectApply(ns.getGlobalPreloadCode, undefined, []); + validateString(code, 'getGlobalPreloadCode return value'); + ArrayPrototypePush(loader.globalPreloadCode, code); + } + + importedLoaders.push(ns); } - internalBinding('errors').triggerUncaughtException( - err, - true /* fromPromise */ - ); + + let cursor = { + resolve: loader._resolve, + getFormat: loader._getFormat, + getSource: loader._getSource, + }; + for (let i = userLoaders.length - 1; i >= 0; i -= 1) { + const { + resolve, + getFormat, + getSource, + } = importedLoaders[i]; + + const nextLoader = cursor; + cursor = { + resolve: resolve ? async (...args) => { + const result = await resolve(...args, nextLoader.resolve); + if (result === null) { + return nextLoader.resolve(...args); + } + return result; + } : nextLoader.resolve, + getFormat: getFormat ? async (...args) => { + const result = await getFormat(...args, nextLoader.getFormat); + if (result === null) { + return nextLoader.getFormat(...args); + } + return result; + } : nextLoader.getFormat, + getSource: getSource ? async (...args) => { + const result = await getSource(...args, nextLoader.getSource); + if (result === null) { + return nextLoader.getSource(...args); + } + return result; + } : nextLoader.getSource, + }; + } + + loader.hook(cursor); + loader.runGlobalPreloadCode(); + } + + return loader; +}; + +let processLoader; +exports.getLoader = () => { + if (!processLoader) { + processLoader = createLoader(); } + return processLoader; }; diff --git a/lib/internal/process/execution.js b/lib/internal/process/execution.js index 08d6ec8c6ea906..5d250cf1ed8d47 100644 --- a/lib/internal/process/execution.js +++ b/lib/internal/process/execution.js @@ -2,7 +2,6 @@ const { JSONStringify, - PromiseResolve, } = primordials; const path = require('path'); @@ -46,7 +45,7 @@ function evalModule(source, print) { const { log, error } = require('internal/console/global'); const { decorateErrorStack } = require('internal/util'); const asyncESM = require('internal/process/esm_loader'); - PromiseResolve(asyncESM.ESMLoader).then(async (loader) => { + asyncESM.getLoader().then(async (loader) => { const { result } = await loader.eval(source); if (print) { log(result); @@ -91,7 +90,7 @@ function evalScript(name, body, breakFirstLine, print) { displayErrors: true, [kVmBreakFirstLineSymbol]: ${!!breakFirstLine}, async importModuleDynamically (specifier) { - const loader = await asyncESM.ESMLoader; + const loader = await asyncESM.getLoader(); return loader.import(specifier, ${JSONStringify(baseUrl)}); } });\n`; @@ -212,6 +211,14 @@ function readStdin(callback) { }); } +function uncaughtException(error, fromPromise = false) { + if (hasUncaughtExceptionCaptureCallback()) { + process._fatalException(error); + return; + } + internalBinding('errors').triggerUncaughtException(error, fromPromise); +} + module.exports = { readStdin, tryGetCwd, @@ -219,5 +226,6 @@ module.exports = { evalScript, onGlobalUncaughtException: createOnGlobalUncaughtException(), setUncaughtExceptionCaptureCallback, - hasUncaughtExceptionCaptureCallback + hasUncaughtExceptionCaptureCallback, + uncaughtException, }; diff --git a/lib/internal/validators.js b/lib/internal/validators.js index deab53d4b8221e..68ddf5447a9f7b 100644 --- a/lib/internal/validators.js +++ b/lib/internal/validators.js @@ -131,6 +131,11 @@ function validateBoolean(value, name) { throw new ERR_INVALID_ARG_TYPE(name, 'boolean', value); } +function validateFunction(value, name) { + if (typeof value !== 'function') + throw new ERR_INVALID_ARG_TYPE(name, 'function', value); +} + const validateObject = hideStackFrames( (value, name, { nullable = false } = {}) => { if ((!nullable && value === null) || @@ -208,6 +213,7 @@ module.exports = { validateBoolean, validateBuffer, validateEncoding, + validateFunction, validateInt32, validateInteger, validateNumber, diff --git a/lib/repl.js b/lib/repl.js index cc5bf069c31295..9a35e832adf67e 100644 --- a/lib/repl.js +++ b/lib/repl.js @@ -399,7 +399,8 @@ function REPLServer(prompt, filename: file, displayErrors: true, importModuleDynamically: async (specifier) => { - return asyncESM.ESMLoader.import(specifier, parentURL); + const loader = await asyncESM.getLoader(); + return loader.import(specifier, parentURL); } }); } catch (e) { diff --git a/node.gyp b/node.gyp index 497c06c6bc0a6c..229033b4db6cc9 100644 --- a/node.gyp +++ b/node.gyp @@ -176,7 +176,6 @@ 'lib/internal/modules/esm/module_job.js', 'lib/internal/modules/esm/module_map.js', 'lib/internal/modules/esm/resolve.js', - 'lib/internal/modules/esm/transform_source.js', 'lib/internal/modules/esm/translators.js', 'lib/internal/net.js', 'lib/internal/options.js', diff --git a/src/node_options.cc b/src/node_options.cc index 93a2268da41363..7900d72733cbcf 100644 --- a/src/node_options.cc +++ b/src/node_options.cc @@ -298,7 +298,7 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() { kAllowedInEnvironment); AddOption("--experimental-loader", "use the specified module as a custom loader", - &EnvironmentOptions::userland_loader, + &EnvironmentOptions::userland_loaders, kAllowedInEnvironment); AddAlias("--loader", "--experimental-loader"); AddOption("--experimental-modules", diff --git a/src/node_options.h b/src/node_options.h index 7f8c223f755a24..db74285ff98f90 100644 --- a/src/node_options.h +++ b/src/node_options.h @@ -149,7 +149,7 @@ class EnvironmentOptions : public Options { bool trace_uncaught = false; bool trace_warnings = false; std::string unhandled_rejections; - std::string userland_loader; + std::vector userland_loaders; bool syntax_check_only = false; bool has_eval_string = false; diff --git a/test/es-module/test-loader-chaining.mjs b/test/es-module/test-loader-chaining.mjs new file mode 100644 index 00000000000000..467d2f7a51443d --- /dev/null +++ b/test/es-module/test-loader-chaining.mjs @@ -0,0 +1,8 @@ +import '../common/index.mjs'; +import { strictEqual } from 'assert'; + +// Flags: --experimental-loader ./test/fixtures/es-module-loaders/loader-chain-a.mjs --experimental-loader ./test/fixtures/es-module-loaders/loader-chain-b.mjs + +// loader-chain-a.mjs changes AAA to BBB +// loader-chain-b.mjs changes BBB to CCC +strictEqual('AAA', 'CCC'); diff --git a/test/fixtures/es-module-loaders/builtin-named-exports-loader.mjs b/test/fixtures/es-module-loaders/builtin-named-exports-loader.mjs index f476c676cdea5b..cf5c251b4bdd7f 100644 --- a/test/fixtures/es-module-loaders/builtin-named-exports-loader.mjs +++ b/test/fixtures/es-module-loaders/builtin-named-exports-loader.mjs @@ -3,7 +3,8 @@ import module from 'module'; const GET_BUILTIN = `$__get_builtin_hole_${Date.now()}`; export function getGlobalPreloadCode() { - return `Object.defineProperty(globalThis, ${JSON.stringify(GET_BUILTIN)}, { + return `\ +Object.defineProperty(globalThis, ${JSON.stringify(GET_BUILTIN)}, { value: (builtinName) => { return getBuiltin(builtinName); }, @@ -13,8 +14,8 @@ export function getGlobalPreloadCode() { `; } -export function resolve(specifier, context, defaultResolve) { - const def = defaultResolve(specifier, context); +export async function resolve(specifier, context, nextResolve) { + const def = await nextResolve(specifier, context); if (def.url.startsWith('nodejs:')) { return { url: `custom-${def.url}`, @@ -23,7 +24,7 @@ export function resolve(specifier, context, defaultResolve) { return def; } -export function getSource(url, context, defaultGetSource) { +export function getSource(url) { if (url.startsWith('custom-nodejs:')) { const urlObj = new URL(url); return { @@ -31,14 +32,14 @@ export function getSource(url, context, defaultGetSource) { format: 'module', }; } - return defaultGetSource(url, context); + return null; } -export function getFormat(url, context, defaultGetFormat) { +export function getFormat(url) { if (url.startsWith('custom-nodejs:')) { return { format: 'module' }; } - return defaultGetFormat(url, context, defaultGetFormat); + return null; } function generateBuiltinModule(builtinName) { diff --git a/test/fixtures/es-module-loaders/example-loader.mjs b/test/fixtures/es-module-loaders/example-loader.mjs index 1ed18bda51070d..e5024987154f0c 100644 --- a/test/fixtures/es-module-loaders/example-loader.mjs +++ b/test/fixtures/es-module-loaders/example-loader.mjs @@ -8,7 +8,7 @@ const JS_EXTENSIONS = new Set(['.js', '.mjs']); const baseURL = new URL('file://'); baseURL.pathname = process.cwd() + '/'; -export function resolve(specifier, { parentURL = baseURL }, defaultResolve) { +export function resolve(specifier, { parentURL = baseURL, ...context }, nextResolve) { if (builtinModules.includes(specifier)) { return { url: 'nodejs:' + specifier @@ -16,7 +16,7 @@ export function resolve(specifier, { parentURL = baseURL }, defaultResolve) { } if (/^\.{1,2}[/]/.test(specifier) !== true && !specifier.startsWith('file:')) { // For node_modules support: - // return defaultResolve(specifier, {parentURL}, defaultResolve); + // return nextResolve(specifier, { parentURL, ...context }); throw new Error( `imports must be URLs or begin with './', or '../'; '${specifier}' does not`); } @@ -26,7 +26,7 @@ export function resolve(specifier, { parentURL = baseURL }, defaultResolve) { }; } -export function getFormat(url, context, defaultGetFormat) { +export function getFormat(url) { if (url.startsWith('nodejs:') && builtinModules.includes(url.slice(7))) { return { format: 'builtin' diff --git a/test/fixtures/es-module-loaders/get-source.mjs b/test/fixtures/es-module-loaders/get-source.mjs index e5a9c65201aa28..c1c2f6b7f1b1fc 100644 --- a/test/fixtures/es-module-loaders/get-source.mjs +++ b/test/fixtures/es-module-loaders/get-source.mjs @@ -1,10 +1,9 @@ -export async function getSource(url, { format }, defaultGetSource) { +export async function getSource(url, { format }) { if (url.endsWith('fixtures/es-modules/message.mjs')) { // Oh, I’ve got that one in my cache! return { source: `export const message = 'Woohoo!'.toUpperCase();` } - } else { - return defaultGetSource(url, {format}, defaultGetSource); } + return null; } diff --git a/test/fixtures/es-module-loaders/js-loader.mjs b/test/fixtures/es-module-loaders/js-loader.mjs index 2f79475e77e269..79e77250f91aae 100644 --- a/test/fixtures/es-module-loaders/js-loader.mjs +++ b/test/fixtures/es-module-loaders/js-loader.mjs @@ -1,9 +1,9 @@ -export function getFormat(url, context, defaultGetFormat) { +export function getFormat(url, context) { // Load all .js files as ESM, regardless of package scope if (url.endsWith('.js')) { return { format: 'module' } } - return defaultGetFormat(url, context, defaultGetFormat); + return null; } diff --git a/test/fixtures/es-module-loaders/loader-chain-a.mjs b/test/fixtures/es-module-loaders/loader-chain-a.mjs new file mode 100644 index 00000000000000..70874b2c11d91e --- /dev/null +++ b/test/fixtures/es-module-loaders/loader-chain-a.mjs @@ -0,0 +1,8 @@ +export async function transformSource(source, { url }) { + if (url.endsWith('test-loader-chaining.mjs')) { + return { + source: source.toString().replace(/A/g, 'B'), + }; + } + return { source }; +} diff --git a/test/fixtures/es-module-loaders/loader-chain-b.mjs b/test/fixtures/es-module-loaders/loader-chain-b.mjs new file mode 100644 index 00000000000000..b8b9299283732a --- /dev/null +++ b/test/fixtures/es-module-loaders/loader-chain-b.mjs @@ -0,0 +1,8 @@ +export async function transformSource(source, { url }) { + if (url.endsWith('test-loader-chaining.mjs')) { + return { + source: source.toString().replace(/B/g, 'C'), + }; + } + return { source }; +} diff --git a/test/fixtures/es-module-loaders/loader-get-format.mjs b/test/fixtures/es-module-loaders/loader-get-format.mjs index 7ade70fca0ebe6..4b6c2f3802f382 100644 --- a/test/fixtures/es-module-loaders/loader-get-format.mjs +++ b/test/fixtures/es-module-loaders/loader-get-format.mjs @@ -1,4 +1,4 @@ -export async function getFormat(url, context, defaultGetFormat) { +export async function getFormat(url) { try { if (new URL(url).pathname.endsWith('.unknown')) { return { @@ -6,5 +6,5 @@ export async function getFormat(url, context, defaultGetFormat) { }; } } catch {} - return defaultGetFormat(url, context, defaultGetFormat); + return null; } diff --git a/test/fixtures/es-module-loaders/loader-invalid-format.mjs b/test/fixtures/es-module-loaders/loader-invalid-format.mjs index 55ae1cec8ee926..7e3692ca3c7212 100644 --- a/test/fixtures/es-module-loaders/loader-invalid-format.mjs +++ b/test/fixtures/es-module-loaders/loader-invalid-format.mjs @@ -1,17 +1,17 @@ -export async function resolve(specifier, { parentURL }, defaultResolve) { +export async function resolve(specifier, { parentURL }) { if (parentURL && specifier === '../fixtures/es-modules/test-esm-ok.mjs') { return { url: 'file:///asdf' }; } - return defaultResolve(specifier, {parentURL}, defaultResolve); + return null; } -export function getFormat(url, context, defaultGetFormat) { +export function getFormat(url) { if (url === 'file:///asdf') { return { format: 'esm' } } - return defaultGetFormat(url, context, defaultGetFormat); + return null; } diff --git a/test/fixtures/es-module-loaders/loader-invalid-url.mjs b/test/fixtures/es-module-loaders/loader-invalid-url.mjs index e7de0d4ed92378..ea3b87929cf918 100644 --- a/test/fixtures/es-module-loaders/loader-invalid-url.mjs +++ b/test/fixtures/es-module-loaders/loader-invalid-url.mjs @@ -1,9 +1,9 @@ /* eslint-disable node-core/required-modules */ -export async function resolve(specifier, { parentURL }, defaultResolve) { +export async function resolve(specifier, { parentURL }) { if (parentURL && specifier === '../fixtures/es-modules/test-esm-ok.mjs') { return { url: specifier }; } - return defaultResolve(specifier, {parentURL}, defaultResolve); + return null; } diff --git a/test/fixtures/es-module-loaders/loader-shared-dep.mjs b/test/fixtures/es-module-loaders/loader-shared-dep.mjs index 3576c074d52cec..45c764c94603b6 100644 --- a/test/fixtures/es-module-loaders/loader-shared-dep.mjs +++ b/test/fixtures/es-module-loaders/loader-shared-dep.mjs @@ -5,7 +5,7 @@ import {createRequire} from '../../common/index.mjs'; const require = createRequire(import.meta.url); const dep = require('./loader-dep.js'); -export function resolve(specifier, { parentURL }, defaultResolve) { +export function resolve() { assert.strictEqual(dep.format, 'module'); - return defaultResolve(specifier, {parentURL}, defaultResolve); + return null; } diff --git a/test/fixtures/es-module-loaders/loader-unknown-builtin-module.mjs b/test/fixtures/es-module-loaders/loader-unknown-builtin-module.mjs index e976343e47e9bc..f4a2f9c3697cca 100644 --- a/test/fixtures/es-module-loaders/loader-unknown-builtin-module.mjs +++ b/test/fixtures/es-module-loaders/loader-unknown-builtin-module.mjs @@ -1,17 +1,17 @@ -export async function resolve(specifier, { parentURL }, defaultResolve) { +export async function resolve(specifier) { if (specifier === 'unknown-builtin-module') { return { url: 'nodejs:unknown-builtin-module' }; } - return defaultResolve(specifier, {parentURL}, defaultResolve); + return null; } -export async function getFormat(url, context, defaultGetFormat) { +export async function getFormat(url) { if (url === 'nodejs:unknown-builtin-module') { return { format: 'builtin' }; } - return defaultGetFormat(url, context, defaultGetFormat); + return null; } diff --git a/test/fixtures/es-module-loaders/loader-with-custom-condition.mjs b/test/fixtures/es-module-loaders/loader-with-custom-condition.mjs index 78ffd75e6be27b..2a6d82fab7d1cb 100644 --- a/test/fixtures/es-module-loaders/loader-with-custom-condition.mjs +++ b/test/fixtures/es-module-loaders/loader-with-custom-condition.mjs @@ -1,9 +1,9 @@ -import {ok, deepStrictEqual} from 'assert'; +import { ok, deepStrictEqual } from 'assert'; -export async function resolve(specifier, context, defaultResolve) { +export async function resolve(specifier, context, nextResolve) { ok(Array.isArray(context.conditions), 'loader receives conditions array'); deepStrictEqual([...context.conditions].sort(), ['import', 'node']); - return defaultResolve(specifier, { + return nextResolve(specifier, { ...context, conditions: ['custom-condition', ...context.conditions], }); diff --git a/test/fixtures/es-module-loaders/loader-with-dep.mjs b/test/fixtures/es-module-loaders/loader-with-dep.mjs index da7d44ae793e22..34e93ef81a5877 100644 --- a/test/fixtures/es-module-loaders/loader-with-dep.mjs +++ b/test/fixtures/es-module-loaders/loader-with-dep.mjs @@ -3,9 +3,10 @@ import {createRequire} from '../../common/index.mjs'; const require = createRequire(import.meta.url); const dep = require('./loader-dep.js'); -export function resolve (specifier, { parentURL }, defaultResolve) { +export async function resolve(specifier, context, nextResolve) { + const { url } = await nextResolve(specifier, context); return { - url: defaultResolve(specifier, {parentURL}, defaultResolve).url, - format: dep.format + url, + format: dep.format, }; } diff --git a/test/fixtures/es-module-loaders/not-found-assert-loader.mjs b/test/fixtures/es-module-loaders/not-found-assert-loader.mjs index 2130bad5f52698..c28c878f2c91aa 100644 --- a/test/fixtures/es-module-loaders/not-found-assert-loader.mjs +++ b/test/fixtures/es-module-loaders/not-found-assert-loader.mjs @@ -3,19 +3,15 @@ import assert from 'assert'; // a loader that asserts that the defaultResolve will throw "not found" // (skipping the top-level main of course) let mainLoad = true; -export async function resolve(specifier, { parentURL }, defaultResolve) { +export async function resolve(specifier, context, nextResolve) { if (mainLoad) { mainLoad = false; - return defaultResolve(specifier, {parentURL}, defaultResolve); + return null; } - try { - await defaultResolve(specifier, {parentURL}, defaultResolve); - } - catch (e) { - assert.strictEqual(e.code, 'ERR_MODULE_NOT_FOUND'); - return { - url: 'nodejs:fs' - }; - } - assert.fail(`Module resolution for ${specifier} should be throw ERR_MODULE_NOT_FOUND`); + await assert.rejects(async () => { + await nextResolve(specifier, context); + }, { + code: 'ERR_MODULE_NOT_FOUND', + }); + return { url: 'nodejs:fs' }; } diff --git a/test/fixtures/es-module-loaders/string-sources.mjs b/test/fixtures/es-module-loaders/string-sources.mjs index 3133231208ce1e..f46f175776a02f 100644 --- a/test/fixtures/es-module-loaders/string-sources.mjs +++ b/test/fixtures/es-module-loaders/string-sources.mjs @@ -10,21 +10,21 @@ const SOURCES = { 'test:Uint8Array': new Uint8Array(0), 'test:undefined': undefined, } -export function resolve(specifier, context, defaultFn) { +export function resolve(specifier) { if (specifier.startsWith('test:')) { return { url: specifier }; } - return defaultFn(specifier, context); + return null; } -export function getFormat(href, context, defaultFn) { +export function getFormat(href) { if (href.startsWith('test:')) { return { format: 'module' }; } - return defaultFn(href, context); + return null; } -export function getSource(href, context, defaultFn) { +export function getSource(href) { if (href.startsWith('test:')) { return { source: SOURCES[href] }; } - return defaultFn(href, context); + return null; } diff --git a/test/fixtures/es-module-loaders/transform-source.mjs b/test/fixtures/es-module-loaders/transform-source.mjs index 25d983b64e62ca..743cb4d8652aa8 100644 --- a/test/fixtures/es-module-loaders/transform-source.mjs +++ b/test/fixtures/es-module-loaders/transform-source.mjs @@ -1,5 +1,4 @@ -export async function transformSource( - source, { url, format }, defaultTransformSource) { +export async function transformSource(source, { url, format }) { if (format === 'module') { if (typeof source !== 'string') { source = new TextDecoder().decode(source); @@ -7,8 +6,6 @@ export async function transformSource( return { source: source.replace(`'A message';`, `'A message'.toUpperCase();`) }; - } else { // source could be a buffer, e.g. for WASM - return defaultTransformSource( - source, {url, format}, defaultTransformSource); } + return { source }; } diff --git a/test/message/esm_display_syntax_error_import.out b/test/message/esm_display_syntax_error_import.out index fe174d54a5c49f..387a63a734b512 100644 --- a/test/message/esm_display_syntax_error_import.out +++ b/test/message/esm_display_syntax_error_import.out @@ -5,4 +5,3 @@ SyntaxError: The requested module '../fixtures/es-module-loaders/module-named-ex at ModuleJob._instantiate (internal/modules/esm/module_job.js:*:*) at async ModuleJob.run (internal/modules/esm/module_job.js:*:*) at async Loader.import (internal/modules/esm/loader.js:*:*) - at async Object.loadESM (internal/process/esm_loader.js:*:*) diff --git a/test/message/esm_display_syntax_error_import_module.out b/test/message/esm_display_syntax_error_import_module.out index d220627bd02654..ae8b99d55fef20 100644 --- a/test/message/esm_display_syntax_error_import_module.out +++ b/test/message/esm_display_syntax_error_import_module.out @@ -5,4 +5,3 @@ SyntaxError: The requested module './module-named-exports.mjs' does not provide at ModuleJob._instantiate (internal/modules/esm/module_job.js:*:*) at async ModuleJob.run (internal/modules/esm/module_job.js:*:*) at async Loader.import (internal/modules/esm/loader.js:*:*) - at async Object.loadESM (internal/process/esm_loader.js:*:*) diff --git a/test/message/esm_loader_not_found.out b/test/message/esm_loader_not_found.out index c9429dad40a072..d0b7b878183deb 100644 --- a/test/message/esm_loader_not_found.out +++ b/test/message/esm_loader_not_found.out @@ -1,18 +1,18 @@ (node:*) ExperimentalWarning: --experimental-loader is an experimental feature. This feature could change at any time (Use `node --trace-warnings ...` to show where the warning was created) -internal/process/esm_loader.js:* - internalBinding('errors').triggerUncaughtException( - ^ +internal/process/execution.js:* + internalBinding('errors').triggerUncaughtException(error, fromPromise); + ^ Error [ERR_MODULE_NOT_FOUND]: Cannot find package 'i-dont-exist' imported from * at new NodeError (internal/errors.js:*:*) at packageResolve (internal/modules/esm/resolve.js:*:*) at moduleResolve (internal/modules/esm/resolve.js:*:*) - at Loader.defaultResolve [as _resolve] (internal/modules/esm/resolve.js:*:*) - at Loader.resolve (internal/modules/esm/loader.js:*:*) - at Loader.getModuleJob (internal/modules/esm/loader.js:*:*) - at Loader.import (internal/modules/esm/loader.js:*:*) - at internal/process/esm_loader.js:*:* - at initializeLoader (internal/process/esm_loader.js:*:*) - at Object.loadESM (internal/process/esm_loader.js:*:*) { + at defaultResolve (internal/modules/esm/resolve.js:*:*) + at Loader.importLoader (internal/modules/esm/loader.js:*:*) + at createLoader (internal/process/esm_loader.js:*:*) + at exports.getLoader (internal/process/esm_loader.js:*:*) + at runMainESM (internal/modules/run_main.js:*:*) + at Function.executeUserEntryPoint [as runMain] (internal/modules/run_main.js:*:*) + at internal/main/run_main_module.js:*:* { code: 'ERR_MODULE_NOT_FOUND' } diff --git a/test/message/esm_loader_not_found_cjs_hint_bare.out b/test/message/esm_loader_not_found_cjs_hint_bare.out index 6063709859573b..3ca0f46c18f767 100644 --- a/test/message/esm_loader_not_found_cjs_hint_bare.out +++ b/test/message/esm_loader_not_found_cjs_hint_bare.out @@ -1,6 +1,6 @@ -internal/process/esm_loader.js:* - internalBinding('errors').triggerUncaughtException( - ^ +internal/process/execution.js:* + internalBinding('errors').triggerUncaughtException(error, fromPromise); + ^ Error [ERR_MODULE_NOT_FOUND]: Cannot find module '*test*fixtures*node_modules*some_module*obj' imported from *test*fixtures*esm_loader_not_found_cjs_hint_bare.mjs Did you mean to import some_module/obj.js? diff --git a/test/message/esm_loader_not_found_cjs_hint_relative.out b/test/message/esm_loader_not_found_cjs_hint_relative.out index 820cb575c4aef1..1a678b9ab50427 100644 --- a/test/message/esm_loader_not_found_cjs_hint_relative.out +++ b/test/message/esm_loader_not_found_cjs_hint_relative.out @@ -1,20 +1,20 @@ (node:*) ExperimentalWarning: --experimental-loader is an experimental feature. This feature could change at any time (Use `node --trace-warnings ...` to show where the warning was created) -internal/process/esm_loader.js:* - internalBinding('errors').triggerUncaughtException( - ^ +internal/process/execution.js:* + internalBinding('errors').triggerUncaughtException(error, fromPromise); + ^ Error [ERR_MODULE_NOT_FOUND]: Cannot find module '*test*common*fixtures' imported from * Did you mean to import ./test/common/fixtures.js? at new NodeError (internal/errors.js:*:*) at finalizeResolution (internal/modules/esm/resolve.js:*:*) at moduleResolve (internal/modules/esm/resolve.js:*:*) - at Loader.defaultResolve [as _resolve] (internal/modules/esm/resolve.js:*:*) - at Loader.resolve (internal/modules/esm/loader.js:*:*) - at Loader.getModuleJob (internal/modules/esm/loader.js:*:*) - at Loader.import (internal/modules/esm/loader.js:*:*) - at internal/process/esm_loader.js:*:* - at initializeLoader (internal/process/esm_loader.js:*:*) - at Object.loadESM (internal/process/esm_loader.js:*:*) { + at defaultResolve (internal/modules/esm/resolve.js:*:*) + at Loader.importLoader (internal/modules/esm/loader.js:*:*) + at createLoader (internal/process/esm_loader.js:*:*) + at exports.getLoader (internal/process/esm_loader.js:*:*) + at runMainESM (internal/modules/run_main.js:*:*) + at Function.executeUserEntryPoint [as runMain] (internal/modules/run_main.js:*:*) + at internal/main/run_main_module.js:*:* { code: 'ERR_MODULE_NOT_FOUND' } diff --git a/test/parallel/test-bootstrap-modules.js b/test/parallel/test-bootstrap-modules.js index 735f2e6394bd20..e6677f1504e5ab 100644 --- a/test/parallel/test-bootstrap-modules.js +++ b/test/parallel/test-bootstrap-modules.js @@ -61,7 +61,6 @@ const expectedModules = new Set([ 'NativeModule internal/modules/esm/module_job', 'NativeModule internal/modules/esm/module_map', 'NativeModule internal/modules/esm/resolve', - 'NativeModule internal/modules/esm/transform_source', 'NativeModule internal/modules/esm/translators', 'NativeModule internal/process/esm_loader', 'NativeModule internal/options',