esm: loader chaining

lib/internal/main/repl.js

@@ -9,7 +9,8 @@ const {
 
 const esmLoader = require('internal/process/esm_loader');
 const {
-  evalScript
+  evalScript,
+  uncaughtException,
 } = require('internal/process/execution');
 
 const console = require('internal/console/global');
@@ -33,7 +34,7 @@ if (process.env.NODE_REPL_EXTERNAL_MODULE) {
     process.exit(1);
   }
 
-  esmLoader.loadESM(() => {
+  esmLoader.getLoader().then(() => {
     console.log(`Welcome to Node.js ${process.version}.\n` +
       'Type ".help" for more information.');
 
@@ -61,5 +62,7 @@ if (process.env.NODE_REPL_EXTERNAL_MODULE) {
                  getOptionValue('--inspect-brk'),
                  getOptionValue('--print'));
     }
+  }).catch((e) => {
+    uncaughtException(e, true /* fromPromise */);
   });
 }

lib/internal/modules/cjs/loader.js

@@ -102,6 +102,13 @@ const {
 const { validateString } = require('internal/validators');
 const pendingDeprecation = getOptionValue('--pending-deprecation');
 
+const originalModuleExports = new SafeWeakMap();
+module.exports = {
+  wrapSafe, Module, toRealPath, readPackageScope,
+  originalModuleExports,
+  get hasLoadedAnyUserCJSModule() { return hasLoadedAnyUserCJSModule; }
+};
+
 const {
   CHAR_FORWARD_SLASH,
   CHAR_BACKWARD_SLASH,
@@ -113,8 +120,6 @@ const {
 } = require('internal/util/types');
 
 const asyncESM = require('internal/process/esm_loader');
-const ModuleJob = require('internal/modules/esm/module_job');
-const { ModuleWrap, kInstantiated } = internalBinding('module_wrap');
 const {
   encodedSepRegEx,
   packageInternalResolve
@@ -1119,30 +1124,7 @@ Module.prototype.load = function(filename) {
   Module._extensions[extension](this, filename);
   this.loaded = true;
 
-  const ESMLoader = asyncESM.ESMLoader;
-  const url = `${pathToFileURL(filename)}`;
-  const module = ESMLoader.moduleMap.get(url);
-  // Create module entry at load time to snapshot exports correctly
-  const exports = this.exports;
-  // Called from cjs translator
-  if (module !== undefined && module.module !== undefined) {
-    if (module.module.getStatus() >= kInstantiated)
-      module.module.setExport('default', exports);
-  } else {
-    // Preemptively cache
-    // We use a function to defer promise creation for async hooks.
-    ESMLoader.moduleMap.set(
-      url,
-      // Module job creation will start promises.
-      // We make it a function to lazily trigger those promises
-      // for async hooks compatibility.
-      () => new ModuleJob(ESMLoader, url, () =>
-        new ModuleWrap(url, undefined, ['default'], function() {
-          this.setExport('default', exports);
-        })
-      , false /* isMain */, false /* inspectBrk */)
-    );
-  }
+  originalModuleExports.set(this, this.exports);
 };
 
 
@@ -1176,7 +1158,7 @@ function wrapSafe(filename, content, cjsModuleInstance) {
       lineOffset: 0,
       displayErrors: true,
       importModuleDynamically: async (specifier) => {
-        const loader = asyncESM.ESMLoader;
+        const loader = await asyncESM.getLoader();
         return loader.import(specifier, normalizeReferrerURL(filename));
       },
     });
@@ -1209,7 +1191,7 @@ function wrapSafe(filename, content, cjsModuleInstance) {
   const { callbackMap } = internalBinding('module_wrap');
   callbackMap.set(compiled.cacheKey, {
     importModuleDynamically: async (specifier) => {
-      const loader = asyncESM.ESMLoader;
+      const loader = await asyncESM.getLoader();
       return loader.import(specifier, normalizeReferrerURL(filename));
     }
   });

lib/internal/modules/esm/get_format.js

@@ -33,7 +33,7 @@ if (experimentalWasmModules)
 if (experimentalJsonModules)
   extensionFormatMap['.json'] = legacyExtensionFormatMap['.json'] = 'json';
 
-function defaultGetFormat(url, context, defaultGetFormatUnused) {
+function defaultGetFormat(url, context, nextGetFormat) {
   if (StringPrototypeStartsWith(url, 'nodejs:')) {
     return { format: 'builtin' };
   }

lib/internal/modules/esm/loader.js

@@ -6,15 +6,14 @@ require('internal/modules/cjs/loader');
 const {
   FunctionPrototypeBind,
   ObjectSetPrototypeOf,
-  SafeMap,
 } = primordials;
 
 const {
   ERR_INVALID_ARG_VALUE,
   ERR_INVALID_RETURN_PROPERTY,
   ERR_INVALID_RETURN_PROPERTY_VALUE,
   ERR_INVALID_RETURN_VALUE,
-  ERR_UNKNOWN_MODULE_FORMAT
+  ERR_UNKNOWN_MODULE_FORMAT,
 } = require('internal/errors').codes;
 const { URL, pathToFileURL } = require('internal/url');
 const { validateString } = require('internal/validators');
@@ -28,11 +27,30 @@ const {
 const { defaultGetFormat } = require('internal/modules/esm/get_format');
 const { defaultGetSource } = require(
   'internal/modules/esm/get_source');
-const { defaultTransformSource } = require(
-  'internal/modules/esm/transform_source');
 const { translators } = require(
   'internal/modules/esm/translators');
 const { getOptionValue } = require('internal/options');
+const {
+  isArrayBufferView,
+  isAnyArrayBuffer,
+} = require('internal/util/types');
+
+let cwd; // Initialized in importLoader
+
+function validateSource(source, hookName, allowString) {
+  if (allowString && typeof source === 'string') {
+    return;
+  }
+  if (isArrayBufferView(source) || isAnyArrayBuffer(source)) {
+    return;
+  }
+  throw new ERR_INVALID_RETURN_PROPERTY_VALUE(
+    `${allowString ? 'string, ' : ''}array buffer, or typed array`,
+    hookName,
+    'source',
+    source,
+  );
+}
 
 /* A Loader instance is used as the main entry point for loading ES modules.
  * Currently, this is a singleton -- there is only one used for loading
@@ -46,33 +64,16 @@ class Loader {
     // Registry of loaded modules, akin to `require.cache`
     this.moduleMap = new ModuleMap();
 
-    // Map of already-loaded CJS modules to use
-    this.cjsCache = new SafeMap();
-
-    // This hook is called before the first root module is imported. It's a
-    // function that returns a piece of code that runs as a sloppy-mode script.
-    // The script may evaluate to a function that can be called with a
-    // `getBuiltin` helper that can be used to retrieve builtins.
-    // If the hook returns `null` instead of a source string, it opts out of
-    // running any preload code.
-    // The preload code runs as soon as the hook module has finished evaluating.
-    this._getGlobalPreloadCode = null;
-    // The resolver has the signature
-    //   (specifier : string, parentURL : string, defaultResolve)
-    //       -> Promise<{ url : string }>
-    // where defaultResolve is ModuleRequest.resolve (having the same
-    // signature itself).
+    // Preload code is provided by loaders to be run after hook initialization.
+    this.globalPreloadCode = [];
+    // Loader resolve hook.
     this._resolve = defaultResolve;
-    // This hook is called after the module is resolved but before a translator
-    // is chosen to load it; the format returned by this function is the name
-    // of a translator.
+    // Loader getFormat hook.
     this._getFormat = defaultGetFormat;
-    // This hook is called just before the source code of an ES module file
-    // is loaded.
+    // Loader getSource hook.
     this._getSource = defaultGetSource;
-    // This hook is called just after the source code of an ES module file
-    // is loaded, but before anything is done with the string.
-    this._transformSource = defaultTransformSource;
+    // Transform source hooks.
+    this.transformSourceHooks = [];
     // The index for assigning unique URLs to anonymous module evaluation
     this.evalIndex = 0;
   }
@@ -83,7 +84,7 @@ class Loader {
       validateString(parentURL, 'parentURL');
 
     const resolveResponse = await this._resolve(
-      specifier, { parentURL, conditions: DEFAULT_CONDITIONS }, defaultResolve);
+      specifier, { parentURL, conditions: DEFAULT_CONDITIONS });
     if (typeof resolveResponse !== 'object') {
       throw new ERR_INVALID_RETURN_VALUE(
         'object', 'loader resolve', resolveResponse);
@@ -98,8 +99,7 @@ class Loader {
   }
 
   async getFormat(url) {
-    const getFormatResponse = await this._getFormat(
-      url, {}, defaultGetFormat);
+    const getFormatResponse = await this._getFormat(url, {});
     if (typeof getFormatResponse !== 'object') {
       throw new ERR_INVALID_RETURN_VALUE(
         'object', 'loader getFormat', getFormatResponse);
@@ -137,6 +137,22 @@ class Loader {
     return format;
   }
 
+  async getSource(url, format) {
+    const { source: originalSource } = await this._getSource(url, { format });
+
+    const allowString = format !== 'wasm';
+    validateSource(originalSource, 'getSource', allowString);
+
+    let source = originalSource;
+    for (let i = 0; i < this.transformSourceHooks.length; i += 1) {
+      const hook = this.transformSourceHooks[i];
+      ({ source } = await hook(source, { url, format, originalSource }));
+      validateSource(source, 'transformSource', allowString);
+    }
+
+    return source;
+  }
+
   async eval(
     source,
     url = pathToFileURL(`${process.cwd()}/[eval${++this.evalIndex}]`).href
@@ -166,72 +182,84 @@ class Loader {
     return module.getNamespace();
   }
 
+  async importLoader(specifier) {
+    if (cwd === undefined) {
+      try {
+        // `process.cwd()` can fail.
+        cwd = process.cwd() + '/';
+      } catch {
+        cwd = 'file:///';
+      }
+      cwd = pathToFileURL(cwd).href;
+    }
+
+    const { url } = await defaultResolve(specifier, cwd,
+                                         { conditions: DEFAULT_CONDITIONS });
+    const { format } = await defaultGetFormat(url, {});
+
+    // !!! CRITICAL SECTION !!!
+    // NO AWAIT OPS BETWEEN HERE AND SETTING JOB IN MODULE MAP!
+    // YIELDING CONTROL COULD RESULT IN MAP BEING OVERRIDDEN!
+    let job = this.moduleMap.get(url);
+    if (job === undefined) {
+      if (!translators.has(format))
+        throw new ERR_UNKNOWN_MODULE_FORMAT(format);
+
+      const loaderInstance = translators.get(format);
+
+      job = new ModuleJob(this, url, loaderInstance, false, false);
+      this.moduleMap.set(url, job);
+      // !!! END CRITICAL SECTION !!!
+    }
+
+    const { module } = await job.run();
+    return module.getNamespace();
+  }
+
   hook(hooks) {
     const {
       resolve,
-      dynamicInstantiate,
       getFormat,
       getSource,
-      transformSource,
-      getGlobalPreloadCode,
     } = hooks;
 
     // Use .bind() to avoid giving access to the Loader instance when called.
     if (resolve !== undefined)
       this._resolve = FunctionPrototypeBind(resolve, null);
-    if (dynamicInstantiate !== undefined) {
-      process.emitWarning(
-        'The dynamicInstantiate loader hook has been removed.');
-    }
     if (getFormat !== undefined) {
       this._getFormat = FunctionPrototypeBind(getFormat, null);
     }
     if (getSource !== undefined) {
       this._getSource = FunctionPrototypeBind(getSource, null);
     }
-    if (transformSource !== undefined) {
-      this._transformSource = FunctionPrototypeBind(transformSource, null);
-    }
-    if (getGlobalPreloadCode !== undefined) {
-      this._getGlobalPreloadCode =
-        FunctionPrototypeBind(getGlobalPreloadCode, null);
-    }
   }
 
   runGlobalPreloadCode() {
-    if (!this._getGlobalPreloadCode) {
-      return;
-    }
-    const preloadCode = this._getGlobalPreloadCode();
-    if (preloadCode === null) {
-      return;
-    }
+    for (let i = 0; i < this.globalPreloadCode.length; i += 1) {
+      const preloadCode = this.globalPreloadCode[i];
 
-    if (typeof preloadCode !== 'string') {
-      throw new ERR_INVALID_RETURN_VALUE(
-        'string', 'loader getGlobalPreloadCode', preloadCode);
+      const { compileFunction } = require('vm');
+      const preloadInit = compileFunction(preloadCode, ['getBuiltin'], {
+        filename: '<preload>',
+      });
+      const { NativeModule } = require('internal/bootstrap/loaders');
+
+      preloadInit.call(globalThis, (builtinName) => {
+        if (NativeModule.canBeRequiredByUsers(builtinName)) {
+          return require(builtinName);
+        }
+        throw new ERR_INVALID_ARG_VALUE('builtinName', builtinName);
+      });
     }
-    const { compileFunction } = require('vm');
-    const preloadInit = compileFunction(preloadCode, ['getBuiltin'], {
-      filename: '<preload>',
-    });
-    const { NativeModule } = require('internal/bootstrap/loaders');
-
-    preloadInit.call(globalThis, (builtinName) => {
-      if (NativeModule.canBeRequiredByUsers(builtinName)) {
-        return require(builtinName);
-      }
-      throw new ERR_INVALID_ARG_VALUE('builtinName', builtinName);
-    });
   }
 
   async getModuleJob(specifier, parentURL) {
     const url = await this.resolve(specifier, parentURL);
     const format = await this.getFormat(url);
+
+    // !!! CRITICAL SECTION !!!
+    // NO AWAIT OPS BETWEEN HERE AND SETTING JOB IN MODULE MAP
     let job = this.moduleMap.get(url);
-    // CommonJS will set functions for lazy job evaluation.
-    if (typeof job === 'function')
-      this.moduleMap.set(url, job = job());
     if (job !== undefined)
       return job;
 
@@ -245,6 +273,8 @@ class Loader {
     job = new ModuleJob(this, url, loaderInstance, parentURL === undefined,
                         inspectBrk);
     this.moduleMap.set(url, job);
+    // !!! END CRITICAL SECTION !!!
+
     return job;
   }
 }

lib/internal/modules/esm/resolve.js

@@ -764,8 +764,7 @@ function resolveAsCommonJS(specifier, parentURL) {
   }
 }
 
-function defaultResolve(specifier, context = {}, defaultResolveUnused) {
-  let { parentURL, conditions } = context;
+function defaultResolve(specifier, { parentURL, conditions } = {}) {
   let parsed;
   try {
     parsed = new URL(specifier);