From 39965cb8f3f4fd828ed3a4cfd04229bc51870bb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20Nison?= Date: Fri, 3 Dec 2021 17:09:10 +0100 Subject: [PATCH 1/2] esm: implement the getFileSystem hook --- demo/foo/index.mjs | 1 + demo/index.mjs | 5 ++ demo/loader-sha512.mjs | 50 ++++++++++++++ lib/internal/modules/esm/get_file_system.js | 69 +++++++++++++++++++ lib/internal/modules/esm/get_source.js | 6 +- lib/internal/modules/esm/loader.js | 74 +++++++++++++++++++++ lib/internal/modules/esm/resolve.js | 54 ++++++++------- lib/internal/process/esm_loader.js | 3 + 8 files changed, 231 insertions(+), 31 deletions(-) create mode 100644 demo/foo/index.mjs create mode 100644 demo/index.mjs create mode 100644 demo/loader-sha512.mjs create mode 100644 lib/internal/modules/esm/get_file_system.js diff --git a/demo/foo/index.mjs b/demo/foo/index.mjs new file mode 100644 index 00000000000000..ac0f28823d44b1 --- /dev/null +++ b/demo/foo/index.mjs @@ -0,0 +1 @@ +console.log(`foo`); diff --git a/demo/index.mjs b/demo/index.mjs new file mode 100644 index 00000000000000..2ac218b4a53d9b --- /dev/null +++ b/demo/index.mjs @@ -0,0 +1,5 @@ +import './foo/index.mjs'; +import hash from './foo/index-sha512.mjs'; + +console.log(`demo`); +console.log(`demo hash:`, hash); diff --git a/demo/loader-sha512.mjs b/demo/loader-sha512.mjs new file mode 100644 index 00000000000000..a0b060b958e21e --- /dev/null +++ b/demo/loader-sha512.mjs @@ -0,0 +1,50 @@ +import crypto from 'crypto'; +import path from 'path'; + +const shaRegExp = /-sha512(\.mjs)$/; + +function getSourcePath(p) { + if (p.protocol !== `file:`) + return p; + + const pString = p.toString(); + const pFixed = pString.replace(shaRegExp, `$1`); + if (pFixed === pString) + return p; + + return new URL(pFixed); +} + +export function getFileSystem(defaultGetFileSystem) { + const fileSystem = defaultGetFileSystem(); + + return { + readFileSync(p) { + const fixedP = getSourcePath(p); + if (fixedP === p) + return fileSystem.readFileSync(p); + + const content = fileSystem.readFileSync(fixedP); + const hash = crypto.createHash(`sha512`).update(content).digest(`hex`); + + return Buffer.from(`export default ${JSON.stringify(hash)};`); + }, + + statEntrySync(p) { + const fixedP = getSourcePath(p); + return fileSystem.statEntrySync(fixedP); + }, + + realpathSync(p) { + const fixedP = getSourcePath(p); + if (fixedP === p) + return fileSystem.realpathSync(p); + + const realpath = fileSystem.realpathSync(fixedP); + if (path.extname(realpath) !== `.mjs`) + throw new Error(`Paths must be .mjs extension to go through the sha512 loader`); + + return realpath.replace(/\.mjs$/, `-sha512.mjs`); + }, + }; +} diff --git a/lib/internal/modules/esm/get_file_system.js b/lib/internal/modules/esm/get_file_system.js new file mode 100644 index 00000000000000..99c7428cffb384 --- /dev/null +++ b/lib/internal/modules/esm/get_file_system.js @@ -0,0 +1,69 @@ +'use strict'; + +const { + FunctionPrototypeBind, + ObjectCreate, + ObjectKeys, + SafeMap, +} = primordials; + +const realpathCache = new SafeMap(); + +const internalFS = require('internal/fs/utils'); +const fs = require('fs'); +const fsPromises = require('internal/fs/promises').exports; +const packageJsonReader = require('internal/modules/package_json_reader'); +const { fileURLToPath } = require('url'); +const { internalModuleStat } = internalBinding('fs'); + +const implementation = { + async readFile(p) { + return fsPromises.readFile(p); + }, + + async statEntry(p) { + return internalModuleStat(fileURLToPath(p)); + }, + + async readJson(p) { + return packageJsonReader.read(fileURLToPath(p)); + }, + + async realpath(p) { + return fsPromises.realpath(p, { + [internalFS.realpathCacheKey]: realpathCache + }); + }, + + readFileSync(p) { + return fs.readFileSync(p); + }, + + statEntrySync(p) { + return internalModuleStat(fileURLToPath(p)); + }, + + readJsonSync(p) { + return packageJsonReader.read(fileURLToPath(p)); + }, + + realpathSync(p) { + return fs.realpathSync(p, { + [internalFS.realpathCacheKey]: realpathCache + }); + }, +}; + +function defaultGetFileSystem(defaultGetFileSystem) { + const copy = ObjectCreate(null); + + const keys = ObjectKeys(implementation); + for (let t = 0; t < keys.length; ++t) + copy[keys[t]] = FunctionPrototypeBind(implementation[keys[t]], null); + + return copy; +} + +module.exports = { + defaultGetFileSystem, +}; diff --git a/lib/internal/modules/esm/get_source.js b/lib/internal/modules/esm/get_source.js index 8281a8e4876aa0..6b990f38cbb0f6 100644 --- a/lib/internal/modules/esm/get_source.js +++ b/lib/internal/modules/esm/get_source.js @@ -5,6 +5,8 @@ const { decodeURIComponent, } = primordials; const { getOptionValue } = require('internal/options'); +const esmLoader = require('internal/process/esm_loader'); + // Do not eagerly grab .manifest, it may be in TDZ const policy = getOptionValue('--experimental-policy') ? require('internal/process/policy') : @@ -12,13 +14,11 @@ const policy = getOptionValue('--experimental-policy') ? const { Buffer } = require('buffer'); -const fs = require('internal/fs/promises').exports; const { URL } = require('internal/url'); const { ERR_INVALID_URL, ERR_INVALID_URL_SCHEME, } = require('internal/errors').codes; -const readFileAsync = fs.readFile; const DATA_URL_PATTERN = /^[^/]+\/[^,;]+(?:[^,]*?)(;base64)?,([\s\S]*)$/; @@ -26,7 +26,7 @@ async function defaultGetSource(url, { format } = {}, defaultGetSource) { const parsed = new URL(url); let source; if (parsed.protocol === 'file:') { - source = await readFileAsync(parsed); + source = await esmLoader.getFileSystem().readFile(parsed); } else if (parsed.protocol === 'data:') { const match = RegExpPrototypeExec(DATA_URL_PATTERN, parsed.pathname); if (!match) { diff --git a/lib/internal/modules/esm/loader.js b/lib/internal/modules/esm/loader.js index 91f570297be341..0995034f7c1a34 100644 --- a/lib/internal/modules/esm/loader.js +++ b/lib/internal/modules/esm/loader.js @@ -6,17 +6,21 @@ require('internal/modules/cjs/loader'); const { Array, ArrayIsArray, + ArrayPrototypeFilter, ArrayPrototypeJoin, ArrayPrototypePush, FunctionPrototypeBind, FunctionPrototypeCall, ObjectAssign, ObjectCreate, + ObjectKeys, + ObjectPrototypeHasOwnProperty, ObjectSetPrototypeOf, PromiseAll, RegExpPrototypeExec, SafeArrayIterator, SafeWeakMap, + StringPrototypeEndsWith, globalThis, } = primordials; const { MessageChannel } = require('internal/worker/io'); @@ -45,6 +49,9 @@ const { initializeImportMeta } = require('internal/modules/esm/initialize_import_meta'); const { defaultLoad } = require('internal/modules/esm/load'); +const { + defaultGetFileSystem +} = require('internal/modules/esm/get_file_system'); const { translators } = require( 'internal/modules/esm/translators'); const { getOptionValue } = require('internal/options'); @@ -81,6 +88,14 @@ class ESMLoader { defaultResolve, ]; + /** + * @private + * @property {Function[]} resolvers First-in-first-out list of resolver hooks + */ + #fileSystemBuilders = [ + defaultGetFileSystem, + ]; + #importMetaInitializer = initializeImportMeta; /** @@ -107,6 +122,7 @@ class ESMLoader { globalPreload, resolve, load, + getFileSystem, // obsolete hooks: dynamicInstantiate, getFormat, @@ -159,10 +175,17 @@ class ESMLoader { if (load) { acceptedHooks.loader = FunctionPrototypeBind(load, null); } + if (getFileSystem) { + acceptedHooks.getFileSystem = FunctionPrototypeBind(getFileSystem, null); + } return acceptedHooks; } + constructor() { + this.buildFileSystem(); + } + /** * Collect custom/user-defined hook(s). After all hooks have been collected, * calls global preload hook(s). @@ -180,6 +203,7 @@ class ESMLoader { globalPreloader, resolver, loader, + getFileSystem, } = ESMLoader.pluckHooks(exports); if (globalPreloader) ArrayPrototypePush( @@ -194,13 +218,63 @@ class ESMLoader { this.#loaders, FunctionPrototypeBind(loader, null), // [1] ); + if (getFileSystem) ArrayPrototypePush( + this.#fileSystemBuilders, + FunctionPrototypeBind(getFileSystem, null), // [1] + ); } // [1] ensure hook function is not bound to ESMLoader instance + this.buildFileSystem(); this.preload(); } + buildFileSystem() { + // Note: makes assumptions as to how chaining will work to demonstrate + // the capability; subject to change once chaining's API is finalized. + const fileSystemFactories = [...this.#fileSystemBuilders]; + + const defaultFileSystemFactory = fileSystemFactories[0]; + let finalFileSystem = + defaultFileSystemFactory(); + + const asyncKeys = ArrayPrototypeFilter( + ObjectKeys(finalFileSystem), + (name) => !StringPrototypeEndsWith(name, 'Sync'), + ); + + for (let i = 1; i < fileSystemFactories.length; ++i) { + const currentFileSystem = finalFileSystem; + const fileSystem = fileSystemFactories[i](() => currentFileSystem); + + // If the loader specifies a sync hook but omits the async one we + // leverage the sync version by default, so that most hook authors + // don't have to write their implementations twice. + for (let j = 0; j < asyncKeys.length; ++j) { + const asyncKey = asyncKeys[j]; + const syncKey = `${asyncKey}Sync`; + + if ( + !ObjectPrototypeHasOwnProperty(fileSystem, asyncKey) && + ObjectPrototypeHasOwnProperty(fileSystem, syncKey) + ) { + fileSystem[asyncKey] = async (...args) => { + return fileSystem[syncKey](...args); + }; + } + } + + finalFileSystem = ObjectAssign( + ObjectCreate(null), + currentFileSystem, + fileSystem, + ); + } + + this.fileSystem = finalFileSystem; + } + async eval( source, url = pathToFileURL(`${process.cwd()}/[eval${++this.evalIndex}]`).href diff --git a/lib/internal/modules/esm/resolve.js b/lib/internal/modules/esm/resolve.js index de8fa349022700..032bda18d87165 100644 --- a/lib/internal/modules/esm/resolve.js +++ b/lib/internal/modules/esm/resolve.js @@ -25,13 +25,7 @@ const { StringPrototypeSplit, StringPrototypeStartsWith, } = primordials; -const internalFS = require('internal/fs/utils'); const { NativeModule } = require('internal/bootstrap/loaders'); -const { - realpathSync, - statSync, - Stats, -} = require('fs'); const { getOptionValue } = require('internal/options'); // Do not eagerly grab .manifest, it may be in TDZ const policy = getOptionValue('--experimental-policy') ? @@ -42,6 +36,7 @@ const preserveSymlinks = getOptionValue('--preserve-symlinks'); const preserveSymlinksMain = getOptionValue('--preserve-symlinks-main'); const typeFlag = getOptionValue('--input-type'); const { URL, pathToFileURL, fileURLToPath } = require('internal/url'); +const esmLoader = require('internal/process/esm_loader'); const { ERR_INPUT_TYPE_NOT_ALLOWED, ERR_INVALID_ARG_VALUE, @@ -57,7 +52,6 @@ const { } = require('internal/errors').codes; const { Module: CJSModule } = require('internal/modules/cjs/loader'); -const packageJsonReader = require('internal/modules/package_json_reader'); const userConditions = getOptionValue('--conditions'); const noAddons = getOptionValue('--no-addons'); const addonConditions = noAddons ? [] : ['node-addons']; @@ -149,16 +143,8 @@ function getConditionsSet(conditions) { return DEFAULT_CONDITIONS_SET; } -const realpathCache = new SafeMap(); const packageJSONCache = new SafeMap(); /* string -> PackageConfig */ -/** - * @param {string | URL} path - * @returns {import('fs').Stats} - */ -const tryStatSync = - (path) => statSync(path, { throwIfNoEntry: false }) ?? new Stats(); - /** * @param {string} path * @param {string} specifier @@ -170,7 +156,9 @@ function getPackageConfig(path, specifier, base) { if (existing !== undefined) { return existing; } - const source = packageJsonReader.read(path).string; + + const source = esmLoader.getFileSystem() + .readJsonSync(pathToFileURL(path)).string; if (source === undefined) { const packageConfig = { pjsonPath: path, @@ -257,7 +245,11 @@ function getPackageScopeConfig(resolved) { * @returns {boolean} */ function fileExists(url) { - return statSync(url, { throwIfNoEntry: false })?.isFile() ?? false; + return esmLoader.getFileSystem().statEntrySync( + typeof url === 'string' ? + pathToFileURL(url) : + url + ) === 0; } /** @@ -345,7 +337,8 @@ function resolveDirectoryEntry(search) { const dirPath = fileURLToPath(search); const pkgJsonPath = resolve(dirPath, 'package.json'); if (fileExists(pkgJsonPath)) { - const pkgJson = packageJsonReader.read(pkgJsonPath); + const pkgJson = esmLoader.getFileSystem() + .readJsonSync(pathToFileURL(pkgJsonPath)); if (pkgJson.containsKeys) { const { main } = JSONParse(pkgJson.string); if (main != null) { @@ -384,21 +377,23 @@ function finalizeResolution(resolved, base, preserveSymlinks) { resolved.pathname, fileURLToPath(base), 'module'); } - const stats = tryStatSync(StringPrototypeEndsWith(path, '/') ? - StringPrototypeSlice(path, -1) : path); - if (stats.isDirectory()) { + const stats = esmLoader.getFileSystem().statEntrySync( + StringPrototypeEndsWith(path, '/') ? + pathToFileURL(StringPrototypeSlice(path, -1)) : + resolved + ); + + if (stats === 1) { const err = new ERR_UNSUPPORTED_DIR_IMPORT(path, fileURLToPath(base)); err.url = String(resolved); throw err; - } else if (!stats.isFile()) { + } else if (stats < 0) { throw new ERR_MODULE_NOT_FOUND( path || resolved.pathname, base && fileURLToPath(base), 'module'); } if (!preserveSymlinks) { - const real = realpathSync(path, { - [internalFS.realpathCacheKey]: realpathCache - }); + const real = esmLoader.getFileSystem().realpathSync(resolved); const { search, hash } = resolved; resolved = pathToFileURL(real + (StringPrototypeEndsWith(path, sep) ? '/' : '')); @@ -836,9 +831,12 @@ function packageResolve(specifier, base, conditions) { let packageJSONPath = fileURLToPath(packageJSONUrl); let lastPath; do { - const stat = tryStatSync(StringPrototypeSlice(packageJSONPath, 0, - packageJSONPath.length - 13)); - if (!stat.isDirectory()) { + const stat = esmLoader.fileSystem.statEntrySync( + pathToFileURL( + StringPrototypeSlice(packageJSONPath, 0, packageJSONPath.length - 13) + ) + ); + if (stat !== 1) { lastPath = packageJSONPath; packageJSONUrl = new URL((isScoped ? '../../../../node_modules/' : '../../../node_modules/') + diff --git a/lib/internal/process/esm_loader.js b/lib/internal/process/esm_loader.js index 73385a85b4e106..3a605a98a45be2 100644 --- a/lib/internal/process/esm_loader.js +++ b/lib/internal/process/esm_loader.js @@ -43,6 +43,9 @@ const esmLoader = new ESMLoader(); exports.esmLoader = esmLoader; +exports.getFileSystem = () => + esmLoader.fileSystem; + /** * Causes side-effects: user-defined loader hooks are added to esmLoader. * @returns {void} From bd6b20b1212ed48653742e4a3c32525176b0d2fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20Nison?= Date: Mon, 13 Dec 2021 15:45:54 +0100 Subject: [PATCH 2/2] Feedback pass --- lib/internal/errors.js | 1 + lib/internal/modules/esm/get_file_system.js | 13 +++---------- lib/internal/modules/esm/loader.js | 19 +++++++++++++------ lib/internal/modules/esm/resolve.js | 4 ++-- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/lib/internal/errors.js b/lib/internal/errors.js index 309f6bb3105341..a0ddd918618ded 100644 --- a/lib/internal/errors.js +++ b/lib/internal/errors.js @@ -1342,6 +1342,7 @@ E('ERR_IPC_CHANNEL_CLOSED', 'Channel closed', Error); E('ERR_IPC_DISCONNECTED', 'IPC channel is already disconnected', Error); E('ERR_IPC_ONE_PIPE', 'Child process can have only one IPC pipe', Error); E('ERR_IPC_SYNC_FORK', 'IPC cannot be used with synchronous forks', Error); +E('ERR_LOADER_MISSING_SYNC_FS', 'Missing synchronous filesystem implementation of a loader', Error); E('ERR_MANIFEST_ASSERT_INTEGRITY', (moduleURL, realIntegrities) => { let msg = `The content of "${ diff --git a/lib/internal/modules/esm/get_file_system.js b/lib/internal/modules/esm/get_file_system.js index 99c7428cffb384..2dce0da2ad15f7 100644 --- a/lib/internal/modules/esm/get_file_system.js +++ b/lib/internal/modules/esm/get_file_system.js @@ -1,9 +1,8 @@ 'use strict'; const { - FunctionPrototypeBind, + ObjectAssign, ObjectCreate, - ObjectKeys, SafeMap, } = primordials; @@ -16,7 +15,7 @@ const packageJsonReader = require('internal/modules/package_json_reader'); const { fileURLToPath } = require('url'); const { internalModuleStat } = internalBinding('fs'); -const implementation = { +const defaultFileSystem = { async readFile(p) { return fsPromises.readFile(p); }, @@ -55,13 +54,7 @@ const implementation = { }; function defaultGetFileSystem(defaultGetFileSystem) { - const copy = ObjectCreate(null); - - const keys = ObjectKeys(implementation); - for (let t = 0; t < keys.length; ++t) - copy[keys[t]] = FunctionPrototypeBind(implementation[keys[t]], null); - - return copy; + return ObjectAssign(ObjectCreate(null), defaultFileSystem); } module.exports = { diff --git a/lib/internal/modules/esm/loader.js b/lib/internal/modules/esm/loader.js index 0995034f7c1a34..05262857f1fe6b 100644 --- a/lib/internal/modules/esm/loader.js +++ b/lib/internal/modules/esm/loader.js @@ -9,6 +9,7 @@ const { ArrayPrototypeFilter, ArrayPrototypeJoin, ArrayPrototypePush, + ArrayPrototypeSlice, FunctionPrototypeBind, FunctionPrototypeCall, ObjectAssign, @@ -31,7 +32,8 @@ const { ERR_INVALID_MODULE_SPECIFIER, ERR_INVALID_RETURN_PROPERTY_VALUE, ERR_INVALID_RETURN_VALUE, - ERR_UNKNOWN_MODULE_FORMAT + ERR_UNKNOWN_MODULE_FORMAT, + ERR_LOADER_MISSING_SYNC_FS } = require('internal/errors').codes; const { pathToFileURL, isURLInstance } = require('internal/url'); const { @@ -90,7 +92,7 @@ class ESMLoader { /** * @private - * @property {Function[]} resolvers First-in-first-out list of resolver hooks + * @property {Function[]} fileSystemBuilders First-in-first-out list of file system utilities compositors */ #fileSystemBuilders = [ defaultGetFileSystem, @@ -119,10 +121,10 @@ class ESMLoader { translators = translators; static pluckHooks({ + getFileSystem, globalPreload, resolve, load, - getFileSystem, // obsolete hooks: dynamicInstantiate, getFormat, @@ -233,7 +235,7 @@ class ESMLoader { buildFileSystem() { // Note: makes assumptions as to how chaining will work to demonstrate // the capability; subject to change once chaining's API is finalized. - const fileSystemFactories = [...this.#fileSystemBuilders]; + const fileSystemFactories = ArrayPrototypeSlice(this.#fileSystemBuilders); const defaultFileSystemFactory = fileSystemFactories[0]; let finalFileSystem = @@ -255,13 +257,18 @@ class ESMLoader { const asyncKey = asyncKeys[j]; const syncKey = `${asyncKey}Sync`; + const hasAsync = ObjectPrototypeHasOwnProperty(fileSystem, asyncKey); + const hasSync = ObjectPrototypeHasOwnProperty(fileSystem, syncKey); + if ( - !ObjectPrototypeHasOwnProperty(fileSystem, asyncKey) && - ObjectPrototypeHasOwnProperty(fileSystem, syncKey) + !hasAsync && + hasSync ) { fileSystem[asyncKey] = async (...args) => { return fileSystem[syncKey](...args); }; + } else if (!hasSync && hasAsync) { + throw new ERR_LOADER_MISSING_SYNC_FS(); } } diff --git a/lib/internal/modules/esm/resolve.js b/lib/internal/modules/esm/resolve.js index 032bda18d87165..d6963759268f54 100644 --- a/lib/internal/modules/esm/resolve.js +++ b/lib/internal/modules/esm/resolve.js @@ -158,7 +158,7 @@ function getPackageConfig(path, specifier, base) { } const source = esmLoader.getFileSystem() - .readJsonSync(pathToFileURL(path)).string; + .readPackageJsonSync(pathToFileURL(path)).string; if (source === undefined) { const packageConfig = { pjsonPath: path, @@ -338,7 +338,7 @@ function resolveDirectoryEntry(search) { const pkgJsonPath = resolve(dirPath, 'package.json'); if (fileExists(pkgJsonPath)) { const pkgJson = esmLoader.getFileSystem() - .readJsonSync(pathToFileURL(pkgJsonPath)); + .readPackageJsonSync(pathToFileURL(pkgJsonPath)); if (pkgJson.containsKeys) { const { main } = JSONParse(pkgJson.string); if (main != null) {