feat: use node's native import() to load modules

target is now node16, which means we still transpile our code to cjs (no "type": "module" yet). we now retain the dynamic import() calls as-is, rather than transpiling them to require() calls. this will make it easier for user projects to move to native esm.
wixplosives · Aug 15, 2024 · 516ed0c · 516ed0c
1 parent 5d0b0f2
commit 516ed0c
Show file tree

Hide file tree

Showing 18 changed files with 99 additions and 40 deletions.
diff --git a/packages/engine-cli/src/cli.ts b/packages/engine-cli/src/cli.ts
@@ -1,5 +1,4 @@
 import { cli, command } from 'cleye';
-import { analyzeCommand } from './analyze-command';
 import type { EngineConfig } from '@wixc3/engine-scripts';
 import { generateFeature } from './feature-generator';
 import fs from '@file-services/node';
@@ -179,6 +178,7 @@ async function engine() {
 
     const rootDir = process.cwd();
     if (argv.command === 'analyze') {
+        const { analyzeCommand } = await import('./analyze-command.js');
         await analyzeCommand({ rootDir, feature: argv.flags.feature, engineConfig });
     } else if (argv.command === 'generate') {
         const featureName = argv._.featureName;

diff --git a/packages/engine-cli/src/load-config-file.ts b/packages/engine-cli/src/load-config-file.ts
@@ -1,9 +1,9 @@
+import { getOriginalModule } from '@wixc3/engine-runtime-node';
 import { pathToFileURL } from 'node:url';
-import { dynamicImport } from '@wixc3/engine-runtime-node';
 
 export async function loadConfigFile(filePath: string): Promise<object> {
     try {
-        const configModuleValue = (await dynamicImport(pathToFileURL(filePath))).default;
+        const configModuleValue = getOriginalModule(await import(pathToFileURL(filePath).href));
         const config = (configModuleValue as { default: unknown }).default ?? configModuleValue;
         if (!config || typeof config !== 'object') {
             throw new Error(`config file: ${filePath} must export an object`);

diff --git a/packages/engine-cli/src/start-dev-server.ts b/packages/engine-cli/src/start-dev-server.ts
@@ -1,6 +1,7 @@
 import cors from 'cors';
 import { safeListeningHttpServer } from 'create-listening-server';
 import express from 'express';
+import type http from 'node:http';
 import io from 'socket.io';
 
 const noContentHandler: express.RequestHandler = (_req, res) => {
@@ -27,7 +28,7 @@ export async function launchServer({
     close: () => Promise<void>;
     port: number;
     app: express.Express;
-    httpServer: import('http').Server;
+    httpServer: http.Server;
     socketServer: io.Server;
 }> {
     const app = express();

diff --git a/packages/engineer/src/cli-commands.ts b/packages/engineer/src/cli-commands.ts
@@ -9,6 +9,7 @@ import { parseCliArguments } from '@wixc3/engine-runtime-node';
 import { Application } from '@wixc3/engine-scripts/dist/application/index.js';
 import type { Command } from 'commander';
 import { resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
 import open from 'open';
 import type { ServerListeningHandler } from './feature/dev-server.types.js';
 import { startDevServer } from './utils.js';
@@ -265,7 +266,7 @@ async function preRequire(pathsToRequire: string[], basePath: string) {
         if (!resolvedFile) {
             throw new Error(`Cannot resolve "${request}"`);
         }
-        await import(resolvedFile);
+        await import(pathToFileURL(resolvedFile).href);
     }
 }
 

diff --git a/packages/engineer/src/feature/gui.dev-server.env.ts b/packages/engineer/src/feature/gui.dev-server.env.ts
@@ -1,8 +1,9 @@
 import { nodeFs as fs } from '@file-services/node';
-import type { IConfigDefinition } from '@wixc3/engine-runtime-node';
+import { getOriginalModule, type IConfigDefinition } from '@wixc3/engine-runtime-node';
 import { createMainEntrypoint, createVirtualEntries } from '@wixc3/engine-scripts';
 import { SetMultiMap } from '@wixc3/patterns';
 import HtmlWebpackPlugin from 'html-webpack-plugin';
+import { pathToFileURL } from 'node:url';
 import type webpack from 'webpack';
 import { devServerEnv } from './dev-server.feature.js';
 import guiFeature, { mainDashboardEnv } from './gui.feature.js';
@@ -25,7 +26,11 @@ guiFeature.setup(
             const baseConfigPath = fs.findClosestFileSync(selfDirectoryPath, 'webpack.config.js');
             const baseConfig =
                 typeof baseConfigPath === 'string'
-                    ? ((await import(baseConfigPath)) as { default: webpack.Configuration }).default
+                    ? (
+                          getOriginalModule(await import(pathToFileURL(baseConfigPath).href)) as {
+                              default: webpack.Configuration;
+                          }
+                      ).default
                     : {};
             const virtualModules: Record<string, string> = {};
 

diff --git a/packages/engineer/src/utils.ts b/packages/engineer/src/utils.ts
@@ -3,6 +3,7 @@ import { defaults } from '@wixc3/common';
 import { BaseHost, RuntimeEngine, RuntimeFeature } from '@wixc3/engine-core';
 import { runNodeEnvironment } from '@wixc3/engine-runtime-node';
 import { isFeatureFile, loadFeaturesFromPaths, type EngineConfig } from '@wixc3/engine-scripts';
+import { pathToFileURL } from 'node:url';
 import { TargetApplication } from './application-proxy-service.js';
 import devServerFeature, { devServerEnv } from './feature/dev-server.feature.js';
 import type { DevServerConfig } from './feature/dev-server.types.js';
@@ -91,6 +92,6 @@ function asDevConfig(options: DStartOptions, engineConfig: DEngineConfig): Parti
 async function preRequire(pathsToRequire: string[], basePath: string) {
     for (const request of pathsToRequire) {
         const resolvedRequest = require.resolve(request, { paths: [basePath] });
-        await import(resolvedRequest);
+        await import(pathToFileURL(resolvedRequest).href);
     }
 }
diff --git a/packages/runtime-node/src/dynamic-import.ts b/packages/runtime-node/src/dynamic-import.ts
diff --git a/packages/runtime-node/src/import-modules.ts b/packages/runtime-node/src/import-modules.ts
@@ -1,5 +1,4 @@
 import { pathToFileURL } from 'node:url';
-import { dynamicImport } from './dynamic-import';
 
 /**
  * Dynamically imports required modules using the specified base path.
@@ -10,7 +9,7 @@ import { dynamicImport } from './dynamic-import';
 export async function importModules(basePath: string, requiredModules: string[]): Promise<void> {
     for (const requiredModule of requiredModules) {
         try {
-            await dynamicImport(pathToFileURL(require.resolve(requiredModule, { paths: [basePath] })));
+            await import(pathToFileURL(require.resolve(requiredModule, { paths: [basePath] })).href);
         } catch (ex) {
             throw new Error(`failed importing: ${requiredModule}`, { cause: ex });
         }

diff --git a/packages/runtime-node/src/index.ts b/packages/runtime-node/src/index.ts
@@ -4,13 +4,16 @@ export * from './core-node/ipc-host.js';
 export * from './core-node/local-env-inititializer.js';
 export * from './core-node/parent-port-host.js';
 export * from './core-node/ws-node-host.js';
-export * from './dynamic-import.js';
 export * from './environments.js';
 export * from './forked-process.js';
 export * from './import-modules.js';
 export * from './ipc-environment.js';
 export * from './launch-http-server.js';
 export * from './load-top-level-config.js';
+export * from './metadata-files.js';
+export * from './metrics-utils.js';
+export * from './micro-rpc.js';
+export * from './module-interop.js';
 export * from './node-env-manager.js';
 export * from './node-environment.js';
 export * from './node-environments-manager.js';
@@ -20,6 +23,3 @@ export * from './types.js';
 export * from './worker-thread-initializer.js';
 export * from './worker-thread-initializer2.js';
 export * from './ws-environment.js';
-export * from './metrics-utils.js';
-export * from './micro-rpc.js';
-export * from './metadata-files.js';
diff --git a/packages/runtime-node/src/load-top-level-config.ts b/packages/runtime-node/src/load-top-level-config.ts
@@ -1,5 +1,7 @@
+import type { ConfigModule, TopLevelConfig } from '@wixc3/engine-core';
 import type { SetMultiMap } from '@wixc3/patterns';
-import type { TopLevelConfig } from '@wixc3/engine-core';
+import { pathToFileURL } from 'node:url';
+import { getOriginalModule } from './module-interop';
 import type { IConfigDefinition } from './types';
 
 export async function loadTopLevelConfigs(
@@ -24,11 +26,19 @@ export async function loadTopLevelConfigs(
                     if (envName) {
                         if (!definition.envName || definition.envName === envName) {
                             config.push(
-                                ...((await import(definition.filePath)) as { default: TopLevelConfig }).default,
+                                ...(
+                                    getOriginalModule(
+                                        await import(pathToFileURL(definition.filePath).href),
+                                    ) as ConfigModule
+                                ).default,
                             );
                         }
                     } else {
-                        config.push(...((await import(definition.filePath)) as { default: TopLevelConfig }).default);
+                        config.push(
+                            ...(
+                                getOriginalModule(await import(pathToFileURL(definition.filePath).href)) as ConfigModule
+                            ).default,
+                        );
                     }
                 } catch (e) {
                     throw new Error(`failed importing: ${definition.filePath}`, { cause: e });

diff --git a/packages/runtime-node/src/module-interop.ts b/packages/runtime-node/src/module-interop.ts
@@ -0,0 +1,18 @@
+/**
+ * When using native dynamic `import()` to evaluate a esm-as-cjs module,
+ * node messes up the module's shape, and the default export is nested under `default` property.
+ * This function is used to get the original module shape.
+ */
+export function getOriginalModule(moduleExports: unknown): unknown {
+    return isObjectLike(moduleExports) &&
+        'default' in moduleExports &&
+        isObjectLike(moduleExports.default) &&
+        '__esModule' in moduleExports.default &&
+        !!moduleExports.default.__esModule
+        ? moduleExports.default
+        : moduleExports;
+}
+
+function isObjectLike(value: unknown): value is object {
+    return typeof value === 'object' && value !== null;
+}
diff --git a/packages/runtime-node/src/node-environment.ts b/packages/runtime-node/src/node-environment.ts
@@ -8,6 +8,8 @@ import {
     type IFeatureLoader,
     type IPreloadModule,
 } from '@wixc3/engine-core';
+import { pathToFileURL } from 'node:url';
+import { getOriginalModule } from './module-interop.js';
 import type { IEnvironmentDescriptor, IStaticFeatureDefinition, StartEnvironmentOptions } from './types.js';
 
 export async function runNodeEnvironment<ENV extends AnyEnvironment>({
@@ -92,15 +94,19 @@ export function createFeatureLoaders(
                     const contextPreloadFilePath = preloadFilePaths[`${env.env}/${childEnvName}`];
 
                     if (contextPreloadFilePath) {
-                        const preloadedContextModule = (await import(contextPreloadFilePath)) as IPreloadModule;
+                        const preloadedContextModule = getOriginalModule(
+                            await import(pathToFileURL(contextPreloadFilePath).href),
+                        ) as IPreloadModule;
                         if (preloadedContextModule.init) {
                             initFunctions.push(preloadedContextModule.init);
                         }
                     }
                 }
                 const preloadFilePath = preloadFilePaths[env.env];
                 if (preloadFilePath) {
-                    const preloadedModule = (await import(preloadFilePath)) as IPreloadModule;
+                    const preloadedModule = getOriginalModule(
+                        await import(pathToFileURL(preloadFilePath).href),
+                    ) as IPreloadModule;
                     if (preloadedModule.init) {
                         initFunctions.push(preloadedModule.init);
                     }
@@ -111,16 +117,17 @@ export function createFeatureLoaders(
                 if (childEnvName && currentContext[env.env] === childEnvName) {
                     const contextFilePath = contextFilePaths[`${env.env}/${childEnvName}`];
                     if (contextFilePath) {
-                        await import(contextFilePath);
+                        await import(pathToFileURL(contextFilePath).href);
                     }
                 }
                 for (const { env: envName } of new Set([env, ...env.dependencies])) {
                     const envFilePath = envFilePaths[envName];
                     if (envFilePath) {
-                        await import(envFilePath);
+                        await import(pathToFileURL(envFilePath).href);
                     }
                 }
-                return ((await import(filePath)) as { default: FeatureClass }).default;
+                return (getOriginalModule(await import(pathToFileURL(filePath).href)) as { default: FeatureClass })
+                    .default;
             },
             depFeatures: dependencies,
             resolvedContexts,

diff --git a/packages/scripts/src/analyze-feature/load-features-from-paths.ts b/packages/scripts/src/analyze-feature/load-features-from-paths.ts
@@ -1,9 +1,10 @@
 import type { IFileSystemSync } from '@file-services/types';
 import { concat, getValue, isPlainObject, map } from '@wixc3/common';
 import type { FeatureClass } from '@wixc3/engine-core';
-import type { IConfigDefinition } from '@wixc3/engine-runtime-node';
+import { getOriginalModule, type IConfigDefinition } from '@wixc3/engine-runtime-node';
 import { SetMultiMap } from '@wixc3/patterns';
 import type { INpmPackage } from '@wixc3/resolve-directory-context';
+import { pathToFileURL } from 'node:url';
 import {
     isFeatureFile,
     parseConfigFileName,
@@ -119,7 +120,7 @@ function setEnvPath(
 }
 
 async function analyzeFeature(filePath: string, featurePackage: IPackageDescriptor): Promise<AnalyzedFeatureModule> {
-    const moduleExports = await import(filePath);
+    const moduleExports = getOriginalModule(await import(pathToFileURL(filePath).href));
     const module = analyzeFeatureModule(filePath, moduleExports);
     const scopedName = scopeToPackage(featurePackage.simplifiedName, module.name);
     return {

diff --git a/packages/scripts/src/application/application.ts b/packages/scripts/src/application/application.ts
@@ -2,6 +2,7 @@ import { nodeFs as fs } from '@file-services/node';
 import { defaults } from '@wixc3/common';
 import { type TopLevelConfig } from '@wixc3/engine-core';
 import {
+    getOriginalModule,
     launchEngineHttpServer,
     NodeEnvironmentsManager,
     resolveEnvironments,
@@ -10,6 +11,7 @@ import {
 } from '@wixc3/engine-runtime-node';
 import { createDisposables, SetMultiMap } from '@wixc3/patterns';
 import express from 'express';
+import { pathToFileURL } from 'node:url';
 import webpack from 'webpack';
 import { analyzeFeatures } from '../analyze-feature';
 import { ENGINE_CONFIG_FILE_NAME } from '../build-constants';
@@ -222,7 +224,11 @@ export class Application {
         if (engineConfigFilePath) {
             try {
                 return {
-                    config: ((await import(engineConfigFilePath)) as { default: EngineConfig }).default,
+                    config: (
+                        getOriginalModule(await import(pathToFileURL(engineConfigFilePath).href)) as {
+                            default: EngineConfig;
+                        }
+                    ).default,
                     path: engineConfigFilePath,
                 };
             } catch (ex) {
@@ -239,7 +245,7 @@ export class Application {
     protected async importModules(requiredModules: string[]) {
         for (const requiredModule of requiredModules) {
             try {
-                await import(require.resolve(requiredModule, { paths: [this.basePath] }));
+                await import(pathToFileURL(require.resolve(requiredModule, { paths: [this.basePath] })).href);
             } catch (ex) {
                 throw new Error(`failed importing: ${requiredModule}`, { cause: ex });
             }
@@ -409,7 +415,11 @@ export class Application {
             : fs.findClosestFileSync(basePath, 'webpack.config.js');
         const baseConfig =
             typeof baseConfigPath === 'string'
-                ? ((await import(baseConfigPath)) as { default: webpack.Configuration }).default
+                ? (
+                      getOriginalModule(await import(pathToFileURL(baseConfigPath).href)) as {
+                          default: webpack.Configuration;
+                      }
+                  ).default
                 : {};
         const webpackConfigs = createWebpackConfigs({
             baseConfig,

diff --git a/packages/scripts/src/import-fresh.ts b/packages/scripts/src/import-fresh.ts
@@ -1,4 +1,6 @@
+import { getOriginalModule } from '@wixc3/engine-runtime-node';
 import { once } from 'node:events';
+import { pathToFileURL } from 'node:url';
 import { Worker, isMainThread, parentPort, workerData } from 'node:worker_threads';
 
 /**
@@ -28,7 +30,7 @@ if (!isMainThread && isImportWorkerData(workerData)) {
     if (Array.isArray(filePath)) {
         const imported: Promise<any>[] = [];
         for (const path of filePath) {
-            imported.push(import(path));
+            imported.push(import(pathToFileURL(path).href).then(getOriginalModule));
         }
         Promise.all(imported)
             .then((moduleExports) => {
@@ -42,8 +44,9 @@ if (!isMainThread && isImportWorkerData(workerData)) {
                 throw e;
             });
     } else {
-        import(filePath)
-            .then((moduleExports) => parentPort?.postMessage(moduleExports[exportSymbolName]))
+        import(pathToFileURL(filePath).href)
+            .then(getOriginalModule)
+            .then((moduleExports: any) => parentPort?.postMessage(moduleExports[exportSymbolName]))
             .catch((e) => {
                 throw e;
             });

diff --git a/packages/scripts/src/resolve-exec-argv.ts b/packages/scripts/src/resolve-exec-argv.ts
@@ -1,10 +1,16 @@
 import { nodeFs as fs } from '@file-services/node';
-import type { EngineConfig } from './types';
+import { getOriginalModule } from '@wixc3/engine-runtime-node';
+import { pathToFileURL } from 'node:url';
 import { ENGINE_CONFIG_FILE_NAME } from './build-constants';
+import type { EngineConfig } from './types';
 
 export async function resolveExecArgv(basePath: string) {
     const engineConfig = await fs.promises.findClosestFile(basePath, ENGINE_CONFIG_FILE_NAME);
-    const { default: config } = (engineConfig ? await import(engineConfig) : {}) as { default?: EngineConfig };
+    const { default: config } = (
+        engineConfig ? getOriginalModule(await import(pathToFileURL(engineConfig).href)) : {}
+    ) as {
+        default?: EngineConfig;
+    };
 
     const execArgv = [...process.execArgv];
     if (config?.require) {

diff --git a/packages/scripts/src/run-environment.ts b/packages/scripts/src/run-environment.ts
@@ -15,10 +15,12 @@ import {
     IStaticFeatureDefinition,
     METADATA_PROVIDER_ENV_ID,
     MetadataCollectionAPI,
+    getOriginalModule,
     loadTopLevelConfigs,
     metadataApiToken,
     runNodeEnvironment,
 } from '@wixc3/engine-runtime-node';
+import { pathToFileURL } from 'node:url';
 import { findFeatures } from './analyze-feature/index.js';
 import { ENGINE_CONFIG_FILE_NAME } from './build-constants.js';
 import { EngineConfig, IFeatureDefinition } from './types.js';
@@ -218,7 +220,7 @@ export async function getRunningFeature<F extends FeatureClass, ENV extends AnyE
 
 async function importWithProperError(filePath: string): Promise<unknown> {
     try {
-        return ((await import(filePath)) as { default: unknown }).default;
+        return (getOriginalModule(await import(pathToFileURL(filePath).href)) as { default: unknown }).default;
     } catch (ex) {
         throw new Error(`failed importing file: ${filePath}`, { cause: ex });
     }