From 862e5f839294f6ada4840abdcc8fc64e25cd87e5 Mon Sep 17 00:00:00 2001 From: Guy Bedford Date: Tue, 15 Dec 2020 11:34:55 -0800 Subject: [PATCH 1/4] Asset subbundle builds (#625) * feat: support sub-bundle asset builds * new relocator loader * Update to latest warl * Pin dep version * add coverage unit test Co-authored-by: Steven --- package.json | 2 +- src/cli.js | 2 + src/index.js | 68 ++++++++-- src/loaders/relocate-loader.js | 2 +- test/index.test.js | 8 +- test/unit/bundle-subasset/input.js | 9 ++ test/unit/bundle-subasset/output-coverage.js | 112 ++++++++++++++++ test/unit/bundle-subasset/output.js | 112 ++++++++++++++++ test/unit/bundle-subasset/pi-bridge.js | 2 + test/unit/bundle-subasset/pi.ts | 3 + test/unit/bundle-subasset/tsconfig.json | 1 + test/unit/bundle-subasset2/input.ts | 11 ++ test/unit/bundle-subasset2/output-coverage.js | 123 ++++++++++++++++++ test/unit/bundle-subasset2/output.js | 123 ++++++++++++++++++ test/unit/bundle-subasset2/pi-bridge.js | 2 + test/unit/bundle-subasset2/pi.ts | 3 + test/unit/bundle-subasset2/tsconfig.json | 1 + yarn.lock | 10 +- 18 files changed, 575 insertions(+), 19 deletions(-) create mode 100644 test/unit/bundle-subasset/input.js create mode 100644 test/unit/bundle-subasset/output-coverage.js create mode 100644 test/unit/bundle-subasset/output.js create mode 100644 test/unit/bundle-subasset/pi-bridge.js create mode 100644 test/unit/bundle-subasset/pi.ts create mode 100644 test/unit/bundle-subasset/tsconfig.json create mode 100644 test/unit/bundle-subasset2/input.ts create mode 100644 test/unit/bundle-subasset2/output-coverage.js create mode 100644 test/unit/bundle-subasset2/output.js create mode 100644 test/unit/bundle-subasset2/pi-bridge.js create mode 100644 test/unit/bundle-subasset2/pi.ts create mode 100644 test/unit/bundle-subasset2/tsconfig.json diff --git a/package.json b/package.json index bd5cc6c3..0734c251 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "@sentry/node": "^4.3.0", "@slack/web-api": "^5.13.0", "@tensorflow/tfjs-node": "^0.3.0", - "@zeit/webpack-asset-relocator-loader": "0.8.0", + "@vercel/webpack-asset-relocator-loader": "0.9.1", "analytics-node": "^3.3.0", "apollo-server-express": "^2.2.2", "arg": "^4.1.0", diff --git a/src/cli.js b/src/cli.js index 92af2787..30d940fd 100755 --- a/src/cli.js +++ b/src/cli.js @@ -138,6 +138,7 @@ async function runCmd (argv, stdout, stderr) { "-s": "--source-map", "--no-cache": Boolean, "-C": "--no-cache", + "--no-asset-builds": Boolean, "--no-source-map-register": Boolean, "--quiet": Boolean, "-q": "--quiet", @@ -235,6 +236,7 @@ async function runCmd (argv, stdout, stderr) { externals: args["--external"], sourceMap: args["--source-map"] || run, sourceMapRegister: args["--no-source-map-register"] ? false : undefined, + noAssetBuilds: args["--no-asset-builds"] ? true : false, cache: args["--no-cache"] ? false : undefined, watch: args["--watch"], v8cache: args["--v8-cache"], diff --git a/src/index.js b/src/index.js index 2a7996dc..a1cba136 100644 --- a/src/index.js +++ b/src/index.js @@ -30,7 +30,8 @@ const defaultPermissions = 0o666; const relocateLoader = eval('require(__dirname + "/loaders/relocate-loader.js")'); -module.exports = ( +module.exports = ncc; +function ncc ( entry, { cache, @@ -40,16 +41,18 @@ module.exports = ( sourceMap = false, sourceMapRegister = true, sourceMapBasePrefix = '../', + noAssetBuilds = false, watch = false, v8cache = false, filterAssetBase = process.cwd(), + existingAssetNames = [], quiet = false, debugLog = false, transpileOnly = false, license = '', target } = {} -) => { +) { process.env.__NCC_OPTS = JSON.stringify({ quiet }); @@ -69,7 +72,7 @@ module.exports = ( const shebangMatch = fs.readFileSync(resolvedEntry).toString().match(shebangRegEx); const mfs = new MemoryFS(); - const existingAssetNames = [filename]; + existingAssetNames.push(filename); if (sourceMap) { existingAssetNames.push(`${filename}.map`); existingAssetNames.push(`sourcemap-register${ext}`); @@ -296,12 +299,12 @@ module.exports = ( watch.inputFileSystem = compiler.inputFileSystem; } let cachedResult; - watcher = compiler.watch({}, (err, stats) => { + watcher = compiler.watch({}, async (err, stats) => { if (err) return watchHandler({ err }); if (stats.hasErrors()) return watchHandler({ err: stats.toString() }); - const returnValue = finalizeHandler(stats); + const returnValue = await finalizeHandler(stats); if (watchHandler) watchHandler(returnValue); else @@ -334,9 +337,9 @@ module.exports = ( }; } - function finalizeHandler (stats) { + async function finalizeHandler (stats) { const assets = Object.create(null); - getFlatFiles(mfs.data, assets, relocateLoader.getAssetPermissions); + getFlatFiles(mfs.data, assets, relocateLoader.getAssetMeta); // filter symlinks to existing assets const symlinks = Object.create(null); for (const [key, value] of Object.entries(relocateLoader.getSymlinks())) { @@ -344,6 +347,7 @@ module.exports = ( if (resolved in assets) symlinks[key] = value; } + // Webpack only emits sourcemaps for .js files // so we need to adjust the .cjs extension handling delete assets[filename + (ext === '.cjs' ? '.js' : '')]; @@ -428,22 +432,64 @@ module.exports = ( map.mappings = ";" + map.mappings; } + // for each .js / .mjs / .cjs file in the asset list, build that file with ncc itself + if (!noAssetBuilds) { + let assetNames = Object.keys(assets); + assetNames.push(`${filename}${ext === '.cjs' ? '.js' : ''}`); + for (const asset of assetNames) { + if (!asset.endsWith('.js') && !asset.endsWith('.cjs') && !asset.endsWith('.ts') && !asset.endsWith('.mjs') || + asset.endsWith('.cache.js') || asset.endsWith('.cache.cjs') || asset.endsWith('.cache.ts') || asset.endsWith('.cache.mjs')) + continue; + const assetMeta = relocateLoader.getAssetMeta(asset); + if (!assetMeta) + continue; + const path = assetMeta.path; + if (path) { + const { code, assets: subbuildAssets, symlinks: subbuildSymlinks, stats: subbuildStats } = await ncc(path, { + cache, + externals, + filename: asset, + minify, + sourceMap, + sourceMapRegister, + sourceMapBasePrefix, + // dont recursively asset build + // could be supported with seen tracking + noAssetBuilds: true, + v8cache, + filterAssetBase, + existingAssetNames: assetNames, + quiet, + debugLog, + transpileOnly, + license, + target + }); + Object.assign(symlinks, subbuildSymlinks); + Object.assign(stats, subbuildStats); + assets[asset] = { source: code, permissions: assetMeta.permissions }; + Object.assign(assets, subbuildAssets); + } + } + } + return { code, map: map ? JSON.stringify(map) : undefined, assets, symlinks, stats }; } -}; +} // this could be rewritten with actual FS apis / globs, but this is simpler -function getFlatFiles(mfsData, output, getAssetPermissions, curBase = "") { +function getFlatFiles(mfsData, output, getAssetMeta, curBase = "") { for (const path of Object.keys(mfsData)) { const item = mfsData[path]; const curPath = `${curBase}/${path}`; // directory - if (item[""] === true) getFlatFiles(item, output, getAssetPermissions, curPath); + if (item[""] === true) getFlatFiles(item, output, getAssetMeta, curPath); // file else if (!curPath.endsWith("/")) { + const meta = getAssetMeta(curPath.substr(1)) || {}; output[curPath.substr(1)] = { source: mfsData[path], - permissions: getAssetPermissions(curPath.substr(1)) + permissions: meta.permissions }; } } diff --git a/src/loaders/relocate-loader.js b/src/loaders/relocate-loader.js index 2b00eb4f..aa158757 100644 --- a/src/loaders/relocate-loader.js +++ b/src/loaders/relocate-loader.js @@ -1 +1 @@ -module.exports = require('@zeit/webpack-asset-relocator-loader'); +module.exports = require('@vercel/webpack-asset-relocator-loader'); diff --git a/test/index.test.js b/test/index.test.js index 3accedbd..55ad1304 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -35,11 +35,17 @@ for (const unitTest of fs.readdirSync(`${__dirname}/unit`)) { // find the name of the input file (e.g input.ts) const inputFile = fs.readdirSync(testDir).find(file => file.includes("input")); await ncc(`${testDir}/${inputFile}`, Object.assign({ + transpileOnly: true, externals: { + 'piscina': 'piscina', 'externaltest': 'externalmapped' } }, opts)).then( async ({ code, assets, map }) => { + if (unitTest.startsWith('bundle-subasset')) { + expect(assets['pi-bridge.js']).toBeDefined(); + expect(assets['pi-bridge.js'].source.toString()).toContain('Math.PI'); + } const actual = code .trim() // Windows support @@ -138,7 +144,7 @@ for (const integrationTest of fs.readdirSync(__dirname + "/integration")) { const stdout = new StoreStream(); const stderr = new StoreStream(); try { - await nccRun(["run", "--no-cache", `${__dirname}/integration/${integrationTest}`], stdout, stderr); + await nccRun(["run", "--no-cache", "--no-asset-builds", `${__dirname}/integration/${integrationTest}`], stdout, stderr); } catch (e) { if (e.silent) { diff --git a/test/unit/bundle-subasset/input.js b/test/unit/bundle-subasset/input.js new file mode 100644 index 00000000..cda6f974 --- /dev/null +++ b/test/unit/bundle-subasset/input.js @@ -0,0 +1,9 @@ +import path from 'path'; + +const file = path.resolve(__dirname, './pi-bridge.js'); + +const obscureRequire = eval(`function obscureRequire (file) { + require(file); +}`); + +console.log(obscureRequire(file)); diff --git a/test/unit/bundle-subasset/output-coverage.js b/test/unit/bundle-subasset/output-coverage.js new file mode 100644 index 00000000..52b6cfe2 --- /dev/null +++ b/test/unit/bundle-subasset/output-coverage.js @@ -0,0 +1,112 @@ +module.exports = +/******/ (() => { // webpackBootstrap +/******/ "use strict"; +/******/ var __webpack_modules__ = ({ + +/***/ 929: +/***/ ((__unused_webpack_module, __webpack_exports__, __webpack_require__) => { + +__webpack_require__.r(__webpack_exports__); +/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(622); +/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_0___default = /*#__PURE__*/__webpack_require__.n(path__WEBPACK_IMPORTED_MODULE_0__); + + +const file = __webpack_require__.ab + "pi-bridge.js"; + +const obscureRequire = eval(`function obscureRequire (file) { + require(file); +}`); + +console.log(obscureRequire(__webpack_require__.ab + "pi-bridge.js")); + + +/***/ }), + +/***/ 622: +/***/ ((module) => { + +module.exports = require("path"); + +/***/ }) + +/******/ }); +/************************************************************************/ +/******/ // The module cache +/******/ var __webpack_module_cache__ = {}; +/******/ +/******/ // The require function +/******/ function __webpack_require__(moduleId) { +/******/ // Check if module is in cache +/******/ if(__webpack_module_cache__[moduleId]) { +/******/ return __webpack_module_cache__[moduleId].exports; +/******/ } +/******/ // Create a new module (and put it into the cache) +/******/ var module = __webpack_module_cache__[moduleId] = { +/******/ // no module.id needed +/******/ // no module.loaded needed +/******/ exports: {} +/******/ }; +/******/ +/******/ // Execute the module function +/******/ var threw = true; +/******/ try { +/******/ __webpack_modules__[moduleId](module, module.exports, __webpack_require__); +/******/ threw = false; +/******/ } finally { +/******/ if(threw) delete __webpack_module_cache__[moduleId]; +/******/ } +/******/ +/******/ // Return the exports of the module +/******/ return module.exports; +/******/ } +/******/ +/************************************************************************/ +/******/ /* webpack/runtime/compat get default export */ +/******/ (() => { +/******/ // getDefaultExport function for compatibility with non-harmony modules +/******/ __webpack_require__.n = (module) => { +/******/ var getter = module && module.__esModule ? +/******/ () => module['default'] : +/******/ () => module; +/******/ __webpack_require__.d(getter, { a: getter }); +/******/ return getter; +/******/ }; +/******/ })(); +/******/ +/******/ /* webpack/runtime/define property getters */ +/******/ (() => { +/******/ // define getter functions for harmony exports +/******/ __webpack_require__.d = (exports, definition) => { +/******/ for(var key in definition) { +/******/ if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) { +/******/ Object.defineProperty(exports, key, { enumerable: true, get: definition[key] }); +/******/ } +/******/ } +/******/ }; +/******/ })(); +/******/ +/******/ /* webpack/runtime/hasOwnProperty shorthand */ +/******/ (() => { +/******/ __webpack_require__.o = (obj, prop) => Object.prototype.hasOwnProperty.call(obj, prop) +/******/ })(); +/******/ +/******/ /* webpack/runtime/make namespace object */ +/******/ (() => { +/******/ // define __esModule on exports +/******/ __webpack_require__.r = (exports) => { +/******/ if(typeof Symbol !== 'undefined' && Symbol.toStringTag) { +/******/ Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' }); +/******/ } +/******/ Object.defineProperty(exports, '__esModule', { value: true }); +/******/ }; +/******/ })(); +/******/ +/******/ /* webpack/runtime/compat */ +/******/ +/******/ __webpack_require__.ab = __dirname + "/";/************************************************************************/ +/******/ // module exports must be returned from runtime so entry inlining is disabled +/******/ // startup +/******/ // Load entry module and return exports +/******/ return __webpack_require__(929); +/******/ })() +; \ No newline at end of file diff --git a/test/unit/bundle-subasset/output.js b/test/unit/bundle-subasset/output.js new file mode 100644 index 00000000..83b0f0f8 --- /dev/null +++ b/test/unit/bundle-subasset/output.js @@ -0,0 +1,112 @@ +module.exports = +/******/ (() => { // webpackBootstrap +/******/ "use strict"; +/******/ var __webpack_modules__ = ({ + +/***/ 743: +/***/ ((__unused_webpack_module, __webpack_exports__, __webpack_require__) => { + +__webpack_require__.r(__webpack_exports__); +/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(622); +/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_0___default = /*#__PURE__*/__webpack_require__.n(path__WEBPACK_IMPORTED_MODULE_0__); + + +const file = __webpack_require__.ab + "pi-bridge.js"; + +const obscureRequire = eval(`function obscureRequire (file) { + require(file); +}`); + +console.log(obscureRequire(__webpack_require__.ab + "pi-bridge.js")); + + +/***/ }), + +/***/ 622: +/***/ ((module) => { + +module.exports = require("path"); + +/***/ }) + +/******/ }); +/************************************************************************/ +/******/ // The module cache +/******/ var __webpack_module_cache__ = {}; +/******/ +/******/ // The require function +/******/ function __webpack_require__(moduleId) { +/******/ // Check if module is in cache +/******/ if(__webpack_module_cache__[moduleId]) { +/******/ return __webpack_module_cache__[moduleId].exports; +/******/ } +/******/ // Create a new module (and put it into the cache) +/******/ var module = __webpack_module_cache__[moduleId] = { +/******/ // no module.id needed +/******/ // no module.loaded needed +/******/ exports: {} +/******/ }; +/******/ +/******/ // Execute the module function +/******/ var threw = true; +/******/ try { +/******/ __webpack_modules__[moduleId](module, module.exports, __webpack_require__); +/******/ threw = false; +/******/ } finally { +/******/ if(threw) delete __webpack_module_cache__[moduleId]; +/******/ } +/******/ +/******/ // Return the exports of the module +/******/ return module.exports; +/******/ } +/******/ +/************************************************************************/ +/******/ /* webpack/runtime/compat get default export */ +/******/ (() => { +/******/ // getDefaultExport function for compatibility with non-harmony modules +/******/ __webpack_require__.n = (module) => { +/******/ var getter = module && module.__esModule ? +/******/ () => module['default'] : +/******/ () => module; +/******/ __webpack_require__.d(getter, { a: getter }); +/******/ return getter; +/******/ }; +/******/ })(); +/******/ +/******/ /* webpack/runtime/define property getters */ +/******/ (() => { +/******/ // define getter functions for harmony exports +/******/ __webpack_require__.d = (exports, definition) => { +/******/ for(var key in definition) { +/******/ if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) { +/******/ Object.defineProperty(exports, key, { enumerable: true, get: definition[key] }); +/******/ } +/******/ } +/******/ }; +/******/ })(); +/******/ +/******/ /* webpack/runtime/hasOwnProperty shorthand */ +/******/ (() => { +/******/ __webpack_require__.o = (obj, prop) => Object.prototype.hasOwnProperty.call(obj, prop) +/******/ })(); +/******/ +/******/ /* webpack/runtime/make namespace object */ +/******/ (() => { +/******/ // define __esModule on exports +/******/ __webpack_require__.r = (exports) => { +/******/ if(typeof Symbol !== 'undefined' && Symbol.toStringTag) { +/******/ Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' }); +/******/ } +/******/ Object.defineProperty(exports, '__esModule', { value: true }); +/******/ }; +/******/ })(); +/******/ +/******/ /* webpack/runtime/compat */ +/******/ +/******/ __webpack_require__.ab = __dirname + "/";/************************************************************************/ +/******/ // module exports must be returned from runtime so entry inlining is disabled +/******/ // startup +/******/ // Load entry module and return exports +/******/ return __webpack_require__(743); +/******/ })() +; \ No newline at end of file diff --git a/test/unit/bundle-subasset/pi-bridge.js b/test/unit/bundle-subasset/pi-bridge.js new file mode 100644 index 00000000..ca7cf426 --- /dev/null +++ b/test/unit/bundle-subasset/pi-bridge.js @@ -0,0 +1,2 @@ +import Pi from './pi'; +export default Pi; diff --git a/test/unit/bundle-subasset/pi.ts b/test/unit/bundle-subasset/pi.ts new file mode 100644 index 00000000..7004205c --- /dev/null +++ b/test/unit/bundle-subasset/pi.ts @@ -0,0 +1,3 @@ +export default function Pi(num: number) { + return Math.PI * num; +} diff --git a/test/unit/bundle-subasset/tsconfig.json b/test/unit/bundle-subasset/tsconfig.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/test/unit/bundle-subasset/tsconfig.json @@ -0,0 +1 @@ +{} diff --git a/test/unit/bundle-subasset2/input.ts b/test/unit/bundle-subasset2/input.ts new file mode 100644 index 00000000..4d2f4b45 --- /dev/null +++ b/test/unit/bundle-subasset2/input.ts @@ -0,0 +1,11 @@ +import Piscina from 'piscina'; +import * as path from 'path'; + +const piscina = new Piscina({ + filename: path.resolve(__dirname, './pi-bridge.js'), +}); + +(async function() { + const result = await piscina.runTask(2); + console.log(result); +})(); diff --git a/test/unit/bundle-subasset2/output-coverage.js b/test/unit/bundle-subasset2/output-coverage.js new file mode 100644 index 00000000..802caca0 --- /dev/null +++ b/test/unit/bundle-subasset2/output-coverage.js @@ -0,0 +1,123 @@ +module.exports = +/******/ (() => { // webpackBootstrap +/******/ "use strict"; +/******/ var __webpack_modules__ = ({ + +/***/ 448: +/***/ (function(__unused_webpack_module, exports, __webpack_require__) { + + +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (_) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +exports.__esModule = true; +var piscina_1 = __webpack_require__(409); +var path = __webpack_require__(622); +var piscina = new piscina_1["default"]({ + filename: __webpack_require__.ab + "pi-bridge.js" +}); +(function () { + return __awaiter(this, void 0, void 0, function () { + var result; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, piscina.runTask(2)]; + case 1: + result = _a.sent(); + console.log(result); + return [2 /*return*/]; + } + }); + }); +})(); + + +/***/ }), + +/***/ 622: +/***/ ((module) => { + +module.exports = require("path"); + +/***/ }), + +/***/ 409: +/***/ ((module) => { + +module.exports = require("piscina"); + +/***/ }) + +/******/ }); +/************************************************************************/ +/******/ // The module cache +/******/ var __webpack_module_cache__ = {}; +/******/ +/******/ // The require function +/******/ function __webpack_require__(moduleId) { +/******/ // Check if module is in cache +/******/ if(__webpack_module_cache__[moduleId]) { +/******/ return __webpack_module_cache__[moduleId].exports; +/******/ } +/******/ // Create a new module (and put it into the cache) +/******/ var module = __webpack_module_cache__[moduleId] = { +/******/ // no module.id needed +/******/ // no module.loaded needed +/******/ exports: {} +/******/ }; +/******/ +/******/ // Execute the module function +/******/ var threw = true; +/******/ try { +/******/ __webpack_modules__[moduleId].call(module.exports, module, module.exports, __webpack_require__); +/******/ threw = false; +/******/ } finally { +/******/ if(threw) delete __webpack_module_cache__[moduleId]; +/******/ } +/******/ +/******/ // Return the exports of the module +/******/ return module.exports; +/******/ } +/******/ +/************************************************************************/ +/******/ /* webpack/runtime/compat */ +/******/ +/******/ __webpack_require__.ab = __dirname + "/";/************************************************************************/ +/******/ // module exports must be returned from runtime so entry inlining is disabled +/******/ // startup +/******/ // Load entry module and return exports +/******/ return __webpack_require__(448); +/******/ })() +; \ No newline at end of file diff --git a/test/unit/bundle-subasset2/output.js b/test/unit/bundle-subasset2/output.js new file mode 100644 index 00000000..61a31743 --- /dev/null +++ b/test/unit/bundle-subasset2/output.js @@ -0,0 +1,123 @@ +module.exports = +/******/ (() => { // webpackBootstrap +/******/ "use strict"; +/******/ var __webpack_modules__ = ({ + +/***/ 886: +/***/ (function(__unused_webpack_module, exports, __webpack_require__) { + + +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (_) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +exports.__esModule = true; +var piscina_1 = __webpack_require__(409); +var path = __webpack_require__(622); +var piscina = new piscina_1["default"]({ + filename: __webpack_require__.ab + "pi-bridge.js" +}); +(function () { + return __awaiter(this, void 0, void 0, function () { + var result; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, piscina.runTask(2)]; + case 1: + result = _a.sent(); + console.log(result); + return [2 /*return*/]; + } + }); + }); +})(); + + +/***/ }), + +/***/ 622: +/***/ ((module) => { + +module.exports = require("path"); + +/***/ }), + +/***/ 409: +/***/ ((module) => { + +module.exports = require("piscina"); + +/***/ }) + +/******/ }); +/************************************************************************/ +/******/ // The module cache +/******/ var __webpack_module_cache__ = {}; +/******/ +/******/ // The require function +/******/ function __webpack_require__(moduleId) { +/******/ // Check if module is in cache +/******/ if(__webpack_module_cache__[moduleId]) { +/******/ return __webpack_module_cache__[moduleId].exports; +/******/ } +/******/ // Create a new module (and put it into the cache) +/******/ var module = __webpack_module_cache__[moduleId] = { +/******/ // no module.id needed +/******/ // no module.loaded needed +/******/ exports: {} +/******/ }; +/******/ +/******/ // Execute the module function +/******/ var threw = true; +/******/ try { +/******/ __webpack_modules__[moduleId].call(module.exports, module, module.exports, __webpack_require__); +/******/ threw = false; +/******/ } finally { +/******/ if(threw) delete __webpack_module_cache__[moduleId]; +/******/ } +/******/ +/******/ // Return the exports of the module +/******/ return module.exports; +/******/ } +/******/ +/************************************************************************/ +/******/ /* webpack/runtime/compat */ +/******/ +/******/ __webpack_require__.ab = __dirname + "/";/************************************************************************/ +/******/ // module exports must be returned from runtime so entry inlining is disabled +/******/ // startup +/******/ // Load entry module and return exports +/******/ return __webpack_require__(886); +/******/ })() +; \ No newline at end of file diff --git a/test/unit/bundle-subasset2/pi-bridge.js b/test/unit/bundle-subasset2/pi-bridge.js new file mode 100644 index 00000000..ca7cf426 --- /dev/null +++ b/test/unit/bundle-subasset2/pi-bridge.js @@ -0,0 +1,2 @@ +import Pi from './pi'; +export default Pi; diff --git a/test/unit/bundle-subasset2/pi.ts b/test/unit/bundle-subasset2/pi.ts new file mode 100644 index 00000000..7004205c --- /dev/null +++ b/test/unit/bundle-subasset2/pi.ts @@ -0,0 +1,3 @@ +export default function Pi(num: number) { + return Math.PI * num; +} diff --git a/test/unit/bundle-subasset2/tsconfig.json b/test/unit/bundle-subasset2/tsconfig.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/test/unit/bundle-subasset2/tsconfig.json @@ -0,0 +1 @@ +{} diff --git a/yarn.lock b/yarn.lock index 31b9e0b0..5826266a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1902,6 +1902,11 @@ dependencies: "@types/yargs-parser" "*" +"@vercel/webpack-asset-relocator-loader@0.9.1": + version "0.9.1" + resolved "https://registry.yarnpkg.com/@vercel/webpack-asset-relocator-loader/-/webpack-asset-relocator-loader-0.9.1.tgz#b9cd56e7b7de002830410aca708b76eea930a2cd" + integrity sha512-TJXZZa94a3mDB0jdHCDFaWzrFGwsNqpYNqqKeUieznpToN//9XDjQgwkgpvAhG04wwGBFIPvBBfZZdBsvGhLqQ== + "@webassemblyjs/ast@1.9.0": version "1.9.0" resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.9.0.tgz#bd850604b4042459a5a41cd7d338cbed695ed964" @@ -2057,11 +2062,6 @@ resolved "https://registry.yarnpkg.com/@xtuc/long/-/long-4.2.2.tgz#d291c6a4e97989b5c61d9acf396ae4fe133a718d" integrity sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ== -"@zeit/webpack-asset-relocator-loader@0.8.0": - version "0.8.0" - resolved "https://registry.yarnpkg.com/@zeit/webpack-asset-relocator-loader/-/webpack-asset-relocator-loader-0.8.0.tgz#aeba5baefb9908caf3865807fc175a211c4746b0" - integrity sha512-PNx8BUlOv3aLWWKJP7OtHF9hwpc7mhv/iToWmFTF22mwltRMeJT70EmgbYj+70+apY+lDwx5IrDuorzeRCDxiA== - JSONStream@^1.0.3, JSONStream@^1.3.1, JSONStream@^1.3.4, JSONStream@^1.3.5: version "1.3.5" resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.5.tgz#3208c1f08d3a4d99261ab64f92302bc15e111ca0" From feb89cefd99c8550251288f91eac21e153c9b7dd Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 17 Dec 2020 10:44:46 -0500 Subject: [PATCH 2/4] 0.26.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0734c251..bfb4920c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@vercel/ncc", "description": "Simple CLI for compiling a Node.js module into a single file, together with all its dependencies, gcc-style.", - "version": "0.25.1", + "version": "0.26.0", "repository": "vercel/ncc", "license": "MIT", "main": "./dist/ncc/index.js", From 9898d248d2a9682d30237b776bdc1efef51b2bcd Mon Sep 17 00:00:00 2001 From: Guy Bedford Date: Fri, 18 Dec 2020 11:53:01 -0800 Subject: [PATCH 3/4] Ensure separate asset compilation states in subbuilds (#630) Co-authored-by: Joe Haddad --- package.json | 2 +- scripts/build.js | 5 ++- src/index.js | 96 ++++++++++++++++++++++++++++++------------------ yarn.lock | 8 ++-- 4 files changed, 69 insertions(+), 42 deletions(-) diff --git a/package.json b/package.json index bfb4920c..d4555b51 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "@sentry/node": "^4.3.0", "@slack/web-api": "^5.13.0", "@tensorflow/tfjs-node": "^0.3.0", - "@vercel/webpack-asset-relocator-loader": "0.9.1", + "@vercel/webpack-asset-relocator-loader": "1.0.0", "analytics-node": "^3.3.0", "apollo-server-express": "^2.2.2", "arg": "^4.1.0", diff --git a/scripts/build.js b/scripts/build.js index c18895ab..5ee2fae3 100644 --- a/scripts/build.js +++ b/scripts/build.js @@ -54,8 +54,9 @@ async function main() { { filename: "ts-loader.js", minify, - v8cache: true - } + v8cache: true, + noAssetBuilds: true + }, ); checkUnknownAssets('ts-loader', Object.keys(tsLoaderAssets).filter(asset => !asset.startsWith('lib/') && !asset.startsWith('typescript/lib'))); diff --git a/src/index.js b/src/index.js index a1cba136..e1847216 100644 --- a/src/index.js +++ b/src/index.js @@ -135,10 +135,15 @@ function ncc ( let watcher, watchHandler, rebuildHandler; + const compilationStack = []; + var plugins = [ { apply(compiler) { - compiler.hooks.compilation.tap("relocate-loader", compilation => relocateLoader.initAssetCache(compilation)); + compiler.hooks.compilation.tap("relocate-loader", compilation => { + compilationStack.push(compilation); + relocateLoader.initAssetCache(compilation); + }); compiler.hooks.watchRun.tap("ncc", () => { if (rebuildHandler) rebuildHandler(); @@ -289,7 +294,10 @@ function ncc ( }); }); }) - .then(finalizeHandler); + .then(finalizeHandler, function (err) { + compilationStack.pop(); + throw err; + }); } else { if (typeof watch === 'object') { @@ -300,10 +308,14 @@ function ncc ( } let cachedResult; watcher = compiler.watch({}, async (err, stats) => { - if (err) + if (err) { + compilationStack.pop(); return watchHandler({ err }); - if (stats.hasErrors()) + } + if (stats.hasErrors()) { + compilationStack.pop(); return watchHandler({ err: stats.toString() }); + } const returnValue = await finalizeHandler(stats); if (watchHandler) watchHandler(returnValue); @@ -434,45 +446,59 @@ function ncc ( // for each .js / .mjs / .cjs file in the asset list, build that file with ncc itself if (!noAssetBuilds) { - let assetNames = Object.keys(assets); - assetNames.push(`${filename}${ext === '.cjs' ? '.js' : ''}`); - for (const asset of assetNames) { + const compilation = compilationStack[compilationStack.length - 1]; + let existingAssetNames = Object.keys(assets); + existingAssetNames.push(`${filename}${ext === '.cjs' ? '.js' : ''}`); + const subbuildAssets = []; + for (const asset of Object.keys(assets)) { if (!asset.endsWith('.js') && !asset.endsWith('.cjs') && !asset.endsWith('.ts') && !asset.endsWith('.mjs') || - asset.endsWith('.cache.js') || asset.endsWith('.cache.cjs') || asset.endsWith('.cache.ts') || asset.endsWith('.cache.mjs')) + asset.endsWith('.cache.js') || asset.endsWith('.cache.cjs') || asset.endsWith('.cache.ts') || asset.endsWith('.cache.mjs') || asset.endsWith('.d.ts')) { + existingAssetNames.push(asset); continue; - const assetMeta = relocateLoader.getAssetMeta(asset); - if (!assetMeta) + } + const assetMeta = relocateLoader.getAssetMeta(asset, compilation); + if (!assetMeta || !assetMeta.path) { + existingAssetNames.push(asset); continue; + } + subbuildAssets.push(asset); + } + for (const asset of subbuildAssets) { + const assetMeta = relocateLoader.getAssetMeta(asset, compilation); const path = assetMeta.path; - if (path) { - const { code, assets: subbuildAssets, symlinks: subbuildSymlinks, stats: subbuildStats } = await ncc(path, { - cache, - externals, - filename: asset, - minify, - sourceMap, - sourceMapRegister, - sourceMapBasePrefix, - // dont recursively asset build - // could be supported with seen tracking - noAssetBuilds: true, - v8cache, - filterAssetBase, - existingAssetNames: assetNames, - quiet, - debugLog, - transpileOnly, - license, - target - }); - Object.assign(symlinks, subbuildSymlinks); - Object.assign(stats, subbuildStats); - assets[asset] = { source: code, permissions: assetMeta.permissions }; - Object.assign(assets, subbuildAssets); + const { code, assets: subbuildAssets, symlinks: subbuildSymlinks, stats: subbuildStats } = await ncc(path, { + cache, + externals, + filename: asset, + minify, + sourceMap, + sourceMapRegister, + sourceMapBasePrefix, + // dont recursively asset build + // could be supported with seen tracking + noAssetBuilds: true, + v8cache, + filterAssetBase, + existingAssetNames, + quiet, + debugLog, + transpileOnly, + license, + target + }); + Object.assign(symlinks, subbuildSymlinks); + Object.assign(stats, subbuildStats); + for (const subasset of Object.keys(subbuildAssets)) { + assets[subasset] = subbuildAssets[subasset]; + if (!existingAssetNames.includes(subasset)) + existingAssetNames.push(subasset); } + assets[asset] = { source: code, permissions: assetMeta.permissions }; } } + compilationStack.pop(); + return { code, map: map ? JSON.stringify(map) : undefined, assets, symlinks, stats }; } } diff --git a/yarn.lock b/yarn.lock index 5826266a..590209ad 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1902,10 +1902,10 @@ dependencies: "@types/yargs-parser" "*" -"@vercel/webpack-asset-relocator-loader@0.9.1": - version "0.9.1" - resolved "https://registry.yarnpkg.com/@vercel/webpack-asset-relocator-loader/-/webpack-asset-relocator-loader-0.9.1.tgz#b9cd56e7b7de002830410aca708b76eea930a2cd" - integrity sha512-TJXZZa94a3mDB0jdHCDFaWzrFGwsNqpYNqqKeUieznpToN//9XDjQgwkgpvAhG04wwGBFIPvBBfZZdBsvGhLqQ== +"@vercel/webpack-asset-relocator-loader@1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@vercel/webpack-asset-relocator-loader/-/webpack-asset-relocator-loader-1.0.0.tgz#cc3cae3b3cc8f3f631552e1ca97e566f597d5e5b" + integrity sha512-JZGVEFBOR0I7ccwewsWSSOB/ke6wYV4e09qQHtTBLTe/zgWrsjes7SX1Xt9M1UgiMTkhZ/0jyJpdxEqrpaYRMA== "@webassemblyjs/ast@1.9.0": version "1.9.0" From 47dfc48ea363034878eebcd09b5d4eb862401a3e Mon Sep 17 00:00:00 2001 From: Steven Date: Fri, 18 Dec 2020 16:49:27 -0500 Subject: [PATCH 4/4] 0.26.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d4555b51..bd57b93b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@vercel/ncc", "description": "Simple CLI for compiling a Node.js module into a single file, together with all its dependencies, gcc-style.", - "version": "0.26.0", + "version": "0.26.1", "repository": "vercel/ncc", "license": "MIT", "main": "./dist/ncc/index.js",