misc(build): use rollup to build lighthouse-core bundles (#12771)

build/build-bundle.js

@@ -12,25 +12,15 @@
 
 const fs = require('fs');
 const path = require('path');
-const assert = require('assert').strict;
-const mkdir = fs.promises.mkdir;
-const LighthouseRunner = require('../lighthouse-core/runner.js');
-const exorcist = require('exorcist');
-const browserify = require('browserify');
-const terser = require('terser');
-
-const inlineFs = require('./plugins/browserify-inline-fs.js');
+const rollup = require('rollup');
+const rollupPlugins = require('./rollup-plugins.js');
+const Runner = require('../lighthouse-core/runner.js');
+const {LH_ROOT} = require('../root.js');
 
 const COMMIT_HASH = require('child_process')
   .execSync('git rev-parse HEAD')
   .toString().trim();
 
-const audits = LighthouseRunner.getAuditList()
-    .map(f => './lighthouse-core/audits/' + f.replace(/\.js$/, ''));
-
-const gatherers = LighthouseRunner.getGathererList()
-    .map(f => './lighthouse-core/gather/gatherers/' + f.replace(/\.js$/, ''));
-
 // HACK: manually include the lighthouse-plugin-publisher-ads audits.
 /** @type {Array<string>} */
 // @ts-expect-error
@@ -45,144 +35,167 @@ const isLightrider = file => path.basename(file).includes('lightrider');
 // Set to true for source maps.
 const DEBUG = false;
 
+const today = (() => {
+  const date = new Date();
+  const year = new Intl.DateTimeFormat('en', {year: 'numeric'}).format(date);
+  const month = new Intl.DateTimeFormat('en', {month: 'short'}).format(date);
+  const day = new Intl.DateTimeFormat('en', {day: '2-digit'}).format(date);
+  return `${month} ${day} ${year}`;
+})();
+const pkg = JSON.parse(fs.readFileSync(LH_ROOT + '/package.json', 'utf-8'));
+const banner = `
+/**
+ * Lighthouse v${pkg.version} ${COMMIT_HASH} (${today})
+ *
+ * ${pkg.description}
+ *
+ * @homepage ${pkg.homepage}
+ * @author   ${pkg.author}
+ * @license  ${pkg.license}
+ */
+`.trim();
+
 /**
- * Browserify starting at the file at entryPath. Contains entry-point-specific
- * ignores (e.g. for DevTools or the extension) to trim the bundle depending on
- * the eventual use case.
+ * Bundle starting at entryPath, writing the minified result to distPath.
  * @param {string} entryPath
  * @param {string} distPath
+ * @param {{minify: boolean}=} opts
  * @return {Promise<void>}
  */
-async function browserifyFile(entryPath, distPath) {
-  let bundle = browserify(entryPath, {debug: DEBUG});
-
-  bundle
-    .plugin('browserify-banner', {
-      pkg: Object.assign({COMMIT_HASH}, require('../package.json')),
-      file: require.resolve('./banner.txt'),
-    })
-    // Transform `fs.readFileSync`, etc into inline strings.
-    .transform(inlineFs({verbose: DEBUG}))
-    // Strip everything out of package.json includes except for the version.
-    .transform('package-json-versionify');
-
-  // scripts will need some additional transforms, ignores and requires…
-  bundle.ignore('source-map')
-    .ignore('debug/node')
-    .ignore('intl')
-    .ignore('intl-pluralrules')
-    .ignore('raven')
-    .ignore('pako/lib/zlib/inflate.js');
-
-  // Don't include the desktop protocol connection.
-  bundle.ignore(require.resolve('../lighthouse-core/gather/connections/cri.js'));
-
-  // Don't include the stringified report in DevTools - see devtools-report-assets.js
-  // Don't include in Lightrider - HTML generation isn't supported, so report assets aren't needed.
-  if (isDevtools(entryPath) || isLightrider(entryPath)) {
-    bundle.ignore(require.resolve('../report/generator/report-assets.js'));
+async function build(entryPath, distPath, opts = {minify: true}) {
+  if (fs.existsSync(LH_ROOT + '/lighthouse-logger/node_modules')) {
+    throw new Error('delete `lighthouse-logger/node_modules` because it messes up rollup bundle');
   }
 
-  // Don't include locales in DevTools.
-  if (isDevtools(entryPath)) {
-    bundle.ignore(require.resolve('../shared/localization/locales.js'));
-  }
+  // List of paths (absolute / relative to config-helpers.js) to include
+  // in bundle and make accessible via config-helpers.js `requireWrapper`.
+  const dynamicModulePaths = [
+    ...Runner.getGathererList().map(gatherer => `../gather/gatherers/${gatherer}`),
+    ...Runner.getAuditList().map(gatherer => `../audits/${gatherer}`),
+  ];
 
-  // Expose the audits, gatherers, and computed artifacts so they can be dynamically loaded.
-  // Exposed path must be a relative path from lighthouse-core/config/config-helpers.js (where loading occurs).
-  const corePath = './lighthouse-core/';
-  const driverPath = `${corePath}gather/`;
-  audits.forEach(audit => {
-    bundle = bundle.require(audit, {expose: audit.replace(corePath, '../')});
-  });
-  gatherers.forEach(gatherer => {
-    bundle = bundle.require(gatherer, {expose: gatherer.replace(driverPath, '../gather/')});
-  });
-
-  // HACK: manually include the lighthouse-plugin-publisher-ads audits.
+  // Include lighthouse-plugin-publisher-ads.
   if (isDevtools(entryPath) || isLightrider(entryPath)) {
-    bundle.require('lighthouse-plugin-publisher-ads');
+    dynamicModulePaths.push('lighthouse-plugin-publisher-ads');
     pubAdsAudits.forEach(pubAdAudit => {
-      bundle = bundle.require(pubAdAudit);
+      dynamicModulePaths.push(pubAdAudit);
     });
   }
 
-  // browerify's url shim doesn't work with .URL in node_modules,
-  // and within robots-parser, it does `var URL = require('url').URL`, so we expose our own.
-  // @see https://github.com/GoogleChrome/lighthouse/issues/5273
-  const pathToURLShim = require.resolve('../lighthouse-core/lib/url-shim.js');
-  bundle = bundle.require(pathToURLShim, {expose: 'url'});
-
-  let bundleStream = bundle.bundle();
+  const bundledMapEntriesCode = dynamicModulePaths.map(modulePath => {
+    const pathNoExt = modulePath.replace('.js', '');
+    return `['${pathNoExt}', require('${modulePath}')]`;
+  }).join(',\n');
 
-  // Make sure path exists.
-  await mkdir(path.dirname(distPath), {recursive: true});
-  await new Promise((resolve, reject) => {
-    const writeStream = fs.createWriteStream(distPath);
-    writeStream.on('finish', resolve);
-    writeStream.on('error', reject);
+  /** @type {Record<string, string>} */
+  const shimsObj = {};
 
-    // Extract the inline source map to an external file.
-    if (DEBUG) bundleStream = bundleStream.pipe(exorcist(`${distPath}.map`));
-    bundleStream.pipe(writeStream);
-  });
+  const modulesToIgnore = [
+    'intl-pluralrules',
+    'intl',
+    'pako/lib/zlib/inflate.js',
+    'raven',
+    'source-map',
+    'ws',
+    require.resolve('../lighthouse-core/gather/connections/cri.js'),
+  ];
 
-  if (isDevtools(distPath)) {
-    let code = fs.readFileSync(distPath, 'utf-8');
-    // Add a comment for TypeScript, but not if in DEBUG mode so that source maps are not affected.
-    // See lighthouse-cli/test/smokehouse/lighthouse-runners/bundle.js
-    if (!DEBUG) {
-      code = '// @ts-nocheck - Prevent tsc stepping into any required bundles.\n' + code;
-    }
+  // Don't include the stringified report in DevTools - see devtools-report-assets.js
+  // Don't include in Lightrider - HTML generation isn't supported, so report assets aren't needed.
+  if (isDevtools(entryPath) || isLightrider(entryPath)) {
+    modulesToIgnore.push(require.resolve('../report/generator/report-assets.js'));
+  }
 
-    // DevTools build system expects `globalThis` rather than setting global variables.
-    assert.ok(code.includes('\nrequire='), 'missing browserify require stub');
-    code = code.replace('\nrequire=', '\nglobalThis.require=');
-    assert.ok(!code.includes('\nrequire='), 'contained unexpected browserify require stub');
+  // Don't include locales in DevTools.
+  if (isDevtools(entryPath)) {
+    shimsObj['./locales.js'] = 'export default {}';
+  }
 
-    fs.writeFileSync(distPath, code);
+  for (const modulePath of modulesToIgnore) {
+    shimsObj[modulePath] = 'export default {}';
   }
-}
 
-/**
- * Minify a javascript file, in place.
- * @param {string} filePath
- */
-async function minifyScript(filePath) {
-  const code = fs.readFileSync(filePath, 'utf-8');
-  const result = await terser.minify(code, {
-    ecma: 2019,
-    output: {
-      comments: /^!|Prevent tsc/,
-      max_line_len: 1000,
-    },
-    // The config relies on class names for gatherers.
-    keep_classnames: true,
-    // Runtime.evaluate errors if function names are elided.
-    keep_fnames: true,
-    sourceMap: DEBUG && {
-      content: JSON.parse(fs.readFileSync(`${filePath}.map`, 'utf-8')),
-      url: path.basename(`${filePath}.map`),
-    },
+  shimsObj[require.resolve('../package.json')] =
+    `export const version = ${JSON.stringify(require('../package.json').version)}`;
+
+  const bundle = await rollup.rollup({
+    input: entryPath,
+    context: 'globalThis',
+    plugins: [
+      rollupPlugins.replace({
+        delimiters: ['', ''],
+        values: {
+          '/* BUILD_REPLACE_BUNDLED_MODULES */': `[\n${bundledMapEntriesCode},\n]`,
+          '__dirname': (id) => `'${path.relative(LH_ROOT, path.dirname(id))}'`,
+          '__filename': (id) => `'${path.relative(LH_ROOT, id)}'`,
+          // This package exports to default in a way that causes Rollup to get confused,
+          // resulting in MessageFormat being undefined.
+          'require(\'intl-messageformat\').default': 'require(\'intl-messageformat\')',
+          // Rollup doesn't replace this, so let's manually change it to false.
+          'require.main === module': 'false',
+          // TODO: Use globalThis directly.
+          'global.isLightrider': 'globalThis.isLightrider',
+          'global.isDevtools': 'globalThis.isDevtools',
+        },
+      }),
+      rollupPlugins.alias({
+        entries: {
+          'debug': require.resolve('debug/src/browser.js'),
+          'lighthouse-logger': require.resolve('../lighthouse-logger/index.js'),
+          'url': require.resolve('../lighthouse-core/lib/url-shim.js'),
+        },
+      }),
+      rollupPlugins.shim({
+        ...shimsObj,
+        // Allows for plugins to import lighthouse.
+        'lighthouse': `
+          import Audit from '${require.resolve('../lighthouse-core/audits/audit.js')}';
+          export {Audit};
+        `,
+      }),
+      rollupPlugins.json(),
+      rollupPlugins.inlineFs({verbose: false}),
+      rollupPlugins.commonjs({
+        // https://github.com/rollup/plugins/issues/922
+        ignoreGlobal: true,
+      }),
+      rollupPlugins.nodePolyfills(),
+      rollupPlugins.nodeResolve({preferBuiltins: true}),
+      // Rollup sees the usages of these functions in page functions (ex: see AnchorElements)
+      // and treats them as globals. Because the names are "taken" by the global, Rollup renames
+      // the actual functions (getNodeDetails$1). The page functions expect a certain name, so
+      // here we undo what Rollup did.
+      rollupPlugins.postprocess([
+        [/getBoundingClientRect\$1/, 'getBoundingClientRect'],
+        [/getElementsInDocument\$1/, 'getElementsInDocument'],
+        [/getNodeDetails\$1/, 'getNodeDetails'],
+        [/getRectCenterPoint\$1/, 'getRectCenterPoint'],
+        [/isPositionFixed\$1/, 'isPositionFixed'],
+      ]),
+      opts.minify && rollupPlugins.terser({
+        ecma: 2019,
+        output: {
+          comments: (node, comment) => {
+            const text = comment.value;
+            if (text.includes('The Lighthouse Authors') && comment.line > 1) return false;
+            return /@ts-nocheck - Prevent tsc|@preserve|@license|@cc_on|^!/i.test(text);
+          },
+          max_line_len: 1000,
+        },
+        // The config relies on class names for gatherers.
+        keep_classnames: true,
+        // Runtime.evaluate errors if function names are elided.
+        keep_fnames: true,
+      }),
+    ],
   });
 
-  if (result.code) {
-    fs.writeFileSync(filePath, result.code);
-  }
-  if (DEBUG && typeof result.map === 'string') {
-    fs.writeFileSync(`${filePath}.map`, result.map);
-  }
-}
-
-/**
- * Browserify starting at entryPath, writing the minified result to distPath.
- * @param {string} entryPath
- * @param {string} distPath
- * @return {Promise<void>}
- */
-async function build(entryPath, distPath) {
-  await browserifyFile(entryPath, distPath);
-  await minifyScript(distPath);
+  await bundle.write({
+    file: distPath,
+    banner,
+    format: 'iife',
+    sourcemap: DEBUG,
+  });
 }
 
 /**

build/build-lightrider-bundles.js

@@ -23,15 +23,12 @@ const generatorFilename = `./report/generator/report-generator.js`;
 const entrySourceName = 'lightrider-entry.js';
 const entryDistName = 'lighthouse-lr-bundle.js';
 
-fs.mkdirSync(path.dirname(distDir), {recursive: true});
+fs.mkdirSync(distDir, {recursive: true});
 
-/**
- * Browserify and minify entry point.
- */
 function buildEntryPoint() {
   const inFile = `${sourceDir}/${entrySourceName}`;
   const outFile = `${distDir}/${entryDistName}`;
-  return bundleBuilder.build(inFile, outFile);
+  return bundleBuilder.build(inFile, outFile, {minify: false});
 }
 
 /**

build/rollup-plugins.js

@@ -16,9 +16,12 @@ function rollupPluginTypeCoerce(module) {
   return module;
 }
 
+const alias = rollupPluginTypeCoerce(require('@rollup/plugin-alias'));
 const commonjs = rollupPluginTypeCoerce(require('@rollup/plugin-commonjs'));
+const json = rollupPluginTypeCoerce(require('@rollup/plugin-json'));
 const nodePolyfills = rollupPluginTypeCoerce(require('rollup-plugin-polyfill-node'));
 const {nodeResolve} = require('@rollup/plugin-node-resolve');
+const postprocess = require('@stadtlandnetz/rollup-plugin-postprocess');
 const replace = rollupPluginTypeCoerce(require('rollup-plugin-replace'));
 // @ts-expect-error: no published types.
 const shim = require('rollup-plugin-shim');
@@ -28,10 +31,13 @@ const typescript = rollupPluginTypeCoerce(require('@rollup/plugin-typescript'));
 const inlineFs = require('./plugins/rollup-plugin-inline-fs.js');
 
 module.exports = {
+  alias,
   commonjs,
   inlineFs,
+  json,
   nodePolyfills,
   nodeResolve,
+  postprocess,
   replace,
   shim,
   terser,

clients/devtools-entry.js

@@ -5,6 +5,8 @@
  */
 'use strict';
 
+/* global globalThis */
+
 const lighthouse = require('../lighthouse-core/index.js');
 const RawProtocol = require('../lighthouse-core/gather/connections/raw.js');
 const log = require('lighthouse-logger');
@@ -14,6 +16,11 @@ const constants = require('../lighthouse-core/config/constants.js');
 
 /** @typedef {import('../lighthouse-core/gather/connections/connection.js')} Connection */
 
+// Rollup seems to overlook some references to `Buffer`, so it must be made explicit.
+// (`parseSourceMapFromDataUrl` breaks without this)
+/** @type {BufferConstructor} */
+globalThis.Buffer = require('buffer').Buffer;
+
 /**
  * Returns a config, which runs only certain categories.
  * Varies the config to use based on device.