From ada73f982550958d57c5f142c607d4f3115089c9 Mon Sep 17 00:00:00 2001 From: Evan Wallace Date: Mon, 4 Oct 2021 09:23:58 -0400 Subject: [PATCH] fix #1647: add a fallback for "npm --no-optional" --- CHANGELOG.md | 8 ++ lib/npm/node-install.ts | 252 ++++++++++++++++++++++++++++++++------- lib/npm/node-platform.ts | 68 +++++++---- lib/npm/node-shim.ts | 4 +- lib/npm/node.ts | 4 +- 5 files changed, 265 insertions(+), 71 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82472649ad4..7116c5f3c06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,14 @@ With this release, the optimization is now implemented with a [link](https://www.man7.org/linux/man-pages/man2/link.2.html) operation followed by a [rename](https://www.man7.org/linux/man-pages/man2/rename.2.html) operation. This should always leave the package in a working state even if either step fails. +* Add a fallback for `npm install esbuild --no-optional` ([#1647](https://github.com/evanw/esbuild/issues/1647)) + + The installation method for esbuild's platform-specific binary executable was recently changed in version 0.13.0. Before that version esbuild downloaded it in an install script, and after that version esbuild lets the package manager download it using the `optionalDependencies` feature in `package.json`. This change was made because downloading the binary executable in an install script never really fully worked. The reasons are complex but basically there are a variety of edge cases where people people want to install esbuild in environments that they have customized such that downloading esbuild isn't possible. Using `optionalDependencies` instead lets the package manager deal with it instead, which should work fine in all cases (either that or your package manager has a bug, but that's not esbuild's problem). + + There is one case where this new installation method doesn't work: if you pass the `--no-optional` flag to npm to disable the `optionalDependencies` feature. If you do this, you prevent esbuild from being installed. This is not a problem with esbuild because you are manually enabling a flag to change npm's behavior such that esbuild doesn't install correctly. However, people still want to do this. + + With this release, esbuild will now fall back to the old installation method if the new installation method fails. **THIS MAY NOT WORK.** The new `optionalDependencies` installation method is the only supported way to install esbuild with npm. The old downloading installation method was removed because it doesn't always work. The downloading method is only being provided to try to be helpful but it's not the supported installation method. If you pass `--no-optional` and the download fails due to some environment customization you did, the recommended fix is to just remove the `--no-optional` flag. + * Support the new `.mts` and `.cts` TypeScript file extensions The upcoming version 4.5 of TypeScript has two new file extensions: `.mts` and `.cts`. Files with these extensions can be imported using the `.mjs` and `.cjs`, respectively. So the statement `import "./foo.mjs"` in TypeScript can actually succeed even if the file `./foo.mjs` doesn't exist on the file system as long as the file `./foo.mts` does exist. The import path with the `.mjs` extension is automatically re-routed to the corresponding file with the `.mts` extension at type-checking time by the TypeScript compiler. See [the TypeScript 4.5 beta announcement](https://devblogs.microsoft.com/typescript/announcing-typescript-4-5-beta/#new-file-extensions) for details. diff --git a/lib/npm/node-install.ts b/lib/npm/node-install.ts index 14eab3d7af6..668f94249f4 100644 --- a/lib/npm/node-install.ts +++ b/lib/npm/node-install.ts @@ -1,8 +1,10 @@ -import { binPathForCurrentPlatform } from './node-platform'; +import { downloadedBinPath, pkgAndSubpathForCurrentPlatform } from './node-platform'; import fs = require('fs'); import os = require('os'); import path = require('path'); +import zlib = require('zlib'); +import https = require('https'); import child_process = require('child_process'); declare const ESBUILD_VERSION: string; @@ -28,12 +30,104 @@ function isYarn2OrAbove(): boolean { return false; } -// This feature was added to give external code a way to modify the binary -// path without modifying the code itself. Do not remove this because -// external code relies on this (in addition to esbuild's own test suite). -if (process.env.ESBUILD_BINARY_PATH) { +function fetch(url: string): Promise { + return new Promise((resolve, reject) => { + https.get(url, res => { + if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location) + return fetch(res.headers.location).then(resolve, reject); + if (res.statusCode !== 200) + return reject(new Error(`Server responded with ${res.statusCode}`)); + let chunks: Buffer[] = []; + res.on('data', chunk => chunks.push(chunk)); + res.on('end', () => resolve(Buffer.concat(chunks))); + }).on('error', reject); + }); +} + +function extractFileFromTarGzip(buffer: Buffer, subpath: string): Buffer { + try { + buffer = zlib.unzipSync(buffer); + } catch (err: any) { + throw new Error(`Invalid gzip data in archive: ${err && err.message || err}`); + } + let str = (i: number, n: number) => String.fromCharCode(...buffer.subarray(i, i + n)).replace(/\0.*$/, ''); + let offset = 0; + subpath = `package/${subpath}`; + while (offset < buffer.length) { + let name = str(offset, 100); + let size = parseInt(str(offset + 124, 12), 8); + offset += 512; + if (!isNaN(size)) { + if (name === subpath) return buffer.subarray(offset, offset + size); + offset += (size + 511) & ~511; + } + } + throw new Error(`Could not find ${JSON.stringify(subpath)} in archive`); +} + +function installUsingNPM(pkg: string, subpath: string, binPath: string): void { + // Erase "npm_config_global" so that "npm install --global esbuild" works. + // Otherwise this nested "npm install" will also be global, and the install + // will deadlock waiting for the global installation lock. + const env = { ...process.env, npm_config_global: undefined }; + + // Create a temporary directory inside the "esbuild" package with an empty + // "package.json" file. We'll use this to run "npm install" in. + const esbuildLibDir = path.dirname(require.resolve('esbuild')); + const installDir = path.join(esbuildLibDir, 'npm-install'); + fs.mkdirSync(installDir); + try { + fs.writeFileSync(path.join(installDir, 'package.json'), '{}'); + + // Run "npm install" in the temporary directory which should download the + // desired package. Try to avoid unnecessary log output. This uses the "npm" + // command instead of a HTTP request so that it hopefully works in situations + // where HTTP requests are blocked but the "npm" command still works due to, + // for example, a custom configured npm registry and special firewall rules. + child_process.execSync(`npm install --loglevel=error --prefer-offline --no-audit --progress=false ${pkg}@${ESBUILD_VERSION}`, + { cwd: installDir, stdio: 'pipe', env }); + + // Move the downloaded binary executable into place. The destination path + // is the same one that the JavaScript API code uses so it will be able to + // find the binary executable here later. + const installedBinPath = path.join(installDir, 'node_modules', pkg, subpath); + fs.renameSync(installedBinPath, binPath); + } finally { + // Try to clean up afterward so we don't unnecessarily waste file system + // space. Leaving nested "node_modules" directories can also be problematic + // for certain tools that scan over the file tree and expect it to have a + // certain structure. + try { + removeRecursive(installDir); + } catch { + // Removing a file or directory can randomly break on Windows, returning + // EBUSY for an arbitrary length of time. I think this happens when some + // other program has that file or directory open (e.g. an anti-virus + // program). This is fine on Unix because the OS just unlinks the entry + // but keeps the reference around until it's unused. There's nothing we + // can do in this case so we just leave the directory there. + } + } +} + +function removeRecursive(dir: string): void { + for (const entry of fs.readdirSync(dir)) { + const entryPath = path.join(dir, entry); + let stats; + try { + stats = fs.lstatSync(entryPath); + } catch { + continue; // Guard against https://github.com/nodejs/node/issues/4760 + } + if (stats.isDirectory()) removeRecursive(entryPath); + else fs.unlinkSync(entryPath); + } + fs.rmdirSync(dir); +} + +function applyManualBinaryPathOverride(overridePath: string): void { // Patch the CLI use case (the "esbuild" command) - const pathString = JSON.stringify(process.env.ESBUILD_BINARY_PATH); + const pathString = JSON.stringify(overridePath); fs.writeFileSync(toPath, `#!/usr/bin/env node\n` + `require('child_process').execFileSync(${pathString}, process.argv.slice(2), { stdio: 'inherit' });\n`); @@ -43,45 +137,117 @@ if (process.env.ESBUILD_BINARY_PATH) { fs.writeFileSync(libMain, `var ESBUILD_BINARY_PATH = ${pathString};\n${code}`); } -// This package contains a "bin/esbuild" JavaScript file that finds and runs -// the appropriate binary executable. However, this means that running the -// "esbuild" command runs another instance of "node" which is way slower than -// just running the binary executable directly. -// -// Here we optimize for this by replacing the JavaScript file with the binary -// executable at install time. This optimization does not work on Windows -// because on Windows the binary executable must be called "esbuild.exe" -// instead of "esbuild". This also doesn't work with Yarn 2+ because the Yarn -// developers don't think binary modules should be used. See this thread for -// details: https://github.com/yarnpkg/berry/issues/882. This optimization also -// doesn't apply when npm's "--ignore-scripts" flag is used since in that case -// this install script will not be run. -else if (os.platform() !== 'win32' && !isYarn2OrAbove()) { - const bin = binPathForCurrentPlatform(); - const tempPath = path.join(__dirname, 'bin-esbuild'); +function maybeOptimizePackage(binPath: string): void { + // This package contains a "bin/esbuild" JavaScript file that finds and runs + // the appropriate binary executable. However, this means that running the + // "esbuild" command runs another instance of "node" which is way slower than + // just running the binary executable directly. + // + // Here we optimize for this by replacing the JavaScript file with the binary + // executable at install time. This optimization does not work on Windows + // because on Windows the binary executable must be called "esbuild.exe" + // instead of "esbuild". This also doesn't work with Yarn 2+ because the Yarn + // developers don't think binary modules should be used. See this thread for + // details: https://github.com/yarnpkg/berry/issues/882. This optimization also + // doesn't apply when npm's "--ignore-scripts" flag is used since in that case + // this install script will not be run. + if (os.platform() !== 'win32' && !isYarn2OrAbove()) { + const tempPath = path.join(__dirname, 'bin-esbuild'); + try { + // First link the binary with a temporary file. If this fails and throws an + // error, then we'll just end up doing nothing. This uses a hard link to + // avoid taking up additional space on the file system. + fs.linkSync(binPath, tempPath); + + // Then use rename to atomically replace the target file with the temporary + // file. If this fails and throws an error, then we'll just end up leaving + // the temporary file there, which is harmless. + fs.renameSync(tempPath, toPath); + + // If we get here, then we know that the target location is now a binary + // executable instead of a JavaScript file. + isToPathJS = false; + } catch { + // Ignore errors here since this optimization is optional + } + } +} + +async function downloadDirectlyFromNPM(pkg: string, subpath: string, binPath: string): Promise { + // If that fails, the user could have npm configured incorrectly or could not + // have npm installed. Try downloading directly from npm as a last resort. + const url = `https://registry.npmjs.org/${pkg}/-/${pkg}-${ESBUILD_VERSION}.tgz`; + console.error(`[esbuild] Trying to download ${JSON.stringify(url)}`); try { - // First link the binary with a temporary file. If this fails and throws an - // error, then we'll just end up doing nothing. This uses a hard link to - // avoid taking up additional space on the file system. - fs.linkSync(bin, tempPath); - - // Then use rename to atomically replace the target file with the temporary - // file. If this fails and throws an error, then we'll just end up leaving - // the temporary file there, which is harmless. - fs.renameSync(tempPath, toPath); - - // If we get here, then we know that the target location is now a binary - // executable instead of a JavaScript file. - isToPathJS = false; - } catch (e) { - // Ignore errors here since this optimization is optional + fs.writeFileSync(binPath, extractFileFromTarGzip(await fetch(url), subpath)); + fs.chmodSync(binPath, 0o755); + } catch (e: any) { + console.error(`[esbuild] Failed to download ${JSON.stringify(url)}: ${e && e.message || e}`); + throw e; } } -if (isToPathJS) { - // We need "node" before this command since it's a JavaScript file - validateBinaryVersion('node', toPath); -} else { - // This is no longer a JavaScript file so don't run it using "node" - validateBinaryVersion(toPath); +async function checkAndPreparePackage(): Promise { + // This feature was added to give external code a way to modify the binary + // path without modifying the code itself. Do not remove this because + // external code relies on this (in addition to esbuild's own test suite). + if (process.env.ESBUILD_BINARY_PATH) { + applyManualBinaryPathOverride(process.env.ESBUILD_BINARY_PATH); + return; + } + + const { pkg, subpath } = pkgAndSubpathForCurrentPlatform(); + + let binPath: string; + try { + // First check for the binary package from our "optionalDependencies". This + // package should have been installed alongside this package at install time. + binPath = require.resolve(`${pkg}/${subpath}`); + } catch (e) { + console.error(`[esbuild] Failed to find package "${pkg}" on the file system + +This can happen if you use the "--no-optional" flag. The "optionalDependencies" +package.json feature is used by esbuild to install the correct binary executable +for your current platform. This install script will now attempt to work around +this. If that fails, you need to remove the "--no-optional" flag to use esbuild. +`); + + // If that didn't work, then someone probably installed esbuild with the + // "--no-optional" flag. Attempt to compensate for this by downloading the + // package using a nested call to "npm" instead. + // + // THIS MAY NOT WORK. Package installation uses "optionalDependencies" for + // a reason: manually downloading the package has a lot of obscure edge + // cases that fail because people have customized their environment in + // some strange way that breaks downloading. This code path is just here + // to be helpful but it's not the supported way of installing esbuild. + binPath = downloadedBinPath(pkg, subpath); + try { + console.error(`[esbuild] Trying to install package "${pkg}" using npm`); + installUsingNPM(pkg, subpath, binPath); + } catch (e2: any) { + console.error(`[esbuild] Failed to install package "${pkg}" using npm: ${e2 && e2.message || e2}`); + + // If that didn't also work, then something is likely wrong with the "npm" + // command. Attempt to compensate for this by manually downloading the + // package from the npm registry over HTTP as a last resort. + try { + await downloadDirectlyFromNPM(pkg, subpath, binPath); + } catch (e3: any) { + throw new Error(`Failed to install package "${pkg}"`); + } + } + } + + maybeOptimizePackage(binPath); } + +checkAndPreparePackage().then(() => { + if (isToPathJS) { + // We need "node" before this command since it's a JavaScript file + validateBinaryVersion('node', toPath); + } else { + // This is no longer a JavaScript file so don't run it using "node" + validateBinaryVersion(toPath); + } +}); diff --git a/lib/npm/node-platform.ts b/lib/npm/node-platform.ts index 8d829c83ca9..930ae694fb3 100644 --- a/lib/npm/node-platform.ts +++ b/lib/npm/node-platform.ts @@ -29,44 +29,34 @@ export const knownUnixlikePackages: Record = { 'sunos x64 LE': 'esbuild-sunos-64', }; -export function binPathForCurrentPlatform(): string { +export function pkgAndSubpathForCurrentPlatform(): { pkg: string, subpath: string } { let pkg: string; - let bin: string; + let subpath: string; let platformKey = `${process.platform} ${os.arch()} ${os.endianness()}`; if (platformKey in knownWindowsPackages) { pkg = knownWindowsPackages[platformKey]; - bin = `${pkg}/esbuild.exe`; + subpath = 'esbuild.exe'; } else if (platformKey in knownUnixlikePackages) { pkg = knownUnixlikePackages[platformKey]; - bin = `${pkg}/bin/esbuild`; + subpath = 'bin/esbuild'; } else { throw new Error(`Unsupported platform: ${platformKey}`); } - try { - bin = require.resolve(bin); - } catch (e) { - try { - require.resolve(pkg) - } catch { - throw new Error(`The package "${pkg}" could not be found, and is needed by esbuild. - -If you are installing esbuild with npm, make sure that you don't specify the -"--no-optional" flag. The "optionalDependencies" package.json feature is used -by esbuild to install the correct binary executable for your current platform.`) - } - throw e - } + return { pkg, subpath }; +} - return bin; +export function downloadedBinPath(pkg: string, subpath: string): string { + const esbuildLibDir = path.dirname(require.resolve('esbuild')); + return path.join(esbuildLibDir, `downloaded-${pkg}-${path.basename(subpath)}`); } -export function extractedBinPath(): string { +export function generateBinPath(): string { // This feature was added to give external code a way to modify the binary // path without modifying the code itself. Do not remove this because // external code relies on this (in addition to esbuild's own test suite). @@ -74,7 +64,37 @@ export function extractedBinPath(): string { return ESBUILD_BINARY_PATH; } - const bin = binPathForCurrentPlatform(); + const { pkg, subpath } = pkgAndSubpathForCurrentPlatform(); + let binPath: string; + + try { + // First check for the binary package from our "optionalDependencies". This + // package should have been installed alongside this package at install time. + binPath = require.resolve(`${pkg}/${subpath}`); + } catch (e) { + // If that didn't work, then someone probably installed esbuild with the + // "--no-optional" flag. Our install script attempts to compensate for this + // by manually downloading the package instead. Check for that next. + binPath = downloadedBinPath(pkg, subpath); + if (!fs.existsSync(binPath)) { + // If that didn't work too, then we're out of options. This can happen + // when someone installs esbuild with both the "--no-optional" and the + // "--ignore-scripts" flags. The fix for this is to just not do that. + // + // In that case we try to have a nice error message if we think we know + // what's happening. Otherwise we just rethrow the original error message. + try { + require.resolve(pkg); + } catch { + throw new Error(`The package "${pkg}" could not be found, and is needed by esbuild. + +If you are installing esbuild with npm, make sure that you don't specify the +"--no-optional" flag. The "optionalDependencies" package.json feature is used +by esbuild to install the correct binary executable for your current platform.`); + } + throw e; + } + } // The esbuild binary executable can't be used in Yarn 2 in PnP mode because // it's inside a virtual file system and the OS needs it in the real file @@ -88,13 +108,13 @@ export function extractedBinPath(): string { } if (isYarnPnP) { const esbuildLibDir = path.dirname(require.resolve('esbuild')); - const binTargetPath = path.join(esbuildLibDir, 'yarn-pnp-' + path.basename(bin)); + const binTargetPath = path.join(esbuildLibDir, 'yarn-pnp-' + path.basename(subpath)); if (!fs.existsSync(binTargetPath)) { - fs.copyFileSync(bin, binTargetPath); + fs.copyFileSync(binPath, binTargetPath); fs.chmodSync(binTargetPath, 0o755); } return binTargetPath; } - return bin; + return binPath; } diff --git a/lib/npm/node-shim.ts b/lib/npm/node-shim.ts index e029374dd97..72bc65c21ea 100644 --- a/lib/npm/node-shim.ts +++ b/lib/npm/node-shim.ts @@ -1,4 +1,4 @@ #!/usr/bin/env node -import { extractedBinPath } from "./node-platform"; -require('child_process').execFileSync(extractedBinPath(), process.argv.slice(2), { stdio: 'inherit' }); +import { generateBinPath } from "./node-platform"; +require('child_process').execFileSync(generateBinPath(), process.argv.slice(2), { stdio: 'inherit' }); diff --git a/lib/npm/node.ts b/lib/npm/node.ts index b7f5cf9f128..300f7249261 100644 --- a/lib/npm/node.ts +++ b/lib/npm/node.ts @@ -1,6 +1,6 @@ import * as types from "../shared/types"; import * as common from "../shared/common"; -import { extractedBinPath } from "./node-platform"; +import { generateBinPath } from "./node-platform"; import child_process = require('child_process'); import crypto = require('crypto'); @@ -66,7 +66,7 @@ let esbuildCommandAndArgs = (): [string, string[]] => { return ['node', [path.join(__dirname, '..', 'bin', 'esbuild')]]; } - return [extractedBinPath(), []]; + return [generateBinPath(), []]; }; // Return true if stderr is a TTY