From 54ae1eea4a7f227d7a7246c5c3d43cf4b025c921 Mon Sep 17 00:00:00 2001
From: Corentin Mors <mikescops@pixelswap.fr>
Date: Tue, 4 Jun 2024 14:44:23 +0200
Subject: [PATCH] feat: add `ignore` option to ignore files from build (#68)

---
 README.md                                     | 36 ++++++++++----
 lib/index.ts                                  |  3 ++
 lib/options.ts                                | 23 +++++++++
 lib/types.ts                                  |  1 +
 lib/walker.ts                                 | 21 ++++++++-
 package.json                                  |  1 +
 test/test-10-pnpm/main.js                     | 11 +++--
 test/test-11-pnpm/main.js                     | 11 +++--
 test/test-50-ignore-files/.gitignore          |  1 +
 test/test-50-ignore-files/main.js             | 47 +++++++++++++++++++
 .../node_modules/delta/index.js               |  3 ++
 .../node_modules/delta/needed.c               |  1 +
 .../node_modules/delta/package.json           |  5 ++
 .../node_modules/delta/useless.c              |  1 +
 test/test-50-ignore-files/package.json        | 12 +++++
 test/test-50-ignore-files/test-x-index.js     |  5 ++
 test/test-80-compression-node-opcua/main.js   | 20 ++++----
 yarn.lock                                     | 14 ++++++
 18 files changed, 184 insertions(+), 32 deletions(-)
 create mode 100644 lib/options.ts
 create mode 100644 test/test-50-ignore-files/.gitignore
 create mode 100644 test/test-50-ignore-files/main.js
 create mode 100644 test/test-50-ignore-files/node_modules/delta/index.js
 create mode 100644 test/test-50-ignore-files/node_modules/delta/needed.c
 create mode 100644 test/test-50-ignore-files/node_modules/delta/package.json
 create mode 100644 test/test-50-ignore-files/node_modules/delta/useless.c
 create mode 100644 test/test-50-ignore-files/package.json
 create mode 100644 test/test-50-ignore-files/test-x-index.js

diff --git a/README.md b/README.md
index 304f445f3..3726addd5 100644
--- a/README.md
+++ b/README.md
@@ -50,23 +50,23 @@ pkg [options] <input>
 
   Examples:
 
-  – Makes executables for Linux, macOS and Windows
+  - Makes executables for Linux, macOS and Windows
     $ pkg index.js
-  – Takes package.json from cwd and follows 'bin' entry
+  - Takes package.json from cwd and follows 'bin' entry
     $ pkg .
-  – Makes executable for particular target machine
+  - Makes executable for particular target machine
     $ pkg -t node16-win-arm64 index.js
-  – Makes executables for target machines of your choice
+  - Makes executables for target machines of your choice
     $ pkg -t node16-linux,node18-linux,node16-win index.js
-  – Bakes '--expose-gc' and '--max-heap-size=34' into executable
+  - Bakes '--expose-gc' and '--max-heap-size=34' into executable
     $ pkg --options "expose-gc,max-heap-size=34" index.js
-  – Consider packageA and packageB to be public
+  - Consider packageA and packageB to be public
     $ pkg --public-packages "packageA,packageB" index.js
-  – Consider all packages to be public
+  - Consider all packages to be public
     $ pkg --public-packages "*" index.js
-  – Bakes '--expose-gc' into executable
+  - Bakes '--expose-gc' into executable
     $ pkg --options expose-gc index.js
-  – reduce size of the data packed inside the executable with GZip
+  - reduce size of the data packed inside the executable with GZip
     $ pkg --compress GZip index.js
 ```
 
@@ -181,6 +181,22 @@ See also
 [Detecting assets in source code](#detecting-assets-in-source-code) and
 [Snapshot filesystem](#snapshot-filesystem).
 
+### Ignore files
+
+`ignore` is a list of globs. Files matching the paths specified as `ignore`
+will be excluded from the final executable.
+
+This is useful when you want to exclude some files from the final executable,
+like tests, documentation or build files that could have been included by a dependency.
+
+```json
+  "pkg": {
+    "ignore": [ "**/*/dependency-name/build.c" ]
+  }
+```
+
+To see if you have unwanted files in your executable, read the [Exploring virtual file system embedded in debug mode](#exploring-virtual-file-system-embedded-in-debug-mode) section.
+
 ### Options
 
 Node.js application can be called with runtime options
@@ -413,7 +429,7 @@ printenv | grep NODE
 
 ## Advanced
 
-### exploring virtual file system embedded in debug mode
+### Exploring virtual file system embedded in debug mode
 
 When you are using the `--debug` flag when building your executable,
 `pkg` add the ability to display the content of the virtual file system
diff --git a/lib/index.ts b/lib/index.ts
index ea575ce97..dd7beb7af 100644
--- a/lib/index.ts
+++ b/lib/index.ts
@@ -27,6 +27,7 @@ import walk, { Marker, WalkerParams } from './walker';
 import { Target, NodeTarget, SymLinks } from './types';
 import { CompressType } from './compress_type';
 import { patchMachOExecutable, signMachOExecutable } from './mach-o';
+import pkgOptions from './options';
 
 const { version } = JSON.parse(
   readFileSync(path.join(__dirname, '../package.json'), 'utf-8'),
@@ -598,12 +599,14 @@ export async function exec(argv2: string[]) {
   let marker: Marker;
 
   if (configJson) {
+    pkgOptions.set(configJson?.pkg);
     marker = {
       config: configJson,
       base: path.dirname(config),
       configPath: config,
     };
   } else {
+    pkgOptions.set(inputJson?.pkg);
     marker = {
       config: inputJson || {}, // not `inputBin` because only `input`
       base: path.dirname(input), // is the place for `inputJson`
diff --git a/lib/options.ts b/lib/options.ts
new file mode 100644
index 000000000..6e34d13ec
--- /dev/null
+++ b/lib/options.ts
@@ -0,0 +1,23 @@
+import { PkgOptions } from './types';
+
+class Options {
+  private options: PkgOptions;
+
+  constructor() {
+    this.options = {
+      dictionary: {},
+    };
+  }
+
+  public set(options: PkgOptions): void {
+    this.options = options ?? this.options;
+  }
+
+  public get(): PkgOptions {
+    return this.options;
+  }
+}
+
+const options = new Options();
+
+export default options;
diff --git a/lib/types.ts b/lib/types.ts
index 89c41fd62..66039569a 100644
--- a/lib/types.ts
+++ b/lib/types.ts
@@ -36,6 +36,7 @@ export interface PkgOptions {
   scripts?: string[];
   log?: (logger: typeof log, context: Record<string, string>) => void;
   assets?: string[];
+  ignore?: string[];
   deployFiles?: string[];
   patches?: Patches;
   dictionary: ConfigDictionary;
diff --git a/lib/walker.ts b/lib/walker.ts
index 418f5ca84..7daebf91d 100644
--- a/lib/walker.ts
+++ b/lib/walker.ts
@@ -6,6 +6,7 @@ import isCore from 'is-core-module';
 import globby from 'globby';
 import path from 'path';
 import chalk from 'chalk';
+import { minimatch } from 'minimatch';
 
 import {
   ALIAS_AS_RELATIVE,
@@ -33,6 +34,7 @@ import {
   PackageJson,
   SymLinks,
 } from './types';
+import pkgOptions from './options';
 
 export interface Marker {
   hasDictionary?: boolean;
@@ -450,6 +452,19 @@ class Walker {
     assert(typeof task.file === 'string');
     const realFile = toNormalizedRealPath(task.file);
 
+    const { ignore } = pkgOptions.get();
+    if (ignore) {
+      // check if the file matches one of the ignore regex patterns
+      const match = ignore.some((pattern) => minimatch(realFile, pattern));
+
+      if (match) {
+        log.debug(
+          `Ignoring file: ${realFile} due to top level config ignore pattern`,
+        );
+        return;
+      }
+    }
+
     if (realFile === task.file) {
       this.append(task);
       return;
@@ -747,7 +762,11 @@ class Walker {
 
     const catchPackageFilter = (config: PackageJson, base: string) => {
       const newPackage = newPackages[newPackages.length - 1];
-      newPackage.marker = { config, configPath: newPackage.packageJson, base };
+      newPackage.marker = {
+        config,
+        configPath: newPackage.packageJson,
+        base,
+      };
     };
 
     let newFile = '';
diff --git a/package.json b/package.json
index 76f42a453..00a7a5a2e 100644
--- a/package.json
+++ b/package.json
@@ -31,6 +31,7 @@
     "globby": "^11.1.0",
     "into-stream": "^6.0.0",
     "is-core-module": "2.9.0",
+    "minimatch": "9.0.4",
     "minimist": "^1.2.6",
     "multistream": "^4.1.0",
     "prebuild-install": "7.1.1",
diff --git a/test/test-10-pnpm/main.js b/test/test-10-pnpm/main.js
index c1abca6a9..ae06890eb 100644
--- a/test/test-10-pnpm/main.js
+++ b/test/test-10-pnpm/main.js
@@ -13,6 +13,7 @@ if (utils.shouldSkipPnpm()) {
 
 assert(__dirname === process.cwd());
 
+const isWindows = process.platform === 'win32';
 const target = process.argv[2] || 'host';
 const input = './test.js';
 const output = './test-output.exe';
@@ -23,14 +24,14 @@ console.log('target = ', target);
 utils.vacuum.sync('./node_modules');
 utils.vacuum.sync('./pnpm-lock.yaml');
 
+const npmlog = utils.exec.sync('npm install -g pnpm@8');
+console.log('npm log :', npmlog);
+
 // launch `pnpm install`
 const pnpmlog = utils.spawn.sync(
-  path.join(
-    path.dirname(process.argv[0]),
-    'npx' + (process.platform === 'win32' ? '.cmd' : ''),
-  ),
+  path.join(path.dirname(process.argv[0]), 'npx' + (isWindows ? '.cmd' : '')),
   ['pnpm', 'install'],
-  { cwd: path.dirname(output), expect: 0 },
+  { cwd: path.dirname(output), expect: 0, shell: isWindows },
 );
 console.log('pnpm log :', pnpmlog);
 
diff --git a/test/test-11-pnpm/main.js b/test/test-11-pnpm/main.js
index d4821e00d..f708e6b42 100644
--- a/test/test-11-pnpm/main.js
+++ b/test/test-11-pnpm/main.js
@@ -14,6 +14,7 @@ if (utils.shouldSkipPnpm()) {
 console.log(__dirname, process.cwd());
 assert(__dirname === process.cwd());
 
+const isWindows = process.platform === 'win32';
 const target = process.argv[2] || 'host';
 const input = './test.js';
 const output = './test-output.exe';
@@ -24,14 +25,14 @@ console.log('target = ', target);
 utils.vacuum.sync('./node_modules');
 utils.vacuum.sync('./pnpm-lock.yaml');
 
+const npmlog = utils.exec.sync('npm install -g pnpm@8');
+console.log('npm log :', npmlog);
+
 // launch `pnpm install`
 const pnpmlog = utils.spawn.sync(
-  path.join(
-    path.dirname(process.argv[0]),
-    'npx' + (process.platform === 'win32' ? '.cmd' : ''),
-  ),
+  path.join(path.dirname(process.argv[0]), 'npx' + (isWindows ? '.cmd' : '')),
   ['pnpm', 'install'],
-  { cwd: path.dirname(output), expect: 0 },
+  { cwd: path.dirname(output), expect: 0, shell: isWindows },
 );
 console.log('pnpm log :', pnpmlog);
 
diff --git a/test/test-50-ignore-files/.gitignore b/test/test-50-ignore-files/.gitignore
new file mode 100644
index 000000000..736e8ae58
--- /dev/null
+++ b/test/test-50-ignore-files/.gitignore
@@ -0,0 +1 @@
+!node_modules
\ No newline at end of file
diff --git a/test/test-50-ignore-files/main.js b/test/test-50-ignore-files/main.js
new file mode 100644
index 000000000..1db8e2592
--- /dev/null
+++ b/test/test-50-ignore-files/main.js
@@ -0,0 +1,47 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const path = require('path');
+const assert = require('assert');
+const utils = require('../utils.js');
+const standard = 'stdout';
+
+assert(!module.parent);
+assert(__dirname === process.cwd());
+
+const target = process.argv[2] || 'host';
+const output = './test-output.exe';
+
+let left, right;
+utils.mkdirp.sync(path.dirname(output));
+
+left = utils.spawn.sync('node', ['test-x-index.js']);
+
+const inspect =
+  standard === 'stdout'
+    ? ['inherit', 'pipe', 'inherit']
+    : ['inherit', 'inherit', 'pipe'];
+
+const log = utils.pkg.sync(
+  ['--target', target, '--output', output, '.', '--debug'],
+  inspect,
+);
+
+assert(
+  log.indexOf('useless.c due to top level config ignore pattern') > 0,
+  'useless.c file is not ignored',
+);
+assert(
+  log.indexOf(
+    'needed.c is added to queue',
+    'needed.c file is not added to queue',
+  ),
+);
+
+right = utils.spawn.sync('./' + path.basename(output), [], {
+  cwd: path.dirname(output),
+});
+
+assert.strictEqual(left, right);
+utils.vacuum.sync(output);
diff --git a/test/test-50-ignore-files/node_modules/delta/index.js b/test/test-50-ignore-files/node_modules/delta/index.js
new file mode 100644
index 000000000..53a4bbf98
--- /dev/null
+++ b/test/test-50-ignore-files/node_modules/delta/index.js
@@ -0,0 +1,3 @@
+'use strict';
+
+global.FOO = 'bar';
diff --git a/test/test-50-ignore-files/node_modules/delta/needed.c b/test/test-50-ignore-files/node_modules/delta/needed.c
new file mode 100644
index 000000000..045d6c487
--- /dev/null
+++ b/test/test-50-ignore-files/node_modules/delta/needed.c
@@ -0,0 +1 @@
+// Needed file to include in bundle
\ No newline at end of file
diff --git a/test/test-50-ignore-files/node_modules/delta/package.json b/test/test-50-ignore-files/node_modules/delta/package.json
new file mode 100644
index 000000000..1c5ed3a50
--- /dev/null
+++ b/test/test-50-ignore-files/node_modules/delta/package.json
@@ -0,0 +1,5 @@
+{
+  "name": "delta",
+  "main": "index.js",
+  "files": ["*.c"]
+}
diff --git a/test/test-50-ignore-files/node_modules/delta/useless.c b/test/test-50-ignore-files/node_modules/delta/useless.c
new file mode 100644
index 000000000..f527564a3
--- /dev/null
+++ b/test/test-50-ignore-files/node_modules/delta/useless.c
@@ -0,0 +1 @@
+// Some useless files
\ No newline at end of file
diff --git a/test/test-50-ignore-files/package.json b/test/test-50-ignore-files/package.json
new file mode 100644
index 000000000..88f7aa7ae
--- /dev/null
+++ b/test/test-50-ignore-files/package.json
@@ -0,0 +1,12 @@
+{
+  "bin": "test-x-index.js",
+  "license": "MIT",
+  "dependencies": {
+    "delta": "*"
+  },
+  "pkg": {
+    "ignore": [
+      "**/*/node_modules/delta/useless.c"
+    ]
+  }
+}
diff --git a/test/test-50-ignore-files/test-x-index.js b/test/test-50-ignore-files/test-x-index.js
new file mode 100644
index 000000000..259ec4f9b
--- /dev/null
+++ b/test/test-50-ignore-files/test-x-index.js
@@ -0,0 +1,5 @@
+'use strict';
+
+var dataPath = 'delta';
+require(dataPath);
+console.log(global.FOO);
diff --git a/test/test-80-compression-node-opcua/main.js b/test/test-80-compression-node-opcua/main.js
index c93558611..60a060f46 100644
--- a/test/test-80-compression-node-opcua/main.js
+++ b/test/test-80-compression-node-opcua/main.js
@@ -14,6 +14,7 @@ const assert = require('assert');
 const utils = require('../utils.js');
 const pkgJson = require('./package.json');
 
+const isWindows = process.platform === 'win32';
 const buildDir = 'build';
 
 assert(!module.parent);
@@ -32,14 +33,14 @@ function clean() {
 // remove any possible left-over
 clean();
 
+const npmlog = utils.exec.sync('npm install -g pnpm@8');
+console.log('npm log :', npmlog);
+
 // launch `pnpm install`
 const pnpmlog = utils.spawn.sync(
-  path.join(
-    path.dirname(process.argv[0]),
-    'npx' + (process.platform === 'win32' ? '.cmd' : ''),
-  ),
+  path.join(path.dirname(process.argv[0]), 'npx' + (isWindows ? '.cmd' : '')),
   ['pnpm', 'install'],
-  { cwd: path.dirname(__filename), expect: 0 },
+  { cwd: path.dirname(__filename), expect: 0, shell: isWindows },
 );
 console.log('pnpm log :', pnpmlog);
 
@@ -54,7 +55,7 @@ assert(
 /* eslint-disable no-unused-vars */
 const input = 'package.json';
 const target = process.argv[2] || 'host';
-const ext = process.platform === 'win32' ? '.exe' : '';
+const ext = isWindows ? '.exe' : '';
 const outputRef = path.join(buildDir, 'test-output-empty' + ext);
 const outputNone = path.join(buildDir, 'test-output-None' + ext);
 const outputGZip = path.join(buildDir, 'test-output-GZip' + ext);
@@ -95,10 +96,7 @@ function pkgCompress(compressMode, output) {
 
 function esbuildBuild(entryPoint) {
   const log = utils.spawn.sync(
-    path.join(
-      path.dirname(process.argv[0]),
-      'npx' + (process.platform === 'win32' ? '.cmd' : ''),
-    ),
+    path.join(path.dirname(process.argv[0]), 'npx' + (isWindows ? '.cmd' : '')),
     [
       'esbuild',
       entryPoint,
@@ -106,7 +104,7 @@ function esbuildBuild(entryPoint) {
       '--outfile=' + path.join(buildDir, pkgJson.main),
       '--platform=node',
     ],
-    { cwd: __dirname, expect: 0 },
+    { cwd: __dirname, expect: 0, shell: isWindows },
   );
 
   console.log(log);
diff --git a/yarn.lock b/yarn.lock
index c76bae004..0b62d8f88 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -947,6 +947,13 @@ brace-expansion@^1.1.7:
     balanced-match "^1.0.0"
     concat-map "0.0.1"
 
+brace-expansion@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae"
+  integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==
+  dependencies:
+    balanced-match "^1.0.0"
+
 braces@^3.0.2:
   version "3.0.2"
   resolved "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz"
@@ -3199,6 +3206,13 @@ mimic-response@^4.0.0:
   resolved "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz"
   integrity sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==
 
+minimatch@9.0.4:
+  version "9.0.4"
+  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.4.tgz#8e49c731d1749cbec05050ee5145147b32496a51"
+  integrity sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==
+  dependencies:
+    brace-expansion "^2.0.1"
+
 minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2:
   version "3.1.2"
   resolved "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz"