Support mocha --watch with monkey-patching (#138)

* Create MochaCtx in watch mode

* Test earljs/mocha with Mocha --watch mode

* Add test for earljs/mocha in --parallel mode

* Monkey-patch Mocha.prototype.ui instead of .rootHooks

* Try to get CI to work... it passes locally

* Increase timeout

* Add a few eslint-disable-next-line comments for CI

* -.-

* ...

* Setup debugging session

* Run CI only on ubuntu for now

* Create a proper test-e2e project

* Try using SIGKILL

* Add --exit flag to e2e mocha

* Go back to normal CI setup

* Add changeset

.changeset/unlucky-pumas-tease.md

@@ -0,0 +1,5 @@
+---
+'earljs': patch
+---
+
+Earl no longer crashes in mocha --watch mode

.gitignore

@@ -1,3 +1,4 @@
 node_modules
 dist
-packages/earljs/README.md
+packages/earljs/README.md
+**/tsconfig.tsbuildinfo

package.json

@@ -33,6 +33,7 @@
     "@typescript-eslint/parser": "4.15.1",
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
+    "cross-spawn-with-kill": "^1.0.0",
     "eslint": "^7.29.0",
     "eslint-config-typestrict": "^1.0.2",
     "eslint-plugin-import": "^2.23.4",

packages/earljs/src/test-runners/mocha.ts

@@ -1,52 +1,110 @@
 import debug from 'debug'
+import mocha, { Context, Runner, Suite } from 'mocha'
 import { assert } from 'ts-essentials'
 
 import { setTestRunnerIntegration } from '../testRunnerCtx'
 import { TestInfo, TestRunnerCtx, TestRunnerHook } from './TestRunnerCtx'
 
 const d = debug('earljs:mocha')
 
-exports.mochaGlobalSetup = async function () {
+/**
+ * Needed in Mocha --watch mode. Mocha doesn't export hooks before mocha.ui() is called
+ */
+function main() {
+  d('earljs/mocha integration is being registered...')
+
+  for (const module of findMochaInstances()) {
+    if (!module || (module as any).__earljs_integrated) {
+      continue
+    }
+    ;(module as any).__earljs_integrated = true
+
+    d('Monkey-patching Mocha.prototype.ui')
+    const { ui } = module.prototype
+    module.prototype.ui = function (...args: Parameters<typeof module.prototype.ui>) {
+      setTestRunnerIntegration(new MochaCtx(this.suite))
+      return ui.apply(this, args)
+    }
+  }
+}
+
+/**
+ * In mocha run mode, we use the suite with hooks already assigned to Mocha's exports.
+ */
+exports.mochaGlobalSetup = function (this: Runner) {
   d('Integrating earl with mocha...')
 
-  setTestRunnerIntegration(new MochaCtx())
+  if ((mocha as Partial<MochaHooks>).beforeEach) {
+    setTestRunnerIntegration(new MochaCtx(mocha))
+  }
+}
+
+function findMochaInstances(): (typeof mocha | undefined)[] {
+  const mochaFromWindow = (globalThis as any).window?.Mocha
+  if (mochaFromWindow) {
+    return [mochaFromWindow]
+  }
+
+  const req = require as typeof require | undefined
+  if (typeof req === 'function') {
+    // require can be undefined in Node ESM and browser contexts
+    // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+    const cache = req.cache || {}
+    return Object.keys(cache)
+      .filter(function (child) {
+        var val = cache[child]?.exports
+        return typeof val === 'function' && val.name === 'Mocha'
+      })
+      .map(function (child) {
+        return cache[child]?.exports
+      })
+  }
+
+  return []
 }
 
 export class MochaCtx implements TestRunnerCtx {
   testInfo!: TestInfo
 
-  constructor() {
+  constructor(private readonly _hooks: MochaHooks) {
     const self = this
 
     d('Installing beforeEach hook to get testInfo before each test')
-    globalThis.beforeEach(function () {
+    _hooks.beforeEach(function () {
       assert(this.currentTest, "Current test not set by mocha. This shouldn't happen.")
       assert(this.currentTest.file, "Current test file path not set by mocha. This shouldn't happen.")
       assert(this.currentTest.parent, "Current test has no parent set by mocha. This shouldn't happen.")
 
       self.testInfo = {
-        suitName: makeSuitName(this.currentTest.parent),
+        suitName: makeSuiteName(this.currentTest.parent),
         testName: this.currentTest.title,
         testFilePath: this.currentTest.file,
       }
     })
   }
 
   afterTestCase(fn: TestRunnerHook) {
-    globalThis.beforeEach(fn)
+    this._hooks.afterEach(fn)
   }
 
   beforeTestCase(fn: TestRunnerHook) {
-    globalThis.afterEach(fn)
+    this._hooks.beforeEach(fn)
   }
 }
 
-function makeSuitName(testCtx: Mocha.Suite): string[] {
+interface MochaHooks {
+  beforeEach(fn: (this: Context) => void): void
+  afterEach(fn: (this: Context) => void): void
+}
+
+function makeSuiteName(testCtx: Suite): string[] {
   if (testCtx.parent) {
-    return [...makeSuitName(testCtx.parent), testCtx.title]
+    return [...makeSuiteName(testCtx.parent), testCtx.title]
   }
   if (testCtx.title) {
     return [testCtx.title]
   }
   return []
 }
+
+main()

packages/example-plugin/package.json

@@ -15,7 +15,7 @@
     "typecheck": "tsc --noEmit",
     "clean": "rimraf dist",
     "build": "yarn clean && tsc -p ./tsconfig.json",
-    "test": "true",
+    "test": "echo ok",
     "test:fix": "yarn lint:fix && yarn format:fix && yarn test && yarn typecheck"
   },
   "devDependencies": {

packages/test-e2e/.mocharc.js

@@ -0,0 +1,14 @@
+process.env.NODE_ENV = 'test'
+
+process.on('unhandledRejection', (reason, promise) => {
+  console.error('Unhandled Rejection during test execution:', promise, 'reason:', reason)
+  process.exit(1)
+})
+
+module.exports = {
+  require: ['ts-node/register/transpile-only'],
+  file: ['./test/setup.ts'],
+  extension: 'ts',
+  watchExtensions: 'ts',
+  spec: 'test/**/*.test.ts',
+}

packages/test-e2e/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "test-e2e",
+  "private": true,
+  "license": "MIT",
+  "version": "9.9.9",
+  "scripts": {
+    "format": "prettier --ignore-path ../../.prettierignore --check .",
+    "format:fix": "prettier --ignore-path ../../.prettierignore --write .",
+    "lint": "eslint --ext .ts test",
+    "lint:fix": "yarn lint --fix",
+    "typecheck": "tsc --noEmit",
+    "test": "mocha --exit",
+    "test:fix": "yarn lint:fix && yarn format:fix && yarn typecheck && yarn test"
+  },
+  "dependencies": {
+    "earljs": "link:../earljs",
+    "@earljs/published": "npm:earljs@latest"
+  },
+  "peerDependencies": {
+    "debug": "^4"
+  }
+}

packages/test-e2e/test/__snapshots__/mocha-integration.tested.snap

@@ -0,0 +1,5 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`earljs/mocha failing test 1`] = `0.685109454492407`;
+
+exports[`earljs/mocha successful test 1`] = `2`;

packages/test-e2e/test/mocha-config.tested.js

@@ -0,0 +1,6 @@
+module.exports = {
+  require: ['ts-node/register/transpile-only', 'earljs/mocha'],
+  extension: ['ts'],
+  watchExtensions: ['ts'],
+  spec: ['*.tested.ts'],
+}

packages/test-e2e/test/mocha-integration.test.ts

@@ -0,0 +1,122 @@
+import { expect } from '@earljs/published'
+// @ts-expect-error missing typings, but they match child_process.spawn
+import _spawn from 'cross-spawn-with-kill'
+import debug from 'debug'
+import { stderr } from 'process'
+
+const d = debug('test:mocha-integration')
+
+const spawn = _spawn as typeof import('child_process').spawn
+
+// There might be a way to handle it in a more elegant manner...
+const PASSING_TESTS = 1
+const FAILING_TESTS = 1
+const expected = { passing: PASSING_TESTS, failing: FAILING_TESTS }
+
+describe('earljs/mocha end-to-end tests', function () {
+  this.timeout(10000)
+
+  it('works in run mode', async () => {
+    const res = await runMocha({})
+
+    expect({ passing: res.passing, failing: res.failing }, errorMessage(res)).toEqual(expected)
+  })
+
+  it('works in parallel run mode', async () => {
+    const res = await runMocha({ parallel: true })
+
+    expect({ passing: res.passing, failing: res.failing }, errorMessage(res)).toEqual(expected)
+  })
+
+  it('works in watch mode', async () => {
+    const res = await runMocha({ watch: true })
+
+    expect({ passing: res.passing, failing: res.failing }, errorMessage(res)).toEqual(expected)
+  })
+
+  it('works in parallel watch mode', async () => {
+    const res = await runMocha({ parallel: true, watch: true })
+
+    expect({ passing: res.passing, failing: res.failing }, errorMessage(res)).toEqual(expected)
+  })
+})
+
+function errorMessage(results: TestResults) {
+  return {
+    extraMessage:
+      `Expected to pass ${PASSING_TESTS} instead of ${results.passing} and fail ${FAILING_TESTS} instead of ${results.failing}.\n` +
+      `\nSTDOUT:\n\`${results.stdout}\`` +
+      `\nSTDERR:\n\`${results.stderr}\`\n\n`,
+  }
+}
+
+interface TestResults {
+  passing: number
+  failing: number
+  stdout: string
+  stderr: string
+}
+
+function runMocha(modes: { watch?: boolean; parallel?: boolean }) {
+  return new Promise<TestResults>((resolve, reject) => {
+    const child = spawn(
+      'mocha',
+      ['--config', './mocha-config.tested.js', modes.watch && '--watch', modes.parallel && '--parallel'].filter(
+        (x): x is string => !!x,
+      ),
+      { env: process.env, cwd: __dirname },
+    )
+
+    const result = { passing: NaN, failing: NaN, stdout: '', stderr: '' }
+
+    const fail = (err: string | Error) => {
+      child.kill('SIGKILL')
+      reject(typeof err === 'string' ? new Error(err) : err)
+    }
+    const succeed = () => {
+      child.kill('SIGKILL')
+      resolve(result)
+    }
+
+    child.stderr.on('data', (data) => {
+      const str = String(data)
+      result.stderr += str
+
+      const ERROR_PREFIX = '\n× \x1B[31mERROR:\x1B[39m Error:'
+      if (str.startsWith(ERROR_PREFIX)) {
+        fail('Process crashed.' + str)
+      }
+
+      d(`stderr: ${str}`)
+
+      if (modes.watch) {
+        if (str.includes('[mocha] waiting for changes...')) {
+          succeed()
+        }
+      }
+    })
+    child.stdout.on('data', (data) => {
+      const str = String(data)
+      result.stdout += str
+
+      d(`stdout: ${str}`)
+
+      const passing = str.match(/(\d+) passing/)
+      if (passing) {
+        result.passing = parseInt(passing[1])
+      }
+      const failing = str.match(/(\d+) failing/)
+      if (failing) {
+        result.failing = parseInt(failing[1])
+        if (!modes.watch) {
+          succeed()
+        }
+      }
+    })
+    child.on('error', (err) => {
+      // eslint-disable-next-line no-console
+      console.error({ stderr })
+      fail(err)
+    })
+  })
+}

packages/test-e2e/test/mocha-integration.tested.ts

@@ -0,0 +1,10 @@
+import { expect } from 'earljs'
+
+describe('earljs/mocha', () => {
+  it('successful test', () => {
+    expect(2).toMatchSnapshot()
+  })
+  it('failing test', () => {
+    expect(Math.random()).toMatchSnapshot()
+  })
+})

packages/test-e2e/test/setup.ts

@@ -0,0 +1,48 @@
+/* eslint-disable no-console */
+import { execSync, ExecSyncOptions } from 'child_process'
+import { resolve } from 'path'
+
+before(function () {
+  const isCI = !!process.env.CI && process.env.CI !== 'false'
+  const skipBuild = isCI || ['1', 'true'].includes(process.env.SKIP_EARL_BUILD as string)
+
+  if (!skipBuild) {
+    // These are end-to-end tests, so we need local EarlJS to be built and present
+    // in node_modules, exactly as the user would have it.
+    // We skip installation on CI, because the packages will be already built.
+    // The developer, however, can repeatedly change the package and rerun end-to-end tests.
+    this.timeout(20000)
+    const testPkgDir = resolve(__dirname, '..')
+    const relativeEarlPkgDir = '../earljs'
+    const earlPkgDir = resolve(testPkgDir, relativeEarlPkgDir)
+
+    console.log('🔨 Building earljs...')
+    exec('yarn build', { cwd: earlPkgDir, errorMsg: 'Failed to build Earl.' })
+
+    console.log('🔧 Linking local earljs in end-to-end tests project...')
+
+    // Using `link` instead of `file:` protocol helps keeping package.json unchanged
+    // and performs a bit faster, albeit it doesn't support aliases like "@earljs/local".
+    exec(`yarn link`, { cwd: earlPkgDir })
+    exec(`yarn link earljs`, { cwd: testPkgDir })
+
+    console.log('🧪 Running end-to-end tests...\n')
+  }
+})
+
+function exec(command: string, { errorMsg, ...options }: ExecSyncOptions & { errorMsg?: string }) {
+  try {
+    execSync(command, {
+      ...options,
+      stdio: 'pipe',
+      encoding: 'utf-8',
+    })
+  } catch (err) {
+    console.error(...['🔥', errorMsg && red(errorMsg), (err as Error).message].filter(Boolean).join(' '))
+    process.exit(1)
+  }
+}
+
+function red(msg: string) {
+  return `\x1b[31m${msg}\x1b[0m`
+}

packages/test-e2e/tsconfig.json

@@ -0,0 +1,4 @@
+{
+  "extends": "../../tsconfig.json",
+  "include": ["src", "test"]
+}