feat(test-runner): Support lifecycle events (#125)

Add support for test runner lifecycle events: init and dispose. These are called once in every test runner life cycle (if implemented).  If they return a promise, we will wait for those promises to be resolved before proceding. We will only wait for max 2 seconds with the dispose before forcing the child process to be killed.

package.json

@@ -69,12 +69,12 @@
     "mocha-sinon": "^1.1.4",
     "sinon": "^1.17.2",
     "sinon-chai": "^2.8.0",
-    "stryker-api": "^0.1.0",
+    "stryker-api": "^0.1.1",
     "stryker-karma-runner": "^0.1.0",
     "typescript": "^1.8.9",
     "typings": "^0.7.11"
   },
   "peerDependencies": {
-    "stryker-api": "^0.1.0"
+    "stryker-api": "^0.1.1"
   }
 }

src/TestRunnerOrchestrator.ts

@@ -1,9 +1,9 @@
 import {StrykerOptions, InputFile} from 'stryker-api/core';
-import {RunResult, RunnerOptions, TestResult} from 'stryker-api/test_runner';
+import {TestRunner, RunResult, RunnerOptions, TestResult} from 'stryker-api/test_runner';
 import {TestSelector} from 'stryker-api/test_selector';
 import StrykerTempFolder from './utils/StrykerTempFolder';
-import IsolatedTestRunnerAdapter from './isolated-runner/IsolatedTestRunnerAdapter';
 import IsolatedTestRunnerAdapterFactory from './isolated-runner/IsolatedTestRunnerAdapterFactory';
+import IsolatedTestRunnerAdapter from './isolated-runner/IsolatedTestRunnerAdapter';
 import * as path from 'path';
 import * as os from 'os';
 import * as _ from 'lodash';
@@ -21,7 +21,7 @@ interface FileMap {
 
 interface TestRunnerSandbox {
   index: number;
-  runnerAdapter: IsolatedTestRunnerAdapter;
+  runner: IsolatedTestRunnerAdapter;
   fileMap: FileMap;
   testSelectionFilePath: string;
 }
@@ -49,17 +49,14 @@ export default class TestRunnerOrchestrator {
 
   private initialRunWithTestSelector() {
     let testSelectionFilePath = this.createTestSelectorFileName(this.createTempFolder());
-    let runnerAdapter = this.createTestRunner(this.files, true, testSelectionFilePath);
+    let runner = this.createTestRunner(this.files, true, testSelectionFilePath);
     let sandbox: TestRunnerSandbox = {
-      runnerAdapter,
+      runner,
       fileMap: null,
       testSelectionFilePath,
       index: 0
     };
-    return this.runSingleTestsRecursive(sandbox, [], 0).then((testResults) => {
-      runnerAdapter.dispose();
-      return testResults;
-    });
+    return this.runSingleTestsRecursive(sandbox, [], 0).then((testResults) => runner.dispose().then(() => testResults));
   }
 
   runMutations(mutants: Mutant[]): Promise<MutantResult[]> {
@@ -75,7 +72,7 @@ export default class TestRunnerOrchestrator {
             let sandbox = sandboxes.pop();
             let sourceFileCopy = sandbox.fileMap[mutant.filename];
             return Promise.all([mutant.save(sourceFileCopy), this.selectTestsIfPossible(sandbox, mutant.scopedTestIds)])
-              .then(() => sandbox.runnerAdapter.run({ timeout: this.calculateTimeout(mutant.timeSpentScopedTests) }))
+              .then(() => sandbox.runner.run({ timeout: this.calculateTimeout(mutant.timeSpentScopedTests) }))
               .then((runResult) => {
                 let result = this.collectFrozenMutantResult(mutant, runResult);
                 results.push(result);
@@ -92,8 +89,8 @@ export default class TestRunnerOrchestrator {
       }
       return new PromisePool(promiseProducer, sandboxes.length)
         .start()
-        .then(() => sandboxes.forEach(testRunner => testRunner.runnerAdapter.dispose()))
         .then(() => this.reportAllMutantsTested(results))
+        .then(() => Promise.all(sandboxes.map(testRunner => testRunner.runner.dispose())))
         .then(() => results);
     });
   }
@@ -153,7 +150,7 @@ export default class TestRunnerOrchestrator {
 
     return new Promise<RunResult[]>(resolve => {
       this.selectTestsIfPossible(sandbox, [currentTestIndex])
-        .then(() => sandbox.runnerAdapter.run({ timeout: 10000 }))
+        .then(() => sandbox.runner.run({ timeout: 10000 }))
         .then(runResult => {
           if (runResult.result === TestResult.Complete && runResult.succeeded > 0 || runResult.failed > 0) {
             runResults[currentTestIndex] = runResult;
@@ -163,7 +160,7 @@ export default class TestRunnerOrchestrator {
               // If this was iteration n+1 (n = number of tests), the runResult.result will be Complete, so we don't record it
               runResults[currentTestIndex] = runResult;
             }
-            sandbox.runnerAdapter.dispose();
+            sandbox.runner.dispose();
             resolve(runResults);
           }
         });
@@ -176,35 +173,37 @@ export default class TestRunnerOrchestrator {
     let allPromises: Promise<any>[] = [];
     log.info(`Creating ${cpuCount} test runners (based on cpu count)`);
     for (let i = 0; i < cpuCount; i++) {
-      allPromises.push(this.createSandbox(i).then(sandbox => testRunnerSandboxes.push(sandbox)));
+      allPromises.push(this.createInitializedSandbox(i));
     }
-    return Promise.all(allPromises).then(() => testRunnerSandboxes);
+    return Promise.all(allPromises);
+
   }
 
   private selectTestsIfPossible(sandbox: TestRunnerSandbox, ids: number[]): Promise<void> {
     if (this.testSelector) {
       let fileContent = this.testSelector.select(ids);
       return StrykerTempFolder.writeFile(sandbox.testSelectionFilePath, fileContent);
-    }else{
+    } else {
       return Promise.resolve(void 0);
     }
   }
 
-  private createSandbox(index: number): Promise<TestRunnerSandbox> {
+  private createInitializedSandbox(index: number): Promise<TestRunnerSandbox> {
     var tempFolder = this.createTempFolder();
     return this.copyAllFilesToFolder(tempFolder).then(fileMap => {
       let runnerFiles: InputFile[] = [];
       let testSelectionFilePath: string = null;
-      if(this.testSelector){
+      if (this.testSelector) {
         testSelectionFilePath = this.createTestSelectorFileName(tempFolder);
       }
       this.files.forEach(originalFile => runnerFiles.push({ path: fileMap[originalFile.path], shouldMutate: originalFile.shouldMutate }));
-      return {
+      let runner = this.createTestRunner(runnerFiles, false, testSelectionFilePath, index);
+      return runner.init().then(() => ({
         index,
         fileMap,
-        runnerAdapter: this.createTestRunner(runnerFiles, false, testSelectionFilePath, index),
+        runner,
         testSelectionFilePath
-      };
+      }));
     });
   }
 

src/isolated-runner/IsolatedTestRunnerAdapter.ts

@@ -1,14 +1,15 @@
 import {TestRunner, RunResult, RunOptions, RunnerOptions, TestResult} from 'stryker-api/test_runner';
 import {StrykerOptions} from 'stryker-api/core';
 import {fork, ChildProcess} from 'child_process';
-import Message, {MessageType} from './Message';
+import Message, {MessageType } from './Message';
 import StartMessageBody from './StartMessageBody';
 import RunMessageBody from './RunMessageBody';
 import ResultMessageBody from './ResultMessageBody';
 import * as _ from 'lodash';
 import * as log4js from 'log4js';
 
 const log = log4js.getLogger('IsolatedTestRunnerAdapter');
+const MAX_WAIT_FOR_DISPOSE = 2000;
 
 /**
  * Runs the given test runner in a child process and forwards reports about test results
@@ -17,10 +18,15 @@ const log = log4js.getLogger('IsolatedTestRunnerAdapter');
 export default class TestRunnerChildProcessAdapter implements TestRunner {
 
   private workerProcess: ChildProcess;
-  private currentPromiseFulfillmentCallback: (result: RunResult) => void;
-  private currentPromise: Promise<RunResult>;
+  private initPromiseFulfillmentCallback: () => void;
+  private runPromiseFulfillmentCallback: (result: RunResult) => void;
+  private disposePromiseFulfillmentCallback: () => void;
+  private initPromise: Promise<void>;
+  private runPromise: Promise<RunResult>;
+  private disposingPromise: Promise<any>;
   private currentTimeoutTimer: NodeJS.Timer;
   private currentRunStartedTimestamp: Date;
+  private isDisposing: boolean;
 
   constructor(private realTestRunnerName: string, private options: RunnerOptions) {
     this.startWorker();
@@ -56,7 +62,15 @@ export default class TestRunnerChildProcessAdapter implements TestRunner {
       this.clearCurrentTimer();
       switch (message.type) {
         case MessageType.Result:
-          this.handleResultMessage(message);
+          if (!this.isDisposing) {
+            this.handleResultMessage(message);
+          }
+          break;
+        case MessageType.InitDone:
+          this.initPromiseFulfillmentCallback();
+          break;
+        case MessageType.DisposeDone:
+          this.disposePromiseFulfillmentCallback();
           break;
         default:
           log.error(`Retrieved unrecognized message from child process: ${JSON.stringify(message)}`)
@@ -65,22 +79,41 @@ export default class TestRunnerChildProcessAdapter implements TestRunner {
     });
   }
 
+  init(): Promise<any> {
+    this.initPromise = new Promise<void>(resolve => this.initPromiseFulfillmentCallback = resolve);
+    this.sendInitCommand();
+    return this.initPromise;
+  }
+
   run(options: RunOptions): Promise<RunResult> {
     this.clearCurrentTimer();
     if (options.timeout) {
       this.markNoResultTimeout(options.timeout);
     }
-    this.currentPromise = new Promise<RunResult>(resolve => {
-      this.currentPromiseFulfillmentCallback = resolve;
+    this.runPromise = new Promise<RunResult>(resolve => {
+      this.runPromiseFulfillmentCallback = resolve;
       this.sendRunCommand(options);
       this.currentRunStartedTimestamp = new Date();
     });
-    return this.currentPromise;
+    return this.runPromise;
   }
 
-  dispose() {
-    this.clearCurrentTimer();
-    this.workerProcess.kill();
+  dispose(): Promise<any> {
+    if (this.isDisposing) {
+      return this.disposingPromise;
+    } else {
+      this.isDisposing = true;
+      this.disposingPromise = new Promise(resolve => this.disposePromiseFulfillmentCallback = resolve)
+        .then(() => {
+          clearTimeout(timer);
+          this.workerProcess.kill();
+          this.isDisposing = false;
+        });
+      this.clearCurrentTimer();
+      this.sendDisposeCommand();
+      let timer = setTimeout(this.disposePromiseFulfillmentCallback, MAX_WAIT_FOR_DISPOSE);
+      return this.disposingPromise;
+    }
   }
 
   private sendRunCommand(options: RunOptions) {
@@ -104,9 +137,17 @@ export default class TestRunnerChildProcessAdapter implements TestRunner {
     this.workerProcess.send(startMessage);
   }
 
+  private sendInitCommand() {
+    this.workerProcess.send(this.emptyMessage(MessageType.Init));
+  }
+
+  private sendDisposeCommand() {
+    this.workerProcess.send(this.emptyMessage(MessageType.Dispose));
+  }
+
   private handleResultMessage(message: Message<ResultMessageBody>) {
     message.body.result.timeSpent = (new Date().getTime() - this.currentRunStartedTimestamp.getTime());
-    this.currentPromiseFulfillmentCallback(message.body.result);
+    this.runPromiseFulfillmentCallback(message.body.result);
   }
 
   private clearCurrentTimer() {
@@ -122,8 +163,13 @@ export default class TestRunnerChildProcessAdapter implements TestRunner {
   }
 
   private handleTimeout() {
-    this.workerProcess.kill();
-    this.startWorker();
-    this.currentPromiseFulfillmentCallback({ result: TestResult.Timeout });
+    this.dispose()
+      .then(() => this.startWorker())
+      .then(() => this.init())
+      .then(() => this.runPromiseFulfillmentCallback({ result: TestResult.Timeout }))
+  }
+
+  private emptyMessage(type: MessageType): Message<void> {
+    return { type };
   }
 }

src/isolated-runner/IsolatedTestRunnerAdapterWorker.ts

@@ -4,10 +4,14 @@ import StartMessageBody from './StartMessageBody';
 import RunMessageBody from './RunMessageBody';
 import ResultMessageBody from './ResultMessageBody';
 import PluginLoader from '../PluginLoader';
+import * as log4js from 'log4js';
+import {isPromise} from '../utils/objectUtils';
+
+const log = log4js.getLogger('TestRunnerChildProcessAdapterWorker');
 
 class TestRunnerChildProcessAdapterWorker {
 
-  underlyingTestRunner: TestRunner;
+  private underlyingTestRunner: TestRunner;
 
   constructor() {
     this.listenToMessages();
@@ -22,6 +26,14 @@ class TestRunnerChildProcessAdapterWorker {
         case MessageType.Run:
           this.run(message.body);
           break;
+        case MessageType.Init:
+          this.init();
+          break;
+        case MessageType.Dispose:
+          this.dispose();
+          break;
+        default:
+          log.warn('Received unsupported message: {}', JSON.stringify(message));
       }
     });
   }
@@ -31,6 +43,38 @@ class TestRunnerChildProcessAdapterWorker {
     this.underlyingTestRunner = TestRunnerFactory.instance().create(body.runnerName, body.runnerOptions);
   }
 
+  init() {
+    let initPromise: Promise<any> | void = void 0;
+    if (this.underlyingTestRunner.init) {
+      initPromise = this.underlyingTestRunner.init();
+    }
+    if (isPromise(initPromise)) {
+      initPromise.then(this.sendInitDone);
+    } else {
+      this.sendInitDone();
+    }
+  }
+
+  sendInitDone() {
+    process.send({ type: MessageType.InitDone });
+  }
+
+  dispose() {
+    let disposePromise: Promise<any> | void = void 0;
+    if (this.underlyingTestRunner.dispose) {
+      disposePromise = this.underlyingTestRunner.dispose();
+    }
+    if (isPromise(disposePromise)) {
+      disposePromise.then(this.sendDisposeDone);
+    } else {
+      this.sendDisposeDone();
+    }
+  }
+
+  sendDisposeDone() {
+    process.send({ type: MessageType.DisposeDone });
+  }
+
   run(body: RunMessageBody) {
     this.underlyingTestRunner.run(body.runOptions).then(this.reportResult, this.reportErrorResult);
   }

src/isolated-runner/Message.ts

@@ -1,13 +1,17 @@
 
-export enum MessageType{
+export enum MessageType {
   Start,
+  Init,
+  InitDone,
   Run,
-  Result
+  Result,
+  Dispose,
+  DisposeDone
 }
 
-interface Message<T> {
+export interface Message<T> {
   type: MessageType;
-  body: T;
+  body?: T;
 }
 
 export default Message;

test/helpers/log4jsMock.ts

@@ -1,4 +1,3 @@
-console.log('l4js:', require.resolve('log4js'));
 import * as log4js from 'log4js';
 import * as sinon from 'sinon';