feat(Command test runner): Add command test runner (#1047)

Add the command test runner. This test runner simply runs command from the bash/cmd prompt. It defaults to `npm test`.

It will always report exactly _one_ test result (called `'All tests'`), as it cannot distinguish between the tests. It is implemented differently from other test runners, as we don't want to spawn this one in a separate process (it already uses `require('child_process').exec` to start a test run).

Fixes #768

integrationTest/helpers.ts

@@ -0,0 +1,12 @@
+import * as path from 'path';
+import * as fs from 'mz/fs';
+import { expect } from 'chai';
+import { ScoreResult } from 'stryker-api/report';
+
+export async function readScoreResult(eventResultDirectory = path.resolve('reports', 'mutation', 'events')) {
+    const allReportFiles = await fs.readdir(eventResultDirectory);
+    const scoreResultReportFile = allReportFiles.find(file => !!file.match(/.*onScoreCalculated.*/));
+    expect(scoreResultReportFile).ok;
+    const scoreResultContent = await fs.readFile(path.resolve(eventResultDirectory, scoreResultReportFile || ''), 'utf8');
+    return JSON.parse(scoreResultContent) as ScoreResult;
+}

integrationTest/test/command/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "test-module",
+  "version": "0.0.0",
+  "private": true,
+  "description": "A module to perform an integration test",
+  "main": "index.js",
+  "scripts": {
+    "pretest": "rimraf \"reports\" \"stryker.log\"",
+    "test": "stryker run stryker.conf.js",
+    "mocha": "mocha",
+    "posttest": "mocha --require ts-node/register verify/*.ts"
+  },
+  "author": "",
+  "license": "ISC"
+}

integrationTest/test/command/src/Add.js

@@ -0,0 +1,26 @@
+module.exports.add = function(num1, num2) {
+  return num1 + num2;
+};
+
+module.exports.addOne = function(number) {
+  number++;
+  return number;
+};
+
+module.exports.negate = function(number) {
+  return -number;
+};
+
+module.exports.notCovered = function(number) {
+  return number > 10;
+};
+
+module.exports.isNegativeNumber = function(number) {
+  var isNegative = false;
+  if(number < 0){
+    isNegative = true;
+  }
+  return isNegative;
+};
+
+

integrationTest/test/command/src/Circle.js

@@ -0,0 +1,8 @@
+module.exports.getCircumference = function(radius) {
+  //Function to test multiple math mutations in a single function.
+  return 2 * Math.PI * radius;
+};
+
+module.exports.untestedFunction = function() {
+  var i = 5 / 2 * 3;
+};

integrationTest/test/command/stryker.conf.js

@@ -0,0 +1,15 @@
+module.exports = function (config) {
+  config.set({
+    mutate: ['src/*.js'],
+    testFramework: 'mocha',
+    coverageAnalysis: 'off',
+    reporter: ['clear-text', 'event-recorder'],
+    mutator: 'javascript',
+    maxConcurrentTestRunners: 2,
+    commandRunner: {
+      command: 'npm run mocha'
+    },
+    symlinkNodeModules: false,
+    fileLogLevel: 'info'
+  });
+};

integrationTest/test/command/test/AddSpec.js

@@ -0,0 +1,55 @@
+var expect = require('chai').expect;
+var addModule = require('../src/Add');
+var add = addModule.add;
+var addOne = addModule.addOne;
+var isNegativeNumber = addModule.isNegativeNumber;
+var negate = addModule.negate;
+var notCovered = addModule.notCovered;
+
+console.log(process.env.path);
+
+describe('Add', function() {
+  it('should be able to add two numbers', function() {
+    var num1 = 2;
+    var num2 = 5;
+    var expected = num1 + num2;
+
+    var actual = add(num1, num2);
+
+    expect(actual).to.be.equal(expected);
+  });
+
+  it('should be able 1 to a number', function() {
+    var number = 2;
+    var expected = 3;
+
+    var actual = addOne(number);
+
+    expect(actual).to.be.equal(expected);
+  });
+
+  it('should be able negate a number', function() {
+    var number = 2;
+    var expected = -2;
+
+    var actual = negate(number);
+
+    expect(actual).to.be.equal(expected);
+  });
+
+  it('should be able to recognize a negative number', function() {
+    var number = -2;
+
+    var isNegative = isNegativeNumber(number);
+
+    expect(isNegative).to.be.true;
+  });
+
+  it('should be able to recognize that 0 is not a negative number', function() {
+    var number = 0;
+
+    var isNegative = isNegativeNumber(number);
+
+    expect(isNegative).to.be.false;
+  });
+});

integrationTest/test/command/test/CircleSpec.js

@@ -0,0 +1,14 @@
+var expect = require('chai').expect;
+var circleModule = require('../src/Circle');
+var getCircumference = circleModule.getCircumference;
+
+describe('Circle', function() {
+  it('should have a circumference of 2PI when the radius is 1', function() {
+    var radius = 1;
+    var expectedCircumference = 2 * Math.PI;
+
+    var circumference = getCircumference(radius);
+
+    expect(circumference).to.be.equal(expectedCircumference);
+  });
+});

integrationTest/test/command/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../../tsconfig.json",
+  "compilerOptions": {
+    "declaration": false
+  },
+  "files": [
+    "verify/verify.ts"
+  ]
+}

integrationTest/test/command/verify/verify.ts

@@ -0,0 +1,20 @@
+import * as fs from 'mz/fs';
+import { expect } from 'chai';
+import * as path from 'path';
+import { readScoreResult } from '../../../helpers';
+
+describe('After running stryker with the command test runner', () => {
+  it('should report 69% mutation score', async () => {
+    const scoreResult = await readScoreResult(path.resolve('reports', 'mutation', 'events'));
+    expect(scoreResult.killed).eq(16);
+    expect(scoreResult.noCoverage).eq(0);
+    expect(scoreResult.mutationScore).greaterThan(69).and.lessThan(70);
+  });
+
+  it('should write to a log file', async () => {
+    const strykerLog = await fs.readFile('./stryker.log', 'utf8');
+    expect(strykerLog).contains('INFO InitialTestExecutor Initial test run succeeded. Ran 1 test');
+    expect(strykerLog).matches(/Stryker Done in \d+/);
+    expect(strykerLog).not.contains('ERROR'); 
+  });
+});

integrationTest/test/jasmine-jasmine/verify/verify.ts

@@ -1,15 +1,10 @@
 import * as fs from 'mz/fs';
 import { expect } from 'chai';
-import * as path from 'path';
-import { ScoreResult } from 'stryker-api/report';
+import { readScoreResult } from '../../../helpers';
 
 describe('After running stryker with test runner jasmine, test framework jasmine', () => {
   it('should report 85% mutation score', async () => {
-    const allReportFiles = await fs.readdir(path.resolve('reports', 'mutation', 'events'));
-    const scoreResultReportFile = allReportFiles.find(file => !!file.match(/.*onScoreCalculated.*/));
-    expect(scoreResultReportFile).ok;
-    const scoreResultContent = await fs.readFile(path.resolve('reports', 'mutation', 'events', scoreResultReportFile || ''), 'utf8');
-    const scoreResult = JSON.parse(scoreResultContent) as ScoreResult;
+    const scoreResult = await readScoreResult();
     expect(scoreResult.killed).eq(12);
     expect(scoreResult.noCoverage).eq(1);
     expect(scoreResult.mutationScore).greaterThan(85).and.lessThan(86);
@@ -18,7 +13,7 @@ describe('After running stryker with test runner jasmine, test framework jasmine
   it('should write to a log file', async () => {
     const strykerLog = await fs.readFile('./stryker.log', 'utf8');
     expect(strykerLog).contains('INFO InputFileResolver Found 2 of 10 file(s) to be mutated');
-    expect(strykerLog).matches(/Stryker Done in \d+ seconds/);
+    expect(strykerLog).matches(/Stryker Done in \d+/);
     // TODO, we now have an error because of a memory leak: https://github.com/jasmine/jasmine-npm/issues/134
     // expect(strykerLog).not.contains('ERROR'); 
   });

packages/stryker-api/src/config/Config.ts

@@ -15,7 +15,7 @@ export default class Config implements StrykerOptions {
   port = 9234;
   reporter = ['progress', 'clear-text'];
   coverageAnalysis: 'perTest' | 'all' | 'off' = 'perTest';
-  testRunner: string;
+  testRunner: string = 'command';
   testFramework: string;
   mutator: string | MutatorDescriptor = 'es5';
   transpilers: string[] = [];

packages/stryker/README.md

@@ -54,8 +54,7 @@ module.exports = function(config){
     testFramework: 'mocha',
     testRunner: 'mocha',
     reporter: ['progress', 'clear-text', 'dots', 'html', 'event-recorder'],
-    coverageAnalysis: 'perTest',
-    plugins: ['stryker-mocha-runner', 'stryker-html-reporter']
+    coverageAnalysis: 'perTest'
   });
 }
 ```
@@ -101,11 +100,15 @@ This is optional, as you can choose to not mutate any files at all and perform a
 #### Test runner  
 **Command line:** `--testRunner karma`  
 **Config file:** `testRunner: 'karma'`  
-**Default value:** *none*  
-**Mandatory**: yes  
+**Default value:** `'command'`  
+**Mandatory**: No  
 **Description:**  
-With `testRunner` you specify the test runner to run your tests. This option is required.  
-Make sure the test runner plugin for Stryker is installed. E.g. we need the `stryker-karma-runner` to use `karma` as a test runner. 
+With `testRunner` you specify the test runner that Stryker uses to run your tests. The default value is `command`. The command runner runs a configurable bash/cmd command and bases the result on the exit code of that program (0 for success, otherwise failed). You can configure this command via the config file using the `commandRunner: { command: 'npm run mocha' }`. It uses `npm test` as the command by default. 
+
+The command test runner can be made to work in any use case, but comes with a performance
+penalty, as Stryker cannot do any optimizations and just runs all tests for all mutants. 
+If possible, you should try to use one of the test runner plugins that hook into your test runner of choice. 
+For example: install and use the `stryker-karma-runner` to use `karma` as a test runner. 
 See the [list of plugins](https://stryker-mutator.io/plugins.html) for an up-to-date list of supported test runners and plugins.
 
 #### Test framework  

packages/stryker/src/initializer/StrykerInitializer.ts

@@ -5,6 +5,7 @@ import PromptOption from './PromptOption';
 import { getLogger } from 'stryker-api/logging';
 import { filterEmpty } from '../utils/objectUtils';
 import StrykerConfigWriter from './StrykerConfigWriter';
+import CommandTestRunner from '../test-runner/CommandTestRunner';
 
 const enum PackageManager {
   Npm = 'npm',
@@ -27,7 +28,8 @@ export default class StrykerInitializer {
     configWriter.guardForExistingConfig();
     this.patchProxies();
     const selectedTestRunner = await this.selectTestRunner();
-    const selectedTestFramework = selectedTestRunner ? await this.selectTestFramework(selectedTestRunner) : null;
+    const selectedTestFramework = selectedTestRunner && !CommandTestRunner.is(selectedTestRunner.name) 
+      ? await this.selectTestFramework(selectedTestRunner) : null;
     const selectedMutator = await this.selectMutator();
     const selectedTranspilers = await this.selectTranspilers();
     const selectedReporters = await this.selectReporters();

packages/stryker/src/initializer/StrykerInquirer.ts

@@ -1,5 +1,6 @@
 import * as inquirer from 'inquirer';
 import PromptOption from './PromptOption';
+import CommandTestRunner from '../test-runner/CommandTestRunner';
 
 export interface PromptResult {
   additionalNpmDependencies: string[];
@@ -9,14 +10,17 @@ export interface PromptResult {
 export class StrykerInquirer {
 
   public async promptTestRunners(options: PromptOption[]): Promise<PromptOption> {
+    const choices: inquirer.ChoiceType[] = options.map(_ => _.name);
+    choices.push(new inquirer.Separator());
+    choices.push(CommandTestRunner.runnerName);
     const answers = await inquirer.prompt<{ testRunner: string }>({
       type: 'list',
       name: 'testRunner',
-      message: 'Which test runner do you want to use?',
-      choices: options.map(_ => _.name),
+      message: 'Which test runner do you want to use? If your test runner isn\'t listed here, you can choose "command" (it uses your `npm test` command, but will come with a big performance penalty)',
+      choices,
       default: 'Mocha'
     });
-    return options.filter(_ => _.name === answers.testRunner)[0] || { name: 'mocha', npm: 'stryker-mocha-runner' };
+    return options.filter(_ => _.name === answers.testRunner)[0] || { name: CommandTestRunner.runnerName };
   }
 
   public async promptTestFrameworks(options: PromptOption[]): Promise<PromptOption> {