feat(stryker): add excludedMutations as a config option (#13) (#652)

This adds the ability to exclude specific mutation types (`'BooleanSubstitution'`, `'ArrayLiteral'`, etc.) from being performed during the test run. Can be used via the config file. Defaults to an empty array, meaning all mutation types are used.

Configuration:

```js
mutator: {
  name: 'javascript', 
  excludedMutations: [ 'BooleanSubstitution', 'Block' /*, ... */]
}
```

packages/stryker-api/core.ts

@@ -5,4 +5,5 @@ export { default as Position } from './src/core/Position';
 export { default as Location } from './src/core/Location';
 export { default as Range } from './src/core/Range';
 export { default as InputFileDescriptor } from './src/core/InputFileDescriptor';
+export { default as MutatorDescriptor } from './src/core/MutatorDescriptor';
 export { default as MutationScoreThresholds } from './src/core/MutationScoreThresholds';

packages/stryker-api/src/config/Config.ts

@@ -1,4 +1,4 @@
-import { StrykerOptions, InputFileDescriptor, MutationScoreThresholds } from '../../core';
+import { StrykerOptions, InputFileDescriptor, MutatorDescriptor, MutationScoreThresholds } from '../../core';
 
 export default class Config implements StrykerOptions {
 
@@ -16,7 +16,7 @@ export default class Config implements StrykerOptions {
   coverageAnalysis: 'perTest' | 'all' | 'off' = 'perTest';
   testRunner: string;
   testFramework: string;
-  mutator: string = 'es5';
+  mutator: string | MutatorDescriptor = 'es5';
   transpilers: string[] = [];
   maxConcurrentTestRunners: number = Infinity;
   thresholds: MutationScoreThresholds = {

packages/stryker-api/src/core/MutatorDescriptor.ts

@@ -0,0 +1,7 @@
+
+interface MutatorDescriptor {
+  name: string;
+  excludedMutations: Array<string>;
+}
+
+export default MutatorDescriptor;

packages/stryker-api/src/core/StrykerOptions.ts

@@ -1,5 +1,6 @@
 import InputFileDescriptor from './InputFileDescriptor';
 import MutationScoreThresholds from './MutationScoreThresholds';
+import MutatorDescriptor from './MutatorDescriptor';
 
 interface StrykerOptions {
   // this ensures that custom config for for example 'karma' can be added under the 'karma' key
@@ -49,11 +50,18 @@ interface StrykerOptions {
   testRunner?: string;
 
   /**
-   * The name of the mutant generator to use to generate mutants based on your input file. 
+   * The mutant generator to use to generate mutants based on your input file.
    * This is often dependent on the language of your source files.
-   * For example: 'es5', 'typescript'
-   */
-  mutator?: string;
+   *
+   * This value can be either a string, or an object with 2 properties:
+   * * `string`: The name of the mutant generator to use. For example: 'javascript', 'typescript'
+   * * { name: 'name', excludedMutations: ['mutationType1', 'mutationType2'] }:
+   *    * The `name` property is mandatory and contains the name of the mutant generator to use.
+   *    * For example: 'javascript', 'typescript'
+   *    * The `excludedMutations` property is mandatory and contains the names of the specific mutation types to exclude from testing.
+   *    * The values must match the given names of the mutations. For example: 'BinaryExpression', 'BooleanSubstitution', etc.
+   */
+  mutator?: string | MutatorDescriptor;
 
   /**
    * The names of the transpilers to use (in order). Default: [].

packages/stryker/README.md

@@ -164,11 +164,21 @@ In addition to requiring your test runner to be able to report the code coverage
  (`Mocha` is not yet supported).
 
 #### Mutator 
-**Config file:** `mutator: 'es5'`  
+**Command line:** `--mutator es5`  
+**Config file:** `mutator: { name: 'es5', excludedMutations: ['BooleanSubstitution', 'StringLiteral'] }`  
 **Default value:** `es5`  
 **Mandatory**: no  
 **Description:**  
-With `mutator` you configure which mutator plugin you want to use. This defaults to es5.  
+With `mutator` you configure which mutator plugin you want to use, and optionally, which mutation types to exclude from the test run.  
+The mutator plugin name defaults to `es5` if not specified. The list of excluded mutation types defaults to an empty array, meaning all mutation types will be included in the test.  
+The full list of mutation types varies slightly between mutators (for example, the `es5` mutator will not use the same mutation types as the `typescript` mutator). Mutation type names are case-sensitive, and can be found either in the source code or in a generated Stryker report.  
+
+When using the command line, only the mutator name as a string may be provided.  
+When using the config file, you can provide either a string representing the mutator name, or a `MutatorDescriptor` object, like so:  
+
+* `MutatorDescriptor` object: `{ name: 'name', excludedMutations: ['mutationType1', 'mutationType2', ...] }`:  
+   * The `name` property is mandatory and contains the name of the mutator plugin to use.  
+   * The `excludedMutations` property is mandatory and contains the types of mutations to exclude from the test run.  
 
 #### Transpilers  
 **Config file:** `transpilers: '['typescript']'`  

packages/stryker/src/ConfigValidator.ts

@@ -1,5 +1,5 @@
 import { TestFramework } from 'stryker-api/test_framework';
-import { MutationScoreThresholds } from 'stryker-api/core';
+import { MutatorDescriptor, MutationScoreThresholds } from 'stryker-api/core';
 import { Config } from 'stryker-api/config';
 import { getLogger } from 'log4js';
 
@@ -14,11 +14,11 @@ export default class ConfigValidator {
   validate() {
     this.validateTestFramework();
     this.validateThresholds();
+    this.validateMutator();
     this.validateLogLevel();
     this.validateTimeout();
     this.validateIsNumber('port', this.strykerConfig.port);
     this.validateIsNumber('maxConcurrentTestRunners', this.strykerConfig.maxConcurrentTestRunners);
-    this.validateIsString('mutator', this.strykerConfig.mutator);
     this.validateIsStringArray('plugins', this.strykerConfig.plugins);
     this.validateIsStringArray('reporter', this.strykerConfig.reporter);
     this.validateIsStringArray('transpilers', this.strykerConfig.transpilers);
@@ -33,6 +33,17 @@ export default class ConfigValidator {
     }
   }
 
+  private validateMutator() {
+    const mutator = this.strykerConfig.mutator;
+    if (typeof mutator === 'object') {
+      const mutatorDescriptor = mutator as MutatorDescriptor;
+      this.validateIsString('mutator.name', mutatorDescriptor.name);
+      this.validateIsStringArray('mutator.excludedMutations', mutatorDescriptor.excludedMutations);
+    } else if (typeof mutator !== 'string') {
+      this.invalidate(`Value "${mutator}" is invalid for \`mutator\`. Expected either a string or an object`);
+    }
+  }
+
   private validateThresholds() {
     const thresholds = this.strykerConfig.thresholds;
     this.validateThresholdsValueExists('high', thresholds.high);

packages/stryker/src/MutatorFacade.ts

@@ -1,4 +1,4 @@
-import { File } from 'stryker-api/core';
+import { File, MutatorDescriptor } from 'stryker-api/core';
 import { Config } from 'stryker-api/config';
 import { Mutant, Mutator, MutatorFactory } from 'stryker-api/mutant';
 import ES5Mutator from './mutators/ES5Mutator';
@@ -12,7 +12,15 @@ export default class MutatorFacade implements Mutator {
 
   mutate(inputFiles: File[]): Mutant[] {
     return MutatorFactory.instance()
-      .create(this.config.mutator, this.config)
+      .create(this.getMutatorName(this.config.mutator), this.config)
       .mutate(inputFiles);
   }
+
+  private getMutatorName(mutator: string | MutatorDescriptor) {
+    if (typeof mutator === 'string') {
+      return mutator;
+    } else {
+      return mutator.name;
+    }
+  }
 }

packages/stryker/src/Stryker.ts

@@ -1,7 +1,8 @@
 import { Config, ConfigEditorFactory } from 'stryker-api/config';
-import { StrykerOptions, File } from 'stryker-api/core';
+import { StrykerOptions, MutatorDescriptor, File } from 'stryker-api/core';
 import { MutantResult } from 'stryker-api/report';
 import { TestFramework } from 'stryker-api/test_framework';
+import { Mutant } from 'stryker-api/mutant';
 import ReporterOrchestrator from './ReporterOrchestrator';
 import TestFrameworkOrchestrator from './TestFrameworkOrchestrator';
 import MutantTestMatcher from './MutantTestMatcher';
@@ -68,14 +69,11 @@ export default class Stryker {
 
   private mutate(inputFiles: File[], initialTestRunResult: InitialTestRunResult) {
     const mutator = new MutatorFacade(this.config);
-    const mutants = mutator.mutate(inputFiles);
-    if (mutants.length) {
-      this.log.info(`${mutants.length} Mutant(s) generated`);
-    } else {
-      this.log.info('It\'s a mutant-free world, nothing to test.');
-    }
+    const allMutants = mutator.mutate(inputFiles);
+    const includedMutants = this.removeExcludedMutants(allMutants);
+    this.logMutantCount(includedMutants.length, allMutants.length);
     const mutantRunResultMatcher = new MutantTestMatcher(
-      mutants,
+      includedMutants,
       inputFiles,
       initialTestRunResult.runResult,
       SourceMapper.create(initialTestRunResult.transpiledFiles, this.config),
@@ -85,6 +83,29 @@ export default class Stryker {
     return mutantRunResultMatcher.matchWithMutants();
   }
 
+  private logMutantCount(includedMutantCount: number, totalMutantCount: number) {
+    let mutantCountMessage;
+    if (includedMutantCount) {
+      mutantCountMessage = `${includedMutantCount} Mutant(s) generated`;
+    } else {
+      mutantCountMessage = `It\'s a mutant-free world, nothing to test.`;
+    }
+    const numberExcluded = totalMutantCount - includedMutantCount;
+    if (numberExcluded) {
+      mutantCountMessage += ` (${numberExcluded} Mutant(s) excluded)`;
+    }
+    this.log.info(mutantCountMessage);
+  }
+
+  private removeExcludedMutants(mutants: Mutant[]): Mutant[] {
+    if (typeof this.config.mutator === 'string') {
+      return mutants;
+    } else {
+      const mutatorDescriptor = this.config.mutator as MutatorDescriptor;
+      return mutants.filter(mutant => mutatorDescriptor.excludedMutations.indexOf(mutant.mutatorName) === -1);
+    }
+  }
+
   private loadPlugins() {
     if (this.config.plugins) {
       new PluginLoader(this.config.plugins).load();

packages/stryker/test/helpers/producers.ts

@@ -228,7 +228,8 @@ export const transpileResult = factoryMethod<TranspileResult>(() => ({
 
 export const sourceFile = () => new SourceFile(textFile());
 
-export const testableMutant = (fileName = 'file') => new TestableMutant('1337', mutant({
+export const testableMutant = (fileName = 'file', mutatorName = 'foobarMutator') => new TestableMutant('1337', mutant({
+  mutatorName,
   range: [12, 13],
   replacement: '-',
   fileName

packages/stryker/test/unit/ConfigValidatorSpec.ts

@@ -103,14 +103,6 @@ describe('ConfigValidator', () => {
     expect(log.fatal).calledWith('Value "break" is invalid for `timeoutFactor`. Expected a number');
   });
 
-  it('should be invalid with non-string mutator', () => {
-    let brokenConfig = breakConfig(config, 'mutator', 0);
-    sut = new ConfigValidator(brokenConfig, testFramework());
-    sut.validate();
-    expect(exitStub).calledWith(1);
-    expect(log.fatal).calledWith('Value "0" is invalid for `mutator`. Expected a string');
-  });
-
   describe('plugins', () => {
     it('should be invalid with non-array plugins', () => {
       let brokenConfig = breakConfig(config, 'plugins', 'stryker-typescript');
@@ -129,6 +121,62 @@ describe('ConfigValidator', () => {
     });
   });
 
+  describe('mutator', () => {
+    it('should be invalid with non-string mutator', () => {
+      let brokenConfig = breakConfig(config, 'mutator', 0);
+      sut = new ConfigValidator(brokenConfig, testFramework());
+      sut.validate();
+      expect(exitStub).calledWith(1);
+      expect(log.fatal).calledWith('Value "0" is invalid for `mutator`. Expected either a string or an object');
+    });
+
+    describe('as an object', () => {
+      it('should be valid with string mutator name and string array excluded mutations', () => {
+        let validConfig = breakConfig(config, 'mutator', {
+          name: 'es5',
+          excludedMutations: ['BooleanSubstitution']
+        });
+        sut = new ConfigValidator(validConfig, testFramework());
+        sut.validate();
+        expect(exitStub).not.called;
+        expect(log.fatal).not.called;
+      });
+
+      it('should be invalid with non-string mutator name', () => {
+        let brokenConfig = breakConfig(config, 'mutator', {
+          name: 0,
+          excludedMutations: []
+        });
+        sut = new ConfigValidator(brokenConfig, testFramework());
+        sut.validate();
+        expect(exitStub).calledWith(1);
+        expect(log.fatal).calledWith('Value "0" is invalid for `mutator.name`. Expected a string');
+      });
+
+      it('should be invalid with non-array excluded mutations', () => {
+        let brokenConfig = breakConfig(config, 'mutator', {
+          name: 'es5',
+          excludedMutations: 'BooleanSubstitution'
+        });
+        sut = new ConfigValidator(brokenConfig, testFramework());
+        sut.validate();
+        expect(exitStub).calledWith(1);
+        expect(log.fatal).calledWith('Value "BooleanSubstitution" is invalid for `mutator.excludedMutations`. Expected an array');
+      });
+
+      it('should be invalid with non-string excluded mutation array elements', () => {
+        let brokenConfig = breakConfig(config, 'mutator', {
+          name: 'es5',
+          excludedMutations: ['BooleanSubstitution', 0]
+        });
+        sut = new ConfigValidator(brokenConfig, testFramework());
+        sut.validate();
+        expect(exitStub).calledWith(1);
+        expect(log.fatal).calledWith('Value "0" is an invalid element of `mutator.excludedMutations`. Expected a string');
+      });
+    });
+  });
+
   describe('reporter', () => {
     it('should be invalid with non-array reporter', () => {
       let brokenConfig = breakConfig(config, 'reporter', 'stryker-typescript');

packages/stryker/test/unit/MutatorFacadeSpec.ts

@@ -17,13 +17,26 @@ describe('MutatorFacade', () => {
   });
 
   describe('mutate', () => {
-    it('should create the configured mutant generator', () => {
+    it('should create the configured mutant generator with a string mutator', () => {
       const config = new Config();
       const sut = new MutatorFacade(config);
       const inputFiles = [file()];
       expect(sut.mutate(inputFiles)).deep.eq(['mutant']);
       expect(mutatorMock.mutate).calledWith(inputFiles);
       expect(MutatorFactory.instance().create).calledWith('es5');
     });
+
+    it('should create the configured mutant generator with an object mutator', () => {
+      const config = new Config();
+      config.mutator = {
+        name: 'javascript',
+        excludedMutations: []
+      };
+      const sut = new MutatorFacade(config);
+      const inputFiles = [file()];
+      expect(sut.mutate(inputFiles)).deep.eq(['mutant']);
+      expect(mutatorMock.mutate).calledWith(inputFiles);
+      expect(MutatorFactory.instance().create).calledWith('javascript');
+    });
   });
 });