feat(incremental): add incremental mode

Add incremental mode. When incremental mode is active, Stryker will use the results from the previous run to determine which mutants need to be retested, yielding much faster results.

You can enable incremental mode with `--incremental` (or using `"incremental": true` in the config file),

Results are stored in the `reports/.mutation-incremental.json` file.

packages/api/schema/stryker-core.json

@@ -270,11 +270,6 @@
       "description": "Set the concurrency of workers. Stryker will always run checkers and test runners in parallel by creating worker processes (note, not `worker_threads`). This defaults to `n-1` where `n` is the number of logical CPU cores available on your machine, unless `n <= 4`, in that case it uses `n`. This is a sane default for most use cases.",
       "type": "number"
     },
-    "maxTestRunnerReuse": {
-      "description": "Restart each test runner worker process after `n` runs. Not recommended unless you are experiencing memory leaks that you are unable to resolve. Configuring `0` here means infinite reuse.",
-      "type": "number",
-      "default": 0
-    },
     "commandRunner": {
       "description": "Options used by the command test runner. Note: these options will only be used when the command test runner is activated (this is the default)",
       "$ref": "#/definitions/commandRunnerOptions",
@@ -313,6 +308,11 @@
       "type": "boolean",
       "default": false
     },
+    "incremental": {
+      "description": "Experimental: Enable 'incremental mode'. Stryker will store results in \"reports/.mutation-incremental.json\" and use those to speed up the next --incremental run",
+      "type": "boolean",
+      "default": false
+    },
     "fileLogLevel": {
       "description": "Set the log level that Stryker uses to write to the \"stryker.log\" file",
       "$ref": "#/definitions/logLevel",
@@ -333,6 +333,11 @@
       "type": "number",
       "default": 9007199254740991
     },
+    "maxTestRunnerReuse": {
+      "description": "Restart each test runner worker process after `n` runs. Not recommended unless you are experiencing memory leaks that you are unable to resolve. Configuring `0` here means infinite reuse.",
+      "type": "number",
+      "default": 0
+    },
     "mutate": {
       "description": "With mutate you configure the subset of files to use for mutation testing. Generally speaking, these should be your own source files.",
       "type": "array",

packages/core/package.json

@@ -75,6 +75,7 @@
     "mkdirp": "~1.0.3",
     "mutation-testing-elements": "1.7.10",
     "mutation-testing-metrics": "1.7.10",
+    "mutation-testing-report-schema": "1.7.10",
     "npm-run-path": "~5.1.0",
     "progress": "~2.0.0",
     "rimraf": "~3.0.0",

packages/core/src/di/core-tokens.ts

@@ -26,4 +26,5 @@ export const validationSchema = 'validationSchema';
 export const optionsValidator = 'optionsValidator';
 export const requireFromCwd = 'requireFromCwd';
 export const fs = 'fs';
+export const incrementalResult = 'incrementalResult';
 export const project = 'project';

packages/core/src/fs/project-reader.ts

@@ -2,19 +2,21 @@ import path from 'path';
 import { isDeepStrictEqual } from 'util';
 
 import minimatch, { type IMinimatch } from 'minimatch';
-import { StrykerOptions, FileDescriptions, FileDescription } from '@stryker-mutator/api/core';
+import { StrykerOptions, FileDescriptions, FileDescription, Location, Position } from '@stryker-mutator/api/core';
 import { Logger } from '@stryker-mutator/api/logging';
 import { commonTokens, tokens } from '@stryker-mutator/api/plugin';
-import { I } from '@stryker-mutator/util';
+import { ERROR_CODES, I, isErrnoException } from '@stryker-mutator/util';
+import type { MutationTestResult } from 'mutation-testing-report-schema/api';
+
+import { OpenEndLocation } from 'mutation-testing-report-schema';
 
 import { defaultOptions, FileMatcher } from '../config/index.js';
 import { coreTokens } from '../di/index.js';
 
-import { Project } from './project.js';
+import { INCREMENTAL_REPORT_FILE, Project } from './project.js';
 import { FileSystem } from './file-system.js';
 
 const { Minimatch } = minimatch;
-
 const ALWAYS_IGNORE = Object.freeze(['node_modules', '.git', '/reports', '*.tsbuildinfo', '/stryker.log']);
 
 export const IGNORE_PATTERN_CHARACTER = '!';
@@ -30,17 +32,23 @@ export const MUTATION_RANGE_REGEX = /(.*?):((\d+)(?::(\d+))?-(\d+)(?::(\d+))?)$/
 export class ProjectReader {
   private readonly mutatePatterns: readonly string[];
   private readonly ignoreRules: readonly string[];
+  private readonly incremental: boolean;
 
   public static inject = tokens(coreTokens.fs, commonTokens.logger, commonTokens.options);
-  constructor(private readonly fs: I<FileSystem>, private readonly log: Logger, { mutate, tempDirName, ignorePatterns }: StrykerOptions) {
+  constructor(
+    private readonly fs: I<FileSystem>,
+    private readonly log: Logger,
+    { mutate, tempDirName, ignorePatterns, incremental }: StrykerOptions
+  ) {
     this.mutatePatterns = mutate;
     this.ignoreRules = [...ALWAYS_IGNORE, tempDirName, ...ignorePatterns];
+    this.incremental = incremental;
   }
 
   public async read(): Promise<Project> {
     const inputFileNames = await this.resolveInputFileNames();
     const fileDescriptions = this.resolveFileDescriptions(inputFileNames);
-    const project = new Project(this.fs, fileDescriptions);
+    const project = new Project(this.fs, fileDescriptions, this.incremental ? await this.readIncrementalReport() : undefined);
     project.logFiles(this.log, this.ignoreRules);
     return project;
   }
@@ -182,4 +190,60 @@ export class ProjectReader {
     const files = await crawlDir(process.cwd());
     return files;
   }
+
+  private async readIncrementalReport(): Promise<MutationTestResult | undefined> {
+    try {
+      // TODO: Validate against the schema or stryker version?
+      const result: MutationTestResult = JSON.parse(await this.fs.readFile(INCREMENTAL_REPORT_FILE, 'utf-8'));
+      return {
+        ...result,
+        files: Object.fromEntries(
+          Object.entries(result.files).map(([fileName, file]) => [
+            fileName,
+            { ...file, mutants: file.mutants.map((mutant) => ({ ...mutant, location: reportLocationToStrykerLocation(mutant.location) })) },
+          ])
+        ),
+        testFiles:
+          result.testFiles &&
+          Object.fromEntries(
+            Object.entries(result.testFiles).map(([fileName, file]) => [
+              fileName,
+              {
+                ...file,
+                tests: file.tests.map((test) => ({ ...test, location: test.location && reportOpenEndLocationToStrykerLocation(test.location) })),
+              },
+            ])
+          ),
+      };
+    } catch (err: unknown) {
+      if (isErrnoException(err) && err.code === ERROR_CODES.NoSuchFileOrDirectory) {
+        this.log.info('No incremental result file found, Stryker will perform a full run.');
+        return undefined;
+      }
+      // Whoops, didn't mean to catch this one!
+      throw err;
+    }
+  }
+}
+
+function reportOpenEndLocationToStrykerLocation({ start, end }: OpenEndLocation): OpenEndLocation {
+  return {
+    start: reportPositionToStrykerPosition(start),
+    end: end && reportPositionToStrykerPosition(end),
+  };
+}
+
+function reportLocationToStrykerLocation({ start, end }: Location): Location {
+  return {
+    start: reportPositionToStrykerPosition(start),
+    end: reportPositionToStrykerPosition(end),
+  };
+}
+
+function reportPositionToStrykerPosition({ line, column }: Position): Position {
+  // stryker's positions are 0-based
+  return {
+    line: line - 1,
+    column: column - 1,
+  };
 }

packages/core/src/fs/project.ts

@@ -1,10 +1,15 @@
+import path from 'path';
+
 import { Logger } from '@stryker-mutator/api/logging';
 import { FileDescriptions } from '@stryker-mutator/api/core';
 import { I, normalizeWhitespaces } from '@stryker-mutator/util';
+import { MutationTestResult } from 'mutation-testing-report-schema';
 
 import { FileSystem } from './file-system.js';
 import { ProjectFile } from './project-file.js';
 
+export const INCREMENTAL_REPORT_FILE = path.join('reports', '.mutation-incremental.json');
+
 /**
  * Represents the project that is under test by Stryker users.
  * This represents the files in the current working directory.
@@ -14,7 +19,11 @@ export class Project {
   public readonly files = new Map<string, ProjectFile>();
   public readonly filesToMutate = new Map<string, ProjectFile>();
 
-  constructor(fs: I<FileSystem>, public readonly fileDescriptions: FileDescriptions) {
+  constructor(
+    private readonly fs: I<FileSystem>,
+    public readonly fileDescriptions: FileDescriptions,
+    public readonly incrementalReport?: MutationTestResult
+  ) {
     Object.entries(fileDescriptions).forEach(([name, desc]) => {
       const file = new ProjectFile(fs, name, desc.mutate);
       this.files.set(name, file);
@@ -49,4 +58,9 @@ export class Project {
       }
     }
   }
+
+  public async writeIncrementalReport(report: MutationTestResult): Promise<void> {
+    await this.fs.mkdir(path.dirname(INCREMENTAL_REPORT_FILE), { recursive: true });
+    await this.fs.writeFile(INCREMENTAL_REPORT_FILE, JSON.stringify(report, null, 2), 'utf-8');
+  }
 }

packages/core/src/mutants/mutant-test-planner.ts

@@ -1,3 +1,5 @@
+import path from 'path';
+
 import { CompleteDryRunResult, TestResult } from '@stryker-mutator/api/test-runner';
 import {
   MutantRunPlan,
@@ -9,6 +11,7 @@ import {
   StrykerOptions,
   MutantStatus,
   MutantEarlyResultPlan,
+  MutantResult,
 } from '@stryker-mutator/api/core';
 import { commonTokens, tokens } from '@stryker-mutator/api/plugin';
 import { Logger } from '@stryker-mutator/api/logging';
@@ -19,6 +22,7 @@ import { StrictReporter } from '../reporters/strict-reporter.js';
 import { Sandbox } from '../sandbox/index.js';
 import { objectUtils } from '../utils/object-utils.js';
 import { optionsPath } from '../utils/index.js';
+import { Project } from '../fs/project.js';
 
 /**
  * The factor by which hit count from dry run is multiplied to calculate the hit limit for a mutant.
@@ -38,37 +42,49 @@ export class MutantTestPlanner {
     coreTokens.dryRunResult,
     coreTokens.reporter,
     coreTokens.sandbox,
+    coreTokens.project,
     coreTokens.timeOverheadMS,
     commonTokens.options,
     commonTokens.logger
   );
   private readonly testsByMutantId: Map<string, Set<TestResult>>;
   private readonly hitsByMutantId: Map<string, number>;
   private readonly timeSpentAllTests: number;
+  private readonly previousFiles: Map<string, string> | undefined;
 
   constructor(
     private readonly dryRunResult: CompleteDryRunResult,
     private readonly reporter: StrictReporter,
     private readonly sandbox: I<Sandbox>,
+    private readonly project: I<Project>,
     private readonly timeOverheadMS: number,
     private readonly options: StrykerOptions,
     private readonly logger: Logger
   ) {
     this.testsByMutantId = this.findTestsByMutant(this.dryRunResult.mutantCoverage?.perTest, dryRunResult.tests);
     this.hitsByMutantId = findHitsByMutantId(this.dryRunResult.mutantCoverage);
     this.timeSpentAllTests = calculateTotalTime(this.dryRunResult.tests);
+    const { incrementalReport } = this.project;
+    if (incrementalReport) {
+      const { files, testFiles } = incrementalReport;
+      this.previousFiles = new Map(Object.entries(files).map(([fileName, { source }]) => [fileName, source]));
+      if (testFiles) {
+        Object.entries(testFiles).forEach(([fileName, { source }]) => source && this.previousFiles!.set(fileName, source));
+      }
+    }
   }
 
-  public makePlan(mutants: readonly Mutant[]): readonly MutantTestPlan[] {
-    const mutantPlans = mutants.map((mutant) => this.planMutant(mutant));
+  public async makePlan(mutants: readonly Mutant[]): Promise<readonly MutantTestPlan[]> {
+    const mutantsDiff = await this.incrementalDiff(mutants);
+    const mutantPlans = mutantsDiff.map((mutant) => this.planMutant(mutant));
     this.reporter.onMutationTestingPlanReady({ mutantPlans });
     this.warnAboutSlow(mutantPlans);
     return mutantPlans;
   }
 
   private planMutant(mutant: Mutant): MutantTestPlan {
-    // If mutant was already ignored, pass that along
     const isStatic = this.dryRunResult.mutantCoverage && this.dryRunResult.mutantCoverage.static[mutant.id] > 0;
+
     if (mutant.status) {
       // If this mutant was already ignored, early result
       return this.createMutantEarlyResultPlan(mutant, { isStatic, status: mutant.status, statusReason: mutant.statusReason });
@@ -218,6 +234,66 @@ export class MutantTestPlanner {
       }
     }
   }
+
+  private async incrementalDiff(currentMutants: readonly Mutant[]): Promise<readonly Mutant[]> {
+    const { incrementalReport } = this.project;
+    if (incrementalReport) {
+      let earlyResultCount = 0;
+      const { files, testFiles } = incrementalReport;
+      const previousMutantResults = new Map(
+        Object.entries(files).flatMap(([fileName, { mutants }]) => mutants.map((mutant) => [toUniqueKey(mutant, fileName), mutant] as const))
+      );
+      const fileChanges = new Map<string, boolean>();
+      const previousFiles = new Map(Object.entries(files).map(([fileName, { source }]) => [path.resolve(fileName), source]));
+      if (testFiles) {
+        Object.entries(testFiles).forEach(([fileName, { source }]) => source && previousFiles.set(path.resolve(fileName), source));
+      }
+      const fileIsSame = async (fileName: string): Promise<boolean> => {
+        if (!fileChanges.has(fileName)) {
+          const previousFile = previousFiles.get(fileName);
+          const currentContent = await this.project.files.get(fileName)!.readOriginal();
+          fileChanges.set(fileName, previousFile === currentContent);
+        }
+        return fileChanges.get(fileName)!;
+      };
+
+      const result = await Promise.all(
+        currentMutants.map(async (mutant) => {
+          const previousMutant = previousMutantResults.get(toUniqueKey(mutant, mutant.fileName));
+          if (previousMutant && (await fileIsSame(mutant.fileName))) {
+            const tests = this.testsByMutantId.get(mutant.id);
+            let testFilesAreSame = true;
+            for (const test of tests ?? []) {
+              if (test.fileName) {
+                testFilesAreSame = await fileIsSame(test.fileName);
+                if (!testFilesAreSame) {
+                  break;
+                }
+              }
+            }
+            if (testFilesAreSame) {
+              earlyResultCount++;
+              return {
+                ...mutant,
+                status: previousMutant.status,
+              };
+            }
+          }
+          return mutant;
+        })
+      );
+      this.logger.info(`Incremental mode: ${earlyResultCount}/${currentMutants.length} are reused.`);
+      return result;
+    }
+    return currentMutants;
+  }
+}
+
+function toUniqueKey(
+  { mutatorName, replacement, location: { start, end } }: Pick<Mutant, 'location' | 'mutatorName'> & { replacement?: string },
+  fileName: string
+) {
+  return `${path.relative(process.cwd(), fileName)}@${start.line}:${start.column}-${end.line}:${end.column}\n${mutatorName}: ${replacement}`;
 }
 
 function calculateTotalTime(testResults: Iterable<TestResult>): number {

packages/core/src/process/1-prepare-executor.ts

@@ -56,19 +56,19 @@ export class PrepareExecutor {
     const loggingContext = await LogConfigurator.configureLoggingServer(options.logLevel, options.fileLogLevel, options.allowConsoleColors);
 
     // Resolve input files
-    const inputFileResolverInjector = optionsValidatorInjector
+    const projectFileReaderInjector = optionsValidatorInjector
       .provideValue(commonTokens.options, options)
       .provideClass(coreTokens.temporaryDirectory, TemporaryDirectory)
       .provideClass(coreTokens.fs, FileSystem)
       .provideValue(coreTokens.pluginsByKind, loadedPlugins.pluginsByKind);
-    const project = await inputFileResolverInjector.injectClass(ProjectReader).read();
+    const project = await projectFileReaderInjector.injectClass(ProjectReader).read();
 
     if (project.isEmpty) {
       throw new ConfigError('No input files found.');
     } else {
       // Done preparing, finish up and return
-      await inputFileResolverInjector.resolve(coreTokens.temporaryDirectory).initialize();
-      return inputFileResolverInjector
+      await projectFileReaderInjector.resolve(coreTokens.temporaryDirectory).initialize();
+      return projectFileReaderInjector
         .provideValue(coreTokens.project, project)
         .provideValue(commonTokens.fileDescriptions, project.fileDescriptions)
         .provideClass(coreTokens.pluginCreator, PluginCreator)

packages/core/src/process/4-mutation-test-executor.ts

@@ -64,22 +64,22 @@ export class MutationTestExecutor {
   ) {}
 
   public async execute(): Promise<MutantResult[]> {
-    const mutantTestPlans = this.planner.makePlan(this.mutants);
-    const { ignoredResult$, notIgnoredMutant$ } = this.executeEarlyResult(from(mutantTestPlans));
-    const { passedMutant$, checkResult$ } = this.executeCheck(notIgnoredMutant$);
+    const mutantTestPlans = await this.planner.makePlan(this.mutants);
+    const { earlyResult$, runMutant$ } = this.executeEarlyResult(from(mutantTestPlans));
+    const { passedMutant$, checkResult$ } = this.executeCheck(runMutant$);
     const { coveredMutant$, noCoverageResult$ } = this.executeNoCoverage(passedMutant$);
     const testRunnerResult$ = this.executeRunInTestRunner(coveredMutant$);
-    const results = await lastValueFrom(merge(testRunnerResult$, checkResult$, noCoverageResult$, ignoredResult$).pipe(toArray()));
+    const results = await lastValueFrom(merge(testRunnerResult$, checkResult$, noCoverageResult$, earlyResult$).pipe(toArray()));
     await this.mutationTestReportHelper.reportAll(results);
     await this.reporter.wrapUp();
     this.logDone();
     return results;
   }
 
   private executeEarlyResult(input$: Observable<MutantTestPlan>) {
-    const [ignoredMutant$, notIgnoredMutant$] = partition(input$.pipe(shareReplay()), isEarlyResult);
-    const ignoredResult$ = ignoredMutant$.pipe(map(({ mutant }) => this.mutationTestReportHelper.reportMutantStatus(mutant, MutantStatus.Ignored)));
-    return { ignoredResult$, notIgnoredMutant$ };
+    const [earlyResultMutants$, runMutant$] = partition(input$.pipe(shareReplay()), isEarlyResult);
+    const earlyResult$ = earlyResultMutants$.pipe(map(({ mutant }) => this.mutationTestReportHelper.reportMutantStatus(mutant, mutant.status)));
+    return { earlyResult$, runMutant$ };
   }
 
   private executeNoCoverage(input$: Observable<MutantRunPlan>) {