feat(mutation testing): sort tests to improve performance (#3467)

Sort tests that need to reload the test environment last. This reduces the total amount of time a test runner process needs to reload, which improves the performance.

packages/core/src/process/4-mutation-test-executor.ts

@@ -26,6 +26,16 @@ export interface MutationTestContext extends DryRunContext {
 
 const CHECK_BUFFER_MS = 10_000;
 
+/**
+ * Sorting the tests just before running them can yield a significant performance boost,
+ * because it can reduce the number of times a test runner process needs to be recreated.
+ * However, we need to buffer the results in order to be able to sort them.
+ *
+ * This value is very low, since it would halt the test execution otherwise.
+ * @see https://github.com/stryker-mutator/stryker-js/issues/3462
+ */
+const BUFFER_FOR_SORTING_MS = 0;
+
 export class MutationTestExecutor {
   public static inject = tokens(
     coreTokens.reporter,
@@ -81,7 +91,11 @@ export class MutationTestExecutor {
   }
 
   private executeRunInTestRunner(input$: Observable<MutantRunPlan>): Observable<MutantResult> {
-    return this.testRunnerPool.schedule(input$, async (testRunner, { mutant, runOptions }) => {
+    const sortedPlan$ = input$.pipe(
+      bufferTime(BUFFER_FOR_SORTING_MS),
+      mergeMap((plans) => plans.sort(reloadEnvironmentLast))
+    );
+    return this.testRunnerPool.schedule(sortedPlan$, async (testRunner, { mutant, runOptions }) => {
       const result = await testRunner.mutantRun(runOptions);
       return this.mutationTestReportHelper.reportMutantRunResult(mutant, result);
     });
@@ -150,6 +164,24 @@ export class MutationTestExecutor {
   }
 }
 
+/**
+ * Sorting function that sorts mutant run plans that reload environments last.
+ * This can yield a significant performance boost, because it reduces the times a test runner process needs to restart.
+ * @see https://github.com/stryker-mutator/stryker-js/issues/3462
+ */
+function reloadEnvironmentLast(a: MutantRunPlan, b: MutantRunPlan): number {
+  if (a.plan === PlanKind.Run && b.plan === PlanKind.Run) {
+    if (a.runOptions.reloadEnvironment && !b.runOptions.reloadEnvironment) {
+      return 1;
+    }
+    if (!a.runOptions.reloadEnvironment && b.runOptions.reloadEnvironment) {
+      return -1;
+    }
+    return 0;
+  }
+  return 0;
+}
+
 function isEarlyResult(mutantPlan: MutantTestPlan): mutantPlan is MutantEarlyResultPlan {
   return mutantPlan.plan === PlanKind.EarlyResult;
 }

packages/core/test/unit/process/4-mutation-test-executor.spec.ts

@@ -320,6 +320,25 @@ describe(MutationTestExecutor.name, () => {
       expect(testRunner.mutantRun).calledWithExactly(plan2.runOptions);
     });
 
+    it('should sort the mutants that reload the environment last', async () => {
+      // Arrange
+      arrangeScenario();
+      const plan1 = mutantRunPlan({ id: '1', reloadEnvironment: true });
+      const plan2 = mutantRunPlan({ id: '2', reloadEnvironment: false });
+      const plan3 = mutantRunPlan({ id: '3', reloadEnvironment: true });
+      mutantTestPlans.push(plan1, plan2, plan3);
+
+      // Act
+      await sut.execute();
+
+      // Assert
+      sinon.assert.callOrder(
+        testRunner.mutantRun.withArgs(plan2.runOptions),
+        testRunner.mutantRun.withArgs(plan1.runOptions),
+        testRunner.mutantRun.withArgs(plan3.runOptions)
+      );
+    });
+
     it('should report mutant run results', async () => {
       // Arrange
       const plan = mutantRunPlan({ static: true });