feat(hit limit): infinite loop prevention in jasmine-runner (#3199)

Add infinite loop prevention using a hit counter to `@stryker-mutator/jasmine-runner`.

e2e/test/hit-limit/.mocharc.json

@@ -1,3 +1,3 @@
 {
-  "spec": "test/**.mocha.spec.js"
+  "require": ["test/chai-setup.js"]
 }

e2e/test/hit-limit/karma.conf.js

@@ -3,7 +3,7 @@ module.exports = function(config) {
     frameworks: ['jasmine'],
     files: [
       'src/*.js',
-      'test/*.karma.spec.js'
+      'test/*.spec.js'
     ],
     reporters: ['progress'],
     colors: true,

e2e/test/hit-limit/package.json

@@ -6,6 +6,7 @@
   "main": "index.js",
   "scripts": {
     "test:mocha": "mocha",
+    "test:jasmine": "jasmine",
     "test:karma": "karma start",
     "test": "mocha --no-package --timeout 60000 --require \"../../tasks/ts-node-register.js\" verify/verify.ts"
   },

e2e/test/hit-limit/spec/support/jasmine.json

@@ -0,0 +1,8 @@
+{
+  "spec_dir": "spec",
+  "spec_files": [
+    "../test/*.spec.js"
+  ],
+  "stopSpecOnExpectationFailure": false,
+  "random": true
+}

e2e/test/hit-limit/test/chai-setup.js

@@ -0,0 +1,6 @@
+const chai = require('chai');
+
+chai.util.addMethod(chai.Assertion.prototype, 'toEqual', function (expected) {
+  var obj = chai.util.flag(this, 'object');
+  new chai.Assertion(obj).to.deep.equal(expected);
+});

e2e/test/hit-limit/test/loop.spec.js

@@ -0,0 +1,14 @@
+if (typeof require === 'function') {
+  loop = require('../src/loop').loop;
+}
+if (typeof expect === 'undefined') {
+  globalThis.expect = require('chai').expect;
+}
+
+describe('loop', () => {
+  it('should result in 15 for n=5 and a sum function', () => {
+    let result = 0;
+    loop(5, (n) => (result += n));
+    expect(result).toEqual(15);
+  });
+});

e2e/test/hit-limit/verify/verify.ts

@@ -18,4 +18,12 @@ describe('Limit counter', () => {
     expect(timeoutResults).lengthOf(3);
     timeoutResults.forEach((result) => expect(result.statusReason).eq('Hit limit reached (501/500)'));
   });
+
+  it('should limit infinite loops in the jasmine-runner', async () => {
+    const stryker = new Stryker({ testRunner: 'jasmine' });
+    const results = await stryker.runMutationTest();
+    const timeoutResults = results.filter(res => res.status === MutantStatus.Timeout);
+    expect(timeoutResults).lengthOf(3);
+    timeoutResults.forEach((result) => expect(result.statusReason).eq('Hit limit reached (501/500)'));
+  });
 });

packages/jasmine-runner/src/jasmine-test-runner.ts

@@ -12,6 +12,7 @@ import {
   toMutantRunResult,
   ErrorDryRunResult,
   DryRunOptions,
+  determineHitLimitReached,
 } from '@stryker-mutator/api/test-runner';
 import { errorToString, Task, DirectoryRequireCache, I } from '@stryker-mutator/util';
 
@@ -59,6 +60,8 @@ export class JasmineTestRunner implements TestRunner {
 
   public async mutantRun(options: MutantRunOptions): Promise<MutantRunResult> {
     this.instrumenterContext.activeMutant = options.activeMutant.id;
+    this.instrumenterContext.hitLimit = options.hitLimit;
+    this.instrumenterContext.hitCount = options.hitLimit ? 0 : undefined;
     const runResult = await this.run(options.testFilter, undefined, options.disableBail);
     return toMutantRunResult(runResult, true);
   }
@@ -91,6 +94,11 @@ export class JasmineTestRunner implements TestRunner {
           if (coverageAnalysis === 'all' || coverageAnalysis === 'perTest') {
             mutantCoverage = self.instrumenterContext.mutantCoverage;
           }
+          const timeoutResult = determineHitLimitReached(self.instrumenterContext.hitCount, self.instrumenterContext.hitLimit);
+          if (timeoutResult) {
+            runTask.resolve(timeoutResult);
+            return;
+          }
           runTask.resolve({
             status: DryRunStatus.Complete,
             tests,

packages/jasmine-runner/test/integration/timeout-on-infinite-loop.it.spec.ts

@@ -0,0 +1,56 @@
+import { testInjector, factory, assertions } from '@stryker-mutator/test-helpers';
+import { expect } from 'chai';
+
+import { createJasmineTestRunnerFactory, JasmineTestRunner } from '../../src';
+import { resolveFromRoot, resolveTestResource } from '../helpers/resolve-test-resource';
+
+describe('Infinite loop', () => {
+  let sut: JasmineTestRunner;
+
+  beforeEach(async () => {
+    process.chdir(resolveTestResource('infinite-loop-instrumented'));
+    sut = testInjector.injector.injectFunction(createJasmineTestRunnerFactory('__stryker2__'));
+  });
+  afterEach(async () => {
+    process.chdir(resolveFromRoot());
+    await sut.dispose();
+  });
+
+  it('should be able to recover using a hit counter', async () => {
+    // Arrange
+    const options = factory.mutantRunOptions({
+      activeMutant: factory.mutant({ id: '19' }),
+      testFilter: ['spec2'],
+      hitLimit: 10,
+    });
+
+    // Act
+    const result = await sut.mutantRun(options);
+
+    // Assert
+    assertions.expectTimeout(result);
+    expect(result.reason).contains('Hit limit reached');
+  });
+
+  it('should reset hit counter state correctly between runs', async () => {
+    const firstResult = await sut.mutantRun(
+      factory.mutantRunOptions({
+        activeMutant: factory.mutant({ id: '19' }),
+        testFilter: ['spec2'],
+        hitLimit: 10,
+      })
+    );
+    const secondResult = await sut.mutantRun(
+      factory.mutantRunOptions({
+        // 27 is a 'normal' mutant that should be killed
+        activeMutant: factory.mutant({ id: '22' }),
+        testFilter: ['spec2'],
+        hitLimit: 10,
+      })
+    );
+
+    // Assert
+    assertions.expectTimeout(firstResult);
+    assertions.expectKilled(secondResult);
+  });
+});

packages/jasmine-runner/testResources/infinite-loop-instrumented/lib/infinite-loop.js

@@ -0,0 +1,84 @@
+// This file is generated with tasks/instrument-test-resources.js
+ function stryNS_9fa48() {
+  var g = new Function("return this")();
+  var ns = g.__stryker2__ || (g.__stryker2__ = {});
+
+  if (ns.activeMutant === undefined && g.process && g.process.env && g.process.env.__STRYKER_ACTIVE_MUTANT__) {
+    ns.activeMutant = g.process.env.__STRYKER_ACTIVE_MUTANT__;
+  }
+
+  function retrieveNS() {
+    return ns;
+  }
+
+  stryNS_9fa48 = retrieveNS;
+  return retrieveNS();
+}
+
+stryNS_9fa48();
+
+function stryCov_9fa48() {
+  var ns = stryNS_9fa48();
+  var cov = ns.mutantCoverage || (ns.mutantCoverage = {
+    static: {},
+    perTest: {}
+  });
+
+  function cover() {
+    var c = cov.static;
+
+    if (ns.currentTestId) {
+      c = cov.perTest[ns.currentTestId] = cov.perTest[ns.currentTestId] || {};
+    }
+
+    var a = arguments;
+
+    for (var i = 0; i < a.length; i++) {
+      c[a[i]] = (c[a[i]] || 0) + 1;
+    }
+  }
+
+  stryCov_9fa48 = cover;
+  cover.apply(null, arguments);
+}
+
+function stryMutAct_9fa48(id) {
+  var ns = stryNS_9fa48();
+
+  function isActive(id) {
+    if (ns.activeMutant === id) {
+      if (ns.hitCount !== void 0 && ++ns.hitCount > ns.hitLimit) {
+        throw new Error('Stryker: Hit count limit reached (' + ns.hitCount + ')');
+      }
+
+      return true;
+    }
+
+    return false;
+  }
+
+  stryMutAct_9fa48 = isActive;
+  return isActive(id);
+}
+
+function loop(n, action) {
+  if (stryMutAct_9fa48("15")) {
+    {}
+  } else {
+    stryCov_9fa48("15");
+    let goOn = stryMutAct_9fa48("16") ? false : (stryCov_9fa48("16"), true);
+
+    while (stryMutAct_9fa48("17") ? false : (stryCov_9fa48("17"), goOn)) {
+      if (stryMutAct_9fa48("18")) {
+        {}
+      } else {
+        stryCov_9fa48("18");
+        action(n);
+        stryMutAct_9fa48("19") ? n++ : (stryCov_9fa48("19"), n--);
+        goOn = stryMutAct_9fa48("23") ? n <= 0 : stryMutAct_9fa48("22") ? n >= 0 : stryMutAct_9fa48("21") ? false : stryMutAct_9fa48("20") ? true : (stryCov_9fa48("20", "21", "22", "23"), n > 0);
+      }
+    }
+  }
+}
+
+module.exports = loop;

packages/jasmine-runner/testResources/infinite-loop-instrumented/spec/support/jasmine.json

@@ -0,0 +1,11 @@
+{
+  "spec_dir": "spec",
+  "spec_files": [
+    "**/*[sS]pec.js"
+  ],
+  "helpers": [
+    "helpers/**/*.js"
+  ],
+  "stopSpecOnExpectationFailure": false,
+  "random": false
+}

packages/jasmine-runner/testResources/infinite-loop-instrumented/spec/tests/infinite-loop.spec.js

@@ -0,0 +1,16 @@
+var loop = require('../../lib/infinite-loop');
+
+it('should handle an infinite loop as a timeout', () => {
+  while (true);
+});
+
+it('should be able to recover and test others', () => {});
+
+it('should be able to break out of an infinite loop with a hit counter', () => {
+  let total = 0;
+  loop(5, (n) => {
+    expect(n).not.toBe(0);
+    total += n;
+  });
+  expect(total).toBe(15);
+});

packages/jasmine-runner/testResources/infinite-loop/lib/infinite-loop.js

@@ -0,0 +1,10 @@
+function loop(n, action) {
+  let goOn = true;
+  while (goOn) {
+    action(n);
+    n--;
+    goOn = n > 0;
+  }
+}
+
+module.exports = loop;

packages/jasmine-runner/testResources/infinite-loop/spec/support/jasmine.json

@@ -0,0 +1,11 @@
+{
+  "spec_dir": "spec",
+  "spec_files": [
+    "**/*[sS]pec.js"
+  ],
+  "helpers": [
+    "helpers/**/*.js"
+  ],
+  "stopSpecOnExpectationFailure": false,
+  "random": false
+}

packages/jasmine-runner/testResources/infinite-loop/spec/tests/infinite-loop.spec.js

@@ -0,0 +1,16 @@
+var loop = require('../../lib/infinite-loop');
+
+it('should handle an infinite loop as a timeout', () => {
+  while (true);
+});
+
+it('should be able to recover and test others', () => {});
+
+it('should be able to break out of an infinite loop with a hit counter', () => {
+  let total = 0;
+  loop(5, (n) => {
+    expect(n).not.toBe(0);
+    total += n;
+  });
+  expect(total).toBe(15);
+});

tasks/instrument-test-resources.js

@@ -17,7 +17,8 @@ async function main() {
   }, '__stryker2__');
   await instrument({
     './packages/jasmine-runner/testResources/jasmine-init/lib/jasmine_examples/Player.js': './packages/jasmine-runner/testResources/jasmine-init-instrumented/lib/jasmine_examples/Player.js',
-    './packages/jasmine-runner/testResources/jasmine-init/lib/jasmine_examples/Song.js': './packages/jasmine-runner/testResources/jasmine-init-instrumented/lib/jasmine_examples/Song.js'
+    './packages/jasmine-runner/testResources/jasmine-init/lib/jasmine_examples/Song.js': './packages/jasmine-runner/testResources/jasmine-init-instrumented/lib/jasmine_examples/Song.js',
+    './packages/jasmine-runner/testResources/infinite-loop/lib/infinite-loop.js': './packages/jasmine-runner/testResources/infinite-loop-instrumented/lib/infinite-loop.js',
   }, '__stryker2__');
   await instrument({
     './packages/karma-runner/testResources/sampleProject/src/Add.js': './packages/karma-runner/testResources/instrumented/src/Add.js',