feat: support test retries

.eslintrc.json

@@ -6,7 +6,8 @@
     "plugin:@cypress/dev/general"
   ],
   "rules": {
-    "prefer-spread": "off"
+    "prefer-spread": "off",
+    "prefer-rest-params": "off"
   },
   "settings": {
     "react": {

circle.yml

@@ -129,14 +129,22 @@ commands:
       browser:
         description: browser shortname to target
         type: string
+      percy:
+        description: enable percy
+        type: boolean
+        default: false
     steps:
       - attach_workspace:
           at: ~/
       - run:
           command: |
+            cmd=$([[ <<parameters.percy>> == 'true' ]] && echo 'yarn percy exec --') || true
+
             CYPRESS_KONFIG_ENV=production \
             CYPRESS_RECORD_KEY=$PACKAGES_RECORD_KEY \
-            yarn workspace @packages/runner cypress:run --record --parallel --group runner-integration-<<parameters.browser>> --browser <<parameters.browser>>
+            PERCY_PARALLEL_NONCE=$CIRCLE_WORKFLOW_ID \
+            PERCY_PARALLEL_TOTAL=-1 \
+            $cmd yarn workspace @packages/runner cypress:run --record --parallel --group runner-integration-<<parameters.browser>> --browser <<parameters.browser>>
       - store_test_results:
           path: /tmp/cypress
       - store_artifacts:
@@ -211,6 +219,7 @@ commands:
           command: |
             git clone --depth 1 --no-single-branch https://github.com/cypress-io/<<parameters.repo>>.git /tmp/<<parameters.repo>>
             cd /tmp/<<parameters.repo>> && (git checkout $NEXT_DEV_VERSION || true)
+
   test-binary-against-repo:
     description: |
       Takes the built binary and NPM package, clones given example repo
@@ -337,6 +346,17 @@ commands:
                 path: /tmp/<<parameters.repo>>/cypress/videos
       - store-npm-logs
 
+  wait-on-circle-jobs:
+    description: Polls certain Circle CI jobs until they finish
+    parameters:
+      job-names:
+        description: comma separated list of circle ci job names to wait for
+        type: string
+    steps:
+      - run:
+          name: "Waiting on Circle CI jobs: <<parameters.job-names>>"
+          command: node ./scripts/wait-on-circle-jobs.js --job-names="<<parameters.job-names>>"
+
 jobs:
   ## code checkout and yarn installs
   build:
@@ -400,6 +420,37 @@ jobs:
           command: node cli/bin/cypress info --dev
       - store-npm-logs
 
+  # a special job that keeps polling Circle and when all
+  # individual jobs are finished, it closes the Percy build
+  percy-finalize:
+    <<: *defaults
+    executor: cy-doc
+    parameters:
+      required_env_var:
+        type: env_var_name
+    steps:
+      - attach_workspace:
+          at: ~/
+      - run:
+          # if this is an external pull request, the environment variables
+          # are NOT set for security reasons, thus no need to poll -
+          # and no need to finalize Percy, since there will be no visual tests
+          name: Check if <<parameters.required_env_var>> is set
+          command: |
+            if [[ -v <<parameters.required_env_var>> ]]; then
+              echo "Internal PR, good to go"
+            else
+              echo "This is an external PR, cannot access other services"
+              circleci-agent step halt
+            fi
+      - wait-on-circle-jobs:
+          job-names: >
+            desktop-gui-integration-tests-2x,
+            desktop-gui-component-tests,
+            cli-visual-tests,
+            runner-integration-tests-chrome,
+      - run: npx percy finalize --all
+
   cli-visual-tests:
     <<: *defaults
     parallelism: 1
@@ -701,6 +752,7 @@ jobs:
     steps:
       - run-runner-integration-tests:
           browser: chrome
+          percy: true
 
   runner-integration-tests-firefox:
     <<: *defaults
@@ -879,13 +931,6 @@ jobs:
           command: node index.js
           working_directory: packages/launcher
 
-  percy-finalize:
-    <<: *defaults
-    steps:
-      - run:
-          name: "finalizes percy builds"
-          command: npx percy finalize --all
-
   build-binary:
     <<: *defaults
     shell: /bin/bash --login
@@ -1369,6 +1414,11 @@ jobs:
           command: npm run test:ci
           pull_request_id: 515
           folder: examples/fundamentals__typescript
+      - test-binary-against-repo:
+          repo: cypress-example-recipes
+          command: npm test
+          pull_request_id: 513
+          folder: examples/fundamentals__module-api-wrap
 
   "test-binary-against-kitchensink":
     <<: *defaults
@@ -1493,6 +1543,11 @@ linux-workflow: &linux-workflow
         name: Linux lint
         requires:
           - build
+    - percy-finalize:
+        context: test-runner:poll-circle-workflow
+        required_env_var: PERCY_TOKEN # skips job if not defined (external PR)
+        requires:
+          - build
     - lint-types:
         requires:
           - build
@@ -1624,12 +1679,6 @@ linux-workflow: &linux-workflow
         requires:
           - build
 
-    - percy-finalize:
-        requires:
-          - desktop-gui-integration-tests-2x
-          - desktop-gui-component-tests
-          - cli-visual-tests
-
     # various testing scenarios, like building full binary
     # and testing it on a real project
     - test-against-staging:
@@ -1666,6 +1715,7 @@ linux-workflow: &linux-workflow
           branches:
             only:
               - develop
+              - test-retries
         requires:
           - build-npm-package
     - build-binary:
@@ -1677,6 +1727,7 @@ linux-workflow: &linux-workflow
           branches:
             only:
               - develop
+              - test-retries
         requires:
           - build-binary
     - test-npm-module-on-minimum-node-version:
@@ -1733,6 +1784,7 @@ linux-workflow: &linux-workflow
           branches:
             only:
               - develop
+              - test-retries
         requires:
           - upload-npm-package
           - upload-binary

cli/schema/cypress.schema.json

@@ -243,6 +243,18 @@
       "type": "boolean",
       "default": false,
       "description": "Polyfills `window.fetch` to enable Network spying and stubbing"
+    },
+    "retries": {
+      "type": [
+        "object",
+        "number",
+        "null"
+      ],
+      "default": {
+        "runMode": 0,
+        "openMode": 0
+      },
+      "description": "The number of times to retry a failing. Can be configured to apply only in runMode or openMode"
     }
   }
 }

cli/types/cypress-npm-api.d.ts

@@ -7,6 +7,11 @@
 // but for now describe it as an ambient module
 
 declare namespace CypressCommandLine {
+  interface TestError {
+    name: string
+    message: string
+    stack: string
+  }
   /**
    * All options that one can pass to "cypress.run"
    * @see https://on.cypress.io/module-api#cypress-run
@@ -166,14 +171,16 @@ declare namespace CypressCommandLine {
     title: string[]
     state: string
     body: string
-    /**
-     * Error stack string if there is an error
-     */
-    stack: string | null
-    /**
-     * Error message if there is an error
+     /**
+     * Error string as it's presented in console if the test fails
      */
-    error: string | null
+    displayError: string | null
+    attempts: AttemptResult[]
+  }
+
+  interface AttemptResult {
+    state: string
+    error: TestError | null
     timings: any
     failedFromHookId: hookId | null
     wallClockStartedAt: dateTimeISO
@@ -199,6 +206,7 @@ declare namespace CypressCommandLine {
     name: string
     testId: testId
     takenAt: dateTimeISO
+    testAttemptIndex: number
     /**
      * Absolute path to the saved image
      */

cli/types/cypress.d.ts

@@ -334,6 +334,11 @@ declare namespace Cypress {
      */
     getFirefoxGcInterval(): number | null | undefined
 
+    /**
+     * @returns the number of test retries currently enabled for the run
+     */
+    getTestRetries(): number | null
+
     /**
      * Checks if a variable is a valid instance of `cy` or a `cy` chainable.
      *
@@ -2563,6 +2568,13 @@ declare namespace Cypress {
      * the `includeShadowDom` option to some DOM commands.
      */
     experimentalShadowDomSupport: boolean
+    /**
+     * Number of times to retry a failed test.
+     * If a number is set, tests will retry in both runMode and openMode.
+     * To enable test retries only in runMode, set e.g. `{ openMode: null, runMode: 2 }`
+     * @default null
+     */
+    retries: Nullable<number | {runMode: Nullable<number>, openMode: Nullable<number>}>
   }
 
   interface TestConfigOverrides extends Partial<Pick<ConfigOptions, 'baseUrl' | 'defaultCommandTimeout' | 'taskTimeout' | 'animationDistanceThreshold' | 'waitForAnimations' | 'viewportHeight' | 'viewportWidth' | 'requestTimeout' | 'execTimeout' | 'env' | 'responseTimeout'>> {

package.json

@@ -136,6 +136,7 @@
     "fs-extra": "8.1.0",
     "gift": "0.10.2",
     "globby": "10.0.1",
+    "got": "11.5.1",
     "gulp": "4.0.2",
     "gulp-awspublish": "4.0.0",
     "gulp-debug": "4.0.0",

packages/desktop-gui/cypress/support/index.js

@@ -1,4 +1,4 @@
-require('@percy/cypress')
+require('@packages/ui-components/cypress/support/customPercyCommand')
 require('cypress-react-unit-test/dist/hooks')
 
 const BluebirdPromise = require('bluebird')

packages/driver/cypress/integration/commands/screenshot_spec.js

@@ -19,6 +19,7 @@ describe('src/cy/commands/screenshot', () => {
       takenAt: new Date().toISOString(),
       name: 'name',
       blackout: ['.foo'],
+      testAttemptIndex: 0,
       duration: 100,
     }
 
@@ -49,7 +50,7 @@ describe('src/cy/commands/screenshot', () => {
 
       Cypress.action('runner:runnable:after:run:async', test, runnable)
       .then(() => {
-        expect(Cypress.action).not.to.be.calledWith('cy:test:set:state')
+        expect(Cypress.action).not.to.be.calledWith('test:set:state')
         expect(Cypress.automation).not.to.be.called
       })
       .finally(() => {
@@ -68,7 +69,7 @@ describe('src/cy/commands/screenshot', () => {
 
       Cypress.action('runner:runnable:after:run:async', test, runnable)
       .then(() => {
-        expect(Cypress.action).not.to.be.calledWith('cy:test:set:state')
+        expect(Cypress.action).not.to.be.calledWith('test:set:state')
         expect(Cypress.automation).not.to.be.called
       })
     })
@@ -89,7 +90,7 @@ describe('src/cy/commands/screenshot', () => {
 
       Cypress.action('runner:runnable:after:run:async', test, runnable)
       .then(() => {
-        expect(Cypress.action).not.to.be.calledWith('cy:test:set:state')
+        expect(Cypress.action).not.to.be.calledWith('test:set:state')
         expect(Cypress.automation).not.to.be.called
       })
     })
@@ -137,6 +138,7 @@ describe('src/cy/commands/screenshot', () => {
           waitForCommandSynchronization: true,
           disableTimersAndAnimations: true,
           blackout: [],
+          testAttemptIndex: 0,
         })
 
         expect(Cypress.action).to.be.calledWith('cy:after:screenshot', {
@@ -147,6 +149,7 @@ describe('src/cy/commands/screenshot', () => {
           waitForCommandSynchronization: true,
           disableTimersAndAnimations: true,
           blackout: [],
+          testAttemptIndex: 0,
         })
       })
     })
@@ -183,6 +186,7 @@ describe('src/cy/commands/screenshot', () => {
           testFailure: true,
           blackout: [],
           scaled: true,
+          testAttemptIndex: 0,
         })
       })
     })
@@ -225,6 +229,7 @@ describe('src/cy/commands/screenshot', () => {
             simple: false,
             scaled: true,
             blackout: [],
+            testAttemptIndex: 0,
           })
         })
       })
@@ -264,6 +269,7 @@ describe('src/cy/commands/screenshot', () => {
           testFailure: true,
           scaled: true,
           blackout: [],
+          testAttemptIndex: 0,
         })
       })
     })
@@ -406,6 +412,7 @@ describe('src/cy/commands/screenshot', () => {
             waitForCommandSynchronization: false,
             disableTimersAndAnimations: true,
             blackout: ['.foo'],
+            testAttemptIndex: 0,
           })
         })
       })
@@ -425,6 +432,7 @@ describe('src/cy/commands/screenshot', () => {
             waitForCommandSynchronization: false,
             disableTimersAndAnimations: true,
             blackout: ['.foo'],
+            testAttemptIndex: 0,
           })
         })
       })
@@ -446,6 +454,7 @@ describe('src/cy/commands/screenshot', () => {
             waitForCommandSynchronization: true,
             disableTimersAndAnimations: true,
             blackout: [],
+            testAttemptIndex: 0,
           })
         })
       })
@@ -466,6 +475,7 @@ describe('src/cy/commands/screenshot', () => {
             waitForCommandSynchronization: false,
             disableTimersAndAnimations: true,
             blackout: ['.foo'],
+            testAttemptIndex: 0,
           })
         })
       })