[test] Visual regression testing using backstopjs (#1157)

* [test] Adding studio * [test] Adding backstop visual regression testing * [test] Adding reference files * [test] Move backstop to test folder and run backstop-test-studio on test * [test] Check that docker is installed and give error message if not
sanity-io · Feb 1, 2019 · a89d445 · a89d445
1 parent dbfedfb
commit a89d445
Show file tree

Hide file tree

Showing 75 changed files with 3,986 additions and 2 deletions.
diff --git a/gulpfile.js b/gulpfile.js
@@ -1,4 +1,4 @@
-/* eslint-disable import/no-commonjs, import/no-unassigned-import */
+/* eslint-disable import/no-commonjs, import/no-unassigned-import, max-nested-callbacks */
 // Note: Node 8 compat, please!
 require('hard-rejection/register')
 
@@ -17,6 +17,9 @@ const through = require('through2')
 const chalk = require('chalk')
 const globby = require('globby')
 const mergeStream = require('merge-stream')
+const backstop = require('backstopjs')
+const waitPort = require('wait-port')
+const kill = require('kill-port')
 
 const isWindows = /^win/.test(process.platform)
 
@@ -190,7 +193,8 @@ const STUDIOS = [
   {name: 'movies-studio', port: '3334'},
   {name: 'example-studio', port: '3335'},
   {name: 'blog-studio', port: '3336'},
-  {name: 'ecommerce-studio', port: '3337'}
+  {name: 'ecommerce-studio', port: '3337'},
+  {name: 'backstop-test-studio', port: '5000'}
 ]
 
 STUDIOS.forEach(studio => {
@@ -238,3 +242,84 @@ gulp.task('storybook', ['watch-js', 'watch-ts', 'watch-assets'], () => {
   proc.stdout.pipe(process.stdout)
   proc.stderr.pipe(process.stderr)
 })
+
+gulp.task('backstop', cb => {
+  const {exec} = require('child_process')
+
+  exec('docker -v', (err, stdout, stderr) => {
+    if (err) {
+      throw new gutil.PluginError({
+        plugin: 'backstop',
+        message: gutil.colors.red('Please install Docker on your computer. https://www.docker.com/')
+      })
+    }
+  })
+
+  gulp.start('backstop-test-studio')
+
+  const params = {
+    host: 'localhost',
+    port: 5000,
+    timeout: 100 * 60 * 10
+  }
+
+  waitPort(params)
+    .then(open => {
+      if (open) {
+        backstop('test', {
+          docker: true,
+          config: './test/backstop/backstop.js'
+        })
+          .then(() => {
+            kill(params.port).then(() => {
+              gutil.log(gutil.colors.green('Backstop test success'))
+              // eslint-disable-next-line
+              process.exit(0)
+            })
+          })
+          .catch(() => {
+            kill(params.port)
+            throw new gutil.PluginError({
+              plugin: 'backstop',
+              message: 'Tests failed'
+            })
+          })
+      } else {
+        kill(params.port)
+        throw new gutil.PluginError({
+          plugin: 'backstop',
+          message: 'The backstop-studio did not start'
+        })
+      }
+    })
+    .catch(err => {
+      kill(params.port)
+      throw new gutil.PluginError({
+        plugin: 'backstop',
+        message: `An unknown error occured while waiting for the port: ${err}`
+      })
+    })
+})
+
+gulp.task('backstop:approve', cb => {
+  backstop('approve', {
+    docker: true,
+    config: './test/backstop/backstop.js'
+  })
+})
+
+gulp.task('backstop:reference', cb => {
+  backstop('reference', {
+    docker: true,
+    config: './test/backstop/backstop.js'
+  })
+    .then(() => {
+      gutil.log(gutil.colors.green('References created'))
+    })
+    .catch(() => {
+      throw new gutil.PluginError({
+        plugin: 'backstop',
+        message: 'Making references failed'
+      })
+    })
+})
diff --git a/lerna.json b/lerna.json
@@ -8,6 +8,7 @@
     "packages/create-sanity",
     "packages/ecommerce-studio",
     "packages/eslint-config-sanity",
+    "packages/backstop-test-studio",
     "packages/example-studio",
     "packages/groq",
     "packages/movies-studio",

diff --git a/package.json b/package.json
@@ -20,13 +20,17 @@
     "start": "npm run test-studio",
     "ecommerce-studio": "gulp ecommerce-studio",
     "test-studio": "gulp test-studio",
+    "backstop-test-studio": "gulp backstop-test-studio",
     "example-studio": "gulp example-studio",
     "movies-studio": "gulp movies-studio",
     "blog-studio": "gulp blog-studio",
     "stylelint": "stylelint \"**/*.css\"",
     "stylelint:fix": "stylelint \"**/*.css\" --fix",
     "test": "echo 'Run `npm run test-all` to run `npm test` for every package'",
     "test-all": "lerna run test --stream",
+    "test:backstop": "gulp backstop",
+    "test:backstop:approve": "gulp backstop:approve",
+    "test:backstop:reference": "gulp backstop:reference",
     "updated": "lerna updated",
     "watch": "gulp watch",
     "install-husky": "node node_modules/husky/bin/install",
@@ -53,6 +57,7 @@
     "babel-plugin-css-modules-transform": "^1.6.1",
     "babel-plugin-istanbul": "^5.0.1",
     "babel-plugin-lodash": "^3.3.4",
+    "backstopjs": "^3.8.5",
     "boxen": "^1.3.0",
     "chalk": "^2.3.0",
     "deepmerge": "^2.0.1",
@@ -80,6 +85,7 @@
     "gulp-watch": "^5.0.1",
     "hard-rejection": "^1.0.0",
     "husky": "^0.14.3",
+    "kill-port": "^1.3.2",
     "lerna": "^3.4.0",
     "merge-stream": "^1.0.1",
     "minimist": "^1.2.0",
@@ -93,6 +99,7 @@
     "stylelint-config-standard": "^18.0.0",
     "through2": "^2.0.3",
     "typescript": "^3.2.2",
+    "wait-port": "^0.2.2",
     "yarn": "^1.3.2"
   }
 }
diff --git a/packages/backstop-test-studio/.gitignore b/packages/backstop-test-studio/.gitignore
@@ -0,0 +1,12 @@
+# Logs
+logs
+*.log
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# Dependency directories
+node_modules
+
+# Built assets
+dist
diff --git a/packages/backstop-test-studio/config/.checksums b/packages/backstop-test-studio/config/.checksums
@@ -0,0 +1,8 @@
+{
+  "#": "Used by Sanity to keep track of configuration file checksums, do not delete or modify!",
+  "@sanity/default-layout": "bb034f391ba508a6ca8cd971967cbedeb131c4d19b17b28a0895f32db5d568ea",
+  "@sanity/data-aspects": "ba5c2649cc1b1c39ae92b7daf2661f95fa79d7325073ffd410245d2717b240e9",
+  "@sanity/storybook": "526dea3b461fda217e7150d12395d0ec639cba0155c05a084b85bcf2c44995a3",
+  "@sanity/google-maps-input": "57ae3a403ce6a070b31ec6fa1f3c8339cafa66661eaddba1d4d5ee3cc2197ec2",
+  "@sanity/default-login": "6fb6d3800aa71346e1b84d95bbcaa287879456f2922372bb0294e30b968cd37f"
+}
diff --git a/packages/backstop-test-studio/config/@sanity/data-aspects.json b/packages/backstop-test-studio/config/@sanity/data-aspects.json
@@ -0,0 +1,3 @@
+{
+  "listOptions": {}
+}
diff --git a/packages/backstop-test-studio/config/@sanity/default-layout.json b/packages/backstop-test-studio/config/@sanity/default-layout.json
@@ -0,0 +1,6 @@
+{
+  "toolSwitcher": {
+    "order": [],
+    "hidden": []
+  }
+}
diff --git a/packages/backstop-test-studio/config/@sanity/default-login.json b/packages/backstop-test-studio/config/@sanity/default-login.json
@@ -0,0 +1,7 @@
+{
+  "providers": {
+    "mode": "append",
+    "redirectOnSingle": false,
+    "entries": []
+  }
+}
diff --git a/packages/backstop-test-studio/config/@sanity/google-maps-input.json b/packages/backstop-test-studio/config/@sanity/google-maps-input.json
@@ -0,0 +1,8 @@
+{
+  "apiKey": "AIzaSyDwxpdp6WJwj9VjoVCjEudljhkR1inzhSE",
+  "defaultZoom": 11,
+  "defaultLocation": {
+    "lat": 40.7058254,
+    "lng": -74.1180863
+  }
+}
diff --git a/packages/backstop-test-studio/config/@sanity/storybook.json b/packages/backstop-test-studio/config/@sanity/storybook.json
@@ -0,0 +1,3 @@
+{
+  "port": 9001
+}
diff --git a/packages/backstop-test-studio/data/norway.js b/packages/backstop-test-studio/data/norway.js
@@ -0,0 +1,116 @@
+export default [
+  {
+    type: 'Feature',
+    id: 'NOR',
+    properties: {name: 'Norway'},
+    geometry: {
+      type: 'MultiPolygon',
+      coordinates: [
+        [
+          [
+            [28.165547, 71.185474],
+            [31.293418, 70.453788],
+            [30.005435, 70.186259],
+            [31.101079, 69.55808],
+            [29.399581, 69.156916],
+            [28.59193, 69.064777],
+            [29.015573, 69.766491],
+            [27.732292, 70.164193],
+            [26.179622, 69.825299],
+            [25.689213, 69.092114],
+            [24.735679, 68.649557],
+            [23.66205, 68.891247],
+            [22.356238, 68.841741],
+            [21.244936, 69.370443],
+            [20.645593, 69.106247],
+            [20.025269, 69.065139],
+            [19.87856, 68.407194],
+            [17.993868, 68.567391],
+            [17.729182, 68.010552],
+            [16.768879, 68.013937],
+            [16.108712, 67.302456],
+            [15.108411, 66.193867],
+            [13.55569, 64.787028],
+            [13.919905, 64.445421],
+            [13.571916, 64.049114],
+            [12.579935, 64.066219],
+            [11.930569, 63.128318],
+            [11.992064, 61.800362],
+            [12.631147, 61.293572],
+            [12.300366, 60.117933],
+            [11.468272, 59.432393],
+            [11.027369, 58.856149],
+            [10.356557, 59.469807],
+            [8.382, 58.313288],
+            [7.048748, 58.078884],
+            [5.665835, 58.588155],
+            [5.308234, 59.663232],
+            [4.992078, 61.970998],
+            [5.9129, 62.614473],
+            [8.553411, 63.454008],
+            [10.527709, 64.486038],
+            [12.358347, 65.879726],
+            [14.761146, 67.810642],
+            [16.435927, 68.563205],
+            [19.184028, 69.817444],
+            [21.378416, 70.255169],
+            [23.023742, 70.202072],
+            [24.546543, 71.030497],
+            [26.37005, 70.986262],
+            [28.165547, 71.185474]
+          ]
+        ],
+        [
+          [
+            [24.72412, 77.85385],
+            [22.49032, 77.44493],
+            [20.72601, 77.67704],
+            [21.41611, 77.93504],
+            [20.8119, 78.25463],
+            [22.88426, 78.45494],
+            [23.28134, 78.07954],
+            [24.72412, 77.85385]
+          ]
+        ],
+        [
+          [
+            [18.25183, 79.70175],
+            [21.54383, 78.95611],
+            [19.02737, 78.5626],
+            [18.47172, 77.82669],
+            [17.59441, 77.63796],
+            [17.1182, 76.80941],
+            [15.91315, 76.77045],
+            [13.76259, 77.38035],
+            [14.66956, 77.73565],
+            [13.1706, 78.02493],
+            [11.22231, 78.8693],
+            [10.44453, 79.65239],
+            [13.17077, 80.01046],
+            [13.71852, 79.66039],
+            [15.14282, 79.67431],
+            [15.52255, 80.01608],
+            [16.99085, 80.05086],
+            [18.25183, 79.70175]
+          ]
+        ],
+        [
+          [
+            [25.447625, 80.40734],
+            [27.407506, 80.056406],
+            [25.924651, 79.517834],
+            [23.024466, 79.400012],
+            [20.075188, 79.566823],
+            [19.897266, 79.842362],
+            [18.462264, 79.85988],
+            [17.368015, 80.318896],
+            [20.455992, 80.598156],
+            [21.907945, 80.357679],
+            [22.919253, 80.657144],
+            [25.447625, 80.40734]
+          ]
+        ]
+      ]
+    }
+  }
+]
diff --git a/packages/backstop-test-studio/package.json b/packages/backstop-test-studio/package.json
@@ -0,0 +1,60 @@
+{
+  "name": "backstop-test-studio",
+  "private": true,
+  "version": "0.139.0",
+  "description": "A bloated, ugly Sanity studio with a contrived schema for edge case testing and development",
+  "author": "Sanity.io <[email protected]>",
+  "license": "MIT",
+  "scripts": {
+    "start": "sanity start",
+    "clean": "rimraf dist"
+  },
+  "keywords": [
+    "sanity",
+    "cms",
+    "headless",
+    "realtime",
+    "content",
+    "test-studio"
+  ],
+  "dependencies": {
+    "@sanity/base": "0.139.6",
+    "@sanity/cli": "0.139.6",
+    "@sanity/code-input": "0.139.0",
+    "@sanity/color-input": "0.139.0",
+    "@sanity/components": "0.139.6",
+    "@sanity/core": "0.139.6",
+    "@sanity/default-layout": "0.139.6",
+    "@sanity/default-login": "0.139.0",
+    "@sanity/desk-tool": "0.139.6",
+    "@sanity/form-builder": "0.139.6",
+    "@sanity/google-maps-input": "0.139.6",
+    "@sanity/image-url": "0.139.0",
+    "@sanity/production-preview": "0.139.0",
+    "@sanity/rich-date-input": "0.139.0",
+    "@sanity/storybook": "0.139.3",
+    "@sanity/vision": "0.139.6",
+    "@turf/helpers": "^6.0.1",
+    "@turf/points-within-polygon": "^5.1.5",
+    "bio-pv": "^1.8.1",
+    "date-fns": "^1.29.0",
+    "fetch": "^1.1.0",
+    "get-video-id": "^2.1.4",
+    "humanize-list": "^1.0.1",
+    "prop-types": "^15.6.0",
+    "react": "^16.2.0",
+    "react-dom": "^16.2.0",
+    "react-icons": "^2.2.7"
+  },
+  "devDependencies": {
+    "rimraf": "^2.6.2"
+  },
+  "bugs": {
+    "url": "https://github.com/sanity-io/sanity/issues"
+  },
+  "homepage": "https://www.sanity.io/",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/sanity-io/sanity.git"
+  }
+}