Run aXe on a few pages during CI

README.md

@@ -17,7 +17,7 @@ You will need to have the following installed on your machine before following t
 1. Ruby v2.2.2+, [Installation guides](https://www.ruby-lang.org/en/documentation/installation/)
 1. Node v4.2.3+, [Installation guides](https://nodejs.org/en/download/)
 1. Bundler v1.12.3+, [Installation guides](http://bundler.io/v1.13/guides/using_bundler_in_application.html#getting-started---installing-bundler-and-bundle-init)
-
+1. Chrome v59 or higher (v60 if on Windows)
 
 ### Building the documentation with gulp
 
@@ -43,7 +43,7 @@ Here are a few other utility commands you may find useful:
 
 - `npm run lint`: Runs `eslint` and `sass-lint` against JavaScript and Sass files.
 
-- `npm test`: Runs `npm run lint` and can also be used to run any tests.
+- `npm test`: Runs all tests and linters.
 
 - `npm run watch`: Runs a series of commands that watches for any changes in both the Standards node module and the root level asset folders in this repo.
 

circle.yml

@@ -5,6 +5,11 @@ machine:
 dependencies:
   pre:
     - bundle install
+    # https://discuss.circleci.com/t/using-the-latest-version-of-chrome-on-circleci/871/4
+    - curl -L -o google-chrome.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+    - sudo dpkg -i google-chrome.deb
+    - sudo sed -i 's|HERE/chrome\"|HERE/chrome\" --disable-setuid-sandbox|g' /opt/google/chrome/google-chrome
+    - rm google-chrome.deb
 
 test:
   pre:

config/crawl.js

@@ -4,7 +4,7 @@ const express = require('express');
 const Crawler = require("simplecrawler");
 const chalk = require('chalk');
 
-const app = express();
+const runServer = require('./static-server');
 
 // These pages incorporate content from other files in other repos, so
 // they should be considered "second class" by the link checker, and
@@ -33,17 +33,8 @@ function shouldFetch(item, referrerItem) {
   return true;
 }
 
-if (fs.existsSync(SITE_PATH)) {
-  app.use(express.static(SITE_PATH));
-} else {
-  console.log(`Please build the site before running me.`);
-  process.exit(1);
-}
-
-const listener = app.listen(() => {
-  const port = listener.address().port;
-  const baseURL = `http://127.0.0.1:${port}`;
-  const crawler = new Crawler(`${baseURL}/`);
+runServer().then(server => {
+  const crawler = new Crawler(`${server.url}/`);
   const referrers = {};
   const notFound = [];
 
@@ -65,7 +56,7 @@ const listener = app.listen(() => {
     notFound.push(item);
   });
   crawler.on("complete", () => {
-    listener.close(() => {
+    server.httpServer.close(() => {
       let errors = 0;
       let warnings = 0;
 
@@ -85,7 +76,7 @@ const listener = app.listen(() => {
       });
 
       WARNING_PAGES.forEach(path => {
-        if (!(`${baseURL}${path}` in referrers)) {
+        if (!(`${server.url}${path}` in referrers)) {
           console.log(`${ERROR}: ${path} was not visited!`);
           console.log(`  If this is not an error, please remove the path ` +
                       `from WARNING_PAGES.`);

config/run-axe.js

@@ -0,0 +1,152 @@
+'use strict';
+
+const fs = require('fs');
+const urlParse = require('url').parse;
+const chromeLauncher = require('chrome-launcher');
+const CDP = require('chrome-remote-interface');
+const chalk = require('chalk');
+const runServer = require('./static-server');
+
+const REMOTE_CHROME_URL = process.env['REMOTE_CHROME_URL'];
+const AXE_JS = fs.readFileSync(__dirname + '/../node_modules/axe-core/axe.js');
+const PAGES = [
+  '/',
+  '/page-templates/landing/',
+  '/page-templates/docs/',
+];
+
+function launchChromeLocally(headless=true) {
+  return chromeLauncher.launch({
+    chromeFlags: [
+      '--window-size=412,732',
+      '--disable-gpu',
+      headless ? '--headless' : ''
+    ]
+  });
+}
+
+function getRemoteChrome() {
+  const info = urlParse(REMOTE_CHROME_URL);
+  if (info.protocol !== 'http:')
+    throw new Error(`Unsupported protocol: ${info.protocol}`);
+  return new Promise(resolve => {
+    resolve({
+      host: info.hostname,
+      port: info.port,
+      kill() { return Promise.resolve(); }
+    });
+  });
+}
+
+// This function is only here so it can be easily .toString()'d
+// and run in the context of a web page by Chrome. It will not
+// be run in the node context.
+function runAxe() {
+  return new Promise((resolve, reject) => {
+    window.axe.run((err, results) => {
+      if (err) return reject(err);
+      resolve(JSON.stringify(results.violations));
+    });
+  });
+}
+
+let getChrome = REMOTE_CHROME_URL ? getRemoteChrome : launchChromeLocally;
+
+Promise.all([runServer(), getChrome()]).then(([server, chrome]) => {
+  const chromeHost = chrome.host || 'localhost';
+  console.log(`Static file server is listening at ${server.url}.`);
+  console.log(`Chrome is debuggable on http://${chromeHost}:${chrome.port}.`);
+  console.log(`Running aXe on:`);
+
+  CDP({
+    host: chrome.host,
+    port: chrome.port,
+  }, client => {
+    const {Page, Network, Runtime} = client;
+    const pagesLeft = PAGES.slice();
+    const loadNextPage = () => {
+      if (pagesLeft.length === 0) {
+        console.log(`Finished visiting ${PAGES.length} pages with no errors.`);
+        terminate(0);
+      } else {
+        const page = pagesLeft.pop();
+        const url = `${server.url}${page}`;
+        process.stdout.write(`  ${page} `);
+        Page.navigate({url});
+      }
+    };
+    const terminate = exitCode => {
+      client.close().then(() => {
+        chrome.kill().then(() => {
+          // Note that we're not killing the server; this is because
+          // the remote chrome instance (if we're using one) may be
+          // keeping some network connections to the server alive, which
+          // makes it harder to kill, so it's easier to just terminate.
+          const color = exitCode === 0 ? chalk.green : chalk.red;
+          console.log(color(`Terminating with exit code ${exitCode}.`));
+          process.exit(exitCode);
+        });
+      });
+    };
+
+    process.on('unhandledRejection', (reason, p) => {
+      console.log('Unhandled Rejection at:', p, 'reason:', reason);
+      terminate(1);
+    });
+
+    Promise.all([
+      Page.enable(),
+      Network.enable(),
+    ]).then(() => {
+      Network.responseReceived(({response}) => {
+        if (response.status < 400) return;
+        console.log(`${response.url} returned HTTP ${response.status}!`);
+        terminate(1);
+      });
+      Network.loadingFailed(details => {
+        console.log("A network request failed to load.");
+        console.log(details);
+        terminate(1);
+      });
+      Page.loadEventFired(() => {
+        Runtime.evaluate({
+          expression: AXE_JS + ';(' + runAxe.toString() + ')()',
+          awaitPromise: true,
+        }).then(details => {
+          let errorFound = false;
+          if (details.result.type === 'string') {
+            const viols = JSON.parse(details.result.value);
+            if (viols.length === 0) {
+              console.log(chalk.green('OK'));
+            } else {
+              console.log(chalk.red(`Found ${viols.length} aXe violations.`));
+              console.log(viols);
+              console.log(chalk.cyan(
+                `\nTo debug these violations, install aXe at:\n\n` +
+                `  https://www.deque.com/products/axe/\n`
+              ));
+              errorFound = true;
+            }
+          } else {
+            console.log('Unexpected result from CDP!');
+            console.log(details.result);
+            errorFound = true;
+          }
+          // Note that this means we're terminating on the first error we
+          // find, rather than reporting the error and moving on to the
+          // next page. We're doing this because it's possible that the
+          // error is caused by a layout or include, which may result
+          // in the same errors being reported across multiple pages, so
+          // we'd rather avoid the potential spam and just exit early.
+          if (errorFound) {
+            terminate(1);
+          } else {
+            loadNextPage();
+          }
+        });
+      });
+
+      loadNextPage();
+    });
+  });
+});

config/static-server.js

@@ -0,0 +1,30 @@
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const express = require('express');
+
+const app = express();
+
+const SITE_PATH = path.normalize(`${__dirname}/../_site`);
+
+if (fs.existsSync(SITE_PATH)) {
+  app.use(express.static(SITE_PATH));
+} else {
+  console.log(`Please build the site before running me.`);
+  process.exit(1);
+}
+
+module.exports = () => {
+  return new Promise((resolve, reject) => {
+    const server = app.listen(() => {
+      const hostname = os.hostname().toLowerCase();
+      const port = server.address().port;
+      resolve({
+        hostname,
+        port,
+        url: `http://${hostname}:${port}`,
+        httpServer: server,
+      });
+    });
+  });
+};

package.json

@@ -14,7 +14,8 @@
     "postinstall": "bundle",
     "prestart": "gulp build",
     "start": "bundle exec jekyll serve --drafts --future",
-    "test": "npm run lint && npm run crawl && bundle exec rspec",
+    "test": "npm run axe && npm run lint && npm run crawl && bundle exec rspec",
+    "axe": "node config/run-axe.js",
     "crawl": "node config/crawl.js",
     "watch": "nswatch"
   },
@@ -64,8 +65,11 @@
   },
   "homepage": "https://github.com/18F/web-design-standards-docs#readme",
   "dependencies": {
+    "axe-core": "^2.3.1",
     "browserify": "^13.0.0",
     "chalk": "^1.1.3",
+    "chrome-launcher": "^0.3.1",
+    "chrome-remote-interface": "^0.24.1",
     "del": "^2.2.0",
     "express": "^4.15.3",
     "gulp": "^3.9.0",