split build matrix; add more browsers; closes #2592

.browser-matrix.json

@@ -0,0 +1,72 @@
+{
+  "ie7": {
+    "base": "SauceLabs",
+    "browserName": "internet explorer",
+    "platform": "Windows XP",
+    "version": "7.0"
+  },
+  "ie8": {
+    "base": "SauceLabs",
+    "browserName": "internet explorer",
+    "platform": "Windows 7",
+    "version": "8.0"
+  },
+  "ie9": {
+    "base": "SauceLabs",
+    "browserName": "internet explorer",
+    "platform": "Windows 7",
+    "version": "9.0"
+  },
+  "ie10": {
+    "base": "SauceLabs",
+    "browserName": "internet explorer",
+    "platform": "Windows 8"
+  },
+  "ie11": {
+    "base": "SauceLabs",
+    "browserName": "internet explorer",
+    "platform": "Windows 10",
+    "version": "11.0"
+  },
+  "chrome": {
+    "base": "SauceLabs",
+    "browserName": "chrome",
+    "platform": "Windows 10",
+    "version": "latest"
+  },
+  "edge": {
+    "base": "SauceLabs",
+    "browserName": "MicrosoftEdge",
+    "platform": "Windows 10",
+    "version": "latest"
+  },
+  "firefox": {
+    "base": "SauceLabs",
+    "browserName": "firefox",
+    "platform": "Windows 10",
+    "version": "latest"
+  },
+  "safari8": {
+    "base": "SauceLabs",
+    "browserName": "safari",
+    "platform": "OS X 10.10"
+  },
+  "safari9": {
+    "base": "SauceLabs",
+    "browserName": "safari",
+    "platform": "OS X 10.11",
+    "version": "9.0"
+  },
+  "safari10": {
+    "base": "SauceLabs",
+    "browserName": "safari",
+    "platform": "OS X 10.11",
+    "version": "10.0"
+  },
+  "opera": {
+    "base": "SauceLabs",
+    "browserName": "opera",
+    "platform": "Linux",
+    "version": "latest"
+  }
+}

.travis.yml

@@ -8,6 +8,10 @@ dist: trusty
 
 language: node_js
 
+cache:
+  directories:
+    - node_modules
+
 env:
   global:
     # phantomjs hosts binaries @ bitbucket, which has fairly restrictive
@@ -35,17 +39,35 @@ matrix:
     - node_js: '0.11'
       env: TARGET=test-node
     - node_js: '0.10'
-      env: TARGET=test-node
+      env: TARGET=test-browser MOCHA_BROWSER=ie7 S3=1
+    - node_js: '7'
+      env: TARGET=test-browser MOCHA_BROWSER=ie8 S3=1
+    - node_js: '7'
+      env: TARGET=test-browser MOCHA_BROWSER=ie9 S3=1
+    - node_js: '7'
+      env: TARGET=test-browser MOCHA_BROWSER=ie10 S3=1
+    - node_js: '7'
+      env: TARGET=test-browser MOCHA_BROWSER=ie11 S3=1
+    - node_js: '7'
+      env: TARGET=test-browser MOCHA_BROWSER=chrome S3=1
+    - node_js: '7'
+      env: TARGET=test-browser MOCHA_BROWSER=edge S3=1
+    - node_js: '7'
+      env: TARGET=test-browser MOCHA_BROWSER=firefox S3=1
+    - node_js: '7'
+      env: TARGET=test-browser MOCHA_BROWSER=safari8 S3=1
+    - node_js: '7'
+      env: TARGET=test-browser MOCHA_BROWSER=safari9 S3=1
     - node_js: '7'
-      env: TARGET=lint
+      env: TARGET=test-browser MOCHA_BROWSER=safari10 S3=1
     - node_js: '7'
-      env: TARGET=test-browser S3=1
+      env: TARGET=test-browser MOCHA_BROWSER=opera S3=1
 
 before_install: scripts/travis-before-install.sh
 
 before_script: scripts/travis-before-script.sh
 
-script: make $TARGET
+script: travis_retry make $TARGET
 
 after_script: scripts/travis-after-script.sh
 

karma.conf.js

@@ -5,10 +5,39 @@ var path = require('path');
 var mkdirp = require('mkdirp');
 var baseBundleDirpath = path.join(__dirname, '.karma');
 var osName = require('os-name');
+var browserMatrix = require('./.browser-matrix.json');
+
+/**
+ * Relevant environment variables:
+ *
+ * - CI: If present, will attempt to run tests on SauceLabs.  If neither TRAVIS
+ * nor APPVEYOR are present, then we configure for local execution.
+ * - TRAVIS: If present, configure for Travis-CI
+ * - APPVEYOR: If present, abort; Travis-CI runs browser tests
+ * - SAUCE_ACCESS_KEY, SAUCE_USERNAME: Must be present to run tests on
+ * SauceLabs.  Specify these with CI=1 to run against SauceLabs locally.
+ * - MOCHA_BROWSER: The browser (singular) to run against.  Can be one of
+ * "Chrome", "PhantomJS" (launchers installed by default) any key present in
+ * `.browser-matrix.json`, or any custom Karma launcher you've installed
+ * locally.  Note that PhantomJS 1.9.x is incompatible with macOS Sierra, and
+ * we default to Safari in that case.
+ * - MOCHA_UI: If present, will run integration tests for the particular interface.  One of "tdd", "bdd", or "qunit".  These tests are mutually exclusive with any others.
+ * - S3: If present, upload build artifacts to S3.  This requires AWS credentials; see scripts/travis-after-script.sh for details.  The folder in S3 will correspond to the Travis build number, or simply "local" if not running on Travis.
+ *
+ * Example of running tests against sauce locally:
+ * $ CI=1 SAUCE_ACCESS_KEY=<key> SAUCE_USERNAME=mochajs make test-browser
+ *
+ * Example of running Karma against multiple browsers (whose launchers must be installed) a single time:
+ * $ karma start --browsers Chrome,PhantomJS --single-run
+ *
+ * Example of running Karma on SauceLabs against IE8 in "watch" mode:
+ * $ CI=1 SAUCE_ACCESS_KEY=<key> SAUCE_USERNAME=mochajs MOCHA_BROWSER=ie8 karma start
+ */
 
 module.exports = function (config) {
-  var bundleDirpath;
+  var env = process.env;
   var cfg = {
+    artifactDirpath: path.join(baseBundleDirpath, 'local'),
     frameworks: [
       'browserify',
       'expect',
@@ -38,78 +67,119 @@ module.exports = function (config) {
           .ignore('path')
           .ignore('supports-color')
           .on('bundled', function (err, content) {
-            if (!err && bundleDirpath) {
+            if (!err) {
               // write bundle to directory for debugging
-              fs.writeFileSync(path.join(bundleDirpath,
+              fs.writeFileSync(path.join(cfg.artifactDirpath,
                 'bundle.' + Date.now() + '.js'), content);
             }
           });
       }
     },
     reporters: ['spec'],
     colors: true,
-    browsers: [osName() === 'macOS Sierra' ? 'Chrome' : 'PhantomJS'],
-    logLevel: config.LOG_INFO,
+    browsers: defaultBrowsers(),
+    logLevel: config.LOG_INFO, // use HTML reporter for better debugging
     client: {
       mocha: {
         reporter: 'html'
       }
     }
   };
 
-  // see https://github.com/saucelabs/karma-sauce-example
-  // TO RUN LOCALLY, execute:
-  // `CI=1 SAUCE_USERNAME=<user> SAUCE_ACCESS_KEY=<key> make test-browser`
-  var env = process.env;
-  var sauceConfig;
-
   if (env.CI) {
     console.error('CI mode enabled');
-    if (env.TRAVIS) {
-      console.error('Travis-CI detected');
-      bundleDirpath = path.join(baseBundleDirpath, process.env.TRAVIS_BUILD_ID);
-      if (env.SAUCE_USERNAME && env.SAUCE_ACCESS_KEY) {
-        // correlate build/tunnel with Travis
-        sauceConfig = {
-          build: 'TRAVIS #' + env.TRAVIS_BUILD_NUMBER +
-          ' (' + env.TRAVIS_BUILD_ID + ')',
-          tunnelIdentifier: env.TRAVIS_JOB_NUMBER
-        };
-        console.error('Configured SauceLabs');
-      } else {
-        console.error('No SauceLabs credentials present');
-      }
-    } else if (env.APPVEYOR) {
-      console.error('AppVeyor detected');
-      bundleDirpath = path.join(baseBundleDirpath, process.env.APPVEYOR_BUILD_ID);
+    configureCI(cfg);
+  } else {
+    console.error('CI mode disabled');
+    if (env.MOCHA_BROWSER) {
+      cfg.browsers = [env.MOCHA_BROWSER];
     } else {
-      console.error('Local/unknown environment detected');
-      bundleDirpath = path.join(baseBundleDirpath, 'local');
-      // don't need to run sauce from appveyor b/c travis does it.
-      if (!(env.SAUCE_USERNAME || env.SAUCE_ACCESS_KEY)) {
-        console.error('No SauceLabs credentials present');
-      } else {
-        sauceConfig = {
-          build: require('os').hostname() + ' (' + Date.now() + ')'
-        };
-        console.error('Configured SauceLabs');
-      }
+      cfg.browsers = defaultBrowsers();
     }
-    mkdirp.sync(bundleDirpath);
+  }
+
+  // create dir for build artifacts
+  mkdirp.sync(cfg.artifactDirpath);
+
+  // if MOCHA_UI is present, this will choose the appropriate tests.
+  configureInterface();
+
+  config.set(cfg);
+};
+
+function addSauceTests (cfg, browser) {
+  var browserName = String(browser)
+    .toLowerCase();
+  var browserConfig = browserMatrix[browserName];
+  if (browserConfig) {
+    cfg.customLaunchers[browserName] = browserConfig;
+    cfg.reporters.push('saucelabs');
+    cfg.browsers = Object.keys(cfg.customLaunchers);
+    cfg.sauceLabs = {
+      public: 'public',
+      startConnect: true
+    };
+    // for slow browser booting, ostensibly
+    cfg.captureTimeout = 120000;
+    cfg.browserNoActivityTimeout = 20000;
   } else {
-    console.error('CI mode disabled');
+    console.error('Unknown browser "' + browser + '"; using defaults');
+  }
+}
+
+function configureSauce (cfg, sauceLabs) {
+  var env = process.env;
+  if (isPhantom()) {
+    console.error('Skipping SauceLabs config for PhantomJS');
+  } else {
+    if (env.SAUCE_USERNAME && env.SAUCE_ACCESS_KEY) {
+      cfg.sauceLabs = sauceLabs;
+      addSauceTests(cfg, env.MOCHA_BROWSER);
+      console.error('Configured SauceLabs');
+    } else {
+      console.error('No SauceLabs credentials present; using defaults');
+    }
   }
+}
 
-  if (sauceConfig) {
-    cfg.sauceLabs = sauceConfig;
-    addSauceTests(cfg);
+function defaultBrowsers () {
+  return [osName() === 'macOS Sierra' ? 'Safari' : 'PhantomJS'];
+}
+
+function isPhantom () {
+  return /phantomjs/i.test(process.env.MOCHA_BROWSER);
+}
+
+function configureCI (cfg) {
+  var env = process.env;
+
+  // we don't run browser tests in AppVeyor; Travis does it instead
+  if (env.APPVEYOR) {
+    console.error('AppVeyor detected; nothing to do');
+    process.exit();
+  } else if (env.TRAVIS) {
+    console.error('Travis-CI detected');
+    cfg.artifactDirpath = path.join(baseBundleDirpath, env.TRAVIS_BUILD_ID);
+    configureSauce(cfg, {
+      build: 'TRAVIS #' + env.TRAVIS_BUILD_NUMBER + ' (' + env.TRAVIS_BUILD_ID +
+      ')',
+      tunnelIdentifier: env.TRAVIS_JOB_NUMBER
+    });
+  } else {
+    console.error('Local/unknown CI environment detected');
+    configureSauce(cfg, {
+      build: require('os')
+        .hostname() + ' (' + Date.now() + ')'
+    });
   }
+}
 
+function configureInterface (cfg) {
   // the MOCHA_UI env var will determine if we're running interface-specific
   // tets.  since you can only load one at a time, each must be run separately.
   // each has its own set of acceptance tests and a fixture.
   // the "bdd" fixture is used by default.
-  var ui = env.MOCHA_UI;
+  var ui = process.env.MOCHA_UI;
   if (ui) {
     if (cfg.sauceLabs) {
       cfg.sauceLabs.testName = 'Interface "' + ui + '" integration tests';
@@ -119,62 +189,6 @@ module.exports = function (config) {
       'test/acceptance/interfaces/' + ui + '.spec.js'
     ];
   } else if (cfg.sauceLabs) {
-    cfg.sauceLabs.testName = 'Unit Tests';
+    cfg.sauceLabs.testName = 'Tests';
   }
-
-  config.set(cfg);
-};
-
-function addSauceTests (cfg) {
-  cfg.reporters.push('saucelabs');
-
-  cfg.customLaunchers = {
-    ie8: {
-      base: 'SauceLabs',
-      browserName: 'internet explorer',
-      platform: 'Windows 7',
-      version: '8.0'
-    },
-    ie7: {
-      base: 'SauceLabs',
-      browserName: 'internet explorer',
-      platform: 'Windows XP',
-      version: '7.0'
-    },
-    chrome: {
-      base: 'SauceLabs',
-      browserName: 'chrome',
-      platform: 'Windows 8',
-      version: 'latest'
-    },
-    edge: {
-      base: 'SauceLabs',
-      browserName: 'MicrosoftEdge',
-      platform: 'Windows 10',
-      version: 'latest'
-    },
-    firefox: {
-      base: 'SauceLabs',
-      browserName: 'firefox',
-      platform: 'Windows 8.1',
-      version: 'latest'
-    },
-    safari: {
-      base: 'SauceLabs',
-      browserName: 'safari',
-      platform: 'OS X 10.11',
-      version: 'latest'
-    }
-  };
-
-  cfg.browsers = cfg.browsers.concat(Object.keys(cfg.customLaunchers));
-
-  cfg.sauceLabs = {
-    public: 'public',
-    startConnect: true
-  };
-
-  // for slow browser booting, ostensibly
-  cfg.captureTimeout = 120000;
-  cfg.browserNoActivityTimeout = 20000;
 }