Improve heatmap render tests; support multiple expected images (#6162)

* improve heatmap render tests; support multiple expected

Some platforms on GL native support half-float-textures-based rendering, for which we provide a separate expected image alongside ubyte-based.

The test harness compares with each expected image and picks the one with the least amount of difference.

* heatmap test renames, add missing images

* don't create expected images if missing

* rename heatmap images back

* throw an error if expected are missing

* add missing heatmap test descriptions

* add a clarifying comment in test harness

test/integration/lib/render.js

@@ -5,24 +5,43 @@ const path = require('path');
 const PNG = require('pngjs').PNG;
 const harness = require('./harness');
 const pixelmatch = require('pixelmatch');
+const glob = require('glob');
 
-function compare(path1, path2, diffPath, callback) {
+function compare(actualPath, expectedPaths, diffPath, callback) {
+
+    const actualImg = fs.createReadStream(actualPath).pipe(new PNG()).on('parsed', doneReading);
+
+    const expectedImgs = [];
+    for (const path of expectedPaths) {
+        expectedImgs.push(fs.createReadStream(path).pipe(new PNG()).on('parsed', doneReading));
+    }
 
-    const img1 = fs.createReadStream(path1).pipe(new PNG()).on('parsed', doneReading);
-    const img2 = fs.createReadStream(path2).pipe(new PNG()).on('parsed', doneReading);
     let read = 0;
 
     function doneReading() {
-        if (++read < 2) return;
+        if (++read < expectedPaths.length + 1) return;
 
-        const diff = new PNG({width: img1.width, height: img1.height});
+        // if we have multiple expected images, we'll compare against each one and pick the one with
+        // the least amount of difference; this is useful for covering features that render differently
+        // depending on platform, i.e. heatmaps use half-float textures for improved rendering where supported
 
-        const numPixels = pixelmatch(img1.data, img2.data, diff.data, img1.width, img1.height, {
-            threshold: 0.13
-        });
+        let minNumPixels = Infinity;
+        let minDiff, minIndex;
+
+        for (let i = 0; i < expectedImgs.length; i++) {
+            const diff = new PNG({width: actualImg.width, height: actualImg.height});
+            const numPixels = pixelmatch(actualImg.data, expectedImgs[i].data,
+                diff.data, actualImg.width, actualImg.height, {threshold: 0.13});
 
-        diff.pack().pipe(fs.createWriteStream(diffPath)).on('finish', () => {
-            callback(null, numPixels / (diff.width * diff.height));
+            if (numPixels < minNumPixels) {
+                minNumPixels = numPixels;
+                minDiff = diff;
+                minIndex = i;
+            }
+        }
+
+        minDiff.pack().pipe(fs.createWriteStream(diffPath)).on('finish', () => {
+            callback(null, minNumPixels / (minDiff.width * minDiff.height), expectedPaths[minIndex]);
         });
     }
 }
@@ -125,6 +144,13 @@ exports.run = function (implementation, ignores, render) {
 
             png.data = data;
 
+            // there may be multiple expected images, covering different platforms
+            const expectedPaths = glob.sync(path.join(dir, 'expected*.png'));
+
+            if (expectedPaths.length === 0) {
+                throw new Error('No expected*.png files found; did you mean to run tests with UPDATE=true?');
+            }
+
             if (process.env.UPDATE) {
                 png.pack()
                     .pipe(fs.createWriteStream(expected))
@@ -133,18 +159,7 @@ exports.run = function (implementation, ignores, render) {
                 png.pack()
                     .pipe(fs.createWriteStream(actual))
                     .on('finish', () => {
-
-                        try {
-                            stats = fs.statSync(expected, fs.R_OK | fs.W_OK);
-                            if (!stats.isFile()) throw new Error();
-                        }                        catch (e) {  // no expected.png, create it
-                            png.pack()
-                                .pipe(fs.createWriteStream(expected))
-                                .on('finish', done);
-                            return;
-                        }
-
-                        compare(actual, expected, diff, (err, difference) => {
+                        compare(actual, expectedPaths, diff, (err, difference, expected) => {
                             if (err) return done(err);
 
                             params.difference = difference;

test/integration/package.json

@@ -14,6 +14,7 @@
     "pixelmatch": "^4.0.2",
     "pngjs": "^3.0.0",
     "shuffle-seed": "^1.1.6",
-    "st": "^1.1.0"
+    "st": "^1.1.0",
+    "glob": "^7.0.3"
   }
 }

test/integration/render-tests/heatmap-color/default/style.json

@@ -3,7 +3,8 @@
   "metadata": {
     "test": {
       "height": 64,
-      "width": 256
+      "width": 256,
+      "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },
   "center": [

test/integration/render-tests/heatmap-color/expression/style.json

@@ -3,7 +3,8 @@
   "metadata": {
     "test": {
       "height": 64,
-      "width": 256
+      "width": 256,
+      "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },
   "center": [

test/integration/render-tests/heatmap-intensity/default/style.json

@@ -3,7 +3,8 @@
   "metadata": {
     "test": {
       "height": 128,
-      "width": 256
+      "width": 256,
+      "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },
   "center": [

test/integration/render-tests/heatmap-intensity/function/style.json

@@ -3,7 +3,8 @@
   "metadata": {
     "test": {
       "height": 128,
-      "width": 256
+      "width": 256,
+      "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },
   "center": [

test/integration/render-tests/heatmap-intensity/literal/style.json

@@ -3,7 +3,8 @@
   "metadata": {
     "test": {
       "height": 128,
-      "width": 256
+      "width": 256,
+      "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },
   "center": [

test/integration/render-tests/heatmap-opacity/default/style.json

@@ -3,7 +3,8 @@
   "metadata": {
     "test": {
       "height": 32,
-      "width": 128
+      "width": 128,
+      "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },
   "center": [

test/integration/render-tests/heatmap-opacity/function/style.json

@@ -3,7 +3,8 @@
   "metadata": {
     "test": {
       "height": 32,
-      "width": 128
+      "width": 128,
+      "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },
   "center": [

test/integration/render-tests/heatmap-opacity/literal/style.json

@@ -3,7 +3,8 @@
   "metadata": {
     "test": {
       "height": 32,
-      "width": 128
+      "width": 128,
+      "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },
   "center": [

test/integration/render-tests/heatmap-radius/antimeridian/style.json

@@ -3,7 +3,8 @@
   "metadata": {
     "test": {
       "width": 514,
-      "height": 64
+      "height": 64,
+      "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },
   "center": [

test/integration/render-tests/heatmap-radius/data-expression/style.json

@@ -3,7 +3,8 @@
   "metadata": {
     "test": {
       "height": 64,
-      "width": 64
+      "width": 64,
+      "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },
   "sources": {

test/integration/render-tests/heatmap-radius/default/style.json

@@ -3,7 +3,8 @@
   "metadata": {
     "test": {
       "height": 128,
-      "width": 256
+      "width": 256,
+      "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },
   "center": [

test/integration/render-tests/heatmap-radius/function/style.json

@@ -3,7 +3,8 @@
   "metadata": {
     "test": {
       "height": 128,
-      "width": 256
+      "width": 256,
+      "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },
   "center": [

test/integration/render-tests/heatmap-radius/literal/style.json

@@ -3,7 +3,8 @@
   "metadata": {
     "test": {
       "height": 128,
-      "width": 256
+      "width": 256,
+      "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },
   "center": [

test/integration/render-tests/heatmap-radius/pitch30/style.json

@@ -3,7 +3,8 @@
   "metadata": {
     "test": {
       "height": 128,
-      "width": 256
+      "width": 256,
+      "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },
   "center": [

test/integration/render-tests/heatmap-weight/default/style.json

@@ -3,7 +3,8 @@
   "metadata": {
     "test": {
       "height": 128,
-      "width": 256
+      "width": 256,
+      "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },
   "center": [

test/integration/render-tests/heatmap-weight/identity-property-function/style.json

@@ -3,7 +3,8 @@
   "metadata": {
     "test": {
       "height": 128,
-      "width": 256
+      "width": 256,
+      "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },
   "center": [

test/integration/render-tests/heatmap-weight/literal/style.json

@@ -3,7 +3,8 @@
   "metadata": {
     "test": {
       "height": 128,
-      "width": 256
+      "width": 256,
+      "description": "Contains two expected images (for ubyte-based rendering and half-float-based); one of them should pass depending on platform."
     }
   },
   "center": [