Make puppeteer-based test runner work

Issue #6051

test/index.html

@@ -6,6 +6,10 @@
 
 <link rel="stylesheet" href="../lib/codemirror.css">
 <link rel="stylesheet" href="mode_test.css">
+<script>
+  var errored = []
+  window.onerror = function(e) { errored.push(e) }
+</script>
 <script src="../doc/activebookmark.js"></script>
 <script src="../lib/codemirror.js"></script>
 <script src="../mode/meta.js"></script>
@@ -171,7 +175,8 @@ <h2>Test Suite</h2>
           bad = "",
           running = false, // Flag that states tests are running
           quit = false, // Flag to quit tests ASAP
-          verbose = false; // Adds message for *every* test to output
+          verbose = false, // Adds message for *every* test to output
+          done = false
 
       function runHarness(){
         if (running) {
@@ -197,6 +202,7 @@ <h2>Test Suite</h2>
         setStatus("Loading tests...");
         count = 0;
         failed = 0;
+        done = false;
         skipped = 0;
         bad = "";
         totalTests = countTests();
@@ -257,11 +263,12 @@ <h2>Test Suite</h2>
             message = failed + " failure" + (failed > 1 ? "s" : "");
           } else {
             type += " ok";
-            message = "All passed";
+            message = "All passed"
             if (skipped) {
               message += " (" + skipped + " skipped)";
             }
           }
+          done = true
           progressTotal.nodeValue = '';
           customMessage = true; // Hack to avoid adding to output
         }

test/run.js

@@ -17,30 +17,25 @@ var server = require('http').createServer(function (req, res) {
   throw err;
 }).listen(3000,(async () => {
   const puppeteer = require('puppeteer');
-  const browser = await puppeteer.launch()
+  const browser = await puppeteer.launch({args: ["--no-sandbox", "--disable-setuid-sandbox"]})
   const page = await browser.newPage()
-  await page.goto('http://localhost:3000/test/index.html')
+  page.on('console', msg => console.log("console:", msg.text()))
   page.on('dialog', async dialog => {
     console.log(dialog.message())
     await dialog.dismiss()
   })
-  var ret1 = false;
-  while(1){
-    ret1 = await page.evaluate(() => {
-      var output = document.getElementById('status');
-      if (!output) {
-        return false;
-      }
-	return (/^(\d+ failures?|all passed)/i).test(output.innerText);
-    })
-  if(ret1 === true) { break}
+  page.evaluateOnNewDocument(() => window.automatedTests = true)
+  await page.goto('http://localhost:3000/test/index.html#' + (process.argv[2] || ""))
+  while(1) {
+    if (await page.evaluate(() => window.done)) break
+    await sleep(200)
   }
-  var failed = await page.evaluate(function () { return window.failed; });
-  var output = await page.evaluate(function () {
-    return document.getElementById('output').innerText + "\n" +
-    document.getElementById('status').innerText;
-  });
-  console.log(output);
-  process.exit(failed > 0 ? 1 : 0);
+  let [failed, errors] = await page.evaluate(() => [window.failed, window.errored])
+  for (let error of errors) console.log(error)
+  console.log(await page.evaluate(() => document.getElementById('output').innerText + "\n" +
+                                          document.getElementById('status').innerText))
+  process.exit(failed > 0 || errors.length ? 1 : 0)
   await browser.close()
-})());
+})())
+
+function sleep(n) { return new Promise(acc => setTimeout(acc, n)) }

test/test.js

@@ -29,8 +29,6 @@ function byClassName(elt, cls) {
 var ie_lt8 = /MSIE [1-7]\b/.test(navigator.userAgent);
 var ie_lt9 = /MSIE [1-8]\b/.test(navigator.userAgent);
 var mac = /Mac/.test(navigator.platform);
-var phantom = /PhantomJS/.test(navigator.userAgent);
-var chrome = !edge && /Chrome\//.test(navigator.userAgent);
 var opera = /Opera\/\./.test(navigator.userAgent);
 var opera_version = opera && navigator.userAgent.match(/Version\/(\d+\.\d+)/);
 if (opera_version) opera_version = Number(opera_version);
@@ -256,6 +254,7 @@ testCM("coordsCharBidi", function(cm) {
 }, {lineNumbers: true});
 
 testCM("badBidiOptimization", function(cm) {
+  if (window.automatedTests) return
   var coords = cm.charCoords(Pos(0, 34))
   eqCharPos(cm.coordsChar({left: coords.right, top: coords.top + 2}), Pos(0, 34))
 }, {value: "----------<p class=\"title\">هل يمكنك اختيار مستوى قسط التأمين الذي ترغب بدفعه؟</p>"})
@@ -625,7 +624,6 @@ testCM("bookmarkCursor", function(cm) {
 }, {value: "foo\nbar\n\n\nx\ny"});
 
 testCM("multiBookmarkCursor", function(cm) {
-  if (phantom) return;
   var ms = [], m;
   function add(insertLeft) {
     for (var i = 0; i < 3; ++i) {
@@ -685,7 +683,6 @@ testCM("scrollSnap", function(cm) {
 });
 
 testCM("scrollIntoView", function(cm) {
-  if (phantom) return;
   function test(line, ch, msg) {
     var pos = Pos(line, ch);
     cm.scrollIntoView(pos);
@@ -724,7 +721,7 @@ testCM("selectAllNoScroll", function(cm) {
 });
 
 testCM("selectionPos", function(cm) {
-  if (phantom || cm.getOption("inputStyle") != "textarea") return;
+  if (cm.getOption("inputStyle") != "textarea") return;
   cm.setSize(100, 100);
   addDoc(cm, 200, 100);
   cm.setSelection(Pos(1, 100), Pos(98, 100));
@@ -929,7 +926,6 @@ testCM("everythingFolded", function(cm) {
 });
 
 testCM("structuredFold", function(cm) {
-  if (phantom) return;
   addDoc(cm, 4, 8);
   var range = cm.markText(Pos(1, 2), Pos(6, 2), {
     replacedWith: document.createTextNode("Q")
@@ -1051,7 +1047,6 @@ testCM("wrappingInlineWidget", function(cm) {
   eq(curR.bottom, cur1.bottom);
   cm.replaceRange("", Pos(0, 9), Pos(0));
   curR = cm.cursorCoords(Pos(0, 9));
-  if (phantom) return;
   eq(curR.top, cur1.top);
   eq(curR.bottom, cur1.bottom);
 }, {value: "1 2 3 xxx 4", lineWrapping: true});
@@ -1134,7 +1129,6 @@ testCM("wrappingAndResizing", function(cm) {
 }, null, ie_lt8);
 
 testCM("measureEndOfLine", function(cm) {
-  if (phantom) return;
   cm.setSize(null, "auto");
   var inner = byClassName(cm.getWrapperElement(), "CodeMirror-lines")[0].firstChild;
   var lh = inner.offsetHeight;
@@ -1158,7 +1152,6 @@ testCM("measureEndOfLine", function(cm) {
 }, {mode: "text/html", value: "<!-- foo barrr -->", lineWrapping: true}, ie_lt8 || opera_lt10);
 
 testCM("measureWrappedEndOfLine", function(cm) {
-  if (phantom) return;
   cm.setSize(null, "auto");
   var inner = byClassName(cm.getWrapperElement(), "CodeMirror-lines")[0].firstChild;
   var lh = inner.offsetHeight;
@@ -1195,7 +1188,6 @@ testCM("measureWrappedBidiLevel2", function(cm) {
 }, {value: "foobar إإ إإ إإ إإ 555 بببببب", lineWrapping: true})
 
 testCM("measureWrappedBeginOfLine", function(cm) {
-  if (phantom) return;
   cm.setSize(null, "auto");
   var inner = byClassName(cm.getWrapperElement(), "CodeMirror-lines")[0].firstChild;
   var lh = inner.offsetHeight;
@@ -1277,7 +1269,7 @@ testCM("verticalScroll", function(cm) {
   cm.replaceRange("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaah", Pos(0, 0), Pos(0));
   is(sc.scrollWidth > baseWidth, "scrollbar present");
   cm.replaceRange("foo", Pos(0, 0), Pos(0));
-  if (!phantom) eq(sc.scrollWidth, baseWidth, "scrollbar gone");
+  eq(sc.scrollWidth, baseWidth, "scrollbar gone");
   cm.replaceRange("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaah", Pos(0, 0), Pos(0));
   cm.replaceRange("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbh", Pos(1, 0), Pos(1));
   is(sc.scrollWidth > baseWidth, "present again");
@@ -1414,8 +1406,7 @@ testCM("verticalMovementCommands", function(cm) {
   cm.execCommand("goLineUp");
   eqCharPos(cm.getCursor(), Pos(0, 0));
   cm.execCommand("goLineDown");
-  if (!phantom) // This fails in PhantomJS, though not in a real Webkit
-    eqCharPos(cm.getCursor(), Pos(1, 0));
+  eqCharPos(cm.getCursor(), Pos(1, 0));
   cm.setCursor(Pos(1, 12));
   cm.execCommand("goLineDown");
   eqCharPos(cm.getCursor(), Pos(2, 5));
@@ -1550,7 +1541,7 @@ testCM("lineChangeEvents", function(cm) {
 });
 
 testCM("scrollEntirelyToRight", function(cm) {
-  if (phantom || cm.getOption("inputStyle") != "textarea") return;
+  if (cm.getOption("inputStyle") != "textarea") return;
   addDoc(cm, 500, 2);
   cm.setCursor(Pos(0, 500));
   var wrap = cm.getWrapperElement(), cur = byClassName(wrap, "CodeMirror-cursor")[0];
@@ -1733,7 +1724,6 @@ testCM("getLineNumber", function(cm) {
 });
 
 testCM("jumpTheGap", function(cm) {
-  if (phantom) return;
   var longLine = "abcdef ghiklmnop qrstuvw xyz ";
   longLine += longLine; longLine += longLine; longLine += longLine;
   cm.replaceRange(longLine, Pos(2, 0), Pos(2));
@@ -2570,20 +2560,18 @@ bidiTests.push("Say ا ب جabj\nS");
 bidiTests.push("Sayyy ا ا ب ج");
 */
 
-if (!phantom) {
-  bidiTests.push("Όȝǝڪȉۥ״ۺ׆ɀҩۏ\nҳ");
-  bidiTests.push("ŌӰтقȤ؁ƥ؅٣ĎȺ١\nϚ");
-  bidiTests.push("ٻоҤѕѽΩ־؉ïίքdz\nٵ");
-  bidiTests.push("؅؁ĆՕƿɁǞϮؠȩóć\nď");
-  bidiTests.push("RŨďңŪzϢŎƏԖڇڦ\nӈ");
-  bidiTests.push("ό׊۷٢ԜһОצЉيčǟ\nѩ");
-  bidiTests.push("ۑÚҳҕڬġڹհяųKV\nr");
-  bidiTests.push("źڻғúہ4ם1Ƞc1a\nԁ");
-  bidiTests.push("ҒȨҟփƞ٦ԓȦڰғâƥ\nڤ");
-  bidiTests.push("ϖسՉȏŧΔԛdžĎӟیڡ\nέ");
-  bidiTests.push("۹ؼL۵ĺȧКԙػא7״\nم");
-  bidiTests.push("ن (ي)\u2009أقواس"); // thin space to throw off Firefox 51's broken white-space compressing behavior
-}
+bidiTests.push("Όȝǝڪȉۥ״ۺ׆ɀҩۏ\nҳ");
+if (!window.automatedTests) bidiTests.push("ŌӰтقȤ؁ƥ؅٣ĎȺ١\nϚ");
+bidiTests.push("ٻоҤѕѽΩ־؉ïίքdz\nٵ");
+bidiTests.push("؅؁ĆՕƿɁǞϮؠȩóć\nď");
+bidiTests.push("RŨďңŪzϢŎƏԖڇڦ\nӈ");
+bidiTests.push("ό׊۷٢ԜһОצЉيčǟ\nѩ");
+bidiTests.push("ۑÚҳҕڬġڹհяųKV\nr");
+bidiTests.push("źڻғúہ4ם1Ƞc1a\nԁ");
+bidiTests.push("ҒȨҟփƞ٦ԓȦڰғâƥ\nڤ");
+bidiTests.push("ϖسՉȏŧΔԛdžĎӟیڡ\nέ");
+bidiTests.push("۹ؼL۵ĺȧКԙػא7״\nم");
+bidiTests.push("ن (ي)\u2009أقواس"); // thin space to throw off Firefox 51's broken white-space compressing behavior
 
 bidiTests.push("քմѧǮßپüŢҍҞўڳ\nӧ");
 
@@ -2641,7 +2629,6 @@ testCM("rtl_wrapped_selection", function(cm) {
 }, {value: new Array(10).join(" فتي تم تضمينها فتي تم"), lineWrapping: true})
 
 testCM("bidi_wrapped_selection", function(cm) {
-  if (phantom) return
   cm.setSize(cm.charCoords(Pos(0, 10), "editor").left)
   cm.setSelection(Pos(0, 37), Pos(0, 80))
   var blocks = byClassName(cm.getWrapperElement(), "CodeMirror-selected")
@@ -2660,7 +2647,7 @@ testCM("delete_wrapped", function(cm) {
 }, {value: "12345", lineWrapping: true})
 
 testCM("issue_4878", function(cm) {
-  if (phantom) return
+  if (window.automatedTests) return
   cm.setCursor(Pos(1, 12, "after"));
   cm.moveH(-1, "char");
   eqCursorPos(cm.getCursor(), Pos(0, 113, "before"));

test/vim_test.js

@@ -471,7 +471,6 @@ testVim('j_k_and_gj_gk', function(cm,vim,helpers){
   helpers.assertCursorAt(0, 176);
 },{ lineWrapping:true, value: 'This line is intentially long to test movement of gj and gk over wrapped lines. I will start on the end of this line, then make a step up and back to set the origin for j and k.\nThis line is supposed to be even longer than the previous. I will jump here and make another wiggle with gj and gk, before I jump back to the line above. Both wiggles should not change my cursor\'s target character but both j/k and gj/gk change each other\'s reference position.'});
 testVim('gj_gk', function(cm, vim, helpers) {
-  if (phantom) return;
   cm.setSize(120);
   // Test top of document edge case.
   cm.setCursor(0, 4);