Merge pull request #1444 from UziTech/normalize-tests

Normalize tests

package.json

@@ -30,18 +30,17 @@
   ],
   "devDependencies": {
     "commonmark": "0.x",
-    "eslint": "^5.12.0",
+    "eslint": "^5.15.1",
     "eslint-config-standard": "^12.0.0",
-    "eslint-plugin-import": "^2.14.0",
+    "eslint-plugin-import": "^2.16.0",
     "eslint-plugin-node": "^8.0.1",
     "eslint-plugin-promise": "^4.0.1",
     "eslint-plugin-standard": "^4.0.0",
     "eslint-plugin-vuln-regex-detector": "^1.0.4",
     "front-matter": "^3.0.1",
     "glob-to-regexp": "^0.4.0",
-    "@markedjs/html-differ": "^2.0.0",
+    "@markedjs/html-differ": "^2.0.1",
     "jasmine": "^3.3.1",
-    "jasmine2-custom-message": "^0.9.3",
     "markdown": "0.x",
     "markdown-it": "8.x",
     "uglify-js": "^3.4.9"
@@ -56,6 +55,7 @@
     "test:old": "node test",
     "test:lint": "eslint bin/marked .",
     "test:redos": "eslint --plugin vuln-regex-detector --rule '\"vuln-regex-detector/no-vuln-regex\": 2' lib/marked.js",
+    "test:node4": "npx node@4 ./node_modules/jasmine/bin/jasmine.js --config=jasmine.json",
     "bench": "node test --bench",
     "lint": "eslint --fix bin/marked .",
     "build": "uglifyjs lib/marked.js -cm  --comments /Copyright/ -o marked.min.js",

test/helpers/helpers.js

@@ -1,5 +1,44 @@
-var marked = require('../../lib/marked.js');
+const marked = require('../../');
+const HtmlDiffer = require('@markedjs/html-differ').HtmlDiffer;
+const htmlDiffer = new HtmlDiffer({ignoreSelfClosingSlash: true});
 
-beforeEach(function () {
+const EXCERPT_LENGTH = 30;
+
+beforeEach(() => {
   marked.setOptions(marked.getDefaults());
+
+  jasmine.addMatchers({
+    toRender: () => {
+      return {
+        compare: (spec, expected) => {
+          const result = {};
+          const actual = marked(spec.markdown, spec.options);
+          result.pass = htmlDiffer.isEqual(expected, actual);
+
+          if (result.pass) {
+            result.message = spec.markdown + '\n------\n\nExpected: Should Fail';
+          } else {
+            var expectedHtml = expected.replace(/\s/g, '');
+            var actualHtml = actual.replace(/\s/g, '');
+
+            for (var i = 0; i < expectedHtml.length; i++) {
+              if (actualHtml[i] !== expectedHtml[i]) {
+                actualHtml = actualHtml.substring(
+                  Math.max(i - EXCERPT_LENGTH, 0),
+                  Math.min(i + EXCERPT_LENGTH, actualHtml.length));
+
+                expectedHtml = expectedHtml.substring(
+                  Math.max(i - EXCERPT_LENGTH, 0),
+                  Math.min(i + EXCERPT_LENGTH, expectedHtml.length));
+
+                break;
+              }
+            }
+            result.message = 'Expected:\n' + expectedHtml + '\n\nActual:\n' + actualHtml;
+          }
+          return result;
+        }
+      };
+    }
+  });
 });

test/index.js

@@ -176,6 +176,17 @@ function testFile(engine, file, filename, index) {
 
   l = html.length;
 
+  if (l === 0 && text.length > 0) {
+    text = text.substring(0, Math.min(30, text.length));
+
+    console.log('    failed in %dms at offset %d. Near: "%s".\n', prettyElapsedTime(elapsed), 0, text);
+
+    console.log('\nActual:\n%s\n', text.trim() || text);
+    console.log('\nExpected:\n\n');
+
+    return false;
+  }
+
   for (j = 0; j < l; j++) {
     if (text[j] !== html[j]) {
       text = text.substring(
@@ -188,7 +199,7 @@ function testFile(engine, file, filename, index) {
 
       console.log('    failed in %dms at offset %d. Near: "%s".\n', prettyElapsedTime(elapsed), j, text);
 
-      console.log('\nGot:\n%s\n', text.trim() || text);
+      console.log('\nActual:\n%s\n', text.trim() || text);
       console.log('\nExpected:\n%s\n', html.trim() || html);
 
       return false;
@@ -346,11 +357,11 @@ function time(options) {
  */
 
 function fix() {
-  ['compiled_tests', 'original', 'new'].forEach(function(dir) {
+  ['compiled_tests', 'original', 'new', 'redos'].forEach(function(dir) {
     try {
       fs.mkdirSync(path.resolve(__dirname, dir));
     } catch (e) {
-      ;
+      // directory already exists
     }
   });
 
@@ -393,20 +404,6 @@ function fix() {
       .replace(/&__QUOT__;/g, '"')
       .replace(/&__APOS__;/g, '\'');
 
-    // add heading id's
-    html = html.replace(/<(h[1-6])>([^<]+)<\/\1>/g, function(s, h, text) {
-      var id = text
-        .replace(/&#39;/g, '\'')
-        .replace(/&quot;/g, '"')
-        .replace(/&gt;/g, '>')
-        .replace(/&lt;/g, '<')
-        .replace(/&amp;/g, '&');
-
-      id = id.toLowerCase().replace(/[^\w]+/g, '-');
-
-      return '<' + h + ' id="' + id + '">' + text + '</' + h + '>';
-    });
-
     fs.writeFileSync(file, html);
   });
 
@@ -435,6 +432,12 @@ function fix() {
     fs.writeFileSync(path.resolve(__dirname, 'compiled_tests', file),
       fs.readFileSync(path.resolve(__dirname, 'new', file)));
   });
+
+  // cp redos/* tests/
+  fs.readdirSync(path.resolve(__dirname, 'redos')).forEach(function(file) {
+    fs.writeFileSync(path.resolve(__dirname, 'compiled_tests', file),
+      fs.readFileSync(path.resolve(__dirname, 'redos', file)));
+  });
 }
 
 /**

test/json-to-files.js

@@ -0,0 +1,62 @@
+const path = require('path');
+const fs = require('fs');
+
+const folder = process.argv[2];
+const jsonFile = process.argv[3];
+
+if (!folder || !jsonFile) {
+  console.log('node ./json-to-files.js {path to folder} {path to json file}');
+  process.exit(1);
+}
+
+const specs = require(jsonFile);
+
+const files = specs.reduce((obj, spec) => {
+  if (!obj[spec.section]) {
+    obj[spec.section] = {
+      md: [],
+      html: [],
+      options: {}
+    };
+  }
+
+  obj[spec.section].md.push(spec.markdown);
+  obj[spec.section].html.push(spec.html);
+  Object.assign(obj[spec.section].options, spec.options);
+
+  return obj;
+}, {});
+
+try {
+  fs.mkdirSync(folder, {recursive: true});
+} catch (ex) {
+  // already exists
+}
+
+for (const section in files) {
+  const file = files[section];
+  const name = section.toLowerCase().replace(' ', '_');
+  const frontMatter = Object.keys(file.options).map(opt => {
+    let value = file.options[opt];
+    if (typeof value !== 'string') {
+      value = JSON.stringify(value);
+    }
+    return `${opt}: ${value}`;
+  }).join('\n');
+
+  let markdown = file.md.join('\n\n');
+  if (frontMatter) {
+    markdown = `---\n${frontMatter}\n---\n\n${markdown}`;
+  }
+  const html = file.html.join('\n\n');
+
+  const mdFile = path.resolve(folder, `${name}.md`);
+  const htmlFile = path.resolve(folder, `${name}.html`);
+
+  if (fs.existsSync(mdFile) || fs.existsSync(htmlFile)) {
+    throw new Error(`${name} already exists.`);
+  }
+
+  fs.writeFileSync(mdFile, markdown);
+  fs.writeFileSync(htmlFile, html);
+}

test/new/autolinks.html

@@ -0,0 +1,15 @@
+<p>(See <a href="https://www.example.com/fhqwhgads">https://www.example.com/fhqwhgads</a>.)</p>
+
+<p>((<a href="http://foo.com">http://foo.com</a>))</p>
+
+<p>((<a href="http://foo.com">http://foo.com</a>.))</p>
+
+<p><a href="HTTP://FOO.COM">HTTP://FOO.COM</a></p>
+
+<p><a href="hTtP://fOo.CoM">hTtP://fOo.CoM</a></p>
+
+<p><del><a href="mailto:[email protected]">[email protected]</a></del></p>
+
+<p><strong><a href="mailto:[email protected]">[email protected]</a></strong></p>
+
+<p><strong><a href="mailto:[email protected]">[email protected]</a></strong></p>

test/new/autolinks.md

@@ -0,0 +1,15 @@
+(See https://www.example.com/fhqwhgads.)
+
+((http://foo.com))
+
+((http://foo.com.))
+
+HTTP://FOO.COM
+
+hTtP://fOo.CoM
+
+~~[email protected]~~
+
+**[email protected]**
+
+__[email protected]__

test/new/code_spans.html

@@ -0,0 +1,3 @@
+<p><code>[email protected]</code></p>
+
+<p>``<em>test`</em></p>

test/new/code_spans.md

@@ -0,0 +1,3 @@
+`[email protected]`
+
+``*test`*

test/new/emphasis_extra tests.html

@@ -0,0 +1 @@
+<p><em>test</em>. <em>test</em>: <em>test</em>! <em>test</em>? <em>test</em>-</p>

test/new/emphasis_extra tests.md

@@ -0,0 +1 @@
+_test_. _test_: _test_! _test_? _test_-

test/new/links.html

@@ -1,5 +1,3 @@
-<p>URL</p>
-<p>URL</p>
-<p>URL</p>
-<p>URL</p>
-<p>URL</p>
+<p>Link: <a href="https://example.org/">constructor</a>.</p>
+
+<p><a href="https://example.com/1">One</a> (<a href="https://example.com/2">Two</a>) <a href="https://example.com/3">Three</a></p>

test/new/links.md

@@ -1,12 +1,5 @@
----
-sanitize: true
----
-[URL](javascript:alert)
+Link: [constructor][].
 
-[URL](vbscript:alert)
+[One](https://example.com/1) ([Two](https://example.com/2)) [Three](https://example.com/3)
 
-[URL](javascript:alert(1))
-
-[URL](javascript&#58document;alert(1))
-
-[URL](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)
+[constructor]: https://example.org/

test/new/sanitize_links.html

@@ -0,0 +1,5 @@
+<p>URL</p>
+<p>URL</p>
+<p>URL</p>
+<p>URL</p>
+<p>URL</p>

test/new/sanitize_links.md

@@ -0,0 +1,12 @@
+---
+sanitize: true
+---
+[URL](javascript:alert)
+
+[URL](vbscript:alert)
+
+[URL](javascript:alert(1))
+
+[URL](javascript&#58document;alert(1))
+
+[URL](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K)

test/new/table_cells.html

@@ -0,0 +1,27 @@
+<table><thead><tr><th>1</th></tr></thead><tbody><tr><td>1</td></tr></tbody></table>
+
+<table><thead><tr><th>1</th></tr></thead><tbody><tr><td>|</td></tr></tbody></table>
+
+<table><thead><tr><th>1</th></tr></thead><tbody><tr><td>1\1</td></tr></tbody></table>
+
+<table><thead><tr><th>1</th></tr></thead><tbody><tr><td>\\</td></tr></tbody></table>
+
+<table><thead><tr><th>1</th></tr></thead><tbody><tr><td>\\|</td></tr></tbody></table>
+
+<table><thead><tr><th>1</th><th>2</th></tr></thead><tbody><tr><td></td><td>2</td></tr></tbody></table>
+
+<table><thead><tr><th>1</th><th>2</th></tr></thead><tbody><tr><td>1|\</td><td>2|\</td></tr></tbody></table>
+
+<table><thead><tr><th>1</th><th>2</th></tr></thead><tbody><tr><td></td><td>2</td></tr></tbody></table>
+
+<table><thead><tr><th>1</th><th>2</th></tr></thead><tbody><tr><td>1|\</td><td>2|\</td></tr></tbody></table>
+
+<table><thead><tr><th>1</th><th>2</th></tr></thead><tbody><tr><td></td><td>2</td></tr></tbody></table>
+
+<table><thead><tr><th>1</th><th>2</th></tr></thead><tbody><tr><td>1</td><td>2|</td></tr></tbody></table>
+
+<table><thead><tr><th>1</th><th>2</th></tr></thead><tbody><tr><td>1</td><td>2|</td></tr></tbody></table>
+
+<table><thead><tr><th>1</th><th>2</th></tr></thead><tbody><tr><td>1</td><td>2|</td></tr></tbody></table>
+
+<table><thead><tr><th>1</th><th>2</th></tr></thead><tbody><tr><td>1</td><td>2|</td></tr></tbody></table>

test/new/table_cells.md

@@ -0,0 +1,55 @@
+|1|
+|-|
+|1|
+
+|1|
+|-|
+|\||
+
+|1|
+|-|
+|1\\1|
+
+|1|
+|-|
+|\\\\||
+
+|1|
+|-|
+|\\\\\||
+
+|1|2|
+|-|-|
+||2|
+
+|1|2|
+|-|-|
+|1\|\\|2\|\\|
+
+|1|2|
+|-|-|
+| |2|
+
+1|2
+-|-
+1\|\\|2\|\\
+
+1|2
+-|-
+ |2
+
+1|2
+-|-
+1|2\|
+
+1|2
+-|-
+1|2\|
+
+|1|2|
+|-|-|
+|1|2\||
+
+|1|2|
+|-|-|
+|1|2\||