feat: allow disabling tokenizers (#119)

[<img height=20 src=https://readme.com/static/favicon.ico align=center>][demo] | RM-215 :---:|:---: ## 🧰 Changes Adds the option of disabling tokenizers. This is in service of the editor's markdown decorators. During deserialization, we can leave the inlines as plain text. Then, during the decoration step, we can parse the markdown with just the inline tokenizers. ## Usage ``` const opts = { tokenizerSet: 'blocks' }; markdown.mdast(md, opts); // won't parse inline code, emphasis, etc. ``` [demo]: https://markdown-pr-119.herokuapp.com
readmeio · Feb 12, 2021 · 0418a05 · 0418a05
1 parent 3c823f1
commit 0418a05
Show file tree

Hide file tree

Showing 15 changed files with 356 additions and 27 deletions.
diff --git a/__tests__/__snapshots__/disabling-tokenizers.test.js.snap b/__tests__/__snapshots__/disabling-tokenizers.test.js.snap
@@ -0,0 +1,69 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`disableTokenizers: "blocks" disabling block tokenizer 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "type": "text",
+          "value": "#  heading 1",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`disableTokenizers: "inlines" disabling delete 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "type": "text",
+          "value": "~~strikethrough~~",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`disableTokenizers: "inlines" disabling emphasis 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "type": "text",
+          "value": "*emphatic **strong** emphatic*",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`disableTokenizers: "inlines" disabling inlineCode 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "type": "text",
+          "value": "\`const js = true \`",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
diff --git a/__tests__/__snapshots__/link-parsers.test.js.snap b/__tests__/__snapshots__/link-parsers.test.js.snap
@@ -0,0 +1,122 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`a bare autoLinked url 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "children": Array [
+            Object {
+              "type": "text",
+              "value": "http://www.googl.com",
+            },
+          ],
+          "title": null,
+          "type": "link",
+          "url": "http://www.googl.com",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`a bare autoLinked url with no protocol 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "children": Array [
+            Object {
+              "type": "text",
+              "value": "www.google.com",
+            },
+          ],
+          "title": null,
+          "type": "link",
+          "url": "http://www.google.com",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`a bracketed autoLinked url 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "children": Array [
+            Object {
+              "type": "text",
+              "value": "http://www.google.com",
+            },
+          ],
+          "title": null,
+          "type": "link",
+          "url": "http://www.google.com",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`a link ref 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "children": Array [
+            Object {
+              "type": "text",
+              "value": "link",
+            },
+          ],
+          "identifier": "link",
+          "label": "link",
+          "referenceType": "shortcut",
+          "type": "linkReference",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`a link with label 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "children": Array [
+            Object {
+              "type": "text",
+              "value": "link",
+            },
+          ],
+          "title": null,
+          "type": "link",
+          "url": "http://www.foo.com",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
diff --git a/__tests__/disabling-tokenizers.test.js b/__tests__/disabling-tokenizers.test.js
@@ -0,0 +1,29 @@
+const markdown = require('../index');
+
+describe('disableTokenizers: "inlines"', () => {
+  const opts = { disableTokenizers: 'inlines' };
+
+  it('disabling inlineCode', () => {
+    const md = '`const js = true `';
+    expect(markdown.mdast(md, opts)).toMatchSnapshot();
+  });
+
+  it('disabling emphasis', () => {
+    const md = '*emphatic **strong** emphatic*';
+    expect(markdown.mdast(md, opts)).toMatchSnapshot();
+  });
+
+  it('disabling delete', () => {
+    const md = '~~strikethrough~~';
+    expect(markdown.mdast(md, opts)).toMatchSnapshot();
+  });
+});
+
+describe('disableTokenizers: "blocks"', () => {
+  const opts = { disableTokenizers: 'blocks' };
+
+  it('disabling block tokenizer', () => {
+    const md = '# heading 1';
+    expect(markdown.mdast(md, opts)).toMatchSnapshot();
+  });
+});
diff --git a/__tests__/flavored-compilers.test.js b/__tests__/flavored-compilers.test.js
@@ -7,7 +7,7 @@ const rehypeSanitize = require('rehype-sanitize');
 
 const parsers = Object.values(require('../processor/parse')).map(parser => parser.sanitize(sanitize));
 const compilers = Object.values(require('../processor/compile'));
-const options = require('../options.json').markdownOptions;
+const options = require('../options.js').options.markdownOptions;
 
 const processor = unified()
   .use(remarkParse, options)

diff --git a/__tests__/flavored-parsers.test.js b/__tests__/flavored-parsers.test.js
@@ -4,7 +4,7 @@ const rehypeSanitize = require('rehype-sanitize');
 
 const parseCallouts = require('../processor/parse/flavored/callout');
 const parseCodeTabs = require('../processor/parse/flavored/code-tabs');
-const options = require('../options.json').markdownOptions;
+const options = require('../options.js').options.markdownOptions;
 
 const sanitize = { attributes: [], tagNames: [] };
 const process = (text, opts = options) =>

diff --git a/__tests__/index.test.js b/__tests__/index.test.js
@@ -4,7 +4,7 @@ const BaseUrlContext = require('../contexts/BaseUrl');
 
 const markdown = require('../index');
 const { tableFlattening } = require('../processor/plugin/table-flattening');
-const settings = require('../options.json');
+const { options } = require('../options.js');
 
 test('image', () => {
   expect(mount(markdown.default('![Image](http://example.com/image.png)')).html()).toMatchSnapshot();
@@ -33,7 +33,7 @@ test('magic image', () => {
     }
     [/block]
     `,
-        settings
+        options
       )
     ).html()
   ).toMatchSnapshot();

diff --git a/__tests__/link-parsers.test.js b/__tests__/link-parsers.test.js
@@ -0,0 +1,21 @@
+const markdown = require('../index');
+
+test('a link with label', () => {
+  expect(markdown.mdast('[link](http://www.foo.com)')).toMatchSnapshot();
+});
+
+test('a link ref', () => {
+  expect(markdown.mdast('[link]')).toMatchSnapshot();
+});
+
+test('a bracketed autoLinked url', () => {
+  expect(markdown.mdast('<http://www.google.com>')).toMatchSnapshot();
+});
+
+test('a bare autoLinked url', () => {
+  expect(markdown.mdast('http://www.googl.com')).toMatchSnapshot();
+});
+
+test.skip('a bare autoLinked url with no protocol', () => {
+  expect(markdown.mdast('www.google.com')).toMatchSnapshot();
+});
diff --git a/__tests__/magic-block-parser.test.js b/__tests__/magic-block-parser.test.js
@@ -3,7 +3,7 @@ const remarkParse = require('remark-parse');
 const rehypeSanitize = require('rehype-sanitize');
 
 const parser = require('../processor/parse/magic-block-parser');
-const options = require('../options.json').markdownOptions;
+const options = require('../options.js').options.markdownOptions;
 
 const { silenceConsole } = require('./helpers');
 

diff --git a/__tests__/magic-block-parser/table.test.js b/__tests__/magic-block-parser/table.test.js
@@ -3,7 +3,7 @@ const remarkParse = require('remark-parse');
 const rehypeSanitize = require('rehype-sanitize');
 
 const parser = require('../../processor/parse/magic-block-parser');
-const options = require('../../options.json').markdownOptions;
+const options = require('../../options.js').options.markdownOptions;
 
 const sanitize = { attributes: [] };
 const process = (text, opts = options) =>

diff --git a/docs/getting-started.md b/docs/getting-started.md
@@ -66,6 +66,7 @@ The `utils` export gives you access to various tools and configuration settings:
   - `markdownOptions`—configuration object passed to `remark`
   - `correctnewlines`—flag to toggle newline transformation.
   - `normalize`—auto-normalize magic blocks before processing.
+  - `disableTokenizers`—disable internal `block` or `inline` tokenizers.
 - **`<GlossaryContext/>`** and **`<VariablesContext/>`**
   React provider and consumer wrappers for [user data injection](doc:features#section-data-injection).
 [block:html]

diff --git a/index.js b/index.js
@@ -21,6 +21,7 @@ const remarkStringify = require('remark-stringify');
 const remarkBreaks = require('remark-breaks');
 const remarkSlug = require('remark-slug');
 const remarkFrontmatter = require('remark-frontmatter');
+const remarkDisableTokenizers = require('remark-disable-tokenizers');
 
 // rehype plugins
 const rehypeSanitize = require('rehype-sanitize');
@@ -61,14 +62,14 @@ const tableFlattening = require('./processor/plugin/table-flattening');
 const toPlainText = require('./processor/plugin/plain-text');
 
 // Processor Option Defaults
-const options = require('./options.json');
+const { options, parseOptions } = require('./options.js');
 
 /**
  * Normalize Magic Block Raw Text
  */
 export function setup(blocks, opts = {}) {
   // merge default and user options
-  opts = { ...options, ...opts };
+  opts = parseOptions(opts);
 
   // normalize magic block linebreaks
   if (opts.normalize && blocks) {
@@ -110,6 +111,7 @@ export function processor(opts = {}) {
    * - sanitize and remove any disallowed attributes
    * - output the hast to a React vdom with our custom components
    */
+
   return unified()
     .use(remarkParse, opts.markdownOptions)
     .use(remarkFrontmatter, ['yaml', 'toml'])
@@ -118,6 +120,7 @@ export function processor(opts = {}) {
     .use(!opts.correctnewlines ? remarkBreaks : () => {})
     .use(customParsers)
     .use(remarkSlug)
+    .use(remarkDisableTokenizers, opts.disableTokenizers)
     .use(remarkRehype, { allowDangerousHtml: true })
     .use(rehypeRaw)
     .use(rehypeSanitize, sanitize);