diff --git a/__tests__/__snapshots__/disabling-tokenizers.test.js.snap b/__tests__/__snapshots__/disabling-tokenizers.test.js.snap new file mode 100644 index 000000000..1dabad5d0 --- /dev/null +++ b/__tests__/__snapshots__/disabling-tokenizers.test.js.snap @@ -0,0 +1,69 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`disableTokenizers: "blocks" disabling block tokenizer 1`] = ` +Object { + "children": Array [ + Object { + "children": Array [ + Object { + "type": "text", + "value": "# heading 1", + }, + ], + "type": "paragraph", + }, + ], + "type": "root", +} +`; + +exports[`disableTokenizers: "inlines" disabling delete 1`] = ` +Object { + "children": Array [ + Object { + "children": Array [ + Object { + "type": "text", + "value": "~~strikethrough~~", + }, + ], + "type": "paragraph", + }, + ], + "type": "root", +} +`; + +exports[`disableTokenizers: "inlines" disabling emphasis 1`] = ` +Object { + "children": Array [ + Object { + "children": Array [ + Object { + "type": "text", + "value": "*emphatic **strong** emphatic*", + }, + ], + "type": "paragraph", + }, + ], + "type": "root", +} +`; + +exports[`disableTokenizers: "inlines" disabling inlineCode 1`] = ` +Object { + "children": Array [ + Object { + "children": Array [ + Object { + "type": "text", + "value": "\`const js = true \`", + }, + ], + "type": "paragraph", + }, + ], + "type": "root", +} +`; diff --git a/__tests__/__snapshots__/link-parsers.test.js.snap b/__tests__/__snapshots__/link-parsers.test.js.snap new file mode 100644 index 000000000..4b1afd3d0 --- /dev/null +++ b/__tests__/__snapshots__/link-parsers.test.js.snap @@ -0,0 +1,122 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`a bare autoLinked url 1`] = ` +Object { + "children": Array [ + Object { + "children": Array [ + Object { + "children": Array [ + Object { + "type": "text", + "value": "http://www.googl.com", + }, + ], + "title": null, + "type": "link", + "url": "http://www.googl.com", + }, + ], + "type": "paragraph", + }, + ], + "type": "root", +} +`; + +exports[`a bare autoLinked url with no protocol 1`] = ` +Object { + "children": Array [ + Object { + "children": Array [ + Object { + "children": Array [ + Object { + "type": "text", + "value": "www.google.com", + }, + ], + "title": null, + "type": "link", + "url": "http://www.google.com", + }, + ], + "type": "paragraph", + }, + ], + "type": "root", +} +`; + +exports[`a bracketed autoLinked url 1`] = ` +Object { + "children": Array [ + Object { + "children": Array [ + Object { + "children": Array [ + Object { + "type": "text", + "value": "http://www.google.com", + }, + ], + "title": null, + "type": "link", + "url": "http://www.google.com", + }, + ], + "type": "paragraph", + }, + ], + "type": "root", +} +`; + +exports[`a link ref 1`] = ` +Object { + "children": Array [ + Object { + "children": Array [ + Object { + "children": Array [ + Object { + "type": "text", + "value": "link", + }, + ], + "identifier": "link", + "label": "link", + "referenceType": "shortcut", + "type": "linkReference", + }, + ], + "type": "paragraph", + }, + ], + "type": "root", +} +`; + +exports[`a link with label 1`] = ` +Object { + "children": Array [ + Object { + "children": Array [ + Object { + "children": Array [ + Object { + "type": "text", + "value": "link", + }, + ], + "title": null, + "type": "link", + "url": "http://www.foo.com", + }, + ], + "type": "paragraph", + }, + ], + "type": "root", +} +`; diff --git a/__tests__/disabling-tokenizers.test.js b/__tests__/disabling-tokenizers.test.js new file mode 100644 index 000000000..d7b7d63ba --- /dev/null +++ b/__tests__/disabling-tokenizers.test.js @@ -0,0 +1,29 @@ +const markdown = require('../index'); + +describe('disableTokenizers: "inlines"', () => { + const opts = { disableTokenizers: 'inlines' }; + + it('disabling inlineCode', () => { + const md = '`const js = true `'; + expect(markdown.mdast(md, opts)).toMatchSnapshot(); + }); + + it('disabling emphasis', () => { + const md = '*emphatic **strong** emphatic*'; + expect(markdown.mdast(md, opts)).toMatchSnapshot(); + }); + + it('disabling delete', () => { + const md = '~~strikethrough~~'; + expect(markdown.mdast(md, opts)).toMatchSnapshot(); + }); +}); + +describe('disableTokenizers: "blocks"', () => { + const opts = { disableTokenizers: 'blocks' }; + + it('disabling block tokenizer', () => { + const md = '# heading 1'; + expect(markdown.mdast(md, opts)).toMatchSnapshot(); + }); +}); diff --git a/__tests__/flavored-compilers.test.js b/__tests__/flavored-compilers.test.js index 957b7041c..886d3856c 100644 --- a/__tests__/flavored-compilers.test.js +++ b/__tests__/flavored-compilers.test.js @@ -7,7 +7,7 @@ const rehypeSanitize = require('rehype-sanitize'); const parsers = Object.values(require('../processor/parse')).map(parser => parser.sanitize(sanitize)); const compilers = Object.values(require('../processor/compile')); -const options = require('../options.json').markdownOptions; +const options = require('../options.js').options.markdownOptions; const processor = unified() .use(remarkParse, options) diff --git a/__tests__/flavored-parsers.test.js b/__tests__/flavored-parsers.test.js index 34066be22..03549ddaa 100644 --- a/__tests__/flavored-parsers.test.js +++ b/__tests__/flavored-parsers.test.js @@ -4,7 +4,7 @@ const rehypeSanitize = require('rehype-sanitize'); const parseCallouts = require('../processor/parse/flavored/callout'); const parseCodeTabs = require('../processor/parse/flavored/code-tabs'); -const options = require('../options.json').markdownOptions; +const options = require('../options.js').options.markdownOptions; const sanitize = { attributes: [], tagNames: [] }; const process = (text, opts = options) => diff --git a/__tests__/index.test.js b/__tests__/index.test.js index 462dd17a8..0894088f6 100644 --- a/__tests__/index.test.js +++ b/__tests__/index.test.js @@ -4,7 +4,7 @@ const BaseUrlContext = require('../contexts/BaseUrl'); const markdown = require('../index'); const { tableFlattening } = require('../processor/plugin/table-flattening'); -const settings = require('../options.json'); +const { options } = require('../options.js'); test('image', () => { expect(mount(markdown.default('![Image](http://example.com/image.png)')).html()).toMatchSnapshot(); @@ -33,7 +33,7 @@ test('magic image', () => { } [/block] `, - settings + options ) ).html() ).toMatchSnapshot(); diff --git a/__tests__/link-parsers.test.js b/__tests__/link-parsers.test.js new file mode 100644 index 000000000..64aa06fa8 --- /dev/null +++ b/__tests__/link-parsers.test.js @@ -0,0 +1,21 @@ +const markdown = require('../index'); + +test('a link with label', () => { + expect(markdown.mdast('[link](http://www.foo.com)')).toMatchSnapshot(); +}); + +test('a link ref', () => { + expect(markdown.mdast('[link]')).toMatchSnapshot(); +}); + +test('a bracketed autoLinked url', () => { + expect(markdown.mdast('')).toMatchSnapshot(); +}); + +test('a bare autoLinked url', () => { + expect(markdown.mdast('http://www.googl.com')).toMatchSnapshot(); +}); + +test.skip('a bare autoLinked url with no protocol', () => { + expect(markdown.mdast('www.google.com')).toMatchSnapshot(); +}); diff --git a/__tests__/magic-block-parser.test.js b/__tests__/magic-block-parser.test.js index 72592a46e..7c9398103 100644 --- a/__tests__/magic-block-parser.test.js +++ b/__tests__/magic-block-parser.test.js @@ -3,7 +3,7 @@ const remarkParse = require('remark-parse'); const rehypeSanitize = require('rehype-sanitize'); const parser = require('../processor/parse/magic-block-parser'); -const options = require('../options.json').markdownOptions; +const options = require('../options.js').options.markdownOptions; const { silenceConsole } = require('./helpers'); diff --git a/__tests__/magic-block-parser/table.test.js b/__tests__/magic-block-parser/table.test.js index 625ebd50b..9de69502d 100644 --- a/__tests__/magic-block-parser/table.test.js +++ b/__tests__/magic-block-parser/table.test.js @@ -3,7 +3,7 @@ const remarkParse = require('remark-parse'); const rehypeSanitize = require('rehype-sanitize'); const parser = require('../../processor/parse/magic-block-parser'); -const options = require('../../options.json').markdownOptions; +const options = require('../../options.js').options.markdownOptions; const sanitize = { attributes: [] }; const process = (text, opts = options) => diff --git a/docs/getting-started.md b/docs/getting-started.md index a226fbe70..13530c976 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -66,6 +66,7 @@ The `utils` export gives you access to various tools and configuration settings: - `markdownOptions`—configuration object passed to `remark` - `correctnewlines`—flag to toggle newline transformation. - `normalize`—auto-normalize magic blocks before processing. + - `disableTokenizers`—disable internal `block` or `inline` tokenizers. - **``** and **``** React provider and consumer wrappers for [user data injection](doc:features#section-data-injection). [block:html] diff --git a/index.js b/index.js index 61e725da3..2a1fbcaf2 100644 --- a/index.js +++ b/index.js @@ -21,6 +21,7 @@ const remarkStringify = require('remark-stringify'); const remarkBreaks = require('remark-breaks'); const remarkSlug = require('remark-slug'); const remarkFrontmatter = require('remark-frontmatter'); +const remarkDisableTokenizers = require('remark-disable-tokenizers'); // rehype plugins const rehypeSanitize = require('rehype-sanitize'); @@ -61,14 +62,14 @@ const tableFlattening = require('./processor/plugin/table-flattening'); const toPlainText = require('./processor/plugin/plain-text'); // Processor Option Defaults -const options = require('./options.json'); +const { options, parseOptions } = require('./options.js'); /** * Normalize Magic Block Raw Text */ export function setup(blocks, opts = {}) { // merge default and user options - opts = { ...options, ...opts }; + opts = parseOptions(opts); // normalize magic block linebreaks if (opts.normalize && blocks) { @@ -110,6 +111,7 @@ export function processor(opts = {}) { * - sanitize and remove any disallowed attributes * - output the hast to a React vdom with our custom components */ + return unified() .use(remarkParse, opts.markdownOptions) .use(remarkFrontmatter, ['yaml', 'toml']) @@ -118,6 +120,7 @@ export function processor(opts = {}) { .use(!opts.correctnewlines ? remarkBreaks : () => {}) .use(customParsers) .use(remarkSlug) + .use(remarkDisableTokenizers, opts.disableTokenizers) .use(remarkRehype, { allowDangerousHtml: true }) .use(rehypeRaw) .use(rehypeSanitize, sanitize); diff --git a/options.js b/options.js new file mode 100644 index 000000000..94ebdd632 --- /dev/null +++ b/options.js @@ -0,0 +1,89 @@ +const options = { + compatibilityMode: false, + copyButtons: true, + correctnewlines: false, + markdownOptions: { + fences: true, + commonmark: true, + gfm: true, + ruleSpaces: false, + listItemIndent: '1', + spacedTable: true, + paddedTable: true, + setext: true, + }, + normalize: true, + settings: { + position: false, + }, +}; + +/** + * @note disabling `newline`, `paragraph`, or `text` tokenizers trips Remark into an infinite loop! + */ +const blocks = [ + // 'newline', + 'indentedCode', + 'fencedCode', + 'blockquote', + 'atxHeading', + 'thematicBreak', + 'list', + 'setextHeading', + 'html', + 'footnote', + 'definition', + 'table', + // 'paragraph', +]; + +const inlines = [ + 'escape', + 'autoLink', + 'url', + 'html', + 'link', + 'reference', + 'strong', + 'emphasis', + 'deletion', + 'code', + 'break', + // 'text', +]; + +const toBeDecorated = { + inlines: inlines.filter(i => !['link', 'reference'].includes(i)), + blocks: [], +}; + +const disableTokenizers = { + inlines: { + disableTokenizers: { + inline: toBeDecorated.inlines, + block: toBeDecorated.blocks, + }, + }, + blocks: { + disableTokenizers: { + inline: inlines.filter(i => !toBeDecorated.inlines.includes(i)), + block: blocks.filter(b => !toBeDecorated.blocks.includes(b)), + }, + }, +}; + +const parseOptions = (userOpts = {}) => { + let opts = { ...options, ...userOpts }; + + if (opts.disableTokenizers in disableTokenizers) { + opts = { ...opts, ...disableTokenizers[opts.disableTokenizers] }; + } else if (opts.disableTokenizers) { + throw new Error( + `opts.disableTokenizers "${opts.disableTokenizers}" not one of "${Object.keys(disableTokenizers)}"` + ); + } + + return opts; +}; + +export { options, parseOptions }; diff --git a/options.json b/options.json deleted file mode 100644 index 7520b0173..000000000 --- a/options.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "compatibilityMode": false, - "copyButtons": true, - "correctnewlines": false, - "markdownOptions": { - "fences": true, - "commonmark": true, - "gfm": true, - "ruleSpaces": false, - "listItemIndent": "1", - "spacedTable": true, - "paddedTable": true, - "setext": true - }, - "normalize": true, - "settings": { - "position": false - } -} diff --git a/package-lock.json b/package-lock.json index bfd871aa2..1ce82ad01 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7010,6 +7010,11 @@ } } }, + "clone": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", + "integrity": "sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18=" + }, "clone-deep": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", @@ -22973,6 +22978,14 @@ "resolved": "https://registry.npmjs.org/remark-breaks/-/remark-breaks-1.0.5.tgz", "integrity": "sha512-lr8+TlJI273NjEqL27eUthPYPTCgXEj4NaLbnazS3bQaQL2FySlsbtgo52gE36fE1gWeQgkn1VdmWsoT+uA7FA==" }, + "remark-disable-tokenizers": { + "version": "1.0.24", + "resolved": "https://registry.npmjs.org/remark-disable-tokenizers/-/remark-disable-tokenizers-1.0.24.tgz", + "integrity": "sha512-HsAmBY5cNliHYAzba4zuskZzkDdp6sG+tRelDb4AoPo2YHNGHnxYsatShzTIsnRNLgCbsxycW5Ge6KigHn701A==", + "requires": { + "clone": "^2.1.2" + } + }, "remark-frontmatter": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/remark-frontmatter/-/remark-frontmatter-2.0.0.tgz", diff --git a/package.json b/package.json index bb9a0daa6..5165a6b61 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "rehype-sanitize": "^3.0.1", "rehype-stringify": "^6.0.0", "remark-breaks": "^1.0.0", + "remark-disable-tokenizers": "^1.0.24", "remark-frontmatter": "^2.0.0", "remark-parse": "^7.0.2", "remark-rehype": "^7.0.0",