Skip to content

Commit

Permalink
fix: only convert leading tabs to spaces (#1559) (#2434)
Browse files Browse the repository at this point in the history
* fix: non leading-tabs in markdown content (#1559)

Only replaces tabs at the beginning of a block construct. Tabs in the
middle of the item are unaffected.

All tests passing. Tabs in both GFM and CommonMark at 100%

fixes #1559

* update new/html_comments.html to preserve tab

* combine redundant if condition

* add test for tab immediately after blockquote character
  • Loading branch information
rossipedia authored Apr 11, 2022
1 parent 3dc35bb commit 7d19665
Show file tree
Hide file tree
Showing 6 changed files with 14 additions and 8 deletions.
10 changes: 7 additions & 3 deletions src/Lexer.js
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,7 @@ export class Lexer {
*/
lex(src) {
src = src
.replace(/\r\n|\r/g, '\n')
.replace(/\t/g, ' ');
.replace(/\r\n|\r/g, '\n');

this.blockTokens(src, this.tokens);

Expand All @@ -133,8 +132,13 @@ export class Lexer {
*/
blockTokens(src, tokens = []) {
if (this.options.pedantic) {
src = src.replace(/^ +$/gm, '');
src = src.replace(/\t/g, ' ').replace(/^ +$/gm, '');
} else {
src = src.replace(/^( *)(\t+)/gm, (_, leading, tabs) => {
return leading + ' '.repeat(tabs.length);
});
}

let token, lastToken, cutSrc, lastParagraphClipped;

while (src) {
Expand Down
4 changes: 2 additions & 2 deletions src/Tokenizer.js
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ export class Tokenizer {
blockquote(src) {
const cap = this.rules.block.blockquote.exec(src);
if (cap) {
const text = cap[0].replace(/^ *> ?/gm, '');
const text = cap[0].replace(/^ *>[ \t]?/gm, '');

return {
type: 'blockquote',
Expand Down Expand Up @@ -187,7 +187,7 @@ export class Tokenizer {
}

// Get next list item
const itemRegex = new RegExp(`^( {0,3}${bull})((?: [^\\n]*)?(?:\\n|$))`);
const itemRegex = new RegExp(`^( {0,3}${bull})((?:[\t ][^\\n]*)?(?:\\n|$))`);

// Check if current bullet point can start a new List Item
while (src) {
Expand Down
4 changes: 2 additions & 2 deletions src/rules.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@ export const block = {
newline: /^(?: *(?:\n|$))+/,
code: /^( {4}[^\n]+(?:\n(?: *(?:\n|$))*)?)+/,
fences: /^ {0,3}(`{3,}(?=[^`\n]*\n)|~{3,})([^\n]*)\n(?:|([\s\S]*?)\n)(?: {0,3}\1[~`]* *(?=\n|$)|$)/,
hr: /^ {0,3}((?:- *){3,}|(?:_ *){3,}|(?:\* *){3,})(?:\n+|$)/,
hr: /^ {0,3}((?:-[\t ]*){3,}|(?:_[ \t]*){3,}|(?:\*[ \t]*){3,})(?:\n+|$)/,
heading: /^ {0,3}(#{1,6})(?=\s|$)(.*)(?:\n+|$)/,
blockquote: /^( {0,3}> ?(paragraph|[^\n]*)(?:\n|$))+/,
list: /^( {0,3}bull)( [^\n]+?)?(?:\n|$)/,
list: /^( {0,3}bull)([ \t][^\n]+?)?(?:\n|$)/,
html: '^ {0,3}(?:' // optional indentation
+ '<(script|pre|style|textarea)[\\s>][\\s\\S]*?(?:</\\1>[^\\n]*\\n+|$)' // (1)
+ '|comment[^\\n]*(\\n+|$)' // (2)
Expand Down
2 changes: 1 addition & 1 deletion test/specs/new/html_comments.html
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ <h3 id="example-9">Example 9</h3>
<h3 id="example-10">Example 10</h3>

<!-- multi
line
line
comment
-->

Expand Down
1 change: 1 addition & 0 deletions test/specs/new/tab_after_blockquote.html
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
<blockquote><p>test</p></blockquote>
1 change: 1 addition & 0 deletions test/specs/new/tab_after_blockquote.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
> test

2 comments on commit 7d19665

@vercel
Copy link

@vercel vercel bot commented on 7d19665 Apr 11, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

markedjs – ./

markedjs-markedjs-legacy.vercel.app
markedjs-git-master-markedjs-legacy.vercel.app

@vercel
Copy link

@vercel vercel bot commented on 7d19665 Apr 11, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.