From 8c9e63f990a8daadbd2d942bffcd34f051f601d7 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Mon, 11 Nov 2019 16:34:31 -0300 Subject: [PATCH 01/12] Convert EOL to UNIX-style to render MD properly --- modules/markup/markdown/markdown.go | 4 ++- modules/util/util.go | 33 +++++++++++++++++++ modules/util/util_test.go | 51 +++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 1 deletion(-) diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index ff78d7ea3a8e4..e68b4ccae1b13 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -157,7 +157,9 @@ func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte { exts |= blackfriday.HardLineBreak } - body = blackfriday.Run(body, blackfriday.WithRenderer(renderer), blackfriday.WithExtensions(exts)) + // Need to normalize EOL to UNIX LF to have consistent results in rendering + // @zeripath please remove this line to co-author this PR ;) + body = blackfriday.Run(util.NormalizeEOL(body), blackfriday.WithRenderer(renderer), blackfriday.WithExtensions(exts)) return markup.SanitizeBytes(body) } diff --git a/modules/util/util.go b/modules/util/util.go index 4203b5eb51732..a5d29516bf9d5 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -5,6 +5,7 @@ package util import ( + "bytes" "strings" ) @@ -63,3 +64,35 @@ func Min(a, b int) int { func IsEmptyString(s string) bool { return len(strings.TrimSpace(s)) == 0 } + +// NormalizeEOL will convert Windows (CRLF) and Mac (CR) EOLs to UNIX (LF) +func NormalizeEOL(input []byte) []byte { + tmp := make([]byte, len(input)) + left := 0 + pos := 0 + for input[left] == '\n' { + left++ + } + if left != 0 { + copy(tmp[pos:pos+left], input[0:left]) + pos = left + } + + for left < len(input) { + if input[left] == '\n' { + left++ + } + right := bytes.Index(input[left:], []byte{'\r'}) + if right == -1 { + copy(tmp[pos:], input[left:]) + pos = pos + len(input) - left + break + } + copy(tmp[pos:pos+right], input[left:left+right]) + pos = pos + right + tmp[pos] = '\n' + left = left + right + 1 + pos++ + } + return tmp[:pos] +} diff --git a/modules/util/util_test.go b/modules/util/util_test.go index 2475065059e88..9955c68b86442 100644 --- a/modules/util/util_test.go +++ b/modules/util/util_test.go @@ -5,6 +5,7 @@ package util import ( + "strings" "testing" "code.gitea.io/gitea/modules/setting" @@ -94,3 +95,53 @@ func TestIsEmptyString(t *testing.T) { assert.Equal(t, v.expected, IsEmptyString(v.s)) } } + +func Test_NormalizeEOL(t *testing.T) { + data1 := []string{ + "", + "This text starts with empty lines", + "another", + "", + "", + "", + "Some other empty lines in the middle", + "more.", + "And more.", + "Ends with empty lines too.", + "", + "", + "", + } + + data2 := []string{ + "This text does not start with empty lines", + "another", + "", + "", + "", + "Some other empty lines in the middle", + "more.", + "And more.", + "Ends without EOLtoo.", + } + + buildEOLData := func(data []string, eol string) []byte { + return []byte(strings.Join(data, eol)) + } + + dos := buildEOLData(data1, "\r\n") + unix := buildEOLData(data1, "\n") + mac := buildEOLData(data1, "\r") + + assert.Equal(t, unix, NormalizeEOL(dos)) + assert.Equal(t, unix, NormalizeEOL(mac)) + assert.Equal(t, unix, NormalizeEOL(unix)) + + dos = buildEOLData(data2, "\r\n") + unix = buildEOLData(data2, "\n") + mac = buildEOLData(data2, "\r") + + assert.Equal(t, unix, NormalizeEOL(dos)) + assert.Equal(t, unix, NormalizeEOL(mac)) + assert.Equal(t, unix, NormalizeEOL(unix)) +} From 612bda644009ceafa7224a2b11e3565946d56d11 Mon Sep 17 00:00:00 2001 From: guillep2k <18600385+guillep2k@users.noreply.github.com> Date: Mon, 11 Nov 2019 16:38:41 -0300 Subject: [PATCH 02/12] Update modules/markup/markdown/markdown.go Co-Authored-By: zeripath --- modules/markup/markdown/markdown.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index e68b4ccae1b13..fc704243e2d7f 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -158,7 +158,6 @@ func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte { } // Need to normalize EOL to UNIX LF to have consistent results in rendering - // @zeripath please remove this line to co-author this PR ;) body = blackfriday.Run(util.NormalizeEOL(body), blackfriday.WithRenderer(renderer), blackfriday.WithExtensions(exts)) return markup.SanitizeBytes(body) } From 067980829bafe1f2c2b6a270cfe0ab1d518ec873 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Mon, 11 Nov 2019 16:42:41 -0300 Subject: [PATCH 03/12] Fix lint optimization --- modules/util/util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/util/util.go b/modules/util/util.go index a5d29516bf9d5..b91a3750deb0e 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -89,9 +89,9 @@ func NormalizeEOL(input []byte) []byte { break } copy(tmp[pos:pos+right], input[left:left+right]) - pos = pos + right + pos += right tmp[pos] = '\n' - left = left + right + 1 + left += right + 1 pos++ } return tmp[:pos] From f3e371ea00c5133ccfc979db76aea02ffe9c9e31 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Mon, 11 Nov 2019 16:50:58 -0300 Subject: [PATCH 04/12] Check for empty content before conversion --- modules/util/util.go | 3 +++ modules/util/util_test.go | 2 ++ 2 files changed, 5 insertions(+) diff --git a/modules/util/util.go b/modules/util/util.go index b91a3750deb0e..e664adfe35ead 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -67,6 +67,9 @@ func IsEmptyString(s string) bool { // NormalizeEOL will convert Windows (CRLF) and Mac (CR) EOLs to UNIX (LF) func NormalizeEOL(input []byte) []byte { + if len(input) == 0 { + return input + } tmp := make([]byte, len(input)) left := 0 pos := 0 diff --git a/modules/util/util_test.go b/modules/util/util_test.go index 9955c68b86442..291b3edc3150f 100644 --- a/modules/util/util_test.go +++ b/modules/util/util_test.go @@ -144,4 +144,6 @@ func Test_NormalizeEOL(t *testing.T) { assert.Equal(t, unix, NormalizeEOL(dos)) assert.Equal(t, unix, NormalizeEOL(mac)) assert.Equal(t, unix, NormalizeEOL(unix)) + + assert.Equal(t, []byte{}, NormalizeEOL([]byte{})) } From 59cabbc4c99267534447f56cfd7c6e3eb144f96e Mon Sep 17 00:00:00 2001 From: guillep2k <18600385+guillep2k@users.noreply.github.com> Date: Mon, 11 Nov 2019 16:52:05 -0300 Subject: [PATCH 05/12] Update modules/util/util.go Co-Authored-By: zeripath --- modules/util/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/util/util.go b/modules/util/util.go index e664adfe35ead..013cefea3ae03 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -85,7 +85,7 @@ func NormalizeEOL(input []byte) []byte { if input[left] == '\n' { left++ } - right := bytes.Index(input[left:], []byte{'\r'}) + right := bytes.IndexByte(input[left:], '\r') if right == -1 { copy(tmp[pos:], input[left:]) pos = pos + len(input) - left From b6e373a8d3baba0e5cea7319097c148a5068dd8d Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Mon, 11 Nov 2019 16:56:31 -0300 Subject: [PATCH 06/12] Improved checks and tests --- modules/util/util.go | 5 +---- modules/util/util_test.go | 4 ++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/util/util.go b/modules/util/util.go index 013cefea3ae03..68b3fe9664c75 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -67,13 +67,10 @@ func IsEmptyString(s string) bool { // NormalizeEOL will convert Windows (CRLF) and Mac (CR) EOLs to UNIX (LF) func NormalizeEOL(input []byte) []byte { - if len(input) == 0 { - return input - } tmp := make([]byte, len(input)) left := 0 pos := 0 - for input[left] == '\n' { + for left < len(input) && input[left] == '\n' { left++ } if left != 0 { diff --git a/modules/util/util_test.go b/modules/util/util_test.go index 291b3edc3150f..c2e0d43804e0c 100644 --- a/modules/util/util_test.go +++ b/modules/util/util_test.go @@ -145,5 +145,9 @@ func Test_NormalizeEOL(t *testing.T) { assert.Equal(t, unix, NormalizeEOL(mac)) assert.Equal(t, unix, NormalizeEOL(unix)) + assert.Equal(t, []byte("one liner"), NormalizeEOL([]byte("one liner"))) + assert.Equal(t, []byte("\n"), NormalizeEOL([]byte("\n"))) + assert.Equal(t, []byte("\ntwo liner"), NormalizeEOL([]byte("\ntwo liner"))) + assert.Equal(t, []byte("two liner\n"), NormalizeEOL([]byte("two liner\n"))) assert.Equal(t, []byte{}, NormalizeEOL([]byte{})) } From 60dcce36014b0be16a23bd72e3db26fbf72a772f Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Mon, 11 Nov 2019 19:17:41 -0300 Subject: [PATCH 07/12] Add paragraph render test --- modules/markup/markdown/markdown_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go index b29f870ce5268..9213dfc1389a5 100644 --- a/modules/markup/markdown/markdown_test.go +++ b/modules/markup/markdown/markdown_test.go @@ -294,3 +294,25 @@ func TestTotal_RenderString(t *testing.T) { assert.Equal(t, testCases[i+1], line) } } + +func TestRender_RenderParagraphs(t *testing.T) { + test := func(t *testing.T, str string, cnt int) { + unix := []byte(str) + res := string(RenderRaw(unix, "", false)) + assert.Equal(t, strings.Count(res, " Date: Mon, 11 Nov 2019 19:28:12 -0300 Subject: [PATCH 08/12] Improve speed even more, improve tests --- modules/markup/markdown/markdown_test.go | 4 ++-- modules/util/util.go | 29 +++++++++++++++++++----- modules/util/util_test.go | 2 ++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go index 9213dfc1389a5..aab2943bb4e87 100644 --- a/modules/markup/markdown/markdown_test.go +++ b/modules/markup/markdown/markdown_test.go @@ -301,11 +301,11 @@ func TestRender_RenderParagraphs(t *testing.T) { res := string(RenderRaw(unix, "", false)) assert.Equal(t, strings.Count(res, " Date: Mon, 11 Nov 2019 22:26:30 -0300 Subject: [PATCH 09/12] Small improvement by @gary-kim --- modules/util/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/util/util.go b/modules/util/util.go index a21a9f09a8c2a..9f43ac53556dc 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -71,8 +71,8 @@ func NormalizeEOL(input []byte) []byte { if right = bytes.IndexByte(input, '\r'); right == -1 { return input } - tmp := make([]byte, len(input)) length := len(input) + tmp := make([]byte, length) for left < length && input[left] == '\n' { left++ } From 22aa9fdf2264358a673b6f2c76b0ef182962e676 Mon Sep 17 00:00:00 2001 From: guillep2k <18600385+guillep2k@users.noreply.github.com> Date: Mon, 11 Nov 2019 23:28:13 -0300 Subject: [PATCH 10/12] Fix test for DOS --- modules/markup/markdown/markdown_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go index aab2943bb4e87..e80173c6cf322 100644 --- a/modules/markup/markdown/markdown_test.go +++ b/modules/markup/markdown/markdown_test.go @@ -305,7 +305,7 @@ func TestRender_RenderParagraphs(t *testing.T) { res = string(RenderRaw(mac, "", false)) assert.Equal(t, strings.Count(res, " Date: Tue, 12 Nov 2019 09:05:06 -0300 Subject: [PATCH 11/12] More improvements --- modules/util/util.go | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/modules/util/util.go b/modules/util/util.go index 9f43ac53556dc..6d02b5f52fa73 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -73,26 +73,13 @@ func NormalizeEOL(input []byte) []byte { } length := len(input) tmp := make([]byte, length) - for left < length && input[left] == '\n' { - left++ - } - if left != 0 { - copy(tmp[pos:pos+left], input[0:left]) - pos = left - right -= left - } - if left < length { - if input[left] == '\n' { - left++ - right-- - } - copy(tmp[pos:pos+right], input[left:left+right]) - pos += right - tmp[pos] = '\n' - left += right + 1 - pos++ - } + // We know that left < length because otherwise right would be -1 from IndexByte. + copy(tmp[pos:pos+right], input[left:left+right]) + pos += right + tmp[pos] = '\n' + left += right + 1 + pos++ for left < length { if input[left] == '\n' { From dc8036dcc680abab52b342d18181a5ee42f40318 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Tue, 12 Nov 2019 10:12:58 -0300 Subject: [PATCH 12/12] Restart CI