Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Convert EOL to UNIX-style to render MD properly #8925

Merged
merged 19 commits into from
Nov 13, 2019
Merged
Show file tree
Hide file tree
Changes from 16 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion modules/markup/markdown/markdown.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,8 @@ func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte {
exts |= blackfriday.HardLineBreak
}

body = blackfriday.Run(body, blackfriday.WithRenderer(renderer), blackfriday.WithExtensions(exts))
// Need to normalize EOL to UNIX LF to have consistent results in rendering
body = blackfriday.Run(util.NormalizeEOL(body), blackfriday.WithRenderer(renderer), blackfriday.WithExtensions(exts))
return markup.SanitizeBytes(body)
}

Expand Down
22 changes: 22 additions & 0 deletions modules/markup/markdown/markdown_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -294,3 +294,25 @@ func TestTotal_RenderString(t *testing.T) {
assert.Equal(t, testCases[i+1], line)
}
}

func TestRender_RenderParagraphs(t *testing.T) {
test := func(t *testing.T, str string, cnt int) {
unix := []byte(str)
res := string(RenderRaw(unix, "", false))
assert.Equal(t, strings.Count(res, "<p"), cnt)

mac := []byte(strings.ReplaceAll(str, "\n", "\r"))
res = string(RenderRaw(mac, "", false))
assert.Equal(t, strings.Count(res, "<p"), cnt)

dos := []byte(strings.ReplaceAll(str, "\n", "\r\n"))
res = string(RenderRaw(dos, "", false))
assert.Equal(t, strings.Count(res, "<p"), cnt)
}

test(t, "\nOne\nTwo\nThree", 1)
test(t, "\n\nOne\nTwo\nThree", 1)
test(t, "\n\nOne\nTwo\nThree\n\n\n", 1)
test(t, "A\n\nB\nC\n", 2)
test(t, "A\n\n\nB\nC\n", 2)
}
37 changes: 37 additions & 0 deletions modules/util/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
package util

import (
"bytes"
"strings"
)

Expand Down Expand Up @@ -63,3 +64,39 @@ func Min(a, b int) int {
func IsEmptyString(s string) bool {
return len(strings.TrimSpace(s)) == 0
}

// NormalizeEOL will convert Windows (CRLF) and Mac (CR) EOLs to UNIX (LF)
func NormalizeEOL(input []byte) []byte {
var right, left, pos int
if right = bytes.IndexByte(input, '\r'); right == -1 {
return input
}
length := len(input)
guillep2k marked this conversation as resolved.
Show resolved Hide resolved
tmp := make([]byte, length)

// We know that left < length because otherwise right would be -1 from IndexByte.
copy(tmp[pos:pos+right], input[left:left+right])
pos += right
tmp[pos] = '\n'
left += right + 1
pos++

for left < length {
if input[left] == '\n' {
left++
}

right = bytes.IndexByte(input[left:], '\r')
zeripath marked this conversation as resolved.
Show resolved Hide resolved
if right == -1 {
copy(tmp[pos:], input[left:])
pos += length - left
break
}
copy(tmp[pos:pos+right], input[left:left+right])
pos += right
tmp[pos] = '\n'
left += right + 1
pos++
}
return tmp[:pos]
}
59 changes: 59 additions & 0 deletions modules/util/util_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
package util

import (
"strings"
"testing"

"code.gitea.io/gitea/modules/setting"
Expand Down Expand Up @@ -94,3 +95,61 @@ func TestIsEmptyString(t *testing.T) {
assert.Equal(t, v.expected, IsEmptyString(v.s))
}
}

func Test_NormalizeEOL(t *testing.T) {
data1 := []string{
"",
"This text starts with empty lines",
"another",
"",
"",
"",
"Some other empty lines in the middle",
"more.",
"And more.",
"Ends with empty lines too.",
"",
"",
"",
}

data2 := []string{
"This text does not start with empty lines",
"another",
"",
"",
"",
"Some other empty lines in the middle",
"more.",
"And more.",
"Ends without EOLtoo.",
}

buildEOLData := func(data []string, eol string) []byte {
return []byte(strings.Join(data, eol))
}

dos := buildEOLData(data1, "\r\n")
unix := buildEOLData(data1, "\n")
mac := buildEOLData(data1, "\r")

assert.Equal(t, unix, NormalizeEOL(dos))
assert.Equal(t, unix, NormalizeEOL(mac))
assert.Equal(t, unix, NormalizeEOL(unix))

dos = buildEOLData(data2, "\r\n")
unix = buildEOLData(data2, "\n")
mac = buildEOLData(data2, "\r")

assert.Equal(t, unix, NormalizeEOL(dos))
assert.Equal(t, unix, NormalizeEOL(mac))
assert.Equal(t, unix, NormalizeEOL(unix))

assert.Equal(t, []byte("one liner"), NormalizeEOL([]byte("one liner")))
assert.Equal(t, []byte("\n"), NormalizeEOL([]byte("\n")))
assert.Equal(t, []byte("\ntwo liner"), NormalizeEOL([]byte("\ntwo liner")))
assert.Equal(t, []byte("two liner\n"), NormalizeEOL([]byte("two liner\n")))
assert.Equal(t, []byte{}, NormalizeEOL([]byte{}))

assert.Equal(t, []byte("mix\nand\nmatch\n."), NormalizeEOL([]byte("mix\r\nand\rmatch\n.")))
}