Skip to content

Commit

Permalink
Fix parsing issues with Windows end of line characters (#9)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mattmook
mattmook authored Apr 16, 2019
1 parent 6bbb94d commit e27d22e
Show file tree
Hide file tree
Showing 10 changed files with 275 additions and 96 deletions.
28 changes: 28 additions & 0 deletions markdown/src/main/kotlin/com/appmattus/markdown/processing/DocumentExt.kt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
import com.vladsch.flexmark.util.Utils
import com.vladsch.flexmark.util.ast.Document
import com.vladsch.flexmark.util.sequence.BasedSequence

@Suppress("ReturnCount")
fun Document.getLineNumberFixed(offset: Int): Int {
val lineSegments = contentLines
if (lineSegments === BasedSequence.EMPTY_LIST) {
val preText = chars.baseSubSequence(0, Utils.maxLimit(offset, chars.length))
if (preText.isEmpty()) return 0
var lineNumber = 0
var nextLineEnd = preText.endOfLineAnyEOL(0)
val length = preText.length
while (nextLineEnd < length) {
lineNumber++
nextLineEnd = preText.endOfLineAnyEOL(nextLineEnd + preText.eolLength(nextLineEnd))
}
return lineNumber
} else {
val iMax = lineSegments.size
for (i in 0 until iMax) {
if (offset < lineSegments[i].endOffset) {
return i
}
}
return iMax
}
}
3 changes: 2 additions & 1 deletion markdown/src/main/kotlin/com/appmattus/markdown/processing/MarkdownDocument.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import com.vladsch.flexmark.util.ast.Node
import com.vladsch.flexmark.util.ast.NodeVisitor
import com.vladsch.flexmark.util.ast.VisitHandler
import com.vladsch.flexmark.util.sequence.BasedSequence
import getLineNumberFixed
import kotlin.reflect.KClass

class MarkdownDocument(val filename: String, val document: Document) {
Expand Down Expand Up @@ -71,7 +72,7 @@ class MarkdownDocument(val filename: String, val document: Document) {
val chars: BasedSequence by lazy { document.chars }
val lines by lazy { chars.splitIntoLines() }

fun getLineNumber(offset: Int) = document.getLineNumber(offset)
fun getLineNumber(offset: Int) = document.getLineNumberFixed(offset)

fun getColumnNumber(offset: Int) = chars.getColumnAtIndex(offset)

Expand Down
10 changes: 7 additions & 3 deletions markdown/src/main/kotlin/com/appmattus/markdown/rules/BlanksAroundHeadersRule.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package com.appmattus.markdown.rules
import com.appmattus.markdown.dsl.RuleSetup
import com.appmattus.markdown.errors.ErrorReporter
import com.appmattus.markdown.processing.MarkdownDocument
import com.appmattus.markdown.rules.extentions.endLineNumberFixed
import com.appmattus.markdown.rules.extentions.startLineNumberFixed
import com.vladsch.flexmark.ast.ListItem

/**
Expand Down Expand Up @@ -42,10 +44,12 @@ class BlanksAroundHeadersRule(
document.headings.filterNot { it.parent is ListItem }.forEach { heading ->

val prefixLineNotEmpty =
heading.startLineNumber > 0 && !document.lines[heading.startLineNumber - 1].matches(whitespaceRegex)
heading.startLineNumberFixed > 0 && !document.lines[heading.startLineNumberFixed - 1].matches(
whitespaceRegex
)

val suffixLineNotEmpty = heading.endLineNumber < document.lines.size - 2
&& !document.lines[heading.endLineNumber + 1].matches(whitespaceRegex)
val suffixLineNotEmpty = heading.endLineNumberFixed < document.lines.size - 2
&& !document.lines[heading.endLineNumberFixed + 1].matches(whitespaceRegex)

val description = when {
prefixLineNotEmpty && suffixLineNotEmpty ->
Expand Down
14 changes: 8 additions & 6 deletions markdown/src/main/kotlin/com/appmattus/markdown/rules/LineLengthRule.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ package com.appmattus.markdown.rules
import com.appmattus.markdown.dsl.RuleSetup
import com.appmattus.markdown.errors.ErrorReporter
import com.appmattus.markdown.processing.MarkdownDocument
import com.appmattus.markdown.rules.extentions.endLineNumberFixed
import com.appmattus.markdown.rules.extentions.splitIntoLines
import com.appmattus.markdown.rules.extentions.startLineNumberFixed
import com.vladsch.flexmark.ast.Emphasis
import com.vladsch.flexmark.ast.StrongEmphasis
import com.vladsch.flexmark.util.ast.Node
Expand Down Expand Up @@ -65,21 +67,21 @@ class LineLengthRule(
}

if (!headings) {
val codeBlocks = document.headings.map { IntRange(it.startLineNumber, it.endLineNumber) }
val codeBlocks = document.headings.map { IntRange(it.startLineNumberFixed, it.endLineNumberFixed) }
result = result.filter { range ->
(codeBlocks.find { it.contains(document.getLineNumber(range.start)) } == null)
}
}

if (!codeBlocks) {
val codeBlocks = document.codeBlocks.map { IntRange(it.startLineNumber, it.endLineNumber) }
val codeBlocks = document.codeBlocks.map { IntRange(it.startLineNumberFixed, it.endLineNumberFixed) }
result = result.filter { range ->
(codeBlocks.find { it.contains(document.getLineNumber(range.start)) } == null)
}
}

if (!tables) {
val tables = document.tables.map { IntRange(it.startLineNumber, it.endLineNumber) }
val tables = document.tables.map { IntRange(it.startLineNumberFixed, it.endLineNumberFixed) }
result = result.filter { range ->
(tables.find { it.contains(document.getLineNumber(range.start)) } == null)
}
Expand All @@ -101,13 +103,13 @@ class LineLengthRule(
item = item.parent
}

AllowedBlock(item.startOffset, item.endOffset, document.getLineNumber(item.startOffset))
AllowedBlock(item.startOffset, item.endOffset, getLineNumber(item.startOffset))
}
}

private fun MarkdownDocument.allowedImages(): List<AllowedBlock> {
return allImages.map {
AllowedBlock(it.startOffset, it.endOffset, document.getLineNumber(it.startOffset))
AllowedBlock(it.startOffset, it.endOffset, getLineNumber(it.startOffset))
}
}

Expand All @@ -128,7 +130,7 @@ class LineLengthRule(
it.endOffset
}

AllowedBlock(start, end, document.getLineNumber(it.startOffset))
AllowedBlock(start, end, getLineNumber(it.startOffset))
}
}
}
Expand Down
4 changes: 3 additions & 1 deletion markdown/src/main/kotlin/com/appmattus/markdown/rules/NoMultipleBlanksRule.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package com.appmattus.markdown.rules
import com.appmattus.markdown.dsl.RuleSetup
import com.appmattus.markdown.errors.ErrorReporter
import com.appmattus.markdown.processing.MarkdownDocument
import com.appmattus.markdown.rules.extentions.endLineNumberFixed
import com.appmattus.markdown.rules.extentions.startLineNumberFixed

/**
* # Multiple consecutive blank lines
Expand Down Expand Up @@ -33,7 +35,7 @@ class NoMultipleBlanksRule(
private val regex = Regex("^\\s*(\r?\n|\n)\\s*(\r?\n|\n|$)", RegexOption.MULTILINE)

override fun visitDocument(document: MarkdownDocument, errorReporter: ErrorReporter) {
val codeBlocks = document.codeBlocks.map { IntRange(it.startLineNumber, it.endLineNumber) }
val codeBlocks = document.codeBlocks.map { IntRange(it.startLineNumberFixed, it.endLineNumberFixed) }

regex.findAll(document.chars).map { match ->
IntRange(match.range.start, match.range.endInclusive)
Expand Down
52 changes: 7 additions & 45 deletions markdown/src/main/kotlin/com/appmattus/markdown/rules/extentions/BasedSequenceExt.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,55 +6,17 @@ import com.vladsch.flexmark.util.sequence.BasedSequence.WHITESPACE_CHARS
fun BasedSequence.canTrim() =
countLeading(WHITESPACE_CHARS, 0, length) + countTrailing(WHITESPACE_CHARS, 0, length) != 0

fun BasedSequence.splitIntoLines() = split(listOf("\n", "\r", "\r\n"), 0, 0, "")

@Suppress("ComplexMethod", "NestedBlockDepth")
private fun BasedSequence.split(
delimiter: List<String>,
limit: Int,
flags: Int,
trimChars: String?
): Array<BasedSequence> {

val trimCharsInternal = trimChars ?: WHITESPACE_CHARS
val limitInternal = if (limit < 1) Integer.MAX_VALUE else limit

val includeDelimiterParts = flags and BasedSequence.SPLIT_INCLUDE_DELIM_PARTS != 0
val includeDelimiter = if (!includeDelimiterParts && flags and BasedSequence.SPLIT_INCLUDE_DELIMS != 0) 1 else 0
val trimParts = flags and BasedSequence.SPLIT_TRIM_PARTS != 0
val skipEmpty = flags and BasedSequence.SPLIT_SKIP_EMPTY != 0
fun BasedSequence.splitIntoLines(): Array<BasedSequence> {
var lastPos = 0
val items = mutableListOf<BasedSequence>()

var lastPos = 0
val length = length
if (limitInternal > 1) {
while (lastPos < length) {
val pos = delimiter.map { indexOf(it, lastPos) }.filter { it >= 0 }.min() ?: break
while (lastPos < length) {
val pos = endOfLineAnyEOL(lastPos)
val eolLength = eolLength(pos)

if (lastPos < pos || !skipEmpty) {
var item = subSequence(lastPos, pos + includeDelimiter)
if (trimParts) item = item.trim(trimCharsInternal)
if (!item.isEmpty || !skipEmpty) {
items.add(item)
if (includeDelimiterParts) {
items.add(subSequence(pos, pos + 1))
}
if (items.size >= limitInternal - 1) {
lastPos = pos + 1
break
}
}
}
lastPos = pos + 1
}
items += subSequence(lastPos, pos)
lastPos = pos + eolLength
}

if (lastPos < length) {
var item = subSequence(lastPos, length)
if (trimParts) item = item.trim(trimCharsInternal)
if (!item.isEmpty || !skipEmpty) {
items.add(item)
}
}
return items.toTypedArray()
}
12 changes: 12 additions & 0 deletions markdown/src/main/kotlin/com/appmattus/markdown/rules/extentions/NodeExt.kt
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,17 @@
package com.appmattus.markdown.rules.extentions

import com.vladsch.flexmark.util.ast.Node
import getLineNumberFixed

fun Node.indent(): Int = document.chars.getColumnAtIndex(startOffset)

val Node.startLineNumberFixed: Int
get() {
return document!!.getLineNumberFixed(chars.startOffset)
}

val Node.endLineNumberFixed: Int
get() {
val endOffset = chars.endOffset
return document!!.getLineNumberFixed(if (endOffset > 0) endOffset - 1 else endOffset)
}
16 changes: 11 additions & 5 deletions markdown/src/test/kotlin/com/appmattus/markdown/MarkdownDocumentExt.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,24 @@ package com.appmattus.markdown
import com.appmattus.markdown.processing.MarkdownDocument
import com.appmattus.markdown.processing.ParserFactory

fun createDocument(documentText: String) =
private val eolRegex = "(\\r?\\n|\\n)".toRegex()

fun loadDocumentUnixEol(filename: String) =
MarkdownDocument(
"test.md",
ParserFactory.parser.parse(documentText)
filename,
ParserFactory.parser.parse(
MarkdownDocument::class.java.classLoader.getResource(
filename
).readText(Charsets.UTF_8).replace(eolRegex, "\n")
)
)

fun loadDocument(filename: String) =
fun loadDocumentWindowsEol(filename: String) =
MarkdownDocument(
filename,
ParserFactory.parser.parse(
MarkdownDocument::class.java.classLoader.getResource(
filename
).readText(Charsets.UTF_8)
).readText(Charsets.UTF_8).replace(eolRegex, "\r\n")
)
)
83 changes: 48 additions & 35 deletions markdown/src/test/kotlin/com/appmattus/markdown/rules/FeatureBodyExt.kt
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
package com.appmattus.markdown.rules

import com.appmattus.markdown.errors.Error
import com.appmattus.markdown.loadDocument
import com.appmattus.markdown.loadDocumentUnixEol
import com.appmattus.markdown.loadDocumentWindowsEol
import com.appmattus.markdown.processing.MarkdownDocument
import mockDocument
import org.assertj.core.api.Assertions.assertThat
Expand All @@ -18,50 +19,62 @@ fun FeatureBody.FileRuleScenario(
files.toMutableList().apply {
removeAll(exclude)
}.forEach { filename ->
val document = loadDocument(filename)
val document = loadDocumentUnixEol(filename)
val documentWindows = loadDocumentWindowsEol(filename)

val expectedErrorCount =
Regex("\\{${Regex.escape(rule::class.java.simpleName)}(:[0-9]+)?}").findAll(document.chars).count()

Scenario(filename) {
lateinit var ruleErrors: List<Error>
documentScenario("$filename (unix eol)", rule, document, expectedErrorCount)
documentScenario("$filename (windows eol)", rule, documentWindows, expectedErrorCount)
}
}

Given("the file $filename") {}
private fun FeatureBody.documentScenario(
scenarioName: String,
rule: Rule,
document: MarkdownDocument,
expectedErrorCount: Int
) {
Scenario(scenarioName) {
lateinit var ruleErrors: List<Error>

When("we run ${rule.javaClass.simpleName}") {
ruleErrors = rule.processDocument(document)
}
Given("the file $scenarioName") {}

Then("we expect $expectedErrorCount errors") {
assertThat(ruleErrors.size).describedAs("Number of errors")
.withFailMessage(ruleErrors.joinToString("\n")).isEqualTo(expectedErrorCount)
}
When("we run ${rule.javaClass.simpleName}") {
ruleErrors = rule.processDocument(document)
}

Then("we expect $expectedErrorCount errors") {
assertThat(ruleErrors.size).describedAs("Number of errors")
.withFailMessage(ruleErrors.joinToString("\n")).isEqualTo(expectedErrorCount)
}

if (expectedErrorCount > 0) {
And("the location of errors match") {
val failures = mutableListOf<String>()
ruleErrors.forEach { error ->
val errorRange =
IntRange(document.getLineNumber(error.startOffset), document.getLineNumber(error.endOffset))

val hasError = errorRange.any { line ->
document.lines[line].contains("{${rule::class.java.simpleName}}") ||
document.chars.contains("{${rule::class.java.simpleName}:${line + 1}}")
}

if (!hasError) {
println("Unexpected: $error")
failures.add(
document.chars.midSequence(
error.startOffset,
error.endOffset
).toString()
)
}
if (expectedErrorCount > 0) {
And("the location of errors match") {
val failures = mutableListOf<String>()
ruleErrors.forEach { error ->
val errorRange =
IntRange(document.getLineNumber(error.startOffset), document.getLineNumber(error.endOffset))

val hasError = errorRange.any { line ->
document.lines[line].contains("{${rule::class.java.simpleName}}") ||
document.chars.contains("{${rule::class.java.simpleName}:${line + 1}}")
}
if (failures.isNotEmpty()) {
fail(failures.joinToString())

if (!hasError) {
println("Unexpected: $error")
failures.add(
document.chars.midSequence(
error.startOffset,
error.endOffset
).toString()
)
}
}
if (failures.isNotEmpty()) {
fail(failures.joinToString())
}
}
}
}
Expand Down
Loading

0 comments on commit e27d22e

Please sign in to comment.