From 5511097d14086454a986d2d6508c552b1d19fc24 Mon Sep 17 00:00:00 2001 From: Steve Baker Date: Sat, 29 Jan 2022 16:09:07 +0100 Subject: [PATCH] Make rule of 2-spaces to preserve a line break universal It's now in the wrapping code and taken out of each parser. This does though mean that things won't be treated as separate paragraphs from the point-of-view of what to wrap with an empty selection. --- core/Parsers_Markdown.fs | 2 +- core/Parsing.DocComments.fs | 4 +--- core/Parsing.Documents.fs | 1 - core/Parsing.Latex.fs | 3 +-- core/Parsing.Sgml.fs | 2 +- core/Wrapping.fs | 16 +++++++++++----- docs/specs/features/spaces.md | 8 +++++++- 7 files changed, 22 insertions(+), 14 deletions(-) diff --git a/core/Parsers_Markdown.fs b/core/Parsers_Markdown.fs index 156c55a..4a77ef1 100644 --- a/core/Parsers_Markdown.fs +++ b/core/Parsers_Markdown.fs @@ -26,7 +26,7 @@ let private mdMarker marker = regex (@"^ {0,3}" + marker) /// The default paragraph type (fallback for if no other parser matches) let private defaultPara (_ctx: Context) : FirstLineParser = - let lineBreakEnd = regex @"(\\|\s{2}|
)$" + let lineBreakEnd = regex @"(\\|
)$" let rec parseLine (line: Line) : FirstLineRes = // We assume a >3 space indent on the first line would have already been // picked up by another parser, so it's ok to just trim all indent diff --git a/core/Parsing.DocComments.fs b/core/Parsing.DocComments.fs index 5f32769..0aed8d2 100644 --- a/core/Parsing.DocComments.fs +++ b/core/Parsing.DocComments.fs @@ -105,9 +105,7 @@ let ddoc = let godoc settings = let indentedLines = ignoreParser (Nonempty.span (fun line -> line.[0] = ' ' || line.[0] = '\t')) - let textLines = - splitIntoChunks (afterRegex (Regex(" $"))) >> map (Wrap << Wrappable.fromLines ("", "")) - + let textLines = Nonempty.singleton << Wrap << Wrappable.fromLines ("", "") textLines |> takeUntil (tryMany [blankLines; indentedLines]) |> repeatToEnd diff --git a/core/Parsing.Documents.fs b/core/Parsing.Documents.fs index dae7155..3023857 100644 --- a/core/Parsing.Documents.fs +++ b/core/Parsing.Documents.fs @@ -17,7 +17,6 @@ open Parsing_SourceCode let oldPlainText settings = let paragraphs = splitIntoChunks (onIndent settings.tabWidth) - >> Nonempty.concatMap (splitIntoChunks (afterRegex (Regex(" $")))) >> map (indentSeparatedParagraphBlock textBlock) takeUntil blankLines paragraphs |> repeatToEnd diff --git a/core/Parsing.Latex.fs b/core/Parsing.Latex.fs index 889f33f..f29805e 100644 --- a/core/Parsing.Latex.fs +++ b/core/Parsing.Latex.fs @@ -14,8 +14,7 @@ let private markdown = Parsing.Markdown.markdown let private newlineRegex: Regex = Regex ( @"(\\(\\\*?|hline|newline|break|linebreak)(\[.*?\])?(\{.*?\})?\s*$)" - + @"| $|" - + @"([^\\]%)" + + @"|([^\\]%)" ) /// Commands that, when starting a line, should always preserve the line break diff --git a/core/Parsing.Sgml.fs b/core/Parsing.Sgml.fs index bae709b..9b3b17a 100644 --- a/core/Parsing.Sgml.fs +++ b/core/Parsing.Sgml.fs @@ -69,7 +69,7 @@ let sgml let breakBefore line = justBlockEndTag line || beginsWithBlockStartTag line let breakAfter line = - Line.contains (regex @"([""\s]>\s*| )$") line || endsWithBlockTag line + Line.contains (regex @"([""\s]>\s*)$") line || endsWithBlockTag line let paragraphBlocks = splitIntoChunks (splitBefore breakBefore) diff --git a/core/Wrapping.fs b/core/Wrapping.fs index 2beb4ed..a507526 100644 --- a/core/Wrapping.fs +++ b/core/Wrapping.fs @@ -37,7 +37,7 @@ let canBreakBetweenChars c1 c2 = /// When concatenating lines before breaking up again, whether to add a space /// between chars -let addSpaceBetweenChars c1 c2 = if isCJ c1 || isCJ c2 then false else true +let addSpaceBetweenChars c1 c2 = if c1 = 10us || isCJ c1 || isCJ c2 then false else true /// Concatenates lines into a single string, adding the right amount of @@ -47,10 +47,9 @@ let concatLines doubleSentenceSpacing lines = let addLine (acc: string) (line: string) = // Shouldn't be getting empty strings in this function but just in case... if line = String.Empty || acc = String.Empty then acc else - let acc = acc.TrimEnd() + let acc = if acc.EndsWith(" ") then acc + "\n" else acc.TrimEnd() let accEnd = uint16 acc.[acc.Length-1] - let space = - if doubleSentenceSpacing && Array.contains accEnd stops then " " else " " + let space = if doubleSentenceSpacing && Array.contains accEnd stops then " " else " " if addSpaceBetweenChars accEnd (uint16 line.[0]) then acc + space + line else acc + line List.reduce addLine (Nonempty.toList lines) @@ -76,7 +75,9 @@ let breakUpString addLine tabWidth maxWidth (str: string) = let outputLine prefixes pStart pEnd = let prefix, nextPrefixes = popOrPeek prefixes let content = - if pEnd > pStart then str.Substring(pStart, pEnd - pStart).Trim() + if pEnd > pStart then + if str[pEnd-1] = '\n' then str.Substring(pStart, pEnd - pStart - 1) + else str.Substring(pStart, pEnd - pStart).TrimEnd() else str.Substring(pStart) addLine (prefix + content.Replace('\000', ' ')) nextPrefixes @@ -87,6 +88,11 @@ let breakUpString addLine tabWidth maxWidth (str: string) = if pStr >= str.Length then outputLine prefixes lineStart 0 |> ignore else let charCode = uint16 str.[pStr] + // If we come across a LF char then start a new line + if charCode = 10us then + let nextPrefixes = outputLine prefixes lineStart (pStr+1) + loop nextPrefixes (pStr+1) (prefixWidth nextPrefixes) (pStr+1) + else let newWidth = curWidth + Line.charWidth tabWidth curWidth charCode // If current char is whitespace we don't need to wrap yet. Wait until we come across // a non-whitespace char diff --git a/docs/specs/features/spaces.md b/docs/specs/features/spaces.md index 07de3c8..e2431ea 100644 --- a/docs/specs/features/spaces.md +++ b/docs/specs/features/spaces.md @@ -23,7 +23,7 @@ wrapping column. ## At the end of a line Two spaces at the end of a line preserves the line break after it. This comes -from Markdown should work for any content. +from Markdown and should work for any content. > language: "plaintext" @@ -64,3 +64,9 @@ from Markdown should work for any content. Baz foo ¦ Baz foo bar ¦ bar baz. ¦ baz. ¦ #> ¦ #> ¦ + +> language: "rst" + + Foo bar.·· ¦ -> Foo bar.·· ¦ + Baz foo ¦ Baz foo bar ¦ + bar baz. ¦ baz. ¦