Raku: Fix unterminated heredoc fixes #547

- Fix heredoc - Move testdata to raku directory - Add testdata for unterminated heredoc
alecthomas · Sep 18, 2021 · 4d7154e · 4d7154e
1 parent 82795e1
commit 4d7154e
Show file tree

Hide file tree

Showing 5 changed files with 53 additions and 8 deletions.
diff --git a/lexers/r/raku.go b/lexers/r/raku.go
@@ -513,14 +513,24 @@ func rakuRules() Rules {
 
 			adverbre := regexp.MustCompile(`:to\b|:heredoc\b`)
 			var heredocTerminator []rune
+			var endHeredocPos int
 			if adverbre.MatchString(string(adverbs)) {
-				heredocTerminator = text[state.Pos:endPos]
-				if len(heredocTerminator) > 0 {
-					endHeredocPos := indexAt(text[endPos:], heredocTerminator, 0)
+				if endPos != len(text) {
+					heredocTerminator = text[state.Pos:endPos]
 					nChars = len(heredocTerminator)
-					endPos += endHeredocPos
 				} else {
-					endPos = len(text)
+					endPos = state.Pos + 1
+					heredocTerminator = []rune{}
+					nChars = 0
+				}
+
+				if nChars > 0 {
+					endHeredocPos = indexAt(text[endPos:], heredocTerminator, 0)
+					if endHeredocPos > -1 {
+						endPos += endHeredocPos
+					} else {
+						endPos = len(text)
+					}
 				}
 			}
 
@@ -532,18 +542,22 @@ func rakuRules() Rules {
 			case rakuQuote:
 				if len(heredocTerminator) > 0 {
 					// Length of heredoc terminator + closing chars + `;`
-					heredocFristPunctuationLen := len(heredocTerminator) + len(openingChars) + 1
+					heredocFristPunctuationLen := nChars + len(openingChars) + 1
 
 					state.NamedGroups[`opening_delimiters`] = string(openingChars) +
 						string(text[state.Pos:state.Pos+heredocFristPunctuationLen])
 
 					state.NamedGroups[`value`] =
 						string(text[state.Pos+heredocFristPunctuationLen : endPos])
 
-					state.NamedGroups[`closing_delimiters`] = string(heredocTerminator)
+					if endHeredocPos > -1 {
+						state.NamedGroups[`closing_delimiters`] = string(heredocTerminator)
+					}
 				} else {
 					state.NamedGroups[`value`] = textBetweenBrackets
-					state.NamedGroups[`closing_delimiters`] = string(closingChars)
+					if nChars > 0 {
+						state.NamedGroups[`closing_delimiters`] = string(closingChars)
+					}
 				}
 			default:
 				state.Groups = []string{state.Groups[0] + string(text[state.Pos:endPos+nChars])}

diff --git a/lexers/testdata/raku.actual → lexers/testdata/raku/raku.actual b/lexers/testdata/raku.actual → lexers/testdata/raku/raku.actual
diff --git a/lexers/testdata/raku.expected → lexers/testdata/raku/raku.expected b/lexers/testdata/raku.expected → lexers/testdata/raku/raku.expected
diff --git a/lexers/testdata/raku/unterminated_heredoc.actual b/lexers/testdata/raku/unterminated_heredoc.actual
@@ -0,0 +1,8 @@
+say q:to//;
+
+say 'something';
+
+# Unterminated heredoc
+say q:to/Foo/;
+
+say 'something';
diff --git a/lexers/testdata/raku/unterminated_heredoc.expected b/lexers/testdata/raku/unterminated_heredoc.expected
@@ -0,0 +1,23 @@
+[
+  {"type":"NameBuiltin","value":"say"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"q"},
+  {"type":"LiteralStringAffix","value":":to"},
+  {"type":"Punctuation","value":"//"},
+  {"type":"LiteralString","value":";"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"NameBuiltin","value":"say"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"'"},
+  {"type":"LiteralStringSingle","value":"something"},
+  {"type":"Punctuation","value":"';"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"CommentSingle","value":"# Unterminated heredoc"},
+  {"type":"Text","value":"\n"},
+  {"type":"NameBuiltin","value":"say"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"q"},
+  {"type":"LiteralStringAffix","value":":to"},
+  {"type":"Punctuation","value":"/Foo/;"},
+  {"type":"LiteralString","value":"\n\nsay 'something';\n"}
+]