From c527031ef8d9112b12d768f308e768af18fa12f0 Mon Sep 17 00:00:00 2001 From: Liam Miller-Cushon Date: Wed, 23 Mar 2022 14:08:47 -0700 Subject: [PATCH] Initial support for text blocks PiperOrigin-RevId: 436825379 --- .../com/google/turbine/parse/StreamLexer.java | 162 ++++++++++++++++++ .../turbine/lower/LowerIntegrationTest.java | 7 +- .../turbine/lower/testdata/textblock.test | 30 ++++ .../com/google/turbine/parse/LexerTest.java | 27 +++ .../google/turbine/parse/ParseErrorTest.java | 13 ++ 5 files changed, 238 insertions(+), 1 deletion(-) create mode 100644 javatests/com/google/turbine/lower/testdata/textblock.test diff --git a/java/com/google/turbine/parse/StreamLexer.java b/java/com/google/turbine/parse/StreamLexer.java index 2348385f..3d46b907 100644 --- a/java/com/google/turbine/parse/StreamLexer.java +++ b/java/com/google/turbine/parse/StreamLexer.java @@ -17,8 +17,11 @@ package com.google.turbine.parse; import static com.google.common.base.Verify.verify; +import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.turbine.parse.UnicodeEscapePreprocessor.ASCII_SUB; +import static java.lang.Math.min; +import com.google.common.collect.ImmutableList; import com.google.turbine.diag.SourceFile; import com.google.turbine.diag.TurbineError; import com.google.turbine.diag.TurbineError.ErrorKind; @@ -399,6 +402,15 @@ public Token next() { case '"': { eat(); + if (ch == '"') { + eat(); + if (ch != '"') { + saveValue(""); + return Token.STRING_LITERAL; + } + eat(); + return textBlock(); + } readFrom(); StringBuilder sb = new StringBuilder(); STRING: @@ -436,6 +448,156 @@ public Token next() { } } + private Token textBlock() { + OUTER: + while (true) { + switch (ch) { + case ' ': + case '\r': + case '\t': + eat(); + break; + default: + break OUTER; + } + } + switch (ch) { + case '\r': + eat(); + if (ch == '\n') { + eat(); + } + break; + case '\n': + eat(); + break; + default: + throw inputError(); + } + readFrom(); + StringBuilder sb = new StringBuilder(); + while (true) { + switch (ch) { + case '"': + eat(); + if (ch != '"') { + sb.append("\""); + continue; + } + eat(); + if (ch != '"') { + sb.append("\"\""); + continue; + } + eat(); + String value = sb.toString(); + value = stripIndent(value); + value = translateEscapes(value); + saveValue(value); + return Token.STRING_LITERAL; + case ASCII_SUB: + if (reader.done()) { + return Token.EOF; + } + // falls through + default: + sb.appendCodePoint(ch); + eat(); + continue; + } + } + } + + static String stripIndent(String value) { + if (value.isEmpty()) { + return value; + } + ImmutableList lines = value.lines().collect(toImmutableList()); + // the amount of whitespace to strip from the beginning of every line + int strip = Integer.MAX_VALUE; + char last = value.charAt(value.length() - 1); + boolean trailingNewline = last == '\n' || last == '\r'; + if (trailingNewline) { + // If the input contains a trailing newline, we have something like: + // + // |String s = """ + // | foo + // |"""; + // + // Because the final """ is unindented, nothing should be stripped. + strip = 0; + } else { + // find the longest common prefix of whitespace across all non-blank lines + for (int i = 0; i < lines.size(); i++) { + String line = lines.get(i); + int nonWhitespaceStart = nonWhitespaceStart(line); + if (nonWhitespaceStart == line.length()) { + continue; + } + strip = min(strip, nonWhitespaceStart); + } + } + StringBuilder result = new StringBuilder(); + boolean first = true; + for (String line : lines) { + if (!first) { + result.append('\n'); + } + int end = trailingWhitespaceStart(line); + if (strip <= end) { + result.append(line, strip, end); + } + first = false; + } + if (trailingNewline) { + result.append('\n'); + } + return result.toString(); + } + + private static int nonWhitespaceStart(String value) { + int i = 0; + while (i < value.length() && Character.isWhitespace(value.charAt(i))) { + i++; + } + return i; + } + + private static int trailingWhitespaceStart(String value) { + int i = value.length() - 1; + while (i >= 0 && Character.isWhitespace(value.charAt(i))) { + i--; + } + return i + 1; + } + + private static String translateEscapes(String value) { + StreamLexer lexer = + new StreamLexer(new UnicodeEscapePreprocessor(new SourceFile(null, value + ASCII_SUB))); + return lexer.translateEscapes(); + } + + private String translateEscapes() { + readFrom(); + StringBuilder sb = new StringBuilder(); + OUTER: + while (true) { + switch (ch) { + case '\\': + eat(); + sb.append(escape()); + continue; + case ASCII_SUB: + break OUTER; + default: + sb.appendCodePoint(ch); + eat(); + continue; + } + } + return sb.toString(); + } + private char escape() { boolean zeroToThree = false; switch (ch) { diff --git a/javatests/com/google/turbine/lower/LowerIntegrationTest.java b/javatests/com/google/turbine/lower/LowerIntegrationTest.java index 97170caa..7ae9b1ba 100644 --- a/javatests/com/google/turbine/lower/LowerIntegrationTest.java +++ b/javatests/com/google/turbine/lower/LowerIntegrationTest.java @@ -44,7 +44,11 @@ public class LowerIntegrationTest { private static final ImmutableMap SOURCE_VERSION = - ImmutableMap.of("record.test", 16, "record2.test", 16, "sealed.test", 17); + ImmutableMap.of( + "record.test", 16, // + "record2.test", 16, + "sealed.test", 17, + "textblock.test", 15); @Parameters(name = "{index}: {0}") public static Iterable parameters() { @@ -285,6 +289,7 @@ public static Iterable parameters() { "superabstract.test", "supplierfunction.test", "tbound.test", + "textblock.test", "tyanno_inner.test", "tyanno_varargs.test", "typaram.test", diff --git a/javatests/com/google/turbine/lower/testdata/textblock.test b/javatests/com/google/turbine/lower/testdata/textblock.test new file mode 100644 index 00000000..96832960 --- /dev/null +++ b/javatests/com/google/turbine/lower/testdata/textblock.test @@ -0,0 +1,30 @@ +=== TextBlock.java === +class TextBlock { + public static final String hello = """ + hello + world + """; + public static final String escape = """ + hello\nworld\" + \r\t\b + \0123 + \' + \\ + \" + """; + public static final String quotes = """ + " "" ""\" """; + public static final String newline = """ + hello + world"""; + public static final String blank = """ + hello + + + world + """; + public static final String allBlank = """ + + + """; +} diff --git a/javatests/com/google/turbine/parse/LexerTest.java b/javatests/com/google/turbine/parse/LexerTest.java index c3d78048..bf0b3748 100644 --- a/javatests/com/google/turbine/parse/LexerTest.java +++ b/javatests/com/google/turbine/parse/LexerTest.java @@ -17,11 +17,15 @@ package com.google.turbine.parse; import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assume.assumeTrue; import com.google.common.escape.SourceCodeEscapers; +import com.google.common.truth.Expect; import com.google.turbine.diag.SourceFile; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.List; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -29,6 +33,8 @@ @RunWith(JUnit4.class) public class LexerTest { + @Rule public final Expect expect = Expect.create(); + @Test public void testSimple() { assertThat(lex("\nasd dsa\n")).containsExactly("IDENT(asd)", "IDENT(dsa)", "EOF"); @@ -367,4 +373,25 @@ public static List lex(String input) { } while (token != Token.EOF); return tokens; } + + @Test + public void stripIndent() throws Exception { + assumeTrue(Runtime.version().feature() >= 13); + String[] inputs = { + "", + "hello", + "hello\n", + "\nhello", + "\n hello\n world", + "\n hello\n world\n ", + "\n hello\n world\n", + "\n hello\n world\n ", + "\n hello\nworld", + "\n hello\n \nworld\n ", + }; + Method stripIndent = String.class.getMethod("stripIndent"); + for (String input : inputs) { + expect.that(StreamLexer.stripIndent(input)).isEqualTo(stripIndent.invoke(input)); + } + } } diff --git a/javatests/com/google/turbine/parse/ParseErrorTest.java b/javatests/com/google/turbine/parse/ParseErrorTest.java index 2c48b817..0187ce0c 100644 --- a/javatests/com/google/turbine/parse/ParseErrorTest.java +++ b/javatests/com/google/turbine/parse/ParseErrorTest.java @@ -307,6 +307,19 @@ public void notCast() { " ^")); } + @Test + public void singleLineTextBlockRejected() { + String input = "class T { String s = \"\"\" \"\"\"; }"; + TurbineError e = assertThrows(TurbineError.class, () -> Parser.parse(input)); + assertThat(e) + .hasMessageThat() + .isEqualTo( + lines( + "<>:1: error: unexpected input: \"", + "class T { String s = \"\"\" \"\"\"; }", + " ^")); + } + private static String lines(String... lines) { return Joiner.on(System.lineSeparator()).join(lines); }