Initial support for text blocks

PiperOrigin-RevId: 436825379

java/com/google/turbine/parse/StreamLexer.java

@@ -17,8 +17,11 @@
 package com.google.turbine.parse;
 
 import static com.google.common.base.Verify.verify;
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.turbine.parse.UnicodeEscapePreprocessor.ASCII_SUB;
+import static java.lang.Math.min;
 
+import com.google.common.collect.ImmutableList;
 import com.google.turbine.diag.SourceFile;
 import com.google.turbine.diag.TurbineError;
 import com.google.turbine.diag.TurbineError.ErrorKind;
@@ -399,6 +402,15 @@ public Token next() {
         case '"':
           {
             eat();
+            if (ch == '"') {
+              eat();
+              if (ch != '"') {
+                saveValue("");
+                return Token.STRING_LITERAL;
+              }
+              eat();
+              return textBlock();
+            }
             readFrom();
             StringBuilder sb = new StringBuilder();
             STRING:
@@ -436,6 +448,156 @@ public Token next() {
     }
   }
 
+  private Token textBlock() {
+    OUTER:
+    while (true) {
+      switch (ch) {
+        case ' ':
+        case '\r':
+        case '\t':
+          eat();
+          break;
+        default:
+          break OUTER;
+      }
+    }
+    switch (ch) {
+      case '\r':
+        eat();
+        if (ch == '\n') {
+          eat();
+        }
+        break;
+      case '\n':
+        eat();
+        break;
+      default:
+        throw inputError();
+    }
+    readFrom();
+    StringBuilder sb = new StringBuilder();
+    while (true) {
+      switch (ch) {
+        case '"':
+          eat();
+          if (ch != '"') {
+            sb.append("\"");
+            continue;
+          }
+          eat();
+          if (ch != '"') {
+            sb.append("\"\"");
+            continue;
+          }
+          eat();
+          String value = sb.toString();
+          value = stripIndent(value);
+          value = translateEscapes(value);
+          saveValue(value);
+          return Token.STRING_LITERAL;
+        case ASCII_SUB:
+          if (reader.done()) {
+            return Token.EOF;
+          }
+          // falls through
+        default:
+          sb.appendCodePoint(ch);
+          eat();
+          continue;
+      }
+    }
+  }
+
+  static String stripIndent(String value) {
+    if (value.isEmpty()) {
+      return value;
+    }
+    ImmutableList<String> lines = value.lines().collect(toImmutableList());
+    // the amount of whitespace to strip from the beginning of every line
+    int strip = Integer.MAX_VALUE;
+    char last = value.charAt(value.length() - 1);
+    boolean trailingNewline = last == '\n' || last == '\r';
+    if (trailingNewline) {
+      // If the input contains a trailing newline, we have something like:
+      //
+      // |String s = """
+      // |    foo
+      // |""";
+      //
+      // Because the final """ is unindented, nothing should be stripped.
+      strip = 0;
+    } else {
+      // find the longest common prefix of whitespace across all non-blank lines
+      for (int i = 0; i < lines.size(); i++) {
+        String line = lines.get(i);
+        int nonWhitespaceStart = nonWhitespaceStart(line);
+        if (nonWhitespaceStart == line.length()) {
+          continue;
+        }
+        strip = min(strip, nonWhitespaceStart);
+      }
+    }
+    StringBuilder result = new StringBuilder();
+    boolean first = true;
+    for (String line : lines) {
+      if (!first) {
+        result.append('\n');
+      }
+      int end = trailingWhitespaceStart(line);
+      if (strip <= end) {
+        result.append(line, strip, end);
+      }
+      first = false;
+    }
+    if (trailingNewline) {
+      result.append('\n');
+    }
+    return result.toString();
+  }
+
+  private static int nonWhitespaceStart(String value) {
+    int i = 0;
+    while (i < value.length() && Character.isWhitespace(value.charAt(i))) {
+      i++;
+    }
+    return i;
+  }
+
+  private static int trailingWhitespaceStart(String value) {
+    int i = value.length() - 1;
+    while (i >= 0 && Character.isWhitespace(value.charAt(i))) {
+      i--;
+    }
+    return i + 1;
+  }
+
+  private static String translateEscapes(String value) {
+    StreamLexer lexer =
+        new StreamLexer(new UnicodeEscapePreprocessor(new SourceFile(null, value + ASCII_SUB)));
+    return lexer.translateEscapes();
+  }
+
+  private String translateEscapes() {
+    readFrom();
+    StringBuilder sb = new StringBuilder();
+    OUTER:
+    while (true) {
+      switch (ch) {
+        case '\\':
+          eat();
+          sb.append(escape());
+          continue;
+        case ASCII_SUB:
+          break OUTER;
+        default:
+          sb.appendCodePoint(ch);
+          eat();
+          continue;
+      }
+    }
+    return sb.toString();
+  }
+
   private char escape() {
     boolean zeroToThree = false;
     switch (ch) {

javatests/com/google/turbine/lower/LowerIntegrationTest.java

@@ -44,7 +44,11 @@
 public class LowerIntegrationTest {
 
   private static final ImmutableMap<String, Integer> SOURCE_VERSION =
-      ImmutableMap.of("record.test", 16, "record2.test", 16, "sealed.test", 17);
+      ImmutableMap.of(
+          "record.test", 16, //
+          "record2.test", 16,
+          "sealed.test", 17,
+          "textblock.test", 15);
 
   @Parameters(name = "{index}: {0}")
   public static Iterable<Object[]> parameters() {
@@ -285,6 +289,7 @@ public static Iterable<Object[]> parameters() {
       "superabstract.test",
       "supplierfunction.test",
       "tbound.test",
+      "textblock.test",
       "tyanno_inner.test",
       "tyanno_varargs.test",
       "typaram.test",

javatests/com/google/turbine/lower/testdata/textblock.test

@@ -0,0 +1,30 @@
+=== TextBlock.java ===
+class TextBlock {
+  public static final String hello = """
+      hello
+      world
+      """;
+  public static final String escape = """
+      hello\nworld\"
+      \r\t\b
+      \0123
+      \'
+      \\
+      \"
+      """;
+  public static final String quotes = """
+      " "" ""\" """;
+  public static final String newline = """
+      hello
+      world""";
+  public static final String blank = """
+      hello
+
+
+      world
+      """;
+  public static final String allBlank = """
+
+
+      """;
+}

javatests/com/google/turbine/parse/LexerTest.java

@@ -17,18 +17,24 @@
 package com.google.turbine.parse;
 
 import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assume.assumeTrue;
 
 import com.google.common.escape.SourceCodeEscapers;
+import com.google.common.truth.Expect;
 import com.google.turbine.diag.SourceFile;
+import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.List;
+import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 @RunWith(JUnit4.class)
 public class LexerTest {
 
+  @Rule public final Expect expect = Expect.create();
+
   @Test
   public void testSimple() {
     assertThat(lex("\nasd dsa\n")).containsExactly("IDENT(asd)", "IDENT(dsa)", "EOF");
@@ -367,4 +373,25 @@ public static List<String> lex(String input) {
     } while (token != Token.EOF);
     return tokens;
   }
+
+  @Test
+  public void stripIndent() throws Exception {
+    assumeTrue(Runtime.version().feature() >= 13);
+    String[] inputs = {
+      "",
+      "hello",
+      "hello\n",
+      "\nhello",
+      "\n    hello\n    world",
+      "\n    hello\n    world\n    ",
+      "\n    hello\n    world\n",
+      "\n    hello\n     world\n     ",
+      "\n    hello\nworld",
+      "\n    hello\n     \nworld\n     ",
+    };
+    Method stripIndent = String.class.getMethod("stripIndent");
+    for (String input : inputs) {
+      expect.that(StreamLexer.stripIndent(input)).isEqualTo(stripIndent.invoke(input));
+    }
+  }
 }

javatests/com/google/turbine/parse/ParseErrorTest.java

@@ -307,6 +307,19 @@ public void notCast() {
                 "                     ^"));
   }
 
+  @Test
+  public void singleLineTextBlockRejected() {
+    String input = "class T { String s = \"\"\" \"\"\"; }";
+    TurbineError e = assertThrows(TurbineError.class, () -> Parser.parse(input));
+    assertThat(e)
+        .hasMessageThat()
+        .isEqualTo(
+            lines(
+                "<>:1: error: unexpected input: \"",
+                "class T { String s = \"\"\" \"\"\"; }",
+                "                         ^"));
+  }
+
   private static String lines(String... lines) {
     return Joiner.on(System.lineSeparator()).join(lines);
   }