From c81113c49190b390d57ccd70af27ccfc9f1aba88 Mon Sep 17 00:00:00 2001 From: Jamie Thompson Date: Mon, 11 Mar 2024 16:57:31 +0100 Subject: [PATCH] Test -print-tasty output of Java sources --- .../tools/dotc/config/ScalaSettings.scala | 1 + .../dotc/core/tasty/TastyAnsiiPrinter.scala | 5 +- .../tools/dotc/core/tasty/TastyPrinter.scala | 26 ++- .../tools/dotc/core/tasty/TreeUnpickler.scala | 4 +- .../tools/dotc/parsing/JavaParsers.scala | 1 + .../tools/dotc/printing/RefinedPrinter.scala | 5 +- .../dotty/tools/dotc/transform/Pickler.scala | 37 +++- tests/pos/i19806/J.tastycheck | 161 ++++++++++++++++++ tests/pos/i19806/J_SCALA_ONLY.java | 22 +++ tests/pos/i19806/Module.scala | 9 + tests/run/i17255/Module.scala | 1 + 11 files changed, 259 insertions(+), 13 deletions(-) create mode 100644 tests/pos/i19806/J.tastycheck create mode 100644 tests/pos/i19806/J_SCALA_ONLY.java create mode 100644 tests/pos/i19806/Module.scala diff --git a/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala b/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala index 3dafedd8e2e0..0cb508ce1272 100644 --- a/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala +++ b/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala @@ -388,6 +388,7 @@ private sealed trait YSettings: val YshowPrintErrors: Setting[Boolean] = BooleanSetting("-Yshow-print-errors", "Don't suppress exceptions thrown during tree printing.") val YprintTasty: Setting[Boolean] = BooleanSetting("-Yprint-tasty", "Prints the generated TASTY to stdout.") val YtestPickler: Setting[Boolean] = BooleanSetting("-Ytest-pickler", "Self-test for pickling functionality; should be used with -Ystop-after:pickler.") + val YtestPicklerCheck: Setting[Boolean] = BooleanSetting("-Ytest-pickler-check", "Self-test for pickling -print-tasty output; should be used with -Ytest-pickler.") val YcheckReentrant: Setting[Boolean] = BooleanSetting("-Ycheck-reentrant", "Check that compiled program does not contain vars that can be accessed from a global root.") val YdropComments: Setting[Boolean] = BooleanSetting("-Ydrop-docs", "Drop documentation when scanning source files.", aliases = List("-Ydrop-comments")) val YcookComments: Setting[Boolean] = BooleanSetting("-Ycook-docs", "Cook the documentation (type check `@usecase`, etc.)", aliases = List("-Ycook-comments")) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyAnsiiPrinter.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyAnsiiPrinter.scala index d8d72a4e651e..a3d8cedacb4a 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyAnsiiPrinter.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyAnsiiPrinter.scala @@ -2,7 +2,10 @@ package dotty.tools.dotc package core package tasty -class TastyAnsiiPrinter(bytes: Array[Byte]) extends TastyPrinter(bytes) { +class TastyAnsiiPrinter(bytes: Array[Byte], testPickler: Boolean) extends TastyPrinter(bytes, testPickler) { + + def this(bytes: Array[Byte]) = this(bytes, testPickler = false) + override protected def nameStr(str: String): String = Console.MAGENTA + str + Console.RESET override protected def treeStr(str: String): String = Console.YELLOW + str + Console.RESET override protected def lengthStr(str: String): String = Console.CYAN + str + Console.RESET diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala index a74607dbc9d5..a4818e8d6fd5 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala @@ -20,9 +20,12 @@ import dotty.tools.tasty.TastyBuffer.Addr object TastyPrinter: def showContents(bytes: Array[Byte], noColor: Boolean): String = + showContents(bytes, noColor, testPickler = false) + + def showContents(bytes: Array[Byte], noColor: Boolean, testPickler: Boolean = false): String = val printer = - if noColor then new TastyPrinter(bytes) - else new TastyAnsiiPrinter(bytes) + if noColor then new TastyPrinter(bytes, testPickler) + else new TastyAnsiiPrinter(bytes, testPickler) printer.showContents() def main(args: Array[String]): Unit = { @@ -62,7 +65,9 @@ object TastyPrinter: println(line) } -class TastyPrinter(bytes: Array[Byte]) { +class TastyPrinter(bytes: Array[Byte], val testPickler: Boolean) { + + def this(bytes: Array[Byte]) = this(bytes, testPickler = false) class TastyPrinterUnpickler extends TastyUnpickler(bytes) { var namesStart: Addr = uninitialized @@ -84,9 +89,16 @@ class TastyPrinter(bytes: Array[Byte]) { private def printHeader(sb: StringBuilder): Unit = val header = unpickler.header sb.append("Header:\n") - sb.append(s" version: ${header.majorVersion}.${header.minorVersion}.${header.experimentalVersion}\n") - sb.append(" tooling: ").append(header.toolingVersion).append("\n") - sb.append(" UUID: ").append(header.uuid).append("\n") + if testPickler then + // these fields are not stable when the TASTy/compiler versions change, so not useful for testing + sb.append(" version: \n") + sb.append(" tooling: \n") + sb.append(" UUID: \n") + else + sb.append(s" version: ${header.majorVersion}.${header.minorVersion}.${header.experimentalVersion}\n") + sb.append(" tooling: ").append(header.toolingVersion).append("\n") + sb.append(" UUID: ").append(header.uuid).append("\n") + end if sb.append("\n") private def printNames(sb: StringBuilder): Unit = @@ -230,6 +242,8 @@ class TastyPrinter(bytes: Array[Byte]) { import reader.* sb.append(s"\n\nAttributes (${reader.endAddr.index - reader.startAddr.index} bytes, starting from $base):\n") while !isAtEnd do + // TODO: Should we elide attributes under testPickler? (i.e. + // if we add new attributes many check files will need to be updated) val tag = readByte() sb.append(" ").append(attributeTagToString(tag)) if isBooleanAttrTag(tag) then () diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala index b95e4df663a1..4b43b8d97d5b 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala @@ -1085,10 +1085,10 @@ class TreeUnpickler(reader: TastyReader, def complete(denot: SymDenotation)(using Context) = denot.info = tdef.symbol.asType.info.subst(tparamRefs, derivedTparamRefs) } - newSymbol(sym, tdef.name, Flags.JavaDefined | Flags.Param, completer, coord = cls.coord) + newSymbol(sym, tdef.name, Flags.JavaDefined | Flags.Param, completer, coord = coordAt(start)) lazy val derivedTparamRefs: List[Type] = derivedTparamSyms.map(_.typeRef) val vparamSym = - newSymbol(sym, nme.syntheticParamName(1), pflags, defn.UnitType, coord = cls.coord) + newSymbol(sym, nme.syntheticParamName(1), pflags, defn.UnitType, coord = coordAt(start)) val vparamSymss: List[List[Symbol]] = List(vparamSym) :: Nil val paramSymss = if derivedTparamSyms.nonEmpty then derivedTparamSyms :: vparamSymss else vparamSymss diff --git a/compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala b/compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala index 79282b0e5223..8a05cc1d391b 100644 --- a/compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala @@ -894,6 +894,7 @@ object JavaParsers { parents = superclass :: interfaces, stats = canonicalConstructor :: accessors ::: body, tparams = tparams, + // ctorSpan = getSpan(start, nameOffset), needsDummyConstr = true ) ).withMods(mods.withFlags(Flags.JavaDefined | Flags.Final)) diff --git a/compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala b/compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala index 893b34f48396..93e280f8a13c 100644 --- a/compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala +++ b/compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala @@ -1010,7 +1010,10 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) { var modsText = modText(constr.mods, constr.symbol, "", isType = false) if (!modsText.isEmpty) modsText = " " ~ modsText if (constr.mods.hasAnnotations && !constr.mods.hasFlags) modsText = modsText ~~ " this" - withEnclosingDef(constr) { addParamssText(tparamsTxt ~~ modsText, constr.trailingParamss) } + val ctorParamss = + // for fake `(x$1: Unit): Foo` constructor, don't print the param (span is not reconstructed correctly) + if constr.symbol.isAllOf(JavaParsers.fakeFlags) then Nil else constr.trailingParamss + withEnclosingDef(constr) { addParamssText(tparamsTxt ~~ modsText, ctorParamss) } } val parentsText = Text(impl.parents.map(constrText), if (ofNew) keywordStr(" with ") else ", ") val derivedText = Text(impl.derived.map(toText(_)), ", ") diff --git a/compiler/src/dotty/tools/dotc/transform/Pickler.scala b/compiler/src/dotty/tools/dotc/transform/Pickler.scala index 27b5d53e25dc..20403f1acd95 100644 --- a/compiler/src/dotty/tools/dotc/transform/Pickler.scala +++ b/compiler/src/dotty/tools/dotc/transform/Pickler.scala @@ -69,6 +69,7 @@ class Pickler extends Phase { // Maps that keep a record if -Ytest-pickler is set. private val beforePickling = new mutable.HashMap[ClassSymbol, String] + private val beforePicklingPrinted = new mutable.HashMap[ClassSymbol, String] private val pickledBytes = new mutable.HashMap[ClassSymbol, (CompilationUnit, Array[Byte])] /** Drop any elements of this list that are linked module classes of other elements in the list */ @@ -184,7 +185,10 @@ class Pickler extends Phase { else val pickled = computePickled() reportPositionWarnings() - if ctx.settings.YtestPickler.value then pickledBytes(cls) = (unit, pickled) + if ctx.settings.YtestPickler.value then + pickledBytes(cls) = (unit, pickled) + if ctx.settings.YtestPicklerCheck.value then + beforePicklingPrinted(cls) = TastyPrinter.showContents(pickled, noColor = true, testPickler = true) () => pickled unit.pickled += (cls -> demandPickled) @@ -251,15 +255,22 @@ class Pickler extends Phase { private def testUnpickler(using Context): Unit = pickling.println(i"testing unpickler at run ${ctx.runId}") ctx.initialize() + val resolveCheck = ctx.settings.YtestPicklerCheck.value val unpicklers = for ((cls, (unit, bytes)) <- pickledBytes) yield { val unpickler = new DottyUnpickler(unit.source.file, bytes) unpickler.enter(roots = Set.empty) - cls -> (unit, unpickler) + val optCheck = + if resolveCheck then + val resolved = unit.source.file.resolveSibling(s"${cls.name.mangledString}.tastycheck") + if resolved == null then None + else Some(resolved) + else None + cls -> (unit, unpickler, optCheck) } pickling.println("************* entered toplevel ***********") val rootCtx = ctx - for ((cls, (unit, unpickler)) <- unpicklers) do + for ((cls, (unit, unpickler, optCheck)) <- unpicklers) do val testJava = unit.typedAsJava if testJava then if unpickler.unpickler.nameAtRef.contents.exists(_ == nme.FromJavaObject) then @@ -268,6 +279,15 @@ class Pickler extends Phase { val freshUnit = CompilationUnit(rootCtx.compilationUnit.source) freshUnit.needsCaptureChecking = unit.needsCaptureChecking freshUnit.knowsPureFuns = unit.knowsPureFuns + optCheck match + case Some(check) => + import java.nio.charset.StandardCharsets.UTF_8 + val checkContents = String(check.toByteArray, UTF_8) + inContext(rootCtx.fresh.setCompilationUnit(freshUnit)): + testSamePrinted(beforePicklingPrinted(cls), checkContents, cls, check) + case None => + () + inContext(printerContext(testJava)(using rootCtx.fresh.setCompilationUnit(freshUnit))): testSame(i"$unpickled%\n%", beforePickling(cls), cls) @@ -283,4 +303,15 @@ class Pickler extends Phase { | | diff before-pickling.txt after-pickling.txt""") end testSame + + private def testSamePrinted(printed: String, checkContents: String, cls: ClassSymbol, check: AbstractFile)(using Context) = + import java.nio.charset.StandardCharsets.UTF_8 + def normal(s: String) = new String(s.getBytes(UTF_8), UTF_8) + val unequal = printed.length() != checkContents.length() || normal(printed) != normal(checkContents) + if unequal then + output("after-printing.txt", printed) + report.error(em"""TASTy printer difference for $cls in ${cls.source}, for details: + | + | diff ${check.toString} after-printing.txt""") + end testSamePrinted } diff --git a/tests/pos/i19806/J.tastycheck b/tests/pos/i19806/J.tastycheck new file mode 100644 index 000000000000..900530e80965 --- /dev/null +++ b/tests/pos/i19806/J.tastycheck @@ -0,0 +1,161 @@ +Header: + version: + tooling: + UUID: + +Names (217 bytes, starting from 80): + 0: ASTs + 1: p + 2: J + 3: J[ModuleClass] + 4: Object + 5: java + 6: lang + 7: java[Qualified . lang] + 8: _ + 9: + 10: Unit + 11: scala + 12: module2 + 13: Module + 14: Module[ModuleClass] + 15: module + 16: innermodule2 + 17: InnerModule + 18: InnerModule[ModuleClass] + 19: innermodule + 20: T + 21: Nothing + 22: Positions + 23: tests/pos/i19806/J_SCALA_ONLY.java + 24: Comments + 25: Attributes + + +Trees (145 bytes, starting from 300): + 0: PACKAGE(142) + 3: TERMREFpkg 1 [p] + 5: VALDEF(11) 2 [J] + 8: IDENTtpt 3 [J[ModuleClass]] + 10: TYPEREFsymbol 18 + 12: TERMREFpkg 1 [p] + 14: ELIDED + 15: SHAREDtype 10 + 17: OBJECT + 18: TYPEDEF(86) 3 [J[ModuleClass]] + 21: TEMPLATE(82) + 23: TYPEREF 4 [Object] + 25: TERMREFpkg 7 [java[Qualified . lang]] + 27: SELFDEF 8 [_] + 29: SINGLETONtpt + 30: TERMREFsymbol 5 + 32: SHAREDtype 12 + 34: DEFDEF(7) 9 [] + 37: EMPTYCLAUSE + 38: TYPEREF 10 [Unit] + 40: TERMREFpkg 11 [scala] + 42: STABLE + 43: DEFDEF(12) 12 [module2] + 46: EMPTYCLAUSE + 47: IDENTtpt 14 [Module[ModuleClass]] + 49: TYPEREF 14 [Module[ModuleClass]] + 51: SHAREDtype 12 + 53: ELIDED + 54: SHAREDtype 49 + 56: STATIC + 57: DEFDEF(12) 15 [module] + 60: EMPTYCLAUSE + 61: SELECTtpt 14 [Module[ModuleClass]] + 63: SHAREDtype 3 + 65: ELIDED + 66: TYPEREF 14 [Module[ModuleClass]] + 68: SHAREDtype 3 + 70: STATIC + 71: DEFDEF(14) 16 [innermodule2] + 74: EMPTYCLAUSE + 75: SELECTtpt 18 [InnerModule[ModuleClass]] + 77: TERMREF 13 [Module] + 79: SHAREDtype 12 + 81: ELIDED + 82: TYPEREF 18 [InnerModule[ModuleClass]] + 84: SHAREDtype 77 + 86: STATIC + 87: DEFDEF(16) 19 [innermodule] + 90: EMPTYCLAUSE + 91: SELECTtpt 18 [InnerModule[ModuleClass]] + 93: SELECT 13 [Module] + 95: SHAREDtype 3 + 97: ELIDED + 98: TYPEREF 18 [InnerModule[ModuleClass]] + 100: TERMREF 13 [Module] + 102: SHAREDtype 3 + 104: STATIC + 105: OBJECT + 106: TYPEDEF(37) 2 [J] + 109: TEMPLATE(34) + 111: TYPEPARAM(11) 20 [T] + 114: TYPEBOUNDS(6) + 116: TYPEREF 21 [Nothing] + 118: SHAREDtype 40 + 120: SHAREDtype 23 + 122: PRIVATE + 123: LOCAL + 124: SHAREDtype 120 + 126: SPLITCLAUSE + 127: DEFDEF(16) 9 [] + 130: TYPEPARAM(7) 20 [T] + 133: TYPEBOUNDStpt(4) + 135: SHAREDtype 116 + 137: SHAREDtype 120 + 139: EMPTYCLAUSE + 140: SHAREDtype 38 + 142: ELIDED + 143: SHAREDtype 38 + 145: + +Positions (145 bytes, starting from 448): + lines: 23 + line sizes: + 10, 0, 19, 0, 15, 0, 35, 29, 3, 0, 36, 29, 3, 0, 52, 41, 3, 0, 53, 41 + 3, 1, 0 + positions: + 0: 0 .. 394 + 5: 12 .. 12 + 8: 12 .. 12 + 18: 12 .. 394 + 21: 52 .. 392 + 23: 25 .. 25 + 30: 52 .. 52 + 34: 52 .. 52 + 38: 52 .. 52 + 43: 52 .. 119 + 47: 66 .. 73 + 57: 123 .. 191 + 61: 137 .. 146 + 63: 137 .. 138 + 71: 195 .. 291 + 75: 209 .. 228 + 77: 209 .. 215 + 87: 295 .. 392 + 91: 309 .. 330 + 93: 309 .. 317 + 95: 309 .. 310 + 106: 12 .. 394 + 109: 27 .. 48 + 111: 27 .. 28 + 114: 27 .. 27 + 124: 28 .. 28 + 127: 35 .. 48 + 130: 27 .. 28 + 135: 27 .. 27 + 137: 27 .. 27 + 140: 46 .. 46 + + source paths: + 0: 23 [tests/pos/i19806/J_SCALA_ONLY.java] + + +Attributes (4 bytes, starting from 597): + JAVAattr + OUTLINEattr + SOURCEFILEattr 23 [tests/pos/i19806/J_SCALA_ONLY.java] diff --git a/tests/pos/i19806/J_SCALA_ONLY.java b/tests/pos/i19806/J_SCALA_ONLY.java new file mode 100644 index 000000000000..abbc6c0e9d57 --- /dev/null +++ b/tests/pos/i19806/J_SCALA_ONLY.java @@ -0,0 +1,22 @@ +package p; + +public class J { + + public J() {} + + public static Module$ module2() { + return p.Module$.MODULE$; + } + + public static p.Module$ module() { + return p.Module$.MODULE$; + } + + public static Module.InnerModule$ innermodule2() { + return p.Module.InnerModule$.MODULE$; + } + + public static p.Module.InnerModule$ innermodule() { + return p.Module.InnerModule$.MODULE$; + } +} diff --git a/tests/pos/i19806/Module.scala b/tests/pos/i19806/Module.scala new file mode 100644 index 000000000000..d0142fc24682 --- /dev/null +++ b/tests/pos/i19806/Module.scala @@ -0,0 +1,9 @@ +//> using options -Yjava-tasty -Ytest-pickler-check + +package p + +object Module: + object InnerModule + +class Outer: + object InnerModule diff --git a/tests/run/i17255/Module.scala b/tests/run/i17255/Module.scala index 936e78e8e4ce..9b7153edbfd1 100644 --- a/tests/run/i17255/Module.scala +++ b/tests/run/i17255/Module.scala @@ -1,3 +1,4 @@ +// scalajs: --skip package p { object Module { override def toString = "Module"