Allow decoding while debugging Yul sources in Solidity 0.8.21 (and related changes)

packages/codec/lib/ast/types.ts

@@ -59,6 +59,8 @@ export interface AstNode {
   isConstructor?: boolean;
   usedErrors?: number[];
   usedEvents?: number[];
+  code?: AstNode; //exists on YulObject
+  block?: AstNode; //exists on YulCode
   //Note: May need to add more in the future.
   //May also want to create a proper system of AstNode types
   //in the future, but sticking with this for now.

packages/codec/lib/compilations/utils.ts

@@ -286,6 +286,10 @@ function sourceIndexForAst(ast: AstNode): number | undefined {
   if (!ast) {
     return undefined;
   }
+  if (ast.nodeType === "YulObject") {
+    //Yul needs some special handling...
+    ast = ast.code.block;
+  }
   return parseInt(ast.src.split(":")[2]);
   //src is given as start:length:file.
   //we want just the file.

packages/compile-solidity/src/run.ts

@@ -207,6 +207,7 @@ function prepareOutputSelection({ targets = [] }: { targets: Targets }) {
     "": ["legacyAST", "ast"],
     "*": [
       "abi",
+      "ast", //necessary to get Yul ASTs
       "metadata",
       "evm.bytecode.object",
       "evm.bytecode.linkReferences",
@@ -341,7 +342,7 @@ function processAllSources({
   if (!compilerOutput.sources) {
     const entries = Object.entries(sources);
     if (entries.length === 1) {
-      //special case for handling Yul
+      //special case for handling old Yul versions
       const [sourcePath, contents] = entries[0];
       return [
         {
@@ -366,6 +367,15 @@ function processAllSources({
       language
     };
   }
+  //HACK: special case for handling a Yul compilation bug that causes
+  //the ID to be returned as 1 rather than 0
+  if (
+    language === "Yul" &&
+    outputSources.length === 2 &&
+    outputSources[0] === undefined
+  ) {
+    return [outputSources[1]];
+  }
   return outputSources;
 }
 

packages/debugger/lib/data/selectors/index.js

@@ -6,7 +6,12 @@ import jsonpointer from "json-pointer";
 import merge from "lodash/merge";
 import semver from "semver";
 
-import { stableKeccak256, makePath } from "lib/helpers";
+import {
+  stableKeccak256,
+  makePath,
+  topLevelNodeTypes,
+  isTopLevelNode
+} from "lib/helpers";
 
 import trace from "lib/trace/selectors";
 import evm from "lib/evm/selectors";
@@ -32,7 +37,7 @@ function solidityVersionHasNoNow(compiler) {
 }
 
 function findAncestorOfType(node, types, scopes, pointer = null, root = null) {
-  //note: you may want to include "SourceUnit" as a fallback type when using
+  //note: you may want to include "SourceUnit" and "YulObject" as fallback types when using
   //this function for convenience.
   //you only need to pass pointer and root if you want this function to work
   //from Yul.  Otherwise you can omit those and you'll get null if you happen
@@ -961,14 +966,13 @@ const data = createSelectorTree({
 
     /**
      * data.current.contract
-     * warning: may return null or similar, even though SourceUnit is included
-     * as fallback
+     * warning: may return null or similar, even though SourceUnit and YulObject are included
+     * as fallbacks
      */
     contract: createLeaf(
       ["./node", "./scopes/inlined", "./pointer", "./root"],
       (node, scopes, pointer, root) => {
-        const types = ["ContractDefinition", "SourceUnit"];
-        //SourceUnit included as fallback
+        const types = ["ContractDefinition", ...topLevelNodeTypes];
         return findAncestorOfType(node, types, scopes, pointer, root);
       }
     ),
@@ -1017,9 +1021,8 @@ const data = createSelectorTree({
           "FunctionDefinition",
           "ModifierDefinition",
           "ContractDefinition",
-          "SourceUnit"
+          ...topLevelNodeTypes
         ];
-        //SourceUnit included as fallback
         return findAncestorOfType(node, types, scopes, pointer, root);
       }
     ),
@@ -1294,7 +1297,7 @@ const data = createSelectorTree({
         //we cannot rely on the data.next selectors, but also if it is we know
         //we're not about to call a modifier or base constructor!)
         //we also want to return false if we can't find things for whatever
-        //reason
+        //reason (including if we're in Yul)
         if (
           isContextChange ||
           !node ||
@@ -1312,7 +1315,7 @@ const data = createSelectorTree({
         //ensure: current position is in a ModifierInvocation or
         //InheritanceSpecifier (recall that SourceUnit was included as
         //fallback)
-        if (invocation.nodeType === "SourceUnit") {
+        if (isTopLevelNode(invocation)) {
           return false;
         }
 
@@ -1326,7 +1329,7 @@ const data = createSelectorTree({
 
         //ensure: next node is not in the same invocation
         if (
-          nextInvocation.nodeType !== "SourceUnit" &&
+          !isTopLevelNode(nextInvocation) &&
           nextInvocation.id === invocation.id
         ) {
           return false;
@@ -1357,9 +1360,8 @@ const data = createSelectorTree({
         const types = [
           "ModifierInvocation",
           "InheritanceSpecifier",
-          "SourceUnit"
+          ...topLevelNodeTypes
         ];
-        //again, SourceUnit included as fallback
         return findAncestorOfType(node, types, scopes);
       }
     ),
@@ -1372,7 +1374,7 @@ const data = createSelectorTree({
     modifierArgumentIndex: createLeaf(
       ["./scopes", "./node", "./modifierInvocation"],
       (scopes, node, invocation) => {
-        if (!invocation || invocation.nodeType === "SourceUnit") {
+        if (!invocation || isTopLevelNode(invocation)) {
           return undefined;
         }
 
@@ -1400,7 +1402,7 @@ const data = createSelectorTree({
     modifierBeingInvoked: createLeaf(
       ["./modifierInvocation", "./scopes/inlined"],
       (invocation, scopes) => {
-        if (!invocation || invocation.nodeType === "SourceUnit") {
+        if (!invocation || isTopLevelNode(invocation)) {
           return undefined;
         }
 
@@ -1877,9 +1879,9 @@ const data = createSelectorTree({
         const types = [
           "ModifierInvocation",
           "InheritanceSpecifier",
-          "SourceUnit"
+          ...topLevelNodeTypes
         ];
-        //again, SourceUnit included as fallback
+        //again, SourceUnit and YulObject are included as fallbacks
         return findAncestorOfType(node, types, scopes);
       }
     ),
@@ -1894,7 +1896,7 @@ const data = createSelectorTree({
         evm.current.step.isContextChange
       ],
       (invocation, scopes, invalid) => {
-        if (invalid || !invocation || invocation.nodeType === "SourceUnit") {
+        if (invalid || !invocation || isTopLevelNode(invocation)) {
           return undefined;
         }
 

packages/debugger/lib/helpers/index.js

@@ -12,6 +12,12 @@ export function isDeliberatelySkippedNodeType(node) {
   return skippedTypes.includes(node.nodeType);
 }
 
+export const topLevelNodeTypes = ["SourceUnit", "YulObject"];
+
+export function isTopLevelNode(node) {
+  return topLevelNodeTypes.includes(node.nodeType);
+}
+
 //HACK
 //these aren't the only types of skipped nodes, but determining all skipped
 //nodes would be too difficult

packages/debugger/test/data/more-decoding.js

@@ -739,6 +739,7 @@ describe("Further Decoding", function () {
 
   describe("Overflow", function () {
     it("Discards padding on unsigned integers", async function () {
+      this.timeout(6000);
       let instance = await abstractions.OverflowTest.deployed();
       let receipt = await instance.unsignedTest();
       let txHash = receipt.tx;
@@ -769,6 +770,7 @@ describe("Further Decoding", function () {
     });
 
     it("Discards padding on signed integers", async function () {
+      this.timeout(6000);
       let instance = await abstractions.OverflowTest.deployed();
       let receipt = await instance.signedTest();
       let txHash = receipt.tx;
@@ -799,6 +801,7 @@ describe("Further Decoding", function () {
     });
 
     it("Discards padding on static bytestrings", async function () {
+      this.timeout(6000);
       let instance = await abstractions.OverflowTest.deployed();
       let receipt = await instance.rawTest();
       let txHash = receipt.tx;

packages/debugger/test/data/yul-source.js

@@ -0,0 +1,122 @@
+import debugModule from "debug";
+const debug = debugModule("debugger:test:data:yul");
+
+import { assert } from "chai";
+
+import Ganache from "ganache";
+
+import { prepareContracts, lineOf, testBlockGasLimit } from "../helpers";
+import Debugger from "lib/debugger";
+
+import sourcemapping from "lib/sourcemapping/selectors";
+
+import * as Codec from "@truffle/codec";
+
+const __YUL = `
+object "YulTest" {
+  code {
+    let size := datasize("runtime")
+    datacopy(0, dataoffset("runtime"), size)
+    return(0, size)
+  }
+  object "runtime" {
+    code {
+      let a := 1
+      let b := 2 //BREAK #1
+      mstore(0, add(b, a)) //BREAK #2
+      return(0, 0x20)
+    }
+  }
+}
+`;
+
+let sources = {
+  "YulTest.yul": __YUL
+};
+
+describe("Assembly decoding (Yul source)", function () {
+  let provider;
+  let abstractions;
+  let compilations;
+
+  before("Create Provider", async function () {
+    provider = Ganache.provider({
+      seed: "debugger",
+      miner: {
+        instamine: "strict",
+        blockGasLimit: testBlockGasLimit
+      },
+      logging: {
+        quiet: true
+      }
+    });
+  });
+
+  before("Prepare contracts and artifacts", async function () {
+    this.timeout(30000);
+
+    let prepared = await prepareContracts(provider, sources);
+    abstractions = prepared.abstractions;
+    compilations = prepared.compilations;
+  });
+
+  it("Decodes variables in Yul files", async function () {
+    this.timeout(12000);
+
+    let instance = await abstractions.YulTest.deployed();
+    let receipt = await instance.sendTransaction({});
+    let txHash = receipt.tx;
+
+    let bugger = await Debugger.forTx(txHash, { provider, compilations });
+
+    let sourceId = bugger.view(sourcemapping.current.source).id;
+    let source = bugger.view(sourcemapping.current.source).source;
+    await bugger.addBreakpoint({
+      sourceId,
+      line: lineOf("BREAK #1", source)
+    });
+    await bugger.addBreakpoint({
+      sourceId,
+      line: lineOf("BREAK #2", source)
+    });
+    await bugger.addBreakpoint({
+      sourceId,
+      line: lineOf("BREAK #3", source)
+    });
+
+    await bugger.continueUntilBreakpoint();
+
+    const numberize = obj =>
+      Object.assign(
+        {},
+        ...Object.entries(obj).map(([key, value]) => ({ [key]: Number(value) }))
+      );
+
+    let variables = numberize(
+      Codec.Format.Utils.Inspect.unsafeNativizeVariables(
+        await bugger.variables()
+      )
+    );
+
+    let expectedResult = {
+      a: 1
+    };
+
+    assert.deepEqual(variables, expectedResult);
+
+    await bugger.continueUntilBreakpoint();
+
+    variables = numberize(
+      Codec.Format.Utils.Inspect.unsafeNativizeVariables(
+        await bugger.variables()
+      )
+    );
+
+    expectedResult = {
+      a: 1,
+      b: 2
+    };
+
+    assert.deepEqual(variables, expectedResult);
+  });
+});

packages/debugger/test/helpers.js

@@ -34,7 +34,7 @@ export async function prepareContracts(
 
   config.compilers = {
     solc: {
-      version: "0.8.20",
+      version: "0.8.21",
       settings: {
         optimizer: { enabled: false, runs: 200 },
         evmVersion: "shanghai"