Merge pull request #14177 from GiokaMarkella/develop

Add support for Yul ASTs output

Changelog.md

@@ -4,8 +4,12 @@ Language Features:
 
 
 Compiler Features:
+ * Commandline Interface: Add ``--ast-compact-json`` output in assembler mode.
+ * Commandline Interface: Add ``--ir-ast-json`` and ``--ir-optimized-ast-json`` outputs for Solidity input, providing AST in compact JSON format for IR and optimized IR.
  * EWasm: Remove EWasm backend.
  * Parser: Introduce ``pragma experimental solidity``, which will enable an experimental language mode that in particular has no stability guarantees between non-breaking releases and is not suited for production use.
+ * Standard JSON Interface: Add ``ast`` file-level output for Yul input.
+ * Standard JSON Interface: Add ``irAst`` and ``irOptimizedAst`` contract-level outputs for Solidity input, providing AST in compact JSON format for IR and optimized IR.
 
 
 Bugfixes:

docs/using-the-compiler.rst

@@ -390,7 +390,9 @@ Input Description
         //   userdoc - User documentation (natspec)
         //   metadata - Metadata
         //   ir - Yul intermediate representation of the code before optimization
+        //   irAst - AST of Yul intermediate representation of the code before optimization
         //   irOptimized - Intermediate representation after optimization
+        //   irOptimizedAst - AST of intermediate representation after optimization
         //   storageLayout - Slots, offsets and types of the contract's state variables.
         //   evm.assembly - New assembly format
         //   evm.legacyAssembly - Old-style assembly format in JSON
@@ -536,8 +538,14 @@ Output Description
             "userdoc": {},
             // Developer documentation (natspec)
             "devdoc": {},
-            // Intermediate representation (string)
+            // Intermediate representation before optimization (string)
             "ir": "",
+            // AST of intermediate representation before optimization
+            "irAst":  {/* ... */},
+            // Intermediate representation after optimization (string)
+            "irOptimized": "",
+            // AST of intermediate representation after optimization
+            "irOptimizedAst": {/* ... */},
             // See the Storage Layout documentation.
             "storageLayout": {"storage": [/* ... */], "types": {/* ... */} },
             // EVM-related outputs

libsolidity/codegen/ir/IRGenerator.cpp

@@ -40,6 +40,8 @@
 
 #include <liblangutil/SourceReferenceFormatter.h>
 
+#include <json/json.h>
+
 #include <sstream>
 #include <variant>
 
@@ -87,7 +89,7 @@ set<CallableDeclaration const*, ASTNode::CompareByID> collectReachableCallables(
 
 }
 
-pair<string, string> IRGenerator::run(
+tuple<string, Json::Value, string, Json::Value> IRGenerator::run(
 	ContractDefinition const& _contract,
 	bytes const& _cborMetadata,
 	map<ContractDefinition const*, string_view const> const& _otherYulSources
@@ -112,9 +114,11 @@ pair<string, string> IRGenerator::run(
 			);
 		solAssert(false, ir + "\n\nInvalid IR generated:\n" + errorMessage + "\n");
 	}
+	Json::Value irAst = asmStack.astJson();
 	asmStack.optimize();
+	Json::Value irOptAst = asmStack.astJson();
 
-	return {std::move(ir), asmStack.print(m_context.soliditySourceProvider())};
+	return {std::move(ir), std::move(irAst), asmStack.print(m_context.soliditySourceProvider()), std::move(irOptAst)};
 }
 
 string IRGenerator::generate(

libsolidity/codegen/ir/IRGenerator.h

@@ -32,6 +32,8 @@
 #include <liblangutil/CharStreamProvider.h>
 #include <liblangutil/EVMVersion.h>
 
+#include <json/json.h>
+
 #include <string>
 
 namespace solidity::frontend
@@ -70,7 +72,7 @@ class IRGenerator
 
 	/// Generates and returns the IR code, in unoptimized and optimized form
 	/// (or just pretty-printed, depending on the optimizer settings).
-	std::pair<std::string, std::string> run(
+	std::tuple<std::string, Json::Value, std::string, Json::Value> run(
 		ContractDefinition const& _contract,
 		bytes const& _cborMetadata,
 		std::map<ContractDefinition const*, std::string_view const> const& _otherYulSources

libsolidity/interface/CompilerStack.cpp

@@ -883,6 +883,14 @@ string const& CompilerStack::yulIR(string const& _contractName) const
 	return contract(_contractName).yulIR;
 }
 
+Json::Value const& CompilerStack::yulIRAst(string const& _contractName) const
+{
+	if (m_stackState != CompilationSuccessful)
+		solThrow(CompilerError, "Compilation was not successful.");
+
+	return contract(_contractName).yulIRAst;
+}
+
 string const& CompilerStack::yulIROptimized(string const& _contractName) const
 {
 	if (m_stackState != CompilationSuccessful)
@@ -891,6 +899,14 @@ string const& CompilerStack::yulIROptimized(string const& _contractName) const
 	return contract(_contractName).yulIROptimized;
 }
 
+Json::Value const& CompilerStack::yulIROptimizedAst(string const& _contractName) const
+{
+	if (m_stackState != CompilationSuccessful)
+		solThrow(CompilerError, "Compilation was not successful.");
+
+	return contract(_contractName).yulIROptimizedAst;
+}
+
 evmasm::LinkerObject const& CompilerStack::object(string const& _contractName) const
 {
 	if (m_stackState != CompilationSuccessful)
@@ -1445,7 +1461,13 @@ void CompilerStack::generateIR(ContractDefinition const& _contract)
 		m_debugInfoSelection,
 		this
 	);
-	tie(compiledContract.yulIR, compiledContract.yulIROptimized) = generator.run(
+
+	tie(
+		compiledContract.yulIR,
+		compiledContract.yulIRAst,
+		compiledContract.yulIROptimized,
+		compiledContract.yulIROptimizedAst
+	) = generator.run(
 		_contract,
 		createCBORMetadata(compiledContract, /* _forIR */ true),
 		otherYulSources

libsolidity/interface/CompilerStack.h

@@ -276,9 +276,15 @@ class CompilerStack: public langutil::CharStreamProvider
 	/// @returns the IR representation of a contract.
 	std::string const& yulIR(std::string const& _contractName) const;
 
+	/// @returns the IR representation of a contract AST in format.
+	Json::Value const& yulIRAst(std::string const& _contractName) const;
+
 	/// @returns the optimized IR representation of a contract.
 	std::string const& yulIROptimized(std::string const& _contractName) const;
 
+	/// @returns the optimized IR representation of a contract AST in JSON format.
+	Json::Value const& yulIROptimizedAst(std::string const& _contractName) const;
+
 	/// @returns the assembled object for a contract.
 	evmasm::LinkerObject const& object(std::string const& _contractName) const;
 
@@ -380,6 +386,8 @@ class CompilerStack: public langutil::CharStreamProvider
 		evmasm::LinkerObject runtimeObject; ///< Runtime object.
 		std::string yulIR; ///< Yul IR code.
 		std::string yulIROptimized; ///< Optimized Yul IR code.
+		Json::Value yulIRAst; ///< JSON AST of Yul IR code.
+		Json::Value yulIROptimizedAst; ///< JSON AST of optimized Yul IR code.
 		util::LazyInit<std::string const> metadata; ///< The metadata json that will be hashed into the chain.
 		util::LazyInit<Json::Value const> abi;
 		util::LazyInit<Json::Value const> storageLayout;

libsolidity/interface/StandardCompiler.cpp

@@ -179,7 +179,7 @@ bool hashMatchesContent(string const& _hash, string const& _content)
 
 bool isArtifactRequested(Json::Value const& _outputSelection, string const& _artifact, bool _wildcardMatchesExperimental)
 {
-	static set<string> experimental{"ir", "irOptimized"};
+	static set<string> experimental{"ir", "irAst", "irOptimized", "irOptimizedAst"};
 	for (auto const& selectedArtifactJson: _outputSelection)
 	{
 		string const& selectedArtifact = selectedArtifactJson.asString();
@@ -263,7 +263,7 @@ bool isBinaryRequested(Json::Value const& _outputSelection)
 	// This does not include "evm.methodIdentifiers" on purpose!
 	static vector<string> const outputsThatRequireBinaries = vector<string>{
 		"*",
-		"ir", "irOptimized",
+		"ir", "irAst", "irOptimized", "irOptimizedAst",
 		"evm.gasEstimates", "evm.legacyAssembly", "evm.assembly"
 	} + evmObjectComponents("bytecode") + evmObjectComponents("deployedBytecode");
 
@@ -295,7 +295,7 @@ bool isEvmBytecodeRequested(Json::Value const& _outputSelection)
 }
 
 /// @returns true if any Yul IR was requested. Note that as an exception, '*' does not
-/// yet match "ir" or "irOptimized"
+/// yet match "ir", "irAst", "irOptimized" or "irOptimizedAst"
 bool isIRRequested(Json::Value const& _outputSelection)
 {
 	if (!_outputSelection.isObject())
@@ -304,7 +304,12 @@ bool isIRRequested(Json::Value const& _outputSelection)
 	for (auto const& fileRequests: _outputSelection)
 		for (auto const& requests: fileRequests)
 			for (auto const& request: requests)
-				if (request == "ir" || request == "irOptimized")
+				if (
+					request == "ir" ||
+					request == "irAst" ||
+					request == "irOptimized" ||
+					request == "irOptimizedAst"
+				)
 					return true;
 
 	return false;
@@ -1350,8 +1355,12 @@ Json::Value StandardCompiler::compileSolidity(StandardCompiler::InputsAndSetting
 		// IR
 		if (compilationSuccess && isArtifactRequested(_inputsAndSettings.outputSelection, file, name, "ir", wildcardMatchesExperimental))
 			contractData["ir"] = compilerStack.yulIR(contractName);
+		if (compilationSuccess && isArtifactRequested(_inputsAndSettings.outputSelection, file, name, "irAst", wildcardMatchesExperimental))
+			contractData["irAst"] = compilerStack.yulIRAst(contractName);
 		if (compilationSuccess && isArtifactRequested(_inputsAndSettings.outputSelection, file, name, "irOptimized", wildcardMatchesExperimental))
 			contractData["irOptimized"] = compilerStack.yulIROptimized(contractName);
+		if (compilationSuccess && isArtifactRequested(_inputsAndSettings.outputSelection, file, name, "irOptimizedAst", wildcardMatchesExperimental))
+			contractData["irOptimizedAst"] = compilerStack.yulIROptimizedAst(contractName);
 
 		// EVM
 		Json::Value evmData(Json::objectValue);
@@ -1513,6 +1522,13 @@ Json::Value StandardCompiler::compileYul(InputsAndSettings _inputsAndSettings)
 	if (isArtifactRequested(_inputsAndSettings.outputSelection, sourceName, contractName, "ir", wildcardMatchesExperimental))
 		output["contracts"][sourceName][contractName]["ir"] = stack.print();
 
+	if (isArtifactRequested(_inputsAndSettings.outputSelection, sourceName, contractName, "ast", wildcardMatchesExperimental))
+	{
+		Json::Value sourceResult = Json::objectValue;
+		sourceResult["id"] = 1;
+		sourceResult["ast"] = stack.astJson();
+		output["sources"][sourceName] = sourceResult;
+	}
 	stack.optimize();
 
 	MachineAssemblyObject object;

libyul/AsmJsonConverter.cpp

@@ -33,23 +33,23 @@ namespace solidity::yul
 
 Json::Value AsmJsonConverter::operator()(Block const& _node) const
 {
-	Json::Value ret = createAstNode(nativeLocationOf(_node), "YulBlock");
+	Json::Value ret = createAstNode(originLocationOf(_node), nativeLocationOf(_node), "YulBlock");
 	ret["statements"] = vectorOfVariantsToJson(_node.statements);
 	return ret;
 }
 
 Json::Value AsmJsonConverter::operator()(TypedName const& _node) const
 {
 	yulAssert(!_node.name.empty(), "Invalid variable name.");
-	Json::Value ret = createAstNode(nativeLocationOf(_node), "YulTypedName");
+	Json::Value ret = createAstNode(originLocationOf(_node), nativeLocationOf(_node), "YulTypedName");
 	ret["name"] = _node.name.str();
 	ret["type"] = _node.type.str();
 	return ret;
 }
 
 Json::Value AsmJsonConverter::operator()(Literal const& _node) const
 {
-	Json::Value ret = createAstNode(nativeLocationOf(_node), "YulLiteral");
+	Json::Value ret = createAstNode(originLocationOf(_node), nativeLocationOf(_node), "YulLiteral");
 	switch (_node.kind)
 	{
 	case LiteralKind::Number:
@@ -76,15 +76,15 @@ Json::Value AsmJsonConverter::operator()(Literal const& _node) const
 Json::Value AsmJsonConverter::operator()(Identifier const& _node) const
 {
 	yulAssert(!_node.name.empty(), "Invalid identifier");
-	Json::Value ret = createAstNode(nativeLocationOf(_node), "YulIdentifier");
+	Json::Value ret = createAstNode(originLocationOf(_node), nativeLocationOf(_node), "YulIdentifier");
 	ret["name"] = _node.name.str();
 	return ret;
 }
 
 Json::Value AsmJsonConverter::operator()(Assignment const& _node) const
 {
 	yulAssert(_node.variableNames.size() >= 1, "Invalid assignment syntax");
-	Json::Value ret = createAstNode(nativeLocationOf(_node), "YulAssignment");
+	Json::Value ret = createAstNode(originLocationOf(_node), nativeLocationOf(_node), "YulAssignment");
 	for (auto const& var: _node.variableNames)
 		ret["variableNames"].append((*this)(var));
 	ret["value"] = _node.value ? std::visit(*this, *_node.value) : Json::nullValue;
@@ -93,22 +93,22 @@ Json::Value AsmJsonConverter::operator()(Assignment const& _node) const
 
 Json::Value AsmJsonConverter::operator()(FunctionCall const& _node) const
 {
-	Json::Value ret = createAstNode(nativeLocationOf(_node), "YulFunctionCall");
+	Json::Value ret = createAstNode(originLocationOf(_node), nativeLocationOf(_node), "YulFunctionCall");
 	ret["functionName"] = (*this)(_node.functionName);
 	ret["arguments"] = vectorOfVariantsToJson(_node.arguments);
 	return ret;
 }
 
 Json::Value AsmJsonConverter::operator()(ExpressionStatement const& _node) const
 {
-	Json::Value ret = createAstNode(nativeLocationOf(_node), "YulExpressionStatement");
+	Json::Value ret = createAstNode(originLocationOf(_node), nativeLocationOf(_node), "YulExpressionStatement");
 	ret["expression"] = std::visit(*this, _node.expression);
 	return ret;
 }
 
 Json::Value AsmJsonConverter::operator()(VariableDeclaration const& _node) const
 {
-	Json::Value ret = createAstNode(nativeLocationOf(_node), "YulVariableDeclaration");
+	Json::Value ret = createAstNode(originLocationOf(_node), nativeLocationOf(_node), "YulVariableDeclaration");
 	for (auto const& var: _node.variables)
 		ret["variables"].append((*this)(var));
 
@@ -120,7 +120,7 @@ Json::Value AsmJsonConverter::operator()(VariableDeclaration const& _node) const
 Json::Value AsmJsonConverter::operator()(FunctionDefinition const& _node) const
 {
 	yulAssert(!_node.name.empty(), "Invalid function name.");
-	Json::Value ret = createAstNode(nativeLocationOf(_node), "YulFunctionDefinition");
+	Json::Value ret = createAstNode(originLocationOf(_node), nativeLocationOf(_node), "YulFunctionDefinition");
 	ret["name"] = _node.name.str();
 	for (auto const& var: _node.parameters)
 		ret["parameters"].append((*this)(var));
@@ -133,7 +133,7 @@ Json::Value AsmJsonConverter::operator()(FunctionDefinition const& _node) const
 Json::Value AsmJsonConverter::operator()(If const& _node) const
 {
 	yulAssert(_node.condition, "Invalid if condition.");
-	Json::Value ret = createAstNode(nativeLocationOf(_node), "YulIf");
+	Json::Value ret = createAstNode(originLocationOf(_node), nativeLocationOf(_node), "YulIf");
 	ret["condition"] = std::visit(*this, *_node.condition);
 	ret["body"] = (*this)(_node.body);
 	return ret;
@@ -142,7 +142,7 @@ Json::Value AsmJsonConverter::operator()(If const& _node) const
 Json::Value AsmJsonConverter::operator()(Switch const& _node) const
 {
 	yulAssert(_node.expression, "Invalid expression pointer.");
-	Json::Value ret = createAstNode(nativeLocationOf(_node), "YulSwitch");
+	Json::Value ret = createAstNode(originLocationOf(_node), nativeLocationOf(_node), "YulSwitch");
 	ret["expression"] = std::visit(*this, *_node.expression);
 	for (auto const& var: _node.cases)
 		ret["cases"].append((*this)(var));
@@ -151,7 +151,7 @@ Json::Value AsmJsonConverter::operator()(Switch const& _node) const
 
 Json::Value AsmJsonConverter::operator()(Case const& _node) const
 {
-	Json::Value ret = createAstNode(nativeLocationOf(_node), "YulCase");
+	Json::Value ret = createAstNode(originLocationOf(_node), nativeLocationOf(_node), "YulCase");
 	ret["value"] = _node.value ? (*this)(*_node.value) : "default";
 	ret["body"] = (*this)(_node.body);
 	return ret;
@@ -160,7 +160,7 @@ Json::Value AsmJsonConverter::operator()(Case const& _node) const
 Json::Value AsmJsonConverter::operator()(ForLoop const& _node) const
 {
 	yulAssert(_node.condition, "Invalid for loop condition.");
-	Json::Value ret = createAstNode(nativeLocationOf(_node), "YulForLoop");
+	Json::Value ret = createAstNode(originLocationOf(_node), nativeLocationOf(_node), "YulForLoop");
 	ret["pre"] = (*this)(_node.pre);
 	ret["condition"] = std::visit(*this, *_node.condition);
 	ret["post"] = (*this)(_node.post);
@@ -170,27 +170,30 @@ Json::Value AsmJsonConverter::operator()(ForLoop const& _node) const
 
 Json::Value AsmJsonConverter::operator()(Break const& _node) const
 {
-	return createAstNode(nativeLocationOf(_node), "YulBreak");
+	return createAstNode(originLocationOf(_node), nativeLocationOf(_node), "YulBreak");
 }
 
 Json::Value AsmJsonConverter::operator()(Continue const& _node) const
 {
-	return createAstNode(nativeLocationOf(_node), "YulContinue");
+	return createAstNode(originLocationOf(_node), nativeLocationOf(_node), "YulContinue");
 }
 
 Json::Value AsmJsonConverter::operator()(Leave const& _node) const
 {
-	return createAstNode(nativeLocationOf(_node), "YulLeave");
+	return createAstNode(originLocationOf(_node), nativeLocationOf(_node), "YulLeave");
 }
 
-Json::Value AsmJsonConverter::createAstNode(langutil::SourceLocation const& _location, string _nodeType) const
+Json::Value AsmJsonConverter::createAstNode(langutil::SourceLocation const& _originLocation, langutil::SourceLocation const& _nativeLocation, string _nodeType) const
 {
 	Json::Value ret{Json::objectValue};
 	ret["nodeType"] = std::move(_nodeType);
-	int length = -1;
-	if (_location.start >= 0 && _location.end >= 0)
-		length = _location.end - _location.start;
-	ret["src"] = to_string(_location.start) + ":" + to_string(length) + ":" + (m_sourceIndex.has_value() ? to_string(m_sourceIndex.value()) : "-1");
+	auto srcLocation = [&](int start, int end) -> string
+	{
+		int length = (start >= 0 && end >= 0 && end >= start) ? end - start : -1;
+		return to_string(start) + ":" + to_string(length) + ":" + (m_sourceIndex.has_value() ? to_string(m_sourceIndex.value()) : "-1");
+	};
+	ret["src"] = srcLocation(_originLocation.start, _originLocation.end);
+	ret["nativeSrc"] = srcLocation(_nativeLocation.start, _nativeLocation.end);
 	return ret;
 }
 

libyul/AsmJsonConverter.h

@@ -62,7 +62,7 @@ class AsmJsonConverter: public boost::static_visitor<Json::Value>
 	Json::Value operator()(Label const& _node) const;
 
 private:
-	Json::Value createAstNode(langutil::SourceLocation const& _location, std::string _nodeType) const;
+	Json::Value createAstNode(langutil::SourceLocation const& _originLocation, langutil::SourceLocation const& _nativeLocation, std::string _nodeType) const;
 	template <class T>
 	Json::Value vectorOfVariantsToJson(std::vector<T> const& vec) const;