diff --git a/packages/codec/lib/compilations/utils.ts b/packages/codec/lib/compilations/utils.ts index d4a608151d5..89e2296d2d8 100644 --- a/packages/codec/lib/compilations/utils.ts +++ b/packages/codec/lib/compilations/utils.ts @@ -122,7 +122,7 @@ export function shimContracts( source, ast: ast, compiler, - language: inferLanguage(ast, compiler) + language: inferLanguage(ast, compiler, sourcePath) }; //ast needs to be coerced because schema doesn't quite match our types here... @@ -293,7 +293,7 @@ function extractPrimarySource(sourceMap: string | undefined): number { return 0; //in this case (e.g. a Vyper contract with an old-style //source map) we infer that it was compiled by itself } - return parseInt(sourceMap.match(/^[^:]+:[^:]+:([^:]+):/)[1]); + return parseInt(sourceMap.match(/^[^:]*:[^:]*:([^:]*):/)[1] || "0"); } function normalizeGeneratedSources( @@ -338,7 +338,8 @@ function isGeneratedSources( //HACK, maybe? function inferLanguage( ast: Ast.AstNode | undefined, - compiler: Compiler.CompilerVersion + compiler: Compiler.CompilerVersion, + sourcePath: string ): string | undefined { if (ast) { if (ast.nodeType === "SourceUnit") { @@ -354,8 +355,11 @@ function inferLanguage( if (compiler.name === "vyper") { return "Vyper"; } else if (compiler.name === "solc") { - //if it's solc but no AST, just assume it's Solidity - return "Solidity"; + if (sourcePath.endsWith(".yul")) { + return "Yul"; + } else { + return "Solidity"; + } } else { return undefined; } diff --git a/packages/compile-solidity/compileWithPragmaAnalysis.js b/packages/compile-solidity/compileWithPragmaAnalysis.js index f801470a787..741e88f5fd5 100644 --- a/packages/compile-solidity/compileWithPragmaAnalysis.js +++ b/packages/compile-solidity/compileWithPragmaAnalysis.js @@ -2,27 +2,28 @@ const CompilerSupplier = require("./compilerSupplier"); const Config = require("@truffle/config"); const semver = require("semver"); const Profiler = require("./profiler"); -const fse = require("fs-extra"); const { run } = require("./run"); const { reportSources } = require("./reportSources"); const OS = require("os"); const cloneDeep = require("lodash.clonedeep"); const getSemverExpression = source => { - return source.match(/pragma solidity(.*);/)[1] ? - source.match(/pragma solidity(.*);/)[1].trim() : - undefined; + return source.match(/pragma solidity(.*);/)[1] + ? source.match(/pragma solidity(.*);/)[1].trim() + : undefined; }; const getSemverExpressions = sources => { - return sources.map(source => getSemverExpression(source)).filter(expression => expression); + return sources + .map(source => getSemverExpression(source)) + .filter(expression => expression); }; const validateSemverExpressions = semverExpressions => { - const { validRange } = semver; for (const expression of semverExpressions) { if (semver.validRange(expression) === null) { - const message = `Invalid semver expression (${expression}) found in` + + const message = + `Invalid semver expression (${expression}) found in` + `one of your contract's imports.`; throw new Error(message); } @@ -50,6 +51,14 @@ const throwCompilerVersionNotFound = ({ path, semverExpressions }) => { }; const compileWithPragmaAnalysis = async ({ paths, options }) => { + //don't compile if there's yul + const yulPath = paths.find(path => path.endsWith(".yul")); + if (yulPath !== undefined) { + throw new Error( + `Paths to compile includes Yul source ${yulPath}. ` + + `Pragma analysis is not supported when compiling Yul.` + ); + } const filteredPaths = paths.filter( path => path.endsWith(".sol") || path.endsWith(".json") ); @@ -73,7 +82,8 @@ const compileWithPragmaAnalysis = async ({ paths, options }) => { semverExpressions: [getSemverExpression(source)] }); if (!parserVersion) { - const m = `Could not find a pragma expression in ${path}. To use the ` + + const m = + `Could not find a pragma expression in ${path}. To use the ` + `"pragma" compiler setting your contracts must contain a pragma ` + `expression.`; throw new Error(m); @@ -129,10 +139,7 @@ const compileWithPragmaAnalysis = async ({ paths, options }) => { compilationOptions.compilers.solc.version = compilerVersion; const config = Config.default().with(compilationOptions); - const compilation = await run( - versionsAndSources[compilerVersion], - config - ); + const compilation = await run(versionsAndSources[compilerVersion], config); if (compilation.contracts.length > 0) { compilations.push(compilation); } diff --git a/packages/compile-solidity/index.js b/packages/compile-solidity/index.js index 659ab8050ab..3f3d090de67 100644 --- a/packages/compile-solidity/index.js +++ b/packages/compile-solidity/index.js @@ -8,16 +8,66 @@ const { normalizeOptions } = require("./normalizeOptions"); const { compileWithPragmaAnalysis } = require("./compileWithPragmaAnalysis"); const { reportSources } = require("./reportSources"); const expect = require("@truffle/expect"); +const partition = require("lodash.partition"); +const fs = require("fs-extra"); + +async function compileYulPaths(yulPaths, options) { + let yulCompilations = []; + for (const path of yulPaths) { + const yulOptions = options.with({ compilationTargets: [path] }); + //load up Yul sources, since they weren't loaded up earlier + //(we'll just use FS for this rather than going through the resolver, + //for simplicity, since there are no imports to worry about) + const yulSource = fs.readFileSync(path, { encoding: "utf8" }); + debug("Compiling Yul"); + const compilation = await run({ [path]: yulSource }, yulOptions, "Yul"); + debug("Yul compiled successfully"); + + // returns CompilerResult - see @truffle/compile-common + if (compilation.contracts.length > 0) { + yulCompilations.push(compilation); + } + } + if (yulPaths.length > 0 && !options.quiet) { + //replacement for individual Yul warnings + options.logger.log( + "> Warning: Yul is still experimental. Avoid using it in live deployments." + ); + } + return yulCompilations; +} const Compile = { // this takes an object with keys being the name and values being source // material as well as an options object async sources({ sources, options }) { options = Config.default().merge(options); - const compilation = await run(sources, normalizeOptions(options)); - return compilation.contracts.length > 0 - ? { compilations: [compilation] } - : { compilations: [] }; + options = normalizeOptions(options); + //note: "solidity" here includes JSON as well! + const [yulNames, solidityNames] = partition(Object.keys(sources), name => + name.endsWith(".yul") + ); + const soliditySources = Object.assign( + {}, + ...solidityNames.map(name => ({ [name]: sources[name] })) + ); + let solidityCompilations = []; + let yulCompilations = []; + if (solidityNames.length > 0) { + debug("Compiling Solidity (specified sources)"); + const compilation = await run(soliditySources, options); + debug("Compiled Solidity"); + if (compilation.contracts.length > 0) { + solidityCompilations = [compilation]; + } + } + for (const name of yulNames) { + debug("Compiling Yul (specified sources)"); + const compilation = await run({ [name]: sources[name] }, options, "Yul"); + debug("Compiled Yul"); + yulCompilations.push(compilation); + } + return { compilations: [...solidityCompilations, ...yulCompilations] }; }, async all(options) { @@ -63,54 +113,54 @@ const Compile = { ]); options = Config.default().merge(options); + options = normalizeOptions(options); + + //note: solidityPaths here still includes JSON as well! + const [yulPaths, solidityPaths] = partition(paths, path => + path.endsWith(".yul") + ); debug("invoking profiler"); + //only invoke profiler on Solidity, not Yul const { allSources, compilationTargets } = await Profiler.requiredSources( options.with({ - paths, + paths: solidityPaths, base_path: options.contracts_directory, resolver: options.resolver }) ); - debug("allSources: %O", allSources); debug("compilationTargets: %O", compilationTargets); - // we can exit if there are no Solidity files to compile since + // we can exit if there are no Solidity/Yul files to compile since // it indicates that we only have Vyper-related JSON const solidityTargets = compilationTargets.filter(fileName => fileName.endsWith(".sol") ); - if (solidityTargets.length === 0) { + if (solidityTargets.length === 0 && yulPaths.length === 0) { return { compilations: [] }; } - reportSources({ paths: compilationTargets, options }); + reportSources({ paths: [...compilationTargets, ...yulPaths], options }); - // when there are no sources, don't call run - if (Object.keys(allSources).length === 0) { - return { compilations: [] }; + let solidityCompilations = []; + // only call run if there are sources to run on! + if (Object.keys(allSources).length > 0) { + const solidityOptions = options.with({ compilationTargets }); + debug("Compiling Solidity"); + const compilation = await run(allSources, solidityOptions); + debug("Solidity compiled successfully"); + + // returns CompilerResult - see @truffle/compile-common + if (compilation.contracts.length > 0) { + solidityCompilations = [compilation]; + } } - options.compilationTargets = compilationTargets; - const { sourceIndexes, sources, contracts, compiler } = await run( - allSources, - normalizeOptions(options) - ); + const yulCompilations = await compileYulPaths(yulPaths, options); - const { name, version } = compiler; - // returns CompilerResult - see @truffle/compile-common - return contracts.length > 0 - ? { - compilations: [ - { - sourceIndexes, - sources, - contracts, - compiler: { name, version } - } - ] - } - : { compilations: [] }; + return { + compilations: [...solidityCompilations, ...yulCompilations] + }; }, async sourcesWithPragmaAnalysis({ paths, options }) { diff --git a/packages/compile-solidity/package.json b/packages/compile-solidity/package.json index d0f361449c9..aca7627538f 100644 --- a/packages/compile-solidity/package.json +++ b/packages/compile-solidity/package.json @@ -22,6 +22,7 @@ "debug": "^4.3.1", "fs-extra": "^9.1.0", "lodash.clonedeep": "^4.5.0", + "lodash.partition": "^4.6.0", "ora": "^3.4.0", "original-require": "^1.0.1", "request": "^2.85.0", diff --git a/packages/compile-solidity/run.js b/packages/compile-solidity/run.js index fc7dda6784b..5ab02893f07 100644 --- a/packages/compile-solidity/run.js +++ b/packages/compile-solidity/run.js @@ -7,7 +7,7 @@ const CompilerSupplier = require("./compilerSupplier"); // this function returns a Compilation - legacy/index.js and ./index.js // both check to make sure rawSources exist before calling this method // however, there is a check here that returns null if no sources exist -async function run(rawSources, options) { +async function run(rawSources, options, language = "Solidity") { if (Object.keys(rawSources).length === 0) { return null; } @@ -24,6 +24,7 @@ async function run(rawSources, options) { const compilerInput = prepareCompilerInput({ sources, targets, + language, settings: options.compilers.solc.settings, modelCheckerSettings: options.compilers.solc.modelCheckerSettings }); @@ -33,6 +34,7 @@ async function run(rawSources, options) { compilerInput, options }); + debug("compilerOutput: %O", compilerOutput); // handle warnings as errors if options.strict // log if not options.quiet @@ -58,9 +60,12 @@ async function run(rawSources, options) { const outputSources = processAllSources({ sources, compilerOutput, - originalSourcePaths + originalSourcePaths, + language }); - const sourceIndexes = outputSources.map(source => source.sourcePath); + const sourceIndexes = outputSources + ? outputSources.map(source => source.sourcePath) + : undefined; //leave undefined if sources undefined return { sourceIndexes, contracts: processContracts({ @@ -78,11 +83,16 @@ async function run(rawSources, options) { } function orderABI({ abi, contractName, ast }) { - // AST can have multiple contract definitions, make sure we have the - // one that matches our contract + if (!abi) { + return []; //Yul doesn't return ABIs, but we require something + } + if (!ast || !ast.nodes) { return abi; } + + // AST can have multiple contract definitions, make sure we have the + // one that matches our contract const contractDefinition = ast.nodes.find( ({ nodeType, name }) => nodeType === "ContractDefinition" && name === contractName @@ -124,11 +134,12 @@ function orderABI({ abi, contractName, ast }) { function prepareCompilerInput({ sources, targets, + language, settings, modelCheckerSettings }) { return { - language: "Solidity", + language, sources: prepareSources({ sources }), settings: { evmVersion: settings.evmVersion, @@ -234,7 +245,10 @@ function detectErrors({ const rawWarnings = options.strict ? [] // none of those in strict mode - : outputErrors.filter(({ severity }) => severity === "warning"); + : outputErrors.filter(({ severity, message }) => + severity === "warning" && + message !== "Yul is still experimental. Please use the output with care." //filter out Yul warning + ); // extract messages let errors = rawErrors.map(({ formattedMessage }) => formattedMessage).join(); @@ -268,8 +282,21 @@ function detectErrors({ * aggregate source information based on compiled output; * this can include sources that do not define any contracts */ -function processAllSources({ sources, compilerOutput, originalSourcePaths }) { - if (!compilerOutput.sources) return []; +function processAllSources({ sources, compilerOutput, originalSourcePaths, language }) { + if (!compilerOutput.sources) { + const entries = Object.entries(sources); + if (entries.length === 1) { + //special case for handling Yul + const [sourcePath, contents] = entries[0]; + return [{ + sourcePath: originalSourcePaths[sourcePath], + contents, + language + }] + } else { + return []; + } + } let outputSources = []; for (const [sourcePath, { id, ast, legacyAST }] of Object.entries( compilerOutput.sources @@ -279,7 +306,7 @@ function processAllSources({ sources, compilerOutput, originalSourcePaths }) { contents: sources[sourcePath], ast, legacyAST, - language: "Solidity" + language }; } return outputSources; @@ -304,8 +331,9 @@ function processContracts({ contractName, contract, source: { - ast: compilerOutput.sources[sourcePath].ast, - legacyAST: compilerOutput.sources[sourcePath].legacyAST, + //some versions of Yul don't have sources in output + ast: ((compilerOutput.sources || {})[sourcePath] || {}).ast, + legacyAST: ((compilerOutput.sources || {})[sourcePath] || {}).legacyAST, contents: sources[sourcePath], sourcePath } @@ -330,13 +358,7 @@ function processContracts({ generatedSources, object: bytecode }, - deployedBytecode: { - sourceMap: deployedSourceMap, - linkReferences: deployedLinkReferences, - generatedSources: deployedGeneratedSources, - immutableReferences, - object: deployedBytecode - } + deployedBytecode: deployedBytecodeInfo //destructured below }, abi, metadata, @@ -358,7 +380,7 @@ function processContracts({ sourcePath: originalSourcePaths[transformedSourcePath], source, sourceMap, - deployedSourceMap, + deployedSourceMap: (deployedBytecodeInfo || {}).sourceMap, ast, legacyAST, bytecode: zeroLinkReferences({ @@ -366,13 +388,14 @@ function processContracts({ linkReferences: formatLinkReferences(linkReferences) }), deployedBytecode: zeroLinkReferences({ - bytes: deployedBytecode, - linkReferences: formatLinkReferences(deployedLinkReferences) + bytes: (deployedBytecodeInfo || {}).object, + linkReferences: formatLinkReferences((deployedBytecodeInfo || {}).linkReferences) }), - immutableReferences, //ideally this would be part of the deployedBytecode object, + immutableReferences: (deployedBytecodeInfo || {}).immutableReferences, + //ideally immutable references would be part of the deployedBytecode object, //but compatibility makes that impossible generatedSources, - deployedGeneratedSources, + deployedGeneratedSources: (deployedBytecodeInfo || {}).generatedSources, compiler: { name: "solc", version: solcVersion @@ -383,6 +406,10 @@ function processContracts({ } function formatLinkReferences(linkReferences) { + if (!linkReferences) { + return []; + } + // convert to flat list const libraryLinkReferences = Object.values(linkReferences) .map(fileLinks => @@ -403,6 +430,9 @@ function formatLinkReferences(linkReferences) { // takes linkReferences in output format (not Solidity's format) function zeroLinkReferences({ bytes, linkReferences }) { + if (bytes === undefined) { + return undefined; + } // inline link references - start by flattening the offsets const flattenedLinkReferences = linkReferences // map each link ref to array of link refs with only one offset diff --git a/packages/compile-solidity/test/sources/yul/YulSource.yul b/packages/compile-solidity/test/sources/yul/YulSource.yul new file mode 100644 index 00000000000..394013a5fc7 --- /dev/null +++ b/packages/compile-solidity/test/sources/yul/YulSource.yul @@ -0,0 +1,13 @@ +object "YulContract" { + code { + let size := datasize("runtime") + datacopy(0, dataoffset("runtime"), size) + return(0, size) + } + object "runtime" { + code { + mstore(0, 1) + return(0, 0x20) + } + } +} diff --git a/packages/compile-solidity/test/test_yul.js b/packages/compile-solidity/test/test_yul.js new file mode 100644 index 00000000000..ce5abdbb53c --- /dev/null +++ b/packages/compile-solidity/test/test_yul.js @@ -0,0 +1,67 @@ +const debug = require("debug")("compile:test:test_yul"); +const path = require("path"); +const { Compile } = require("@truffle/compile-solidity"); +const assert = require("assert"); +const Resolver = require("@truffle/resolver"); + +describe("Yul compilation", function () { + this.timeout(5000); // solc + + const options = { + working_directory: __dirname, + contracts_directory: path.join(__dirname, "./sources/yul"), + contracts_build_directory: path.join(__dirname, "./does/not/matter"), //nothing is actually written, but resolver demands it + compilers: { + solc: { + version: "0.5.17", + settings: { + optimizer: { + enabled: false, + runs: 200 + } + } + } + }, + quiet: true + }; + options.resolver = new Resolver(options); + + it("Compiles Yul", async function () { + this.timeout(150000); + const paths = [ + "YulSource.yul", + ].map(filePath => path.join(options.contracts_directory, filePath)); + + const { compilations } = await Compile.sourcesWithDependencies({ + paths, + options + }); + + //is there 1 compilation? + assert.equal(compilations.length, 1); + //do all compilations have sources? + assert.ok(compilations[0].sources); + assert.equal(compilations[0].sources.length, 1); + //do all compilations have contracts? + assert.ok(compilations[0].contracts); + assert.equal(compilations[0].contracts.length, 1); + //do they all have compiler? + assert.ok(compilations[0].compiler); + //are they Yul? + assert.equal(compilations[0].sources[0].language, "Yul"); + //do they all have contents and sourcePath? + assert.ok(compilations[0].sources[0].contents); + assert.ok(compilations[0].sources[0].sourcePath); + //do they all have a contract name? + assert.ok(compilations[0].contracts[0].contractName); + //do they all have an ABI? + assert.ok(compilations[0].contracts[0].abi); + //do the Yul sources have empty ABI? + assert.equal(compilations[0].contracts[0].abi.length, 0); + //do they all have constructor bytecode? + assert.ok(compilations[0].contracts[0].bytecode.bytes); + //do they all have source & sourcePath? + assert.ok(compilations[0].contracts[0].source); + assert.ok(compilations[0].contracts[0].sourcePath); + }); +}); diff --git a/packages/debugger/lib/data/sagas/index.js b/packages/debugger/lib/data/sagas/index.js index 621bb1eea2f..eae0320bfe6 100644 --- a/packages/debugger/lib/data/sagas/index.js +++ b/packages/debugger/lib/data/sagas/index.js @@ -163,6 +163,7 @@ export function* decodeReturnValue() { } //at this point, result.value holds the final value debug("done decoding"); + debug("decoded value: %O", result.value); return result.value; } diff --git a/packages/source-map-utils/index.js b/packages/source-map-utils/index.js index b48aad9d219..48769832025 100644 --- a/packages/source-map-utils/index.js +++ b/packages/source-map-utils/index.js @@ -39,7 +39,11 @@ var SourceMapUtils = { getHumanReadableSourceMap: function (sourceMap) { const instructions = sourceMap.split(";"); - let processedInstruction = {}; //persists across instructions for when info doesn't change + let processedInstruction = { + start: 0, + length: 0, + file: 0 + }; //persists across instructions for when info doesn't change let processedSourceMap = []; //JS doesn't have scan, so we'll do this scan manually