Enhancement: Add experimental support for Yul compilation

packages/codec/lib/compilations/utils.ts

@@ -122,7 +122,7 @@ export function shimContracts(
       source,
       ast: <Ast.AstNode>ast,
       compiler,
-      language: inferLanguage(<Ast.AstNode>ast, compiler)
+      language: inferLanguage(<Ast.AstNode>ast, compiler, sourcePath)
     };
     //ast needs to be coerced because schema doesn't quite match our types here...
 
@@ -293,7 +293,7 @@ function extractPrimarySource(sourceMap: string | undefined): number {
     return 0; //in this case (e.g. a Vyper contract with an old-style
     //source map) we infer that it was compiled by itself
   }
-  return parseInt(sourceMap.match(/^[^:]+:[^:]+:([^:]+):/)[1]);
+  return parseInt(sourceMap.match(/^[^:]*:[^:]*:([^:]*):/)[1] || "0");
 }
 
 function normalizeGeneratedSources(
@@ -338,7 +338,8 @@ function isGeneratedSources(
 //HACK, maybe?
 function inferLanguage(
   ast: Ast.AstNode | undefined,
-  compiler: Compiler.CompilerVersion
+  compiler: Compiler.CompilerVersion,
+  sourcePath: string
 ): string | undefined {
   if (ast) {
     if (ast.nodeType === "SourceUnit") {
@@ -354,8 +355,11 @@ function inferLanguage(
     if (compiler.name === "vyper") {
       return "Vyper";
     } else if (compiler.name === "solc") {
-      //if it's solc but no AST, just assume it's Solidity
-      return "Solidity";
+      if (sourcePath.endsWith(".yul")) {
+        return "Yul";
+      } else {
+        return "Solidity";
+      }
     } else {
       return undefined;
     }

packages/compile-solidity/compileWithPragmaAnalysis.js

@@ -2,27 +2,28 @@ const CompilerSupplier = require("./compilerSupplier");
 const Config = require("@truffle/config");
 const semver = require("semver");
 const Profiler = require("./profiler");
-const fse = require("fs-extra");
 const { run } = require("./run");
 const { reportSources } = require("./reportSources");
 const OS = require("os");
 const cloneDeep = require("lodash.clonedeep");
 
 const getSemverExpression = source => {
-  return source.match(/pragma solidity(.*);/)[1] ?
-    source.match(/pragma solidity(.*);/)[1].trim() :
-    undefined;
+  return source.match(/pragma solidity(.*);/)[1]
+    ? source.match(/pragma solidity(.*);/)[1].trim()
+    : undefined;
 };
 
 const getSemverExpressions = sources => {
-  return sources.map(source => getSemverExpression(source)).filter(expression => expression);
+  return sources
+    .map(source => getSemverExpression(source))
+    .filter(expression => expression);
 };
 
 const validateSemverExpressions = semverExpressions => {
-  const { validRange } = semver;
   for (const expression of semverExpressions) {
     if (semver.validRange(expression) === null) {
-      const message = `Invalid semver expression (${expression}) found in` +
+      const message =
+        `Invalid semver expression (${expression}) found in` +
         `one of your contract's imports.`;
       throw new Error(message);
     }
@@ -50,6 +51,14 @@ const throwCompilerVersionNotFound = ({ path, semverExpressions }) => {
 };
 
 const compileWithPragmaAnalysis = async ({ paths, options }) => {
+  //don't compile if there's yul
+  const yulPath = paths.find(path => path.endsWith(".yul"));
+  if (yulPath !== undefined) {
+    throw new Error(
+      `Paths to compile includes Yul source ${yulPath}.  ` +
+        `Pragma analysis is not supported when compiling Yul.`
+    );
+  }
   const filteredPaths = paths.filter(
     path => path.endsWith(".sol") || path.endsWith(".json")
   );
@@ -73,7 +82,8 @@ const compileWithPragmaAnalysis = async ({ paths, options }) => {
       semverExpressions: [getSemverExpression(source)]
     });
     if (!parserVersion) {
-      const m = `Could not find a pragma expression in ${path}. To use the ` +
+      const m =
+        `Could not find a pragma expression in ${path}. To use the ` +
         `"pragma" compiler setting your contracts must contain a pragma ` +
         `expression.`;
       throw new Error(m);
@@ -129,10 +139,7 @@ const compileWithPragmaAnalysis = async ({ paths, options }) => {
     compilationOptions.compilers.solc.version = compilerVersion;
 
     const config = Config.default().with(compilationOptions);
-    const compilation = await run(
-      versionsAndSources[compilerVersion],
-      config
-    );
+    const compilation = await run(versionsAndSources[compilerVersion], config);
     if (compilation.contracts.length > 0) {
       compilations.push(compilation);
     }

packages/compile-solidity/index.js

@@ -8,16 +8,40 @@ const { normalizeOptions } = require("./normalizeOptions");
 const { compileWithPragmaAnalysis } = require("./compileWithPragmaAnalysis");
 const { reportSources } = require("./reportSources");
 const expect = require("@truffle/expect");
+const partition = require("lodash.partition");
+const fs = require("fs-extra");
 
 const Compile = {
   // this takes an object with keys being the name and values being source
   // material as well as an options object
   async sources({ sources, options }) {
     options = Config.default().merge(options);
-    const compilation = await run(sources, normalizeOptions(options));
-    return compilation.contracts.length > 0
-      ? { compilations: [compilation] }
-      : { compilations: [] };
+    options = normalizeOptions(options);
+    //note: "solidity" here includes JSON as well!
+    const [yulNames, solidityNames] = partition(Object.keys(sources), name =>
+      name.endsWith(".yul")
+    );
+    const soliditySources = Object.assign(
+      {},
+      ...solidityNames.map(name => ({ [name]: sources[name] }))
+    );
+    let solidityCompilations = [];
+    let yulCompilations = [];
+    if (solidityNames.length > 0) {
+      debug("Compiling Solidity (specified sources)");
+      const compilation = await run(soliditySources, options);
+      debug("Compiled Solidity");
+      if (compilation.contracts.length > 0) {
+        solidityCompilations = [compilation];
+      }
+    }
+    for (const name of yulNames) {
+      debug("Compiling Yul (specified sources)");
+      const compilation = await run({ [name]: sources[name] }, options, "Yul");
+      debug("Compiled Yul");
+      yulCompilations.push(compilation);
+    }
+    return { compilations: [...solidityCompilations, ...yulCompilations] };
   },
 
   async all(options) {
@@ -63,54 +87,76 @@ const Compile = {
     ]);
 
     options = Config.default().merge(options);
+    options = normalizeOptions(options);
+
+    //note: solidityPaths here still includes JSON as well!
+    const [yulPaths, solidityPaths] = partition(paths, path =>
+      path.endsWith(".yul")
+    );
 
     debug("invoking profiler");
+    //only invoke profiler on Solidity, not Yul
     const { allSources, compilationTargets } = await Profiler.requiredSources(
       options.with({
-        paths,
+        paths: solidityPaths,
         base_path: options.contracts_directory,
         resolver: options.resolver
       })
     );
-    debug("allSources: %O", allSources);
     debug("compilationTargets: %O", compilationTargets);
 
-    // we can exit if there are no Solidity files to compile since
+    // we can exit if there are no Solidity/Yul files to compile since
     // it indicates that we only have Vyper-related JSON
     const solidityTargets = compilationTargets.filter(fileName =>
       fileName.endsWith(".sol")
     );
-    if (solidityTargets.length === 0) {
+    if (solidityTargets.length === 0 && yulPaths.length === 0) {
       return { compilations: [] };
     }
 
-    reportSources({ paths: compilationTargets, options });
+    reportSources({ paths: [...compilationTargets, ...yulPaths], options });
 
-    // when there are no sources, don't call run
-    if (Object.keys(allSources).length === 0) {
-      return { compilations: [] };
+    let solidityCompilations = [];
+    let yulCompilations = [];
+
+    // only call run if there are sources to run on!
+    if (Object.keys(allSources).length > 0) {
+      const solidityOptions = options.with({ compilationTargets });
+      debug("Compiling Solidity");
+      const compilation = await run(allSources, solidityOptions);
+      debug("Solidity compiled successfully");
+
+      // returns CompilerResult - see @truffle/compile-common
+      if (compilation.contracts.length > 0) {
+        solidityCompilations = [compilation];
+      }
     }
 
-    options.compilationTargets = compilationTargets;
-    const { sourceIndexes, sources, contracts, compiler } = await run(
-      allSources,
-      normalizeOptions(options)
-    );
+    for (const path of yulPaths) {
+      const yulOptions = options.with({ compilationTargets: [path] });
+      //load up Yul sources, since they weren't loaded up earlier
+      //(we'll just use FS for this rather than going through the resolver,
+      //for simplicity, since there are no imports to worry about)
+      const yulSource = fs.readFileSync(path, { encoding: "utf8" });
+      debug("Compiling Yul");
+      const compilation = await run({ [path]: yulSource }, yulOptions, "Yul");
+      debug("Yul compiled successfully");
+
+      // returns CompilerResult - see @truffle/compile-common
+      if (compilation.contracts.length > 0) {
+        yulCompilations.push(compilation);
+      }
+    }
+    if (yulPaths.length > 0 && !options.quiet) {
+      //replacement for individual Yul warnings
+      options.logger.log(
+        "> Warning: Yul is still experimental. Avoid using it in live deployments."
+      );
+    }
 
-    const { name, version } = compiler;
-    // returns CompilerResult - see @truffle/compile-common
-    return contracts.length > 0
-      ? {
-          compilations: [
-            {
-              sourceIndexes,
-              sources,
-              contracts,
-              compiler: { name, version }
-            }
-          ]
-        }
-      : { compilations: [] };
+    return {
+      compilations: [...solidityCompilations, ...yulCompilations]
+    };
   },
 
   async sourcesWithPragmaAnalysis({ paths, options }) {

packages/compile-solidity/package.json

@@ -22,6 +22,7 @@
     "debug": "^4.3.1",
     "fs-extra": "^9.1.0",
     "lodash.clonedeep": "^4.5.0",
+    "lodash.partition": "^4.6.0",
     "ora": "^3.4.0",
     "original-require": "^1.0.1",
     "request": "^2.85.0",