diff --git a/Cargo.lock b/Cargo.lock index 1633037fbe72..f352c5cf5f3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2008,6 +2008,7 @@ dependencies = [ "parking_lot 0.12.0", "proptest", "revm", + "semver", "serde", "serde_json", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index 6b6fab33cb86..83cc47c88cd7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,4 +61,4 @@ debug = 0 #ethers-providers = { path = "../ethers-rs/ethers-providers" } #ethers-signers = { path = "../ethers-rs/ethers-signers" } #ethers-etherscan = { path = "../ethers-rs/ethers-etherscan" } -#ethers-solc = { path = "../ethers-rs/ethers-solc" } +#ethers-solc = { path = "../ethers-rs/ethers-solc" } \ No newline at end of file diff --git a/cli/src/cmd/forge/coverage.rs b/cli/src/cmd/forge/coverage.rs new file mode 100644 index 000000000000..e18d8c0b4601 --- /dev/null +++ b/cli/src/cmd/forge/coverage.rs @@ -0,0 +1,268 @@ +//! Coverage command +use crate::{ + cmd::{ + forge::{build::CoreBuildArgs, test::Filter}, + Cmd, + }, + compile::ProjectCompiler, + utils::{self, p_println, FoundryPathExt}, +}; +use cast::trace::identifier::TraceIdentifier; +use clap::{AppSettings, ArgEnum, Parser}; +use ethers::{ + prelude::{Artifact, Project, ProjectCompileOutput}, + solc::{artifacts::contract::CompactContractBytecode, sourcemap::SourceMap, ArtifactId}, +}; +use forge::{ + coverage::{ + CoverageMap, CoverageReporter, DebugReporter, LcovReporter, SummaryReporter, Visitor, + }, + executor::opts::EvmOpts, + result::SuiteResult, + trace::identifier::LocalTraceIdentifier, + MultiContractRunnerBuilder, +}; +use foundry_common::{evm::EvmArgs, fs}; +use foundry_config::{figment::Figment, Config}; +use std::{collections::HashMap, path::PathBuf, sync::mpsc::channel, thread}; + +// Loads project's figment and merges the build cli arguments into it +foundry_config::impl_figment_convert!(CoverageArgs, opts, evm_opts); + +/// Generate coverage reports for your tests. +#[derive(Debug, Clone, Parser)] +#[clap(global_setting = AppSettings::DeriveDisplayOrder)] +pub struct CoverageArgs { + #[clap( + long, + arg_enum, + default_value = "summary", + help = "The report type to use for coverage." + )] + report: CoverageReportKind, + + #[clap(flatten, next_help_heading = "TEST FILTERING")] + filter: Filter, + + #[clap(flatten, next_help_heading = "EVM OPTIONS")] + evm_opts: EvmArgs, + + #[clap(flatten, next_help_heading = "BUILD OPTIONS")] + opts: CoreBuildArgs, +} + +impl CoverageArgs { + /// Returns the flattened [`CoreBuildArgs`] + pub fn build_args(&self) -> &CoreBuildArgs { + &self.opts + } + + /// Returns the currently configured [Config] and the extracted [EvmOpts] from that config + pub fn config_and_evm_opts(&self) -> eyre::Result<(Config, EvmOpts)> { + // Merge all configs + let figment: Figment = self.into(); + let evm_opts = figment.extract()?; + let config = Config::from_provider(figment).sanitized(); + + Ok((config, evm_opts)) + } +} + +impl Cmd for CoverageArgs { + type Output = (); + + fn run(self) -> eyre::Result { + let (config, evm_opts) = self.configure()?; + let (project, output) = self.build(&config)?; + p_println!(!self.opts.silent => "Analysing contracts..."); + let (map, source_maps) = self.prepare(output.clone())?; + + p_println!(!self.opts.silent => "Running tests..."); + self.collect(project, output, source_maps, map, config, evm_opts) + } +} + +/// A map, keyed by artifact ID, to a tuple of the deployment source map and the runtime source map. +type SourceMaps = HashMap; + +// The main flow of the command itself +impl CoverageArgs { + /// Collects and adjusts configuration. + fn configure(&self) -> eyre::Result<(Config, EvmOpts)> { + // Merge all configs + let (config, mut evm_opts) = self.config_and_evm_opts()?; + + // We always want traces + evm_opts.verbosity = 3; + + Ok((config, evm_opts)) + } + + /// Builds the project. + fn build(&self, config: &Config) -> eyre::Result<(Project, ProjectCompileOutput)> { + // Set up the project + let project = { + let mut project = config.ephemeral_no_artifacts_project()?; + + // Disable the optimizer for more accurate source maps + project.solc_config.settings.optimizer.disable(); + + project + }; + + let output = ProjectCompiler::default() + .compile(&project)? + .with_stripped_file_prefixes(project.root()); + + Ok((project, output)) + } + + /// Builds the coverage map. + fn prepare(&self, output: ProjectCompileOutput) -> eyre::Result<(CoverageMap, SourceMaps)> { + // Get sources and source maps + let (artifacts, sources) = output.into_artifacts_with_sources(); + + let source_maps: SourceMaps = artifacts + .into_iter() + .map(|(id, artifact)| (id, CompactContractBytecode::from(artifact))) + .filter_map(|(id, artifact): (ArtifactId, CompactContractBytecode)| { + Some(( + id, + ( + artifact.get_source_map()?.ok()?, + artifact + .get_deployed_bytecode() + .as_ref()? + .bytecode + .as_ref()? + .source_map()? + .ok()?, + ), + )) + }) + .collect(); + + let mut map = CoverageMap::default(); + for (path, versioned_sources) in sources.0.into_iter() { + // TODO: Make these checks robust + // NOTE: We should actually filter out test contracts in the AST + // instead of on a source file level. Repositories like Solmate + // have a lot of abstract contracts that are being tested, and these + // are usually defined in the test files themselves. + let is_test = path.is_sol_test(); + let is_dependency = path.starts_with("lib"); + if is_test || is_dependency { + continue + } + + for mut versioned_source in versioned_sources { + let source = &mut versioned_source.source_file; + if let Some(ast) = source.ast.take() { + let source_maps: HashMap = source_maps + .iter() + .filter(|(id, _)| { + id.version == versioned_source.version && + id.source == PathBuf::from(&path) + }) + .map(|(id, (_, source_map))| { + // TODO: Deploy source map too? + (id.name.clone(), source_map.clone()) + }) + .collect(); + + let items = Visitor::new(source.id, fs::read_to_string(&path)?, source_maps) + .visit_ast(ast)?; + + if items.is_empty() { + continue + } + + map.add_source(path.clone(), versioned_source, items); + } + } + } + + Ok((map, source_maps)) + } + + /// Runs tests, collects coverage data and generates the final report. + fn collect( + self, + project: Project, + output: ProjectCompileOutput, + source_maps: SourceMaps, + mut map: CoverageMap, + config: Config, + evm_opts: EvmOpts, + ) -> eyre::Result<()> { + // Setup the fuzzer + // TODO: Add CLI Options to modify the persistence + let cfg = proptest::test_runner::Config { + failure_persistence: None, + cases: config.fuzz_runs, + max_local_rejects: config.fuzz_max_local_rejects, + max_global_rejects: config.fuzz_max_global_rejects, + ..Default::default() + }; + let fuzzer = proptest::test_runner::TestRunner::new(cfg); + let root = project.paths.root; + + // Build the contract runner + let evm_spec = utils::evm_spec(&config.evm_version); + let mut runner = MultiContractRunnerBuilder::default() + .fuzzer(fuzzer) + .initial_balance(evm_opts.initial_balance) + .evm_spec(evm_spec) + .sender(evm_opts.sender) + .with_fork(utils::get_fork(&evm_opts, &config.rpc_storage_caching)) + .set_coverage(true) + .build(root.clone(), output, evm_opts)?; + let (tx, rx) = channel::<(String, SuiteResult)>(); + + // Set up identifier + let local_identifier = LocalTraceIdentifier::new(&runner.known_contracts); + + // TODO: Coverage for fuzz tests + let handle = thread::spawn(move || runner.test(&self.filter, Some(tx), false).unwrap()); + for mut result in rx.into_iter().flat_map(|(_, suite)| suite.test_results.into_values()) { + if let Some(hit_map) = result.coverage.take() { + for (_, trace) in &mut result.traces { + local_identifier + .identify_addresses(trace.addresses().into_iter().collect()) + .into_iter() + .filter_map(|identity| { + let artifact_id = identity.artifact_id?; + let source_map = source_maps.get(&artifact_id)?; + + Some((artifact_id, source_map, hit_map.get(&identity.address)?)) + }) + .for_each(|(id, source_map, hits)| { + // TODO: Distinguish between creation/runtime in a smart way + map.add_hit_map(id.version.clone(), &source_map.0, hits.clone()); + map.add_hit_map(id.version, &source_map.1, hits.clone()) + }); + } + } + } + + // Reattach the thread + let _ = handle.join(); + + match self.report { + CoverageReportKind::Summary => SummaryReporter::default().report(map), + // TODO: Sensible place to put the LCOV file + CoverageReportKind::Lcov => { + LcovReporter::new(&mut fs::create_file(root.join("lcov.info"))?).report(map) + } + CoverageReportKind::Debug => DebugReporter::default().report(map), + } + } +} + +// TODO: HTML +#[derive(Debug, Clone, ArgEnum)] +pub enum CoverageReportKind { + Summary, + Lcov, + Debug, +} diff --git a/cli/src/cmd/forge/mod.rs b/cli/src/cmd/forge/mod.rs index e14dfc198f68..b01a3057cf41 100644 --- a/cli/src/cmd/forge/mod.rs +++ b/cli/src/cmd/forge/mod.rs @@ -41,6 +41,7 @@ pub mod bind; pub mod build; pub mod cache; pub mod config; +pub mod coverage; pub mod create; pub mod debug; pub mod flatten; diff --git a/cli/src/cmd/forge/script/executor.rs b/cli/src/cmd/forge/script/executor.rs index 6b3cc901d8f4..974774d68a39 100644 --- a/cli/src/cmd/forge/script/executor.rs +++ b/cli/src/cmd/forge/script/executor.rs @@ -159,20 +159,15 @@ impl ScriptArgs { ) .await; - let mut builder = ExecutorBuilder::default() + let executor = ExecutorBuilder::default() .with_cheatcodes(CheatsConfig::new(&script_config.config, &script_config.evm_opts)) .with_config(env) .with_spec(crate::utils::evm_spec(&script_config.config.evm_version)) - .with_gas_limit(script_config.evm_opts.gas_limit()); + .with_gas_limit(script_config.evm_opts.gas_limit()) + .set_tracing(script_config.evm_opts.verbosity >= 3 || self.debug) + .set_debugger(self.debug) + .build(db); - if script_config.evm_opts.verbosity >= 3 { - builder = builder.with_tracing(); - } - - if self.debug { - builder = builder.with_tracing().with_debugger(); - } - - ScriptRunner::new(builder.build(db), script_config.evm_opts.initial_balance, sender) + ScriptRunner::new(executor, script_config.evm_opts.initial_balance, sender) } } diff --git a/cli/src/forge.rs b/cli/src/forge.rs index 592e4fe24614..5b1d95d540a6 100644 --- a/cli/src/forge.rs +++ b/cli/src/forge.rs @@ -35,6 +35,9 @@ fn main() -> eyre::Result<()> { Subcommands::Script(cmd) => { utils::block_on(cmd.run_script())?; } + Subcommands::Coverage(cmd) => { + cmd.run()?; + } Subcommands::Bind(cmd) => { cmd.run()?; } diff --git a/cli/src/opts/forge.rs b/cli/src/opts/forge.rs index 88a4960bdad6..bbef0b2ba9e9 100644 --- a/cli/src/opts/forge.rs +++ b/cli/src/opts/forge.rs @@ -7,7 +7,7 @@ use crate::cmd::forge::{ bind::BindArgs, build::BuildArgs, cache::CacheArgs, - config, + config, coverage, create::CreateArgs, debug::DebugArgs, flatten, @@ -59,7 +59,10 @@ pub enum Subcommands { #[clap(visible_alias = "s")] Script(ScriptArgs), - #[clap(visible_alias = "bi")] + #[clap(about = "Generate coverage reports.")] + Coverage(coverage::CoverageArgs), + + #[clap(alias = "bi")] #[clap(about = "Generate Rust bindings for smart contracts.")] Bind(BindArgs), diff --git a/evm/Cargo.toml b/evm/Cargo.toml index e927523c8d0a..00f9dcb0965b 100644 --- a/evm/Cargo.toml +++ b/evm/Cargo.toml @@ -46,5 +46,8 @@ proptest = "1.0.0" yansi = "0.5.1" url = "2.2.2" +# Coverage +semver = "1.0.5" + [dev-dependencies] tempfile = "3.3.0" diff --git a/evm/src/coverage/mod.rs b/evm/src/coverage/mod.rs new file mode 100644 index 000000000000..abf5ab82a6aa --- /dev/null +++ b/evm/src/coverage/mod.rs @@ -0,0 +1,346 @@ +mod visitor; +pub use visitor::Visitor; + +use ethers::{ + prelude::{sourcemap::SourceMap, sources::VersionedSourceFile}, + types::Address, +}; +use semver::Version; +use std::{ + collections::{BTreeMap, HashMap}, + fmt::Display, + ops::AddAssign, + path::PathBuf, + usize, +}; + +/// A coverage map. +/// +/// The coverage map collects coverage items for sources. It also converts hit data from +/// [HitMap]s into their appropriate coverage items. +/// +/// You **MUST** add all the sources before you start adding hit data. +#[derive(Default, Debug, Clone)] +pub struct CoverageMap { + /// A map of `(version, source id)` -> `source file` + sources: HashMap<(Version, u32), SourceFile>, +} + +impl CoverageMap { + /// Adds coverage items and a source map for the given source. + /// + /// Sources are identified by path, and then by source ID and version. + /// + /// We need both the version and the source ID in case the project has distinct file + /// hierarchies that use different compiler versions, as that will result in multiple compile + /// jobs, and source IDs are only guaranteed to be stable across a single compile job. + pub fn add_source( + &mut self, + path: impl Into, + source: VersionedSourceFile, + items: Vec, + ) { + let VersionedSourceFile { version, source_file: source } = source; + + self.sources.insert((version, source.id), SourceFile { path: path.into(), items }); + } + + /// Processes data from a [HitMap] and sets hit counts for coverage items in this coverage map. + /// + /// This function should only be called *after* all the relevant sources have been processed and + /// added to the map (see [add_source]). + /// + /// NOTE(onbjerg): I've made an assumption here that the coverage items are laid out in + /// sorted order, ordered by their anchors. + /// + /// This assumption is based on the way we process the AST - we start at the root node, + /// and work our way down. If we change up how we process the AST, we *have* to either + /// change this logic to work with unsorted data, or sort the data prior to calling + /// this function. + pub fn add_hit_map( + &mut self, + source_version: Version, + source_map: &SourceMap, + hit_map: HitMap, + ) { + for (ic, instruction_hits) in hit_map.hits.into_iter() { + if instruction_hits == 0 { + continue + } + + // Get the source ID in the source map. + let source_id = + if let Some(source_id) = source_map.get(ic).and_then(|element| element.index) { + source_id + } else { + continue + }; + + // Get the coverage items corresponding to the source ID in the source map. + if let Some(source) = self.sources.get_mut(&(source_version.clone(), source_id)) { + for item in source.items.iter_mut() { + // We've reached a point where we will no longer be able to map this + // instruction to coverage items + /*if ic > item.anchor() { + break; + }*/ + + // We found a matching coverage item, but there may be more + if ic == item.anchor() { + item.increment_hits(instruction_hits); + } + } + } + } + } +} + +impl IntoIterator for CoverageMap { + type Item = SourceFile; + type IntoIter = std::collections::hash_map::IntoValues<(Version, u32), Self::Item>; + + fn into_iter(self) -> Self::IntoIter { + self.sources.into_values() + } +} + +/// A collection of [HitMap]s +pub type HitMaps = HashMap; + +/// Hit data for an address. +/// +/// Contains low-level data about hit counters for the instructions in the bytecode of a contract. +#[derive(Debug, Clone, Default)] +pub struct HitMap { + hits: BTreeMap, +} + +impl HitMap { + /// Increase the hit counter for the given instruction counter. + pub fn hit(&mut self, ic: usize) { + *self.hits.entry(ic).or_default() += 1; + } +} + +/// A source file. +#[derive(Default, Debug, Clone)] +pub struct SourceFile { + pub path: PathBuf, + pub items: Vec, +} + +impl SourceFile { + /// Get a simple summary of the coverage for the file. + pub fn summary(&self) -> CoverageSummary { + self.items.iter().fold(CoverageSummary::default(), |mut summary, item| match item { + CoverageItem::Line { hits, .. } => { + summary.line_count += 1; + if *hits > 0 { + summary.line_hits += 1; + } + summary + } + CoverageItem::Statement { hits, .. } => { + summary.statement_count += 1; + if *hits > 0 { + summary.statement_hits += 1; + } + summary + } + CoverageItem::Branch { hits, .. } => { + summary.branch_count += 1; + if *hits > 0 { + summary.branch_hits += 1; + } + summary + } + CoverageItem::Function { hits, .. } => { + summary.function_count += 1; + if *hits > 0 { + summary.function_hits += 1; + } + summary + } + }) + } +} + +#[derive(Clone, Debug)] +pub enum CoverageItem { + /// An executable line in the code. + Line { + /// The location of the line in the source code. + loc: SourceLocation, + /// The instruction counter that covers this line. + anchor: usize, + /// The number of times this item was hit. + hits: u64, + }, + + /// A statement in the code. + Statement { + /// The location of the statement in the source code. + loc: SourceLocation, + /// The instruction counter that covers this statement. + anchor: usize, + /// The number of times this statement was hit. + hits: u64, + }, + + /// A branch in the code. + Branch { + /// The location of the branch in the source code. + loc: SourceLocation, + /// The instruction counter that covers this branch. + anchor: usize, + /// The ID that identifies the branch. + /// + /// There may be multiple items with the same branch ID - they belong to the same branch, + /// but represent different paths. + branch_id: usize, + /// The path ID for this branch. + /// + /// The first path has ID 0, the next ID 1, and so on. + path_id: usize, + /// The branch kind. + kind: BranchKind, + /// The number of times this item was hit. + hits: u64, + }, + + /// A function in the code. + Function { + /// The location of the function in the source code. + loc: SourceLocation, + /// The instruction counter that covers this function. + anchor: usize, + /// The name of the function. + name: String, + /// The number of times this item was hit. + hits: u64, + }, +} + +impl CoverageItem { + pub fn source_location(&self) -> &SourceLocation { + match self { + Self::Line { loc, .. } | + Self::Statement { loc, .. } | + Self::Branch { loc, .. } | + Self::Function { loc, .. } => loc, + } + } + + pub fn anchor(&self) -> usize { + match self { + Self::Line { anchor, .. } | + Self::Statement { anchor, .. } | + Self::Branch { anchor, .. } | + Self::Function { anchor, .. } => *anchor, + } + } + + pub fn increment_hits(&mut self, delta: u64) { + match self { + Self::Line { hits, .. } | + Self::Statement { hits, .. } | + Self::Branch { hits, .. } | + Self::Function { hits, .. } => *hits += delta, + } + } + + pub fn hits(&self) -> u64 { + match self { + Self::Line { hits, .. } | + Self::Statement { hits, .. } | + Self::Branch { hits, .. } | + Self::Function { hits, .. } => *hits, + } + } +} + +impl Display for CoverageItem { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self { + CoverageItem::Line { loc, anchor, hits } => { + write!(f, "Line (location: {loc}, anchor: {anchor}, hits: {hits})") + } + CoverageItem::Statement { loc, anchor, hits } => { + write!(f, "Statement (location: {loc}, anchor: {anchor}, hits: {hits})") + } + CoverageItem::Branch { loc, anchor, hits, branch_id, path_id, kind } => { + write!(f, "{} Branch (branch: {branch_id}, path: {path_id}) (location: {loc}, anchor: {anchor}, hits: {hits})", match kind { + BranchKind::True => "True", + BranchKind::False => "False", + }) + } + CoverageItem::Function { loc, anchor, hits, name } => { + write!(f, r#"Function "{name}" (location: {loc}, anchor: {anchor}, hits: {hits})"#) + } + } + } +} + +#[derive(Debug, Clone)] +pub struct SourceLocation { + /// Start byte in the source code. + pub start: usize, + /// Number of bytes in the source code. + pub length: Option, + /// The line in the source code. + pub line: usize, +} + +impl Display for SourceLocation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "L{}, C{}-{}", + self.line, + self.start, + self.length.map_or(self.start, |length| self.start + length) + ) + } +} + +#[derive(Debug, Clone)] +pub enum BranchKind { + /// A false branch + True, + /// A true branch + False, +} + +/// Coverage summary for a source file. +#[derive(Default, Debug, Clone)] +pub struct CoverageSummary { + /// The number of executable lines in the source file. + pub line_count: usize, + /// The number of lines that were hit. + pub line_hits: usize, + /// The number of statements in the source file. + pub statement_count: usize, + /// The number of statements that were hit. + pub statement_hits: usize, + /// The number of branches in the source file. + pub branch_count: usize, + /// The number of branches that were hit. + pub branch_hits: usize, + /// The number of functions in the source file. + pub function_count: usize, + /// The number of functions hit. + pub function_hits: usize, +} + +impl AddAssign<&Self> for CoverageSummary { + fn add_assign(&mut self, other: &Self) { + self.line_count += other.line_count; + self.line_hits += other.line_hits; + self.statement_count += other.statement_count; + self.statement_hits += other.statement_hits; + self.branch_count += other.branch_count; + self.branch_hits += other.branch_hits; + self.function_count += other.function_count; + self.function_hits += other.function_hits; + } +} diff --git a/evm/src/coverage/visitor.rs b/evm/src/coverage/visitor.rs new file mode 100644 index 000000000000..cf15722d4221 --- /dev/null +++ b/evm/src/coverage/visitor.rs @@ -0,0 +1,397 @@ +use super::{BranchKind, CoverageItem, SourceLocation}; +use ethers::{ + prelude::sourcemap::SourceMap, + solc::artifacts::ast::{self, Ast, Node, NodeType}, +}; +use std::collections::HashMap; +use tracing::warn; + +#[derive(Debug, Default, Clone)] +pub struct Visitor { + /// The source ID containing the AST being walked. + source_id: u32, + /// The source code that contains the AST being walked. + source: String, + /// Source maps for this specific source file, keyed by the contract name. + source_maps: HashMap, + + /// The contract whose AST we are currently walking + context: String, + /// The current branch ID + // TODO: Does this need to be unique across files? + branch_id: usize, + /// Stores the last line we put in the items collection to ensure we don't push duplicate lines + last_line: usize, + + /// Coverage items + items: Vec, +} + +impl Visitor { + pub fn new(source_id: u32, source: String, source_maps: HashMap) -> Self { + Self { source_id, source, source_maps, ..Default::default() } + } + + pub fn visit_ast(mut self, ast: Ast) -> eyre::Result> { + // Walk AST + for node in ast.nodes.into_iter() { + if !matches!(node.node_type, NodeType::ContractDefinition) { + continue + } + + self.visit_contract(node)?; + } + + Ok(self.items) + } + + fn visit_contract(&mut self, node: Node) -> eyre::Result<()> { + let is_contract = + node.attribute("contractKind").map_or(false, |kind: String| kind == "contract"); + let is_abstract: bool = node.attribute("abstract").unwrap_or_default(); + + // Skip interfaces, libraries and abstract contracts + if !is_contract || is_abstract { + return Ok(()) + } + + // Set the current context + let contract_name: String = + node.attribute("name").ok_or_else(|| eyre::eyre!("contract has no name"))?; + self.context = contract_name; + + // Find all functions and walk their AST + for node in node.nodes { + if node.node_type == NodeType::FunctionDefinition { + self.visit_function_definition(node)?; + } + } + + Ok(()) + } + + fn visit_function_definition(&mut self, mut node: Node) -> eyre::Result<()> { + let name: String = + node.attribute("name").ok_or_else(|| eyre::eyre!("function has no name"))?; + let is_virtual: bool = node.attribute("virtual").unwrap_or_default(); + + // Skip virtual functions + if is_virtual { + return Ok(()) + } + + match node.body.take() { + // Skip virtual functions + Some(body) if !is_virtual => { + self.push_item(CoverageItem::Function { + name, + loc: self.source_location_for(&node.src), + anchor: self.anchor_for(&body.src), + hits: 0, + }); + self.visit_block(*body) + } + _ => Ok(()), + } + } + + fn visit_block(&mut self, node: Node) -> eyre::Result<()> { + let statements: Vec = node.attribute("statements").unwrap_or_default(); + + for statement in statements { + self.visit_statement(statement)?; + } + + Ok(()) + } + fn visit_statement(&mut self, node: Node) -> eyre::Result<()> { + // TODO: YulSwitch, YulForLoop, YulFunctionDefinition, YulVariableDeclaration + match node.node_type { + // Blocks + NodeType::Block | NodeType::UncheckedBlock | NodeType::YulBlock => { + self.visit_block(node) + } + // Inline assembly block + NodeType::InlineAssembly => self.visit_block( + node.attribute("AST") + .ok_or_else(|| eyre::eyre!("inline assembly block with no AST attribute"))?, + ), + // Simple statements + NodeType::Break | + NodeType::Continue | + NodeType::EmitStatement | + NodeType::PlaceholderStatement | + NodeType::Return | + NodeType::RevertStatement | + NodeType::YulAssignment | + NodeType::YulBreak | + NodeType::YulContinue | + NodeType::YulLeave => { + self.push_item(CoverageItem::Statement { + loc: self.source_location_for(&node.src), + anchor: self.anchor_for(&node.src), + hits: 0, + }); + Ok(()) + } + // Variable declaration + NodeType::VariableDeclarationStatement => { + self.push_item(CoverageItem::Statement { + loc: self.source_location_for(&node.src), + anchor: self.anchor_for(&node.src), + hits: 0, + }); + if let Some(expr) = node.attribute("initialValue") { + self.visit_expression(expr)?; + } + Ok(()) + } + // While loops + NodeType::DoWhileStatement | NodeType::WhileStatement => { + self.visit_expression( + node.attribute("condition") + .ok_or_else(|| eyre::eyre!("while statement had no condition"))?, + )?; + + let body = + node.body.ok_or_else(|| eyre::eyre!("while statement had no body node"))?; + self.visit_block_or_statement(*body) + } + // For loops + NodeType::ForStatement => { + if let Some(stmt) = node.attribute("initializationExpression") { + self.visit_statement(stmt)?; + } + if let Some(expr) = node.attribute("condition") { + self.visit_expression(expr)?; + } + if let Some(stmt) = node.attribute("loopExpression") { + self.visit_statement(stmt)?; + } + + let body = + node.body.ok_or_else(|| eyre::eyre!("for statement had no body node"))?; + self.visit_block_or_statement(*body) + } + // Expression statement + NodeType::ExpressionStatement | NodeType::YulExpressionStatement => self + .visit_expression( + node.attribute("expression") + .ok_or_else(|| eyre::eyre!("expression statement had no expression"))?, + ), + // If statement + NodeType::IfStatement => { + self.visit_expression( + node.attribute("condition") + .ok_or_else(|| eyre::eyre!("while statement had no condition"))?, + )?; + + let true_body: Node = node + .attribute("trueBody") + .ok_or_else(|| eyre::eyre!("if statement had no true body"))?; + + // We need to store the current branch ID here since visiting the body of either of + // the if blocks may increase `self.branch_id` in the case of nested if statements. + let branch_id = self.branch_id; + + // We increase the branch ID here such that nested branches do not use the same + // branch ID as we do + self.branch_id += 1; + + self.push_item(CoverageItem::Branch { + branch_id, + path_id: 0, + kind: BranchKind::True, + loc: self.source_location_for(&node.src), + anchor: self.anchor_for(&true_body.src), + hits: 0, + }); + self.visit_block_or_statement(true_body)?; + + let false_body: Option = node.attribute("falseBody"); + if let Some(false_body) = false_body { + self.push_item(CoverageItem::Branch { + branch_id, + path_id: 1, + kind: BranchKind::False, + loc: self.source_location_for(&node.src), + anchor: self.anchor_for(&false_body.src), + hits: 0, + }); + self.visit_block_or_statement(false_body)?; + } + + Ok(()) + } + NodeType::YulIf => { + self.visit_expression( + node.attribute("condition") + .ok_or_else(|| eyre::eyre!("while statement had no condition"))?, + )?; + + let body: Node = node + .attribute("body") + .ok_or_else(|| eyre::eyre!("yul if statement had no body"))?; + + // We need to store the current branch ID here since visiting the body of either of + // the if blocks may increase `self.branch_id` in the case of nested if statements. + let branch_id = self.branch_id; + + // We increase the branch ID here such that nested branches do not use the same + // branch ID as we do + self.branch_id += 1; + + self.push_item(CoverageItem::Branch { + branch_id, + path_id: 0, + kind: BranchKind::True, + loc: self.source_location_for(&node.src), + anchor: self.anchor_for(&node.src), + hits: 0, + }); + self.visit_block(body)?; + + Ok(()) + } + // Try-catch statement + NodeType::TryStatement => { + // TODO: Clauses + // TODO: This is branching, right? + self.visit_expression( + node.attribute("externalCall") + .ok_or_else(|| eyre::eyre!("try statement had no call"))?, + ) + } + _ => { + warn!("unexpected node type, expected a statement: {:?}", node.node_type); + Ok(()) + } + } + } + + fn visit_expression(&mut self, node: Node) -> eyre::Result<()> { + // TODO + // elementarytypenameexpression + // memberaccess + // newexpression + // tupleexpression + // yulfunctioncall + match node.node_type { + NodeType::Assignment | NodeType::UnaryOperation | NodeType::BinaryOperation => { + // TODO: Should we explore the subexpressions? + self.push_item(CoverageItem::Statement { + loc: self.source_location_for(&node.src), + anchor: self.anchor_for(&node.src), + hits: 0, + }); + Ok(()) + } + NodeType::FunctionCall => { + // TODO: Handle assert and require + self.push_item(CoverageItem::Statement { + loc: self.source_location_for(&node.src), + anchor: self.anchor_for(&node.src), + hits: 0, + }); + Ok(()) + } + NodeType::Conditional => { + // TODO: Do we count these as branches? + self.push_item(CoverageItem::Statement { + loc: self.source_location_for(&node.src), + anchor: self.anchor_for(&node.src), + hits: 0, + }); + Ok(()) + } + // Does not count towards coverage + NodeType::FunctionCallOptions | + NodeType::Identifier | + NodeType::IndexAccess | + NodeType::IndexRangeAccess | + NodeType::Literal | + NodeType::YulLiteralValue | + NodeType::YulIdentifier => Ok(()), + _ => { + warn!("unexpected node type, expected an expression: {:?}", node.node_type); + Ok(()) + } + } + } + + fn visit_block_or_statement(&mut self, node: Node) -> eyre::Result<()> { + match node.node_type { + NodeType::Block => self.visit_block(node), + NodeType::Break | + NodeType::Continue | + NodeType::DoWhileStatement | + NodeType::EmitStatement | + NodeType::ExpressionStatement | + NodeType::ForStatement | + NodeType::IfStatement | + NodeType::InlineAssembly | + NodeType::PlaceholderStatement | + NodeType::Return | + NodeType::RevertStatement | + NodeType::TryStatement | + NodeType::VariableDeclarationStatement | + NodeType::WhileStatement => self.visit_statement(node), + _ => { + warn!("unexpected node type, expected block or statement: {:?}", node.node_type); + Ok(()) + } + } + } + + /// Pushes a coverage item to the internal collection, and might push a line item as well. + fn push_item(&mut self, item: CoverageItem) { + let source_location = item.source_location(); + + // Push a line item if we haven't already + if matches!(item, CoverageItem::Statement { .. } | CoverageItem::Branch { .. }) && + self.last_line < source_location.line + { + self.items.push(CoverageItem::Line { + loc: source_location.clone(), + anchor: item.anchor(), + hits: 0, + }); + self.last_line = source_location.line; + } + + self.items.push(item); + } + + fn source_location_for(&self, loc: &ast::SourceLocation) -> SourceLocation { + SourceLocation { + start: loc.start, + length: loc.length, + line: self.source[..loc.start].lines().count() + 1, + } + } + + fn anchor_for(&self, loc: &ast::SourceLocation) -> usize { + self.source_maps + .get(&self.context) + .and_then(|source_map| { + source_map + .iter() + .enumerate() + .find(|(_, element)| { + element + .index + .and_then(|source_id| { + Some( + source_id == self.source_id && + element.offset >= loc.start && + element.offset + element.length <= + loc.start + loc.length?, + ) + }) + .unwrap_or_default() + }) + .map(|(ic, _)| ic) + }) + .unwrap_or(loc.start) + } +} diff --git a/evm/src/executor/builder.rs b/evm/src/executor/builder.rs index 030e5665d179..0ab122d14419 100644 --- a/evm/src/executor/builder.rs +++ b/evm/src/executor/builder.rs @@ -138,17 +138,24 @@ impl ExecutorBuilder { self } - /// Enables tracing + /// Enables or disables tracing #[must_use] - pub fn with_tracing(mut self) -> Self { - self.inspector_config.tracing = true; + pub fn set_tracing(mut self, enable: bool) -> Self { + self.inspector_config.tracing = enable; self } - /// Enables the debugger + /// Enables or disables the debugger #[must_use] - pub fn with_debugger(mut self) -> Self { - self.inspector_config.debugger = true; + pub fn set_debugger(mut self, enable: bool) -> Self { + self.inspector_config.debugger = enable; + self + } + + /// Enables or disables coverage collection + #[must_use] + pub fn set_coverage(mut self, enable: bool) -> Self { + self.inspector_config.coverage = enable; self } diff --git a/evm/src/executor/inspector/coverage.rs b/evm/src/executor/inspector/coverage.rs new file mode 100644 index 000000000000..5c4dbeb0af9b --- /dev/null +++ b/evm/src/executor/inspector/coverage.rs @@ -0,0 +1,153 @@ +use crate::{coverage::HitMaps, executor::inspector::utils::get_create_address}; +use bytes::Bytes; +use ethers::types::Address; +use revm::{ + opcode, spec_opcode_gas, CallInputs, CreateInputs, Database, EVMData, Gas, Inspector, + Interpreter, Return, SpecId, +}; +use std::collections::BTreeMap; + +#[derive(Default, Debug)] +pub struct CoverageCollector { + /// Maps that track instruction hit data. + pub maps: HitMaps, + + /// The execution addresses, with the topmost one being the current address. + context: Vec
, + + /// A mapping of program counters to instruction counters. + /// + /// The program counter keeps track of where we are in the contract bytecode as a whole, + /// including push bytes, while the instruction counter ignores push bytes. + /// + /// The instruction counter is used in Solidity source maps. + pub ic_map: BTreeMap>, +} + +impl CoverageCollector { + /// Builds the instruction counter map for the given bytecode. + // TODO: Some of the same logic is performed in REVM, but then later discarded. We should + // investigate if we can reuse it + // TODO: Duplicate code of the debugger inspector + pub fn build_ic_map(&mut self, spec: SpecId, code: &Bytes) { + if let Some(context) = self.context.last() { + let opcode_infos = spec_opcode_gas(spec); + let mut ic_map: BTreeMap = BTreeMap::new(); + + let mut i = 0; + let mut cumulative_push_size = 0; + while i < code.len() { + let op = code[i]; + ic_map.insert(i, i - cumulative_push_size); + if opcode_infos[op as usize].is_push { + // Skip the push bytes. + // + // For more context on the math, see: https://github.com/bluealloy/revm/blob/007b8807b5ad7705d3cacce4d92b89d880a83301/crates/revm/src/interpreter/contract.rs#L114-L115 + i += (op - opcode::PUSH1 + 1) as usize; + cumulative_push_size += (op - opcode::PUSH1 + 1) as usize; + } + i += 1; + } + + self.ic_map.insert(*context, ic_map); + } + } + + pub fn enter(&mut self, address: Address) { + self.context.push(address); + } + + pub fn exit(&mut self) { + self.context.pop(); + } +} + +impl Inspector for CoverageCollector +where + DB: Database, +{ + fn call( + &mut self, + _: &mut EVMData<'_, DB>, + call: &mut CallInputs, + _: bool, + ) -> (Return, Gas, Bytes) { + self.enter(call.context.code_address); + + (Return::Continue, Gas::new(call.gas_limit), Bytes::new()) + } + + // TODO: Duplicate code of the debugger inspector + fn initialize_interp( + &mut self, + interp: &mut Interpreter, + data: &mut EVMData<'_, DB>, + _: bool, + ) -> Return { + // TODO: This is rebuilt for all contracts every time. We should only run this if the IC + // map for a given address does not exist, *but* we need to account for the fact that the + // code given by the interpreter may either be the contract init code, or the runtime code. + self.build_ic_map(data.env.cfg.spec_id, &interp.contract().code); + Return::Continue + } + + // TODO: Don't collect coverage for test contract if possible + fn step( + &mut self, + interpreter: &mut Interpreter, + _: &mut EVMData<'_, DB>, + _is_static: bool, + ) -> Return { + let pc = interpreter.program_counter(); + if let Some(context) = self.context.last() { + if let Some(ic) = self.ic_map.get(context).and_then(|ic_map| ic_map.get(&pc)) { + let map = self.maps.entry(*context).or_default(); + + map.hit(*ic); + } + } + + Return::Continue + } + + fn call_end( + &mut self, + _: &mut EVMData<'_, DB>, + _: &CallInputs, + gas: Gas, + status: Return, + retdata: Bytes, + _: bool, + ) -> (Return, Gas, Bytes) { + self.exit(); + + (status, gas, retdata) + } + + fn create( + &mut self, + data: &mut EVMData<'_, DB>, + call: &mut CreateInputs, + ) -> (Return, Option
, Gas, Bytes) { + // TODO: Does this increase gas cost? + data.subroutine.load_account(call.caller, data.db); + let nonce = data.subroutine.account(call.caller).info.nonce; + self.enter(get_create_address(call, nonce)); + + (Return::Continue, None, Gas::new(call.gas_limit), Bytes::new()) + } + + fn create_end( + &mut self, + _: &mut EVMData<'_, DB>, + _: &CreateInputs, + status: Return, + address: Option
, + gas: Gas, + retdata: Bytes, + ) -> (Return, Option
, Gas, Bytes) { + self.exit(); + + (status, address, gas, retdata) + } +} diff --git a/evm/src/executor/inspector/mod.rs b/evm/src/executor/inspector/mod.rs index ee3931afadb4..99f84f9a264c 100644 --- a/evm/src/executor/inspector/mod.rs +++ b/evm/src/executor/inspector/mod.rs @@ -10,6 +10,9 @@ pub use tracer::Tracer; mod debugger; pub use debugger::Debugger; +mod coverage; +pub use coverage::CoverageCollector; + mod stack; pub use stack::{InspectorData, InspectorStack}; @@ -38,6 +41,8 @@ pub struct InspectorStackConfig { pub tracing: bool, /// Whether or not the debugger is enabled pub debugger: bool, + /// Whether or not coverage info should be collected + pub coverage: bool, } impl InspectorStackConfig { @@ -57,6 +62,9 @@ impl InspectorStackConfig { if self.debugger { stack.debugger = Some(Debugger::default()); } + if self.coverage { + stack.coverage = Some(CoverageCollector::default()); + } stack } } diff --git a/evm/src/executor/inspector/stack.rs b/evm/src/executor/inspector/stack.rs index 2ee44a0f579c..a16d437a610a 100644 --- a/evm/src/executor/inspector/stack.rs +++ b/evm/src/executor/inspector/stack.rs @@ -1,5 +1,5 @@ -use super::{Cheatcodes, Debugger, LogCollector, Tracer}; -use crate::{debug::DebugArena, trace::CallTraceArena}; +use super::{Cheatcodes, CoverageCollector, Debugger, LogCollector, Tracer}; +use crate::{coverage::HitMaps, debug::DebugArena, trace::CallTraceArena}; use bytes::Bytes; use ethers::types::{Address, Log, H256}; use revm::{db::Database, CallInputs, CreateInputs, EVMData, Gas, Inspector, Interpreter, Return}; @@ -22,6 +22,7 @@ pub struct InspectorData { pub labels: BTreeMap, pub traces: Option, pub debug: Option, + pub coverage: Option, pub cheatcodes: Option, } @@ -35,6 +36,7 @@ pub struct InspectorStack { pub logs: Option, pub cheatcodes: Option, pub debugger: Option, + pub coverage: Option, } impl InspectorStack { @@ -48,6 +50,7 @@ impl InspectorStack { .unwrap_or_default(), traces: self.tracer.map(|tracer| tracer.traces), debug: self.debugger.map(|debugger| debugger.arena), + coverage: self.coverage.map(|coverage| coverage.maps), cheatcodes: self.cheatcodes, } } @@ -65,7 +68,13 @@ where ) -> Return { call_inspectors!( inspector, - [&mut self.debugger, &mut self.tracer, &mut self.logs, &mut self.cheatcodes], + [ + &mut self.debugger, + &mut self.coverage, + &mut self.tracer, + &mut self.logs, + &mut self.cheatcodes + ], { let status = inspector.initialize_interp(interpreter, data, is_static); @@ -87,7 +96,13 @@ where ) -> Return { call_inspectors!( inspector, - [&mut self.debugger, &mut self.tracer, &mut self.logs, &mut self.cheatcodes], + [ + &mut self.debugger, + &mut self.tracer, + &mut self.coverage, + &mut self.logs, + &mut self.cheatcodes + ], { let status = inspector.step(interpreter, data, is_static); @@ -144,7 +159,13 @@ where ) -> (Return, Gas, Bytes) { call_inspectors!( inspector, - [&mut self.debugger, &mut self.tracer, &mut self.logs, &mut self.cheatcodes], + [ + &mut self.debugger, + &mut self.tracer, + &mut self.coverage, + &mut self.logs, + &mut self.cheatcodes + ], { let (status, gas, retdata) = inspector.call(data, call, is_static); @@ -169,7 +190,13 @@ where ) -> (Return, Gas, Bytes) { call_inspectors!( inspector, - [&mut self.debugger, &mut self.tracer, &mut self.logs, &mut self.cheatcodes], + [ + &mut self.debugger, + &mut self.tracer, + &mut self.coverage, + &mut self.logs, + &mut self.cheatcodes + ], { let (new_status, new_gas, new_retdata) = inspector.call_end( data, @@ -198,7 +225,13 @@ where ) -> (Return, Option
, Gas, Bytes) { call_inspectors!( inspector, - [&mut self.debugger, &mut self.tracer, &mut self.logs, &mut self.cheatcodes], + [ + &mut self.debugger, + &mut self.tracer, + &mut self.coverage, + &mut self.logs, + &mut self.cheatcodes + ], { let (status, addr, gas, retdata) = inspector.create(data, call); @@ -223,7 +256,13 @@ where ) -> (Return, Option
, Gas, Bytes) { call_inspectors!( inspector, - [&mut self.debugger, &mut self.tracer, &mut self.logs, &mut self.cheatcodes], + [ + &mut self.debugger, + &mut self.tracer, + &mut self.coverage, + &mut self.logs, + &mut self.cheatcodes + ], { let (new_status, new_address, new_gas, new_retdata) = inspector.create_end( data, diff --git a/evm/src/executor/mod.rs b/evm/src/executor/mod.rs index 8fb61b0d9f59..b7398aebde0f 100644 --- a/evm/src/executor/mod.rs +++ b/evm/src/executor/mod.rs @@ -28,7 +28,8 @@ pub use revm::Env; use self::inspector::{InspectorData, InspectorStackConfig}; use crate::{ - debug::DebugArena, executor::inspector::DEFAULT_CREATE2_DEPLOYER, trace::CallTraceArena, CALLER, + coverage::HitMaps, debug::DebugArena, executor::inspector::DEFAULT_CREATE2_DEPLOYER, + trace::CallTraceArena, CALLER, }; use bytes::Bytes; use ethers::{ @@ -108,6 +109,8 @@ pub struct CallResult { pub labels: BTreeMap, /// The traces of the call pub traces: Option, + /// The coverage info collected during the call + pub coverage: Option, /// The debug nodes of the call pub debug: Option, /// Scripted transactions generated from this call @@ -138,6 +141,8 @@ pub struct RawCallResult { pub labels: BTreeMap, /// The traces of the call pub traces: Option, + /// The coverage info collected during the call + pub coverage: Option, /// The debug nodes of the call pub debug: Option, /// Scripted transactions generated from this call @@ -160,6 +165,7 @@ impl Default for RawCallResult { logs: Vec::new(), labels: BTreeMap::new(), traces: None, + coverage: None, debug: None, transactions: None, state_changeset: None, @@ -297,6 +303,7 @@ where logs, labels, traces, + coverage, debug, transactions, state_changeset, @@ -312,6 +319,7 @@ where logs, labels, traces, + coverage, debug, transactions, state_changeset, @@ -361,7 +369,7 @@ where _ => Bytes::default(), }; - let InspectorData { logs, labels, traces, debug, mut cheatcodes } = + let InspectorData { logs, labels, traces, coverage, debug, mut cheatcodes } = inspector.collect_inspector_states(); // Persist the changed block environment @@ -394,6 +402,7 @@ where stipend, logs, labels, + coverage, traces, debug, transactions, @@ -424,6 +433,7 @@ where logs, labels, traces, + coverage, debug, transactions, state_changeset, @@ -439,6 +449,7 @@ where logs, labels, traces, + coverage, debug, transactions, state_changeset, @@ -488,7 +499,7 @@ where _ => Bytes::default(), }; - let InspectorData { logs, labels, traces, debug, cheatcodes, .. } = + let InspectorData { logs, labels, traces, debug, coverage, cheatcodes, .. } = inspector.collect_inspector_states(); let transactions = if let Some(cheats) = cheatcodes { @@ -510,6 +521,7 @@ where logs: logs.to_vec(), labels, traces, + coverage, debug, transactions, state_changeset: Some(state_changeset), diff --git a/evm/src/lib.rs b/evm/src/lib.rs index 1de1c3a42166..224935654975 100644 --- a/evm/src/lib.rs +++ b/evm/src/lib.rs @@ -1,12 +1,16 @@ /// Decoding helpers pub mod decode; -/// Call trace arena, decoding and formatting +/// Call tracing +/// Contains a call trace arena, decoding and formatting utilities pub mod trace; /// Debugger data structures pub mod debug; +/// Coverage data structures +pub mod coverage; + /// Forge test execution backends pub mod executor; diff --git a/evm/src/trace/identifier/etherscan.rs b/evm/src/trace/identifier/etherscan.rs index 8854a7f5a953..0626c7bf8ef6 100644 --- a/evm/src/trace/identifier/etherscan.rs +++ b/evm/src/trace/identifier/etherscan.rs @@ -65,6 +65,7 @@ impl TraceIdentifier for EtherscanIdentifier { label: Some(label.clone()), contract: Some(label), abi: Some(Cow::Owned(abi)), + artifact_id: None, }) .collect(); diff --git a/evm/src/trace/identifier/local.rs b/evm/src/trace/identifier/local.rs index 5539eb70ff66..74597ad877e3 100644 --- a/evm/src/trace/identifier/local.rs +++ b/evm/src/trace/identifier/local.rs @@ -8,7 +8,7 @@ use std::{borrow::Cow, collections::BTreeMap}; /// A trace identifier that tries to identify addresses using local contracts. pub struct LocalTraceIdentifier { - local_contracts: BTreeMap, (String, Abi)>, + local_contracts: BTreeMap, (ArtifactId, Abi)>, } impl LocalTraceIdentifier { @@ -16,9 +16,7 @@ impl LocalTraceIdentifier { Self { local_contracts: known_contracts .iter() - .map(|(id, (abi, runtime_code))| { - (runtime_code.clone(), (id.name.clone(), abi.clone())) - }) + .map(|(id, (abi, runtime_code))| (runtime_code.clone(), (id.clone(), abi.clone()))) .collect(), } } @@ -38,16 +36,17 @@ impl TraceIdentifier for LocalTraceIdentifier { .into_iter() .filter_map(|(address, code)| { let code = code?; - let (_, (name, abi)) = self + let (_, (id, abi)) = self .local_contracts .iter() .find(|(known_code, _)| diff_score(known_code, code) < 0.1)?; Some(AddressIdentity { address: *address, - contract: Some(name.clone()), - label: Some(name.clone()), + contract: Some(id.identifier()), + label: Some(id.name.clone()), abi: Some(Cow::Borrowed(abi)), + artifact_id: Some(id.clone()), }) }) .collect() diff --git a/evm/src/trace/identifier/mod.rs b/evm/src/trace/identifier/mod.rs index 760327e42f93..67617bf15d8b 100644 --- a/evm/src/trace/identifier/mod.rs +++ b/evm/src/trace/identifier/mod.rs @@ -7,7 +7,10 @@ pub use etherscan::EtherscanIdentifier; mod signatures; pub use signatures::SignaturesIdentifier; -use ethers::abi::{Abi, Address}; +use ethers::{ + abi::{Abi, Address}, + prelude::ArtifactId, +}; use std::borrow::Cow; /// An address identity @@ -22,6 +25,8 @@ pub struct AddressIdentity<'a> { pub contract: Option, /// The ABI of the contract at this address pub abi: Option>, + /// The artifact ID of the contract, if any. + pub artifact_id: Option, } /// Trace identifiers figure out what ABIs and labels belong to all the addresses of the trace. diff --git a/forge/src/coverage.rs b/forge/src/coverage.rs new file mode 100644 index 000000000000..b9eb1a2684c0 --- /dev/null +++ b/forge/src/coverage.rs @@ -0,0 +1,200 @@ +use comfy_table::{Attribute, Cell, Color, Row, Table}; +pub use foundry_evm::coverage::*; +use std::{collections::HashMap, io::Write, path::PathBuf}; + +/// A coverage reporter. +pub trait CoverageReporter { + fn report(self, map: CoverageMap) -> eyre::Result<()>; +} + +/// A simple summary reporter that prints the coverage results in a table. +pub struct SummaryReporter { + /// The summary table. + table: Table, + /// The total coverage of the entire project. + total: CoverageSummary, +} + +impl Default for SummaryReporter { + fn default() -> Self { + let mut table = Table::new(); + table.set_header(&["File", "% Lines", "% Statements", "% Branches", "% Funcs"]); + + Self { table, total: CoverageSummary::default() } + } +} + +impl SummaryReporter { + fn add_row(&mut self, name: impl Into, summary: CoverageSummary) { + let mut row = Row::new(); + row.add_cell(name.into()) + .add_cell(format_cell(summary.line_hits, summary.line_count)) + .add_cell(format_cell(summary.statement_hits, summary.statement_count)) + .add_cell(format_cell(summary.branch_hits, summary.branch_count)) + .add_cell(format_cell(summary.function_hits, summary.function_count)); + self.table.add_row(row); + } +} + +impl CoverageReporter for SummaryReporter { + fn report(mut self, map: CoverageMap) -> eyre::Result<()> { + for file in map { + let summary = file.summary(); + + self.total += &summary; + self.add_row(file.path.to_string_lossy(), summary); + } + + self.add_row("Total", self.total.clone()); + println!("{}", self.table); + Ok(()) + } +} + +fn format_cell(hits: usize, total: usize) -> Cell { + let percentage = if total == 0 { 1. } else { hits as f64 / total as f64 }; + + let mut cell = + Cell::new(format!("{:.2}% ({hits}/{total})", percentage * 100.)).fg(match percentage { + _ if total == 0 => Color::Grey, + _ if percentage < 0.5 => Color::Red, + _ if percentage < 0.75 => Color::Yellow, + _ => Color::Green, + }); + + if total == 0 { + cell = cell.add_attribute(Attribute::Dim); + } + cell +} + +pub struct LcovReporter<'a> { + /// Destination buffer + destination: &'a mut (dyn Write + 'a), +} + +impl<'a> LcovReporter<'a> { + pub fn new(destination: &'a mut (dyn Write + 'a)) -> LcovReporter<'a> { + Self { destination } + } +} + +impl<'a> CoverageReporter for LcovReporter<'a> { + fn report(self, map: CoverageMap) -> eyre::Result<()> { + for file in map { + let summary = file.summary(); + + writeln!(self.destination, "TN:")?; + writeln!(self.destination, "SF:{}", file.path.to_string_lossy())?; + + for item in file.items { + match item { + CoverageItem::Function { + loc: SourceLocation { line, .. }, name, hits, .. + } => { + writeln!(self.destination, "FN:{line},{name}")?; + writeln!(self.destination, "FNDA:{hits},{name}")?; + } + CoverageItem::Line { loc: SourceLocation { line, .. }, hits, .. } => { + writeln!(self.destination, "DA:{line},{hits}")?; + } + CoverageItem::Branch { + loc: SourceLocation { line, .. }, + branch_id, + path_id, + hits, + .. + } => { + writeln!( + self.destination, + "BRDA:{line},{branch_id},{path_id},{}", + if hits == 0 { "-".to_string() } else { hits.to_string() } + )?; + } + // Statements are not in the LCOV format + CoverageItem::Statement { .. } => (), + } + } + + // Function summary + writeln!(self.destination, "FNF:{}", summary.function_count)?; + writeln!(self.destination, "FNH:{}", summary.function_hits)?; + + // Line summary + writeln!(self.destination, "LF:{}", summary.line_count)?; + writeln!(self.destination, "LH:{}", summary.line_hits)?; + + // Branch summary + writeln!(self.destination, "BRF:{}", summary.branch_count)?; + writeln!(self.destination, "BRH:{}", summary.branch_hits)?; + + writeln!(self.destination, "end_of_record")?; + } + + println!("Wrote LCOV report."); + + Ok(()) + } +} + +/// A super verbose reporter for debugging coverage while it is still unstable. +pub struct DebugReporter { + /// The summary table. + table: Table, + /// The total coverage of the entire project. + total: CoverageSummary, + /// Uncovered items + uncovered: HashMap>, +} + +impl Default for DebugReporter { + fn default() -> Self { + let mut table = Table::new(); + table.set_header(&["File", "% Lines", "% Statements", "% Branches", "% Funcs"]); + + Self { table, total: CoverageSummary::default(), uncovered: HashMap::default() } + } +} + +impl DebugReporter { + fn add_row(&mut self, name: impl Into, summary: CoverageSummary) { + let mut row = Row::new(); + row.add_cell(name.into()) + .add_cell(format_cell(summary.line_hits, summary.line_count)) + .add_cell(format_cell(summary.statement_hits, summary.statement_count)) + .add_cell(format_cell(summary.branch_hits, summary.branch_count)) + .add_cell(format_cell(summary.function_hits, summary.function_count)); + self.table.add_row(row); + } +} + +impl CoverageReporter for DebugReporter { + fn report(mut self, map: CoverageMap) -> eyre::Result<()> { + for file in map { + let summary = file.summary(); + + self.total += &summary; + self.add_row(file.path.to_string_lossy(), summary); + + file.items.iter().for_each(|item| { + if item.hits() == 0 { + self.uncovered.entry(file.path.clone()).or_default().push(item.clone()); + } + }) + } + + self.add_row("Total", self.total.clone()); + println!("{}", self.table); + + for (path, items) in self.uncovered { + println!("Uncovered for {}:", path.to_string_lossy()); + items.iter().for_each(|item| { + if item.hits() == 0 { + println!("- {}", item); + } + }); + println!(); + } + Ok(()) + } +} diff --git a/forge/src/lib.rs b/forge/src/lib.rs index 29b18554ed85..a8f203fa6887 100644 --- a/forge/src/lib.rs +++ b/forge/src/lib.rs @@ -1,6 +1,9 @@ /// Gas reports pub mod gas_report; +/// Coverage reports +pub mod coverage; + /// The Forge test runner mod runner; pub use runner::ContractRunner; diff --git a/forge/src/multi_runner.rs b/forge/src/multi_runner.rs index fc9c778fb303..af5ec3e9e8ba 100644 --- a/forge/src/multi_runner.rs +++ b/forge/src/multi_runner.rs @@ -31,6 +31,8 @@ pub struct MultiContractRunnerBuilder { pub fork: Option, /// Additional cheatcode inspector related settings derived from the `Config` pub cheats_config: Option, + /// Whether or not to collect coverage info + pub coverage: bool, } pub type DeployableContracts = BTreeMap)>; @@ -128,6 +130,7 @@ impl MultiContractRunnerBuilder { source_paths, fork: self.fork, cheats_config: self.cheats_config.unwrap_or_default(), + coverage: self.coverage, }) } @@ -166,6 +169,12 @@ impl MultiContractRunnerBuilder { self.cheats_config = Some(cheats_config); self } + + #[must_use] + pub fn set_coverage(mut self, enable: bool) -> Self { + self.coverage = enable; + self + } } /// A multi contract runner receives a set of contracts deployed in an EVM instance and proceeds @@ -192,6 +201,8 @@ pub struct MultiContractRunner { pub fork: Option, /// Additional cheatcode inspector related settings derived from the `Config` pub cheats_config: CheatsConfig, + /// Whether or not to collect coverage info + pub coverage: bool, } impl MultiContractRunner { @@ -271,17 +282,15 @@ impl MultiContractRunner { }) .filter(|(_, (abi, _, _))| abi.functions().any(|func| filter.matches_test(&func.name))) .map(|(id, (abi, deploy_code, libs))| { - let mut builder = ExecutorBuilder::default() + let executor = ExecutorBuilder::default() .with_cheatcodes(self.cheats_config.clone()) .with_config(env.clone()) .with_spec(self.evm_spec) - .with_gas_limit(self.evm_opts.gas_limit()); - - if self.evm_opts.verbosity >= 3 { - builder = builder.with_tracing(); - } + .with_gas_limit(self.evm_opts.gas_limit()) + .set_tracing(self.evm_opts.verbosity >= 3) + .set_coverage(self.coverage) + .build(db.clone()); - let executor = builder.build(db.clone()); let result = self.run_tests( &id.identifier(), abi, diff --git a/forge/src/result.rs b/forge/src/result.rs index 22ac386fff8d..d22aee936fea 100644 --- a/forge/src/result.rs +++ b/forge/src/result.rs @@ -3,6 +3,7 @@ use crate::Address; use ethers::prelude::Log; use foundry_evm::{ + coverage::HitMaps, fuzz::{CounterExample, FuzzedCases}, trace::{CallTraceArena, TraceKind}, }; @@ -63,6 +64,10 @@ pub struct TestResult { /// Traces pub traces: Vec<(TraceKind, CallTraceArena)>, + /// Raw coverage info + #[serde(skip)] + pub coverage: Option, + /// Labeled addresses pub labeled_addresses: BTreeMap, } diff --git a/forge/src/runner.rs b/forge/src/runner.rs index 84e3a8455e03..fa13c5348049 100644 --- a/forge/src/runner.rs +++ b/forge/src/runner.rs @@ -15,7 +15,6 @@ use foundry_evm::{ }; use proptest::test_runner::TestRunner; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; - use std::{collections::BTreeMap, time::Instant}; pub struct ContractRunner<'a, DB: DatabaseRef> { @@ -203,6 +202,7 @@ impl<'a, DB: DatabaseRef + Send + Sync> ContractRunner<'a, DB> { logs: vec![], kind: TestKind::Standard(0), traces: vec![], + coverage: None, labeled_addresses: BTreeMap::new(), }, )] @@ -225,6 +225,7 @@ impl<'a, DB: DatabaseRef + Send + Sync> ContractRunner<'a, DB> { logs: setup.logs, kind: TestKind::Standard(0), traces: setup.traces, + coverage: None, labeled_addresses: setup.labeled_addresses, }, )] @@ -285,44 +286,50 @@ impl<'a, DB: DatabaseRef + Send + Sync> ContractRunner<'a, DB> { // Run unit test let start = Instant::now(); - let (reverted, reason, gas, stipend, execution_traces, state_changeset) = match self - .executor - .call::<(), _, _>(self.sender, address, func.clone(), (), 0.into(), self.errors) - { - Ok(CallResult { - reverted, - gas, - stipend, - logs: execution_logs, - traces: execution_trace, - labels: new_labels, - state_changeset, - .. - }) => { - labeled_addresses.extend(new_labels); - logs.extend(execution_logs); - (reverted, None, gas, stipend, execution_trace, state_changeset) - } - Err(EvmError::Execution { - reverted, - reason, - gas, - stipend, - logs: execution_logs, - traces: execution_trace, - labels: new_labels, - state_changeset, - .. - }) => { - labeled_addresses.extend(new_labels); - logs.extend(execution_logs); - (reverted, Some(reason), gas, stipend, execution_trace, state_changeset) - } - Err(err) => { - tracing::error!(?err); - return Err(err.into()) - } - }; + let (reverted, reason, gas, stipend, execution_traces, coverage, state_changeset) = + match self.executor.call::<(), _, _>( + self.sender, + address, + func.clone(), + (), + 0.into(), + self.errors, + ) { + Ok(CallResult { + reverted, + gas, + stipend, + logs: execution_logs, + traces: execution_trace, + coverage, + labels: new_labels, + state_changeset, + .. + }) => { + labeled_addresses.extend(new_labels); + logs.extend(execution_logs); + (reverted, None, gas, stipend, execution_trace, coverage, state_changeset) + } + Err(EvmError::Execution { + reverted, + reason, + gas, + stipend, + logs: execution_logs, + traces: execution_trace, + labels: new_labels, + state_changeset, + .. + }) => { + labeled_addresses.extend(new_labels); + logs.extend(execution_logs); + (reverted, Some(reason), gas, stipend, execution_trace, None, state_changeset) + } + Err(err) => { + tracing::error!(?err); + return Err(err.into()) + } + }; traces.extend(execution_traces.map(|traces| (TraceKind::Execution, traces)).into_iter()); let success = self.executor.is_success( @@ -346,6 +353,7 @@ impl<'a, DB: DatabaseRef + Send + Sync> ContractRunner<'a, DB> { logs, kind: TestKind::Standard(gas.overflowing_sub(stipend).0), traces, + coverage, labeled_addresses, }) } @@ -387,6 +395,8 @@ impl<'a, DB: DatabaseRef + Send + Sync> ContractRunner<'a, DB> { logs, kind: TestKind::Fuzz(result.cases), traces, + // TODO: Maybe support coverage for fuzz tests + coverage: None, labeled_addresses, }) }