diff --git a/Cargo.lock b/Cargo.lock index fb1aa429..8f800be6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2553,6 +2553,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aac81fa3e28d21450aa4d2ac065992ba96a1d7303efbce51a95f4fd175b67562" +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + [[package]] name = "dotenvy" version = "0.15.7" @@ -4134,6 +4140,31 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" +[[package]] +name = "inkwell" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40fb405537710d51f6bdbc8471365ddd4cd6d3a3c3ad6e0c8291691031ba94b2" +dependencies = [ + "either", + "inkwell_internals", + "libc", + "llvm-sys", + "once_cell", + "thiserror", +] + +[[package]] +name = "inkwell_internals" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd28cfd4cfba665d47d31c08a6ba637eed16770abca2eccbbc3ca831fef1e44" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + [[package]] name = "inout" version = "0.1.3" @@ -4969,6 +5000,20 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "llvm-sys" +version = "180.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "778fa5fa02e32728e718f11eec147e6f134137399ab02fd2c13d32476337affa" +dependencies = [ + "anyhow", + "cc", + "lazy_static", + "libc", + "regex-lite", + "semver 1.0.23", +] + [[package]] name = "lock_api" version = "0.4.12" @@ -7027,6 +7072,7 @@ dependencies = [ "csv", "ctor", "derivative", + "dotenv", "eth-sparse-mpt", "ethereum-consensus", "ethereum_ssz", @@ -7076,6 +7122,11 @@ dependencies = [ "revm", "revm-inspectors", "revm-primitives", + "revmc-build", + "revmc-toolkit-build", + "revmc-toolkit-load", + "revmc-toolkit-sim", + "revmc-toolkit-utils", "secp256k1", "serde", "serde_json", @@ -7193,6 +7244,12 @@ dependencies = [ "regex-syntax 0.8.4", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.6.29" @@ -9584,6 +9641,8 @@ version = "10.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "959ecbc36802de6126852479844737f20194cf8e6718e0c30697d306a2cca916" dependencies = [ + "paste", + "phf 0.11.2", "revm-primitives", "serde", ] @@ -9627,6 +9686,177 @@ dependencies = [ "serde", ] +[[package]] +name = "revmc" +version = "0.1.0" +source = "git+https://github.com/paradigmxyz/revmc#9ad12ebe060ab25818ae1bf89febde70e08ca775" +dependencies = [ + "alloy-primitives 0.8.0", + "bitflags 2.6.0", + "bitvec", + "either", + "revm-interpreter", + "revm-primitives", + "revmc-backend", + "revmc-builtins", + "revmc-context", + "revmc-llvm", + "rustc-hash 2.0.0", + "tracing", +] + +[[package]] +name = "revmc-backend" +version = "0.1.0" +source = "git+https://github.com/paradigmxyz/revmc#9ad12ebe060ab25818ae1bf89febde70e08ca775" +dependencies = [ + "eyre", + "ruint", +] + +[[package]] +name = "revmc-build" +version = "0.1.0" +source = "git+https://github.com/paradigmxyz/revmc#9ad12ebe060ab25818ae1bf89febde70e08ca775" + +[[package]] +name = "revmc-builtins" +version = "0.1.0" +source = "git+https://github.com/paradigmxyz/revmc#9ad12ebe060ab25818ae1bf89febde70e08ca775" +dependencies = [ + "paste", + "revm-interpreter", + "revm-primitives", + "revmc-backend", + "revmc-context", + "tracing", +] + +[[package]] +name = "revmc-context" +version = "0.1.0" +source = "git+https://github.com/paradigmxyz/revmc#9ad12ebe060ab25818ae1bf89febde70e08ca775" +dependencies = [ + "revm-interpreter", + "revm-primitives", +] + +[[package]] +name = "revmc-llvm" +version = "0.1.0" +source = "git+https://github.com/paradigmxyz/revmc#9ad12ebe060ab25818ae1bf89febde70e08ca775" +dependencies = [ + "inkwell", + "revmc-backend", + "rustc-hash 2.0.0", + "tracing", +] + +[[package]] +name = "revmc-toolkit-build" +version = "0.1.0" +source = "git+https://github.com/halo3mic/revmc-toolkit?branch=dev#eb23c23a7498e9cf599fcb3642eb0ce24e66920e" +dependencies = [ + "eyre", + "hex", + "rayon", + "revm", + "revmc", + "serde", + "serde_json", + "tracing", +] + +[[package]] +name = "revmc-toolkit-load" +version = "0.1.0" +source = "git+https://github.com/halo3mic/revmc-toolkit?branch=dev#eb23c23a7498e9cf599fcb3642eb0ce24e66920e" +dependencies = [ + "eyre", + "libloading", + "revm", + "revmc", + "rustc-hash 2.0.0", + "tracing", +] + +[[package]] +name = "revmc-toolkit-sim" +version = "0.1.0" +source = "git+https://github.com/halo3mic/revmc-toolkit?branch=dev#eb23c23a7498e9cf599fcb3642eb0ce24e66920e" +dependencies = [ + "cc", + "clap", + "criterion", + "csv", + "dotenv", + "eyre", + "hex", + "libloading", + "rand 0.8.5", + "rand_chacha 0.3.1", + "rayon", + "reth-chainspec", + "reth-db", + "reth-evm", + "reth-evm-ethereum", + "reth-primitives", + "reth-provider", + "reth-revm", + "reth-rpc-types", + "revm", + "revm-interpreter", + "revmc", + "revmc-build", + "revmc-builtins", + "revmc-context", + "revmc-toolkit-build", + "revmc-toolkit-load", + "revmc-toolkit-utils", + "serde", + "serde_json", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "revmc-toolkit-utils" +version = "0.1.0" +source = "git+https://github.com/halo3mic/revmc-toolkit?branch=dev#eb23c23a7498e9cf599fcb3642eb0ce24e66920e" +dependencies = [ + "cc", + "clap", + "criterion", + "csv", + "dotenv", + "eyre", + "hex", + "libloading", + "rand 0.8.5", + "rand_chacha 0.3.1", + "rayon", + "reth-chainspec", + "reth-db", + "reth-evm", + "reth-evm-ethereum", + "reth-primitives", + "reth-provider", + "reth-revm", + "reth-rpc-types", + "revm", + "revm-interpreter", + "revmc", + "revmc-build", + "revmc-builtins", + "revmc-context", + "revmc-toolkit-build", + "revmc-toolkit-load", + "serde", + "serde_json", + "tracing", + "tracing-subscriber", +] + [[package]] name = "rfc6979" version = "0.3.1" diff --git a/crates/rbuilder/Cargo.toml b/crates/rbuilder/Cargo.toml index 4f7b9490..3051e5f4 100644 --- a/crates/rbuilder/Cargo.toml +++ b/crates/rbuilder/Cargo.toml @@ -126,8 +126,15 @@ async-trait = "0.1.80" eth-sparse-mpt = { git = "https://github.com/flashbots/eth-sparse-mpt", rev = "664759b" } +revmc-toolkit-load = { git = "https://github.com/halo3mic/revmc-toolkit", branch = "dev" } +revmc-toolkit-sim = { git = "https://github.com/halo3mic/revmc-toolkit", branch = "dev" } +revmc-toolkit-build = { git = "https://github.com/halo3mic/revmc-toolkit", branch = "dev" } +revmc-toolkit-utils = { git = "https://github.com/halo3mic/revmc-toolkit", branch = "dev" } +dotenv = "0.15.0" + [build-dependencies] built = { version = "0.7.1", features = ["git2", "chrono"] } +revmc-build = { git = "https://github.com/paradigmxyz/revmc" } [dev-dependencies] tempfile = "3.8" diff --git a/crates/rbuilder/build.rs b/crates/rbuilder/build.rs index d8f91cb9..ccc8bfca 100644 --- a/crates/rbuilder/build.rs +++ b/crates/rbuilder/build.rs @@ -1,3 +1,4 @@ fn main() { built::write_built_file().expect("Failed to acquire build-time information"); + revmc_build::emit(); } diff --git a/crates/rbuilder/src/backtest/backtest_build_block.rs b/crates/rbuilder/src/backtest/backtest_build_block.rs index 8062f565..08be2ad6 100644 --- a/crates/rbuilder/src/backtest/backtest_build_block.rs +++ b/crates/rbuilder/src/backtest/backtest_build_block.rs @@ -7,7 +7,6 @@ use ahash::HashMap; use alloy_primitives::utils::format_ether; - use crate::backtest::restore_landed_orders::{ restore_landed_orders, sim_historical_block, ExecutedBlockTx, ExecutedTxs, SimplifiedOrder, }; @@ -15,7 +14,7 @@ use crate::backtest::OrdersWithTimestamp; use crate::{ backtest::{ execute::{backtest_prepare_ctx_for_block, BacktestBlockInput}, - BlockData, HistoricalDataStorage, + BlockData, HistoricalDataStorage, utils, }, building::builders::BacktestSimulateBlockInput, live_builder::{base_config::load_config_toml_and_env, cli::LiveBuilderConfig}, @@ -91,26 +90,31 @@ pub async fn run_backtest_build_block() -> eyre:: .provider_factory_unchecked(); let chain_spec = config.base_config().chain_spec()?; let sbundle_mergeabe_signers = config.base_config().sbundle_mergeabe_signers(); + let llvm_compiler_fns = config.base_config().load_llvm_compiled_fns()?; if cli.sim_landed_block { let tx_sim_results = sim_historical_block( provider_factory.clone(), chain_spec.clone(), block_data.onchain_block.clone(), + llvm_compiler_fns.clone(), )?; print_onchain_block_data(tx_sim_results, &orders, &block_data); } - let BacktestBlockInput { - ctx, sim_orders, .. - } = backtest_prepare_ctx_for_block( + let ( + BacktestBlockInput {ctx, sim_orders, .. }, + elapsed + ) = utils::timeit!({backtest_prepare_ctx_for_block( block_data.clone(), provider_factory.clone(), chain_spec.clone(), cli.block_building_time_ms, config.base_config().blocklist()?, config.base_config().coinbase_signer()?, - )?; + llvm_compiler_fns, + )?}); + println!("Prepared ctx in: {:?}", elapsed); if cli.show_sim { print_simulated_orders(&sim_orders, &order_and_timestamp, &block_data); diff --git a/crates/rbuilder/src/backtest/backtest_build_range.rs b/crates/rbuilder/src/backtest/backtest_build_range.rs index b54b5f6a..42fa9095 100644 --- a/crates/rbuilder/src/backtest/backtest_build_range.rs +++ b/crates/rbuilder/src/backtest/backtest_build_range.rs @@ -115,6 +115,7 @@ pub async fn run_backtest_build_range Some(ok), Err(err) => { diff --git a/crates/rbuilder/src/backtest/execute.rs b/crates/rbuilder/src/backtest/execute.rs index 3fc77dde..5fd3c88e 100644 --- a/crates/rbuilder/src/backtest/execute.rs +++ b/crates/rbuilder/src/backtest/execute.rs @@ -16,6 +16,7 @@ use reth::providers::ProviderFactory; use reth_chainspec::ChainSpec; use reth_db::{database::Database, DatabaseEnv}; use reth_payload_builder::database::CachedReads; +use revmc_toolkit_load::EvmCompilerFns; use serde::{Deserialize, Serialize}; use std::sync::Arc; @@ -64,6 +65,7 @@ pub fn backtest_prepare_ctx_for_block( build_block_lag_ms: i64, blocklist: HashSet
, builder_signer: Signer, + llvm_compiler_fns: Option, ) -> eyre::Result { let orders = block_data .available_orders @@ -85,6 +87,7 @@ pub fn backtest_prepare_ctx_for_block( builder_signer.address, block_data.winning_bid_trace.proposer_fee_recipient, Some(builder_signer), + llvm_compiler_fns, ); let (sim_orders, sim_errors) = simulate_all_orders_with_sim_tree(provider_factory.clone(), &ctx, &orders, false)?; @@ -105,6 +108,7 @@ pub fn backtest_simulate_block( config: &ConfigType, blocklist: HashSet
, sbundle_mergeabe_signers: &[Address], + llvm_compiler_fns: Option, ) -> eyre::Result { let BacktestBlockInput { ctx, @@ -117,6 +121,7 @@ pub fn backtest_simulate_block( build_block_lag_ms, blocklist, config.base_config().coinbase_signer()?, + llvm_compiler_fns, )?; let filtered_orders_blocklist_count = sim_errors diff --git a/crates/rbuilder/src/backtest/mod.rs b/crates/rbuilder/src/backtest/mod.rs index 21bfb526..8847a8bf 100644 --- a/crates/rbuilder/src/backtest/mod.rs +++ b/crates/rbuilder/src/backtest/mod.rs @@ -7,6 +7,7 @@ pub mod redistribute; pub mod restore_landed_orders; mod results_store; mod store; +mod utils; pub use backtest_build_block::run_backtest_build_block; pub use backtest_build_range::run_backtest_build_range; diff --git a/crates/rbuilder/src/backtest/redistribute/cli/mod.rs b/crates/rbuilder/src/backtest/redistribute/cli/mod.rs index 9a904bbb..53d7f9b7 100644 --- a/crates/rbuilder/src/backtest/redistribute/cli/mod.rs +++ b/crates/rbuilder/src/backtest/redistribute/cli/mod.rs @@ -6,6 +6,7 @@ use crate::live_builder::base_config::load_config_toml_and_env; use crate::live_builder::cli::LiveBuilderConfig; use crate::{backtest::HistoricalDataStorage, live_builder::config::Config}; use alloy_primitives::utils::format_ether; +use revmc_toolkit_load::EvmCompilerFns; use clap::Parser; use csv_output::{CSVOutputRow, CSVResultWriter}; use reth_db::DatabaseEnv; @@ -58,6 +59,7 @@ pub async fn run_backtest_redistribute() -> eyre: .base_config .provider_factory()? .provider_factory_unchecked(); + let llvm_compiler_fns = config.base_config.load_llvm_compiled_fns()?; let mut csv_writer = cli .csv .map(|path| -> io::Result<_> { CSVResultWriter::new(path) }) @@ -77,6 +79,7 @@ pub async fn run_backtest_redistribute() -> eyre: provider_factory.clone(), &config, cli.distribute_to_mempool_txs, + llvm_compiler_fns, )?; } Commands::Range { @@ -94,6 +97,7 @@ pub async fn run_backtest_redistribute() -> eyre: provider_factory.clone(), &config, cli.distribute_to_mempool_txs, + llvm_compiler_fns.clone(), )?; } } @@ -114,6 +118,7 @@ fn process_redisribution( provider_factory: ProviderFactory>, config: &ConfigType, distribute_to_mempool_txs: bool, + llvm_compiler_fns: Option, ) -> eyre::Result<()> { let block_number = block_data.block_number; let block_hash = block_data.onchain_block.header.hash; @@ -123,6 +128,7 @@ fn process_redisribution( config, block_data, distribute_to_mempool_txs, + llvm_compiler_fns, ) { Ok(ok) => ok, Err(err) => { diff --git a/crates/rbuilder/src/backtest/redistribute/mod.rs b/crates/rbuilder/src/backtest/redistribute/mod.rs index aed7f5db..d7320e89 100644 --- a/crates/rbuilder/src/backtest/redistribute/mod.rs +++ b/crates/rbuilder/src/backtest/redistribute/mod.rs @@ -25,6 +25,7 @@ use reth_provider::ProviderFactory; use std::cmp::{max, min}; use std::sync::Arc; use tracing::{debug, info, info_span, trace, warn}; +use revmc_toolkit_load::EvmCompilerFns; #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] @@ -92,6 +93,7 @@ pub fn calc_redistributions( config: &ConfigType, block_data: BlockData, distribute_to_mempool_txs: bool, + llvm_compiler_fns: Option, ) -> eyre::Result { let _block_span = info_span!("block", block = block_data.block_number).entered(); let protect_signers = config.base_config().backtest_protect_bundle_signers.clone(); @@ -106,6 +108,7 @@ pub fn calc_redistributions( config.base_config().chain_spec()?, &block_data, &included_orders_available, + llvm_compiler_fns.clone(), )?; let available_orders = split_orders_by_identities( @@ -116,7 +119,11 @@ pub fn calc_redistributions( ); let results_without_exclusion = - calculate_backtest_without_exclusion(provider_factory.clone(), config, block_data.clone())?; + calculate_backtest_without_exclusion( + provider_factory.clone(), + config, block_data.clone(), + llvm_compiler_fns.clone() + )?; let exclusion_results = calculate_backtest_identity_and_order_exclusion( provider_factory.clone(), @@ -124,6 +131,7 @@ pub fn calc_redistributions( block_data.clone(), &available_orders, &results_without_exclusion, + llvm_compiler_fns.clone(), )?; let exclusion_results = calc_joint_exclusion_results( @@ -133,6 +141,7 @@ pub fn calc_redistributions( &available_orders, &results_without_exclusion, exclusion_results, + llvm_compiler_fns.clone(), )?; let calculated_redistribution_result = apply_redistribution_formula( @@ -238,11 +247,13 @@ fn restore_available_landed_orders( chain_spec: Arc, block_data: &BlockData, included_orders_available: &[OrdersWithTimestamp], + llvm_compiler_fns: Option, ) -> eyre::Result> { let block_txs = sim_historical_block( provider_factory.clone(), chain_spec, block_data.onchain_block.clone(), + llvm_compiler_fns, )? .into_iter() .map(|executed_tx| { @@ -431,6 +442,7 @@ fn calculate_backtest_without_exclusion( provider_factory: ProviderFactory>, config: &ConfigType, block_data: BlockData, + llvm_compiler_fns: Option, ) -> eyre::Result { let ExclusionResult { profit, @@ -447,6 +459,7 @@ fn calculate_backtest_without_exclusion( orders_excluded_before: vec![], profit_before: U256::ZERO, }, + llvm_compiler_fns, )?; Ok(ResultsWithoutExclusion { profit, @@ -492,6 +505,7 @@ fn calculate_backtest_identity_and_order_exclusion, ) -> eyre::Result { let included_orders_exclusion = { let mut result = Vec::new(); @@ -517,6 +531,7 @@ fn calculate_backtest_identity_and_order_exclusion( available_orders: &AvailableOrders, results_without_exclusion: &ResultsWithoutExclusion, mut exclusion_results: ExclusionResults, + llvm_compiler_fns: Option, ) -> eyre::Result { // calculate identities that are possibly connected let mut joint_contribution_todo: Vec<(Address, Address)> = Vec::new(); @@ -619,6 +636,7 @@ fn calc_joint_exclusion_results( config, &block_data, results_without_exclusion.exclusion_input(orders), + llvm_compiler_fns.clone(), ) .map(|ok| ((address1, address2), ok)) }) @@ -869,6 +887,7 @@ fn calc_profit_after_exclusion( config: &ConfigType, block_data: &BlockData, exclusion_input: ExclusionInput, + llvm_compiler_fns: Option, ) -> eyre::Result { let block_data_with_excluded = { let mut block_data = block_data.clone(); @@ -904,6 +923,7 @@ fn calc_profit_after_exclusion( config, base_config.blocklist()?, &base_config.sbundle_mergeabe_signers(), + llvm_compiler_fns, )? .builder_outputs .into_iter() diff --git a/crates/rbuilder/src/backtest/restore_landed_orders/resim_landed_block.rs b/crates/rbuilder/src/backtest/restore_landed_orders/resim_landed_block.rs index bffbdf4f..32760d85 100644 --- a/crates/rbuilder/src/backtest/restore_landed_orders/resim_landed_block.rs +++ b/crates/rbuilder/src/backtest/restore_landed_orders/resim_landed_block.rs @@ -9,6 +9,7 @@ use eyre::Context; use reth_chainspec::ChainSpec; use reth_db::DatabaseEnv; use reth_primitives::{Receipt, TransactionSignedEcRecovered, TxHash}; +use revmc_toolkit_load::EvmCompilerFns; use reth_provider::ProviderFactory; use std::sync::Arc; @@ -30,6 +31,7 @@ pub fn sim_historical_block( provider_factory: ProviderFactory>, chain_spec: Arc, onchain_block: alloy_rpc_types::Block, + llvm_compiler_fns: Option, ) -> eyre::Result> { let mut results = Vec::new(); @@ -46,6 +48,7 @@ pub fn sim_historical_block( coinbase, suggested_fee_recipient, None, + llvm_compiler_fns, ); let state_provider = provider_factory.history_by_block_hash(ctx.attributes.parent)?; diff --git a/crates/rbuilder/src/backtest/utils.rs b/crates/rbuilder/src/backtest/utils.rs new file mode 100644 index 00000000..3cf3612a --- /dev/null +++ b/crates/rbuilder/src/backtest/utils.rs @@ -0,0 +1,10 @@ +macro_rules! timeit { + ($block:expr) => {{ + let t0 = std::time::Instant::now(); + let res = $block; + let elapsed = t0.elapsed(); + (res, elapsed) + }}; +} + +pub(crate) use timeit; \ No newline at end of file diff --git a/crates/rbuilder/src/bin/debug-bench-machine.rs b/crates/rbuilder/src/bin/debug-bench-machine.rs index f2cc7584..6c91c323 100644 --- a/crates/rbuilder/src/bin/debug-bench-machine.rs +++ b/crates/rbuilder/src/bin/debug-bench-machine.rs @@ -72,6 +72,7 @@ async fn main() -> eyre::Result<()> { coinbase, suggested_fee_recipient, None, + None, ); // let signer = Signer::try_from_secret(B256::random())?; diff --git a/crates/rbuilder/src/bin/dummy-builder.rs b/crates/rbuilder/src/bin/dummy-builder.rs index 561a9294..5d40e435 100644 --- a/crates/rbuilder/src/bin/dummy-builder.rs +++ b/crates/rbuilder/src/bin/dummy-builder.rs @@ -99,6 +99,7 @@ async fn main() -> eyre::Result<()> { extra_rpc: RpcModule::new(()), sink_factory: Box::new(TraceBlockSinkFactory {}), builders: vec![Arc::new(DummyBuildingAlgorithm::new(10))], + llvm_compiled_fns: None, }; let ctrlc = tokio::spawn(async move { diff --git a/crates/rbuilder/src/bin/revmc-aot-compile.rs b/crates/rbuilder/src/bin/revmc-aot-compile.rs new file mode 100644 index 00000000..acaba463 --- /dev/null +++ b/crates/rbuilder/src/bin/revmc-aot-compile.rs @@ -0,0 +1,175 @@ +use rbuilder::live_builder::{base_config, config::Config as ConfigType}; +use revmc_toolkit_utils::{evm as evm_utils, build as build_utils}; +use revmc_toolkit_sim::gas_guzzlers::GasGuzzlerConfig; +use revmc_toolkit_build::CompilerOptions; +use std::{path::PathBuf, str::FromStr}; +use revm::primitives::SpecId; +use eyre::{OptionExt, Result}; +use tracing::debug; +use clap::Parser; + +// todo: compiler config in file instead? +#[derive(Parser, Debug)] +struct Cli { + #[clap(flatten)] + compiler_opt: CompilerOptionsCli, + #[clap(long, help = "Config file path", env = "RBUILDER_CONFIG")] + config_path: PathBuf, + #[clap(subcommand)] + commands: Commands, +} + +#[derive(Parser, Debug)] +enum Commands { + #[clap(about = "Fetches contracts specified in config file.")] + FromConfig { build_file_path: PathBuf }, + #[clap(about = "Fetches contracts based on their historical gas usage.")] + GasGuzzlers(GasGuzzlerConfigCli), +} + +#[derive(Parser, Debug)] +struct CompilerOptionsCli { + #[clap(long, value_parser, help = "Output directory")] + out_dir: Option, + #[clap(long, value_parser, help = "Target features")] + target_features: Option, + #[clap(long, value_parser, help = "Target CPU")] + target_cpu: Option, + #[clap(long, value_parser, help = "Target")] + target: Option, + #[clap(long, value_parser, help = "Optimization level")] + opt_level: Option, + #[clap(long, value_parser, help = "No link")] + no_link: Option, + #[clap(long, value_parser, help = "No gas")] + no_gas: Option, + #[clap(long, value_parser, help = "No length checks")] + no_len_checks: Option, + #[clap(long, value_parser, help = "Frame pointers")] + frame_pointers: Option, + #[clap(long, value_parser, help = "Debug assertions")] + debug_assertions: Option, + #[clap(long, value_parser, help = "Label")] + label: Option, + #[clap(long, value_parser, help = "Spec id")] + spec_id: Option, +} + +macro_rules! set_if_some { + ($src:ident, $dst:ident, { $( $field:ident ),+ }) => { + $( + if let Some(value) = $src.$field { + $dst.$field = value; + } + )+ + }; +} + +macro_rules! set_if_some_opt { + ($src:ident, $dst:ident, { $( $field:ident ),+ }) => { + $( + if let Some(value) = $src.$field { + $dst.$field = Some(value); + } + )+ + }; +} + +impl TryFrom for CompilerOptions { + type Error = eyre::Error; + + fn try_from(config_cli: CompilerOptionsCli) -> Result { + let mut config = CompilerOptions::default(); + + set_if_some_opt!(config_cli, config, { target_features, target_cpu, label }); + set_if_some!(config_cli, config, { no_link, no_gas, no_len_checks, frame_pointers, debug_assertions, target }); + + if let Some(out_dir) = config_cli.out_dir { + config.out_dir = PathBuf::from_str(&out_dir)?; + } + config.opt_level = config_cli.opt_level + .map(|opt_id| opt_id.try_into()) + .transpose()? + .unwrap_or_default(); + config.spec_id = config_cli.spec_id + .map(|s_id| SpecId::try_from_u8(s_id).ok_or_eyre("Invalid spec id")) + .transpose()? + .unwrap_or_default(); + + Ok(config) + } +} + +#[derive(Parser, Debug)] +struct GasGuzzlerConfigCli { + #[clap(long, help = "Start block")] + start_block: u64, + #[clap(long, help = "End block")] + end_block: Option, + #[clap(long, help = "Number of top contracts to consider.")] + head_size: usize, + #[clap(long, help = "Size of the sample. If not provided, all blocks will be used.")] + sample_size: Option, + #[clap(long, help = "Seed for generating pseudo-random sample")] + seed: Option, + +} + +impl From for GasGuzzlerConfig { + fn from(config: GasGuzzlerConfigCli) -> Self { + let rnd_seed = config.seed.map(|seed| revm::primitives::keccak256(seed.as_bytes()).0); + Self { + start_block: config.start_block, + end_block: config.end_block, + sample_size: config.sample_size, + seed: rnd_seed, + } + } +} + +fn main() { + if let Err(e) = run() { + eprintln!("Error: {}", e); + std::process::exit(1); + } +} + +fn run() -> Result<()> { + let Cli { compiler_opt, config_path, commands } = Cli::parse(); + let provider_factory = new_provider_factory(config_path)?; + + match commands { + Commands::FromConfig { build_file_path } => { + debug!("Fetching contracts from config at {:?}", build_file_path); + let _: () = build_utils::compile_aot_from_file_path( + &provider_factory.latest()?, + &build_file_path + )?.into_iter().collect::>()?; + } + Commands::GasGuzzlers(gas_guzzlers_config) => { + debug!("Searching for gas guzzlers"); + let head_size = gas_guzzlers_config.head_size; + let bytecodes = GasGuzzlerConfig::from(gas_guzzlers_config) + .find_gas_guzzlers(provider_factory)? + .contract_to_bytecode()? + .into_top_guzzlers(head_size); + let compiler_opt = CompilerOptions::try_from(compiler_opt)?; + debug!("Compiling contracts to {:?}", compiler_opt.out_dir); + let _: () = build_utils::compile_aot_from_codes( + bytecodes, + Some(compiler_opt) + )?.into_iter().collect::>()?; + } + // todo: add remove command + // todo: add command that lists top gas guzzler addresses + } + + Ok(()) +} + +fn new_provider_factory(config_path: PathBuf) -> Result> { + let config: ConfigType = base_config::load_config_toml_and_env(config_path)?; + // todo: get provider_factory straight from the config with create_provider_factory + let reth_path = config.base_config.reth_db_path.expect("RETH_DB_PATH field not set"); + evm_utils::make_provider_factory(&reth_path) +} \ No newline at end of file diff --git a/crates/rbuilder/src/building/evm_inspector.rs b/crates/rbuilder/src/building/evm_inspector.rs index 073c5889..47febca4 100644 --- a/crates/rbuilder/src/building/evm_inspector.rs +++ b/crates/rbuilder/src/building/evm_inspector.rs @@ -40,7 +40,7 @@ enum NextStepAction { } #[derive(Debug)] -struct UsedStateEVMInspector<'a> { +pub(crate) struct UsedStateEVMInspector<'a> { next_step_action: NextStepAction, used_state_trace: &'a mut UsedStateTrace, } diff --git a/crates/rbuilder/src/building/mod.rs b/crates/rbuilder/src/building/mod.rs index 491e51f8..c0e880b7 100644 --- a/crates/rbuilder/src/building/mod.rs +++ b/crates/rbuilder/src/building/mod.rs @@ -10,6 +10,7 @@ pub mod payout_tx; pub mod sim; pub mod testing; pub mod tracers; +pub mod revmc_ext_ctx; pub use block_orders::BlockOrders; use eth_sparse_mpt::SparseTrieSharedCache; use reth_primitives::proofs::calculate_requests_root; @@ -45,6 +46,7 @@ use revm::{ db::states::bundle_state::BundleRetention::{self, PlainState}, primitives::{BlobExcessGasAndPrice, BlockEnv, CfgEnvWithHandlerCfg, SpecId}, }; +use revmc_toolkit_load::EvmCompilerFns; use serde::Deserialize; use std::{hash::Hash, str::FromStr, sync::Arc}; use thiserror::Error; @@ -66,6 +68,7 @@ pub struct BlockBuildingContext { pub initialized_cfg: CfgEnvWithHandlerCfg, pub attributes: EthPayloadBuilderAttributes, pub chain_spec: Arc, + pub llvm_compiled_fns: Option, /// Signer to sign builder payoffs (end of block and mev-share). /// Is Option to avoid any possible bug (losing money!) with payoffs. /// None: coinbase = attributes.suggested_fee_recipient. No payoffs allowed. @@ -92,6 +95,7 @@ impl BlockBuildingContext { prefer_gas_limit: Option, extra_data: Vec, spec_id: Option, + llvm_compiled_fns: Option, ) -> BlockBuildingContext { let attributes = EthPayloadBuilderAttributes::try_new( attributes.data.parent_block_hash, @@ -138,6 +142,7 @@ impl BlockBuildingContext { initialized_cfg, attributes, chain_spec, + llvm_compiled_fns, builder_signer: Some(signer), blocklist, extra_data, @@ -158,6 +163,7 @@ impl BlockBuildingContext { coinbase: Address, suggested_fee_recipient: Address, builder_signer: Option, + llvm_compiled_fns: Option, ) -> BlockBuildingContext { let block_number = onchain_block.header.number; @@ -228,6 +234,7 @@ impl BlockBuildingContext { extra_data: Vec::new(), excess_blob_gas: onchain_block.header.excess_blob_gas.map(|b| b as u64), spec_id, + llvm_compiled_fns, shared_sparse_mpt_cache: Default::default(), } } @@ -244,6 +251,7 @@ impl BlockBuildingContext { Default::default(), Default::default(), Default::default(), + None, ) } diff --git a/crates/rbuilder/src/building/order_commit.rs b/crates/rbuilder/src/building/order_commit.rs index 59f43bda..e131a38c 100644 --- a/crates/rbuilder/src/building/order_commit.rs +++ b/crates/rbuilder/src/building/order_commit.rs @@ -31,6 +31,8 @@ use revm::{ use crate::building::evm_inspector::{RBuilderEVMInspector, UsedStateTrace}; use std::{collections::HashMap, sync::Arc}; use thiserror::Error; +use crate::building::revmc_ext_ctx::RBuilderRevmcInspectorExtCtx; +use revmc_toolkit_load::{revmc_register_handler, RevmcExtCtx}; #[derive(Clone)] pub struct BlockState { @@ -431,14 +433,23 @@ impl<'a, 'b, Tracer: SimulationTracer> PartialBlockFork<'a, 'b, Tracer> { let used_state_tracer = self.tracer.as_mut().and_then(|t| t.get_used_state_tracer()); let mut rbuilder_inspector = RBuilderEVMInspector::new(tx, used_state_tracer); + let rbuilder_inspector_with_revmc = RBuilderRevmcInspectorExtCtx::new( + &mut rbuilder_inspector, + ctx.llvm_compiled_fns.clone() + .map(|fns| RevmcExtCtx::from(fns).with_touch_tracking()), + ); - let mut evm = revm::Evm::builder() + let mut partial_evm = revm::Evm::builder() .with_spec_id(ctx.spec_id) .with_env(Box::new(env)) .with_db(db.as_mut()) - .with_external_context(&mut rbuilder_inspector) - .append_handler_register(inspector_handle_register) - .build(); + .with_external_context(rbuilder_inspector_with_revmc) + .append_handler_register(inspector_handle_register); + if ctx.llvm_compiled_fns.is_some() { + partial_evm = partial_evm.append_handler_register(revmc_register_handler); + } + let mut evm = partial_evm.build(); + let res = match evm.transact() { Ok(res) => res, Err(err) => match err { @@ -451,6 +462,8 @@ impl<'a, 'b, Tracer: SimulationTracer> PartialBlockFork<'a, 'b, Tracer> { | EVMError::Precompile(_) => return Err(err.into()), }, }; + // todo: does it make sense to log how many touches were non-native? + let mut db_context = evm.into_context(); let db = &mut db_context.evm.db; let access_list = rbuilder_inspector.into_access_list(); diff --git a/crates/rbuilder/src/building/revmc_ext_ctx.rs b/crates/rbuilder/src/building/revmc_ext_ctx.rs new file mode 100644 index 00000000..c249a5a0 --- /dev/null +++ b/crates/rbuilder/src/building/revmc_ext_ctx.rs @@ -0,0 +1,92 @@ +use revm::{ + interpreter::{CallInputs, CallOutcome, Interpreter}, + primitives::{B256, U256, Address}, + Database, EvmContext, Inspector, +}; +use revmc_toolkit_load::{ + RevmcExtCtxExtTrait, + RevmcExtCtx, + EvmCompilerFn, + Touches, +}; +use super::evm_inspector::{ + RBuilderEVMInspector, + UsedStateEVMInspector, +}; + + +pub struct RBuilderRevmcInspectorExtCtx<'a, 'b> where 'a: 'b { + evm_inspector: &'b mut RBuilderEVMInspector<'a>, + pub revmc_ext_ctx: Option, +} + +impl<'a, 'b> RBuilderRevmcInspectorExtCtx<'a, 'b> { + pub fn new( + evm_inspector: &'b mut RBuilderEVMInspector<'a>, + revmc_ext_ctx: Option, + ) -> Self { + Self { + evm_inspector, + revmc_ext_ctx, + } + } + +} + +impl<'a, 'b, DB> Inspector for RBuilderRevmcInspectorExtCtx<'a, 'b> +where + DB: Database, + UsedStateEVMInspector<'a>: Inspector, +{ + #[inline] + fn step(&mut self, interp: &mut Interpreter, data: &mut EvmContext) { + self.evm_inspector.step(interp, data); + } + + #[inline] + fn call( + &mut self, + context: &mut EvmContext, + inputs: &mut CallInputs, + ) -> Option { + self.evm_inspector.call(context, inputs) + } + + #[inline] + fn create_end( + &mut self, + context: &mut EvmContext, + inputs: &revm::interpreter::CreateInputs, + outcome: revm::interpreter::CreateOutcome, + ) -> revm::interpreter::CreateOutcome { + self.evm_inspector.create_end(context, inputs, outcome) + } + + #[inline] + fn selfdestruct(&mut self, contract: Address, target: Address, value: U256) { + self.evm_inspector.selfdestruct(contract, target, value) + } +} + +// todo: improve this - move away from if let statements +impl RevmcExtCtxExtTrait for RBuilderRevmcInspectorExtCtx<'_, '_> { + fn get_function(&self, bytecode_hash: B256) -> Option { + if let Some(revmc_ext_ctx) = &self.revmc_ext_ctx { + revmc_ext_ctx.get_function(bytecode_hash) + } else { + None + } + } + fn register_touch(&mut self, address: Address, non_native: bool) { + if let Some(revmc_ext_ctx) = &mut self.revmc_ext_ctx { + revmc_ext_ctx.register_touch(address, non_native); + } + } + fn touches(&self) -> Option<&Touches> { + if let Some(revmc_ext_ctx) = &self.revmc_ext_ctx { + revmc_ext_ctx.touches.as_ref() + } else { + None + } + } +} diff --git a/crates/rbuilder/src/building/testing/test_chain_state.rs b/crates/rbuilder/src/building/testing/test_chain_state.rs index 50832b8f..8f10cee3 100644 --- a/crates/rbuilder/src/building/testing/test_chain_state.rs +++ b/crates/rbuilder/src/building/testing/test_chain_state.rs @@ -319,6 +319,7 @@ impl TestBlockContextBuilder { self.prefer_gas_limit, vec![], Some(SpecId::SHANGHAI), + None, ); if self.use_suggested_fee_recipient_as_coinbase { res.modify_use_suggested_fee_recipient_as_coinbase(); diff --git a/crates/rbuilder/src/live_builder/base_config.rs b/crates/rbuilder/src/live_builder/base_config.rs index 83764560..25d273d8 100644 --- a/crates/rbuilder/src/live_builder/base_config.rs +++ b/crates/rbuilder/src/live_builder/base_config.rs @@ -9,12 +9,13 @@ use crate::{ }; use ahash::HashSet; use alloy_primitives::{Address, B256}; -use eyre::{eyre, Context}; +use eyre::{eyre, Context, OptionExt}; use jsonrpsee::RpcModule; use lazy_static::lazy_static; use reth::tasks::pool::BlockingTaskPool; use reth_chainspec::ChainSpec; use reth_db::DatabaseEnv; +use revmc_toolkit_load::{EvmCompilerFns, EvmCompilerFnLoader}; use reth_node_core::args::utils::chain_value_parser; use reth_primitives::StaticFileSegment; use reth_provider::StaticFileProviderFactory; @@ -101,6 +102,10 @@ pub struct BaseConfig { pub backtest_builders: Vec, pub backtest_results_store_path: PathBuf, pub backtest_protect_bundle_signers: Vec
, + + // llvm compiled fns + pub use_llvm_compiled_fns: bool, + pub aot_compiled_fns_dir: Option, } lazy_static! { @@ -132,6 +137,7 @@ pub fn load_config_toml_and_env( path.as_ref().to_string_lossy() ) })?; + println!("data: {:?}", data); let config: T = toml::from_str(&data).context("Config file parsing")?; Ok(config) @@ -213,9 +219,20 @@ impl BaseConfig { extra_rpc: RpcModule::new(()), sink_factory, builders: Vec::new(), + llvm_compiled_fns: self.load_llvm_compiled_fns()?, }) } + pub fn load_llvm_compiled_fns(&self) -> eyre::Result> { + // todo: consider whitelist/blacklist? + if !self.use_llvm_compiled_fns { + return Ok(None); + } + let dir_path = self.aot_compiled_fns_dir.as_ref() + .ok_or_eyre("no llvm dir specified")?; + EvmCompilerFnLoader::new(dir_path).load_all().map(|fns| Some(fns.into())) + } + pub fn jsonrpc_server_ip(&self) -> Ipv4Addr { parse_ip(&self.jsonrpc_server_ip) } @@ -437,6 +454,8 @@ impl Default for BaseConfig { live_builders: vec!["mgp-ordering".to_string(), "mp-ordering".to_string()], simulation_threads: 1, sbundle_mergeabe_signers: None, + use_llvm_compiled_fns: false, + aot_compiled_fns_dir: None, } } } diff --git a/crates/rbuilder/src/live_builder/mod.rs b/crates/rbuilder/src/live_builder/mod.rs index 7d5b6a24..aca4d5c2 100644 --- a/crates/rbuilder/src/live_builder/mod.rs +++ b/crates/rbuilder/src/live_builder/mod.rs @@ -38,6 +38,7 @@ use time::OffsetDateTime; use tokio::{sync::mpsc, task::spawn_blocking}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; +use revmc_toolkit_load::EvmCompilerFns; /// Time the proposer have to propose a block from the beginning of the slot (https://www.paradigm.xyz/2023/04/mev-boost-ethereum-consensus Slot anatomy) const SLOT_PROPOSAL_DURATION: std::time::Duration = Duration::from_secs(4); @@ -76,6 +77,8 @@ pub struct LiveBuilder { pub sink_factory: Box, pub builders: Vec>>, pub extra_rpc: RpcModule<()>, + + pub llvm_compiled_fns: Option, } impl @@ -215,6 +218,7 @@ impl Some(payload.suggested_gas_limit), self.extra_data.clone(), None, + self.llvm_compiled_fns.clone(), ); builder_pool.start_block_building(