[WIP] Add remote state provider access

config-backtest-example.toml

@@ -2,8 +2,10 @@ log_level = "info,rbuilder=debug"
 
 chain = "mainnet"
 reth_datadir = "/mnt/data/reth"
+relay_secret_key = "5eae315483f028b5cdd5d1090ff0c7618b18737ea9bf3c35047189db22835c48"
+coinbase_secret_key = "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
 
-backtest_fetch_eth_rpc_url = "http://127.0.0.1:8545"
+backtest_fetch_eth_rpc_url = "http://localhost:8545"
 backtest_fetch_eth_rpc_parallel = 400
 backtest_fetch_output_file = "~/.rbuilder/backtest/main.sqlite"
 backtest_fetch_mempool_data_dir = "~/.rbuilder/mempool-data"

crates/rbuilder/Cargo.toml

@@ -56,7 +56,7 @@ ethereum_ssz.workspace = true
 
 test_utils = { path = "src/test_utils" }
 
-reqwest = { version = "0.11.20", features = ["blocking"] }
+reqwest = { version = "0.12", features = ["blocking"] }
 serde_with = { version = "3.8.1", features = ["time_0_3"] }
 primitive-types = "0.12.1"
 url = "2.4.1"

crates/rbuilder/src/backtest/backtest_build_block.rs

@@ -20,6 +20,7 @@ use crate::{
     building::builders::BacktestSimulateBlockInput,
     live_builder::{base_config::load_config_toml_and_env, cli::LiveBuilderConfig},
     primitives::{Order, OrderId, SimulatedOrder},
+    provider::http_provider::HttpProvider,
     utils::timestamp_as_u64,
 };
 use clap::Parser;
@@ -80,15 +81,18 @@ pub async fn run_backtest_build_block<ConfigType: LiveBuilderConfig>() -> eyre::
         .unzip();
 
     println!("Available orders: {}", orders.len());
-
+    println!("A");
     if cli.show_orders {
         print_order_and_timestamp(&block_data.available_orders, &block_data);
     }
 
-    let provider_factory = config.base_config().provider_factory()?;
+    // let provider_factory = config.base_config().provider_factory()?;
+    let provider_factory =
+        HttpProvider::new_with_url(config.base_config().backtest_fetch_eth_rpc_url.as_str());
+
     let chain_spec = config.base_config().chain_spec()?;
     let sbundle_mergeabe_signers = config.base_config().sbundle_mergeabe_signers();
-
+    println!("B");
     if cli.sim_landed_block {
         let tx_sim_results = sim_historical_block(
             provider_factory.clone(),
@@ -97,7 +101,7 @@ pub async fn run_backtest_build_block<ConfigType: LiveBuilderConfig>() -> eyre::
         )?;
         print_onchain_block_data(tx_sim_results, &orders, &block_data);
     }
-
+    println!("B1");
     let BacktestBlockInput {
         ctx, sim_orders, ..
     } = backtest_prepare_ctx_for_block(
@@ -108,16 +112,20 @@ pub async fn run_backtest_build_block<ConfigType: LiveBuilderConfig>() -> eyre::
         config.base_config().blocklist()?,
         config.base_config().coinbase_signer()?,
     )?;
-
+    println!("C");
     if cli.show_sim {
         print_simulated_orders(&sim_orders, &order_and_timestamp, &block_data);
     }
 
+    println!("D");
+
     if !cli.no_block_building {
         let winning_builder = cli
             .builders
             .iter()
             .filter_map(|builder_name: &String| {
+                println!("E");
+
                 let input = BacktestSimulateBlockInput {
                     ctx: ctx.clone(),
                     builder_name: builder_name.clone(),

crates/rbuilder/src/building/mod.rs

@@ -688,6 +688,9 @@ impl<Tracer: SimulationTracer> PartialBlock<Tracer> {
         ctx: &BlockBuildingContext,
         state: &mut BlockState,
     ) -> eyre::Result<()> {
+        println!("SKIP FOR NOW; FIX LATER PRECALL");
+        return Ok(());
+
         let mut db = state.new_db_ref();
         pre_block_beacon_root_contract_call(
             db.as_mut(),

crates/rbuilder/src/building/sim.rs

@@ -206,7 +206,13 @@ impl<Provider: StateProviderFactory> SimTree<Provider> {
 
     pub fn push_orders(&mut self, orders: Vec<Order>) -> Result<(), ProviderError> {
         let state = self.nonce_cache.get_ref()?;
+        let total = orders.len();
+        println!("Nim orders: {:?}", orders.len());
+        let mut count = 0;
         for order in orders {
+            println!("Nim order: {:?}/{:?}", count, total);
+            count += 1;
+
             self.push_order(order, &state)?;
         }
         Ok(())
@@ -311,23 +317,34 @@ pub fn simulate_all_orders_with_sim_tree<Provider: StateProviderFactory + Clone>
     orders: &[Order],
     randomize_insertion: bool,
 ) -> Result<(Vec<SimulatedOrder>, Vec<OrderErr>), CriticalCommitOrderError> {
+    println!("HERE!");
+
     let mut sim_tree = SimTree::new(factory.clone(), ctx.attributes.parent);
 
+    println!("HERE2!");
+
     let mut orders = orders.to_vec();
     let random_insert_size = max(orders.len() / 20, 1);
     if randomize_insertion {
+        println!("HERE2.1!");
+
         let mut rng = rand::thread_rng();
         // shuffle orders
         orders.shuffle(&mut rng);
     } else {
+        println!("HERE2.2!");
+
         sim_tree.push_orders(orders.clone())?;
     }
+    println!("HERE3!");
 
     let mut sim_errors = Vec::new();
     let mut state_for_sim =
         Arc::<dyn StateProvider>::from(factory.history_by_block_hash(ctx.attributes.parent)?);
     let mut cache_reads = Some(CachedReads::default());
     loop {
+        println!("Simulating orders, orders len: {}", orders.len());
+
         // mix new orders into the sim_tree
         if randomize_insertion && !orders.is_empty() {
             let insert_size = min(random_insert_size, orders.len());
@@ -344,6 +361,8 @@ pub fn simulate_all_orders_with_sim_tree<Provider: StateProviderFactory + Clone>
             }
         }
 
+        println!("Simulating orders, sim_tasks len: {}", sim_tasks.len());
+
         let mut sim_results = Vec::new();
         for sim_task in sim_tasks {
             let start_time = Instant::now();