Merge pull request #209 from Vid201/chore/db

chore: use db by default

.github/workflows/ci.yml

@@ -100,6 +100,7 @@ jobs:
         with:
           name: silius
           path: silius-artifact
+
       - run: mv silius-artifact/silius /usr/local/bin/
       - run: chmod a+x /usr/local/bin/silius
       - run: chmod a+x silius/bundler-spec-tests/launcher.sh

Cargo.toml

@@ -23,6 +23,7 @@ rust-version = "1.71.1"
 anyhow = "1"
 async-trait = "0.1"
 ethers = { git = "https://github.com/gakonst/ethers-rs", rev = "fa3017715a298728d9fb341933818a5d0d84c2dc" }
+expanded-pathbuf = "0.1"
 parking_lot = "0.12"
 serde_json = "1"
 tokio = { version = "1.18", features = ["full"] }

bin/silius/Cargo.toml

@@ -17,7 +17,7 @@ anyhow = { workspace = true }
 clap = { version = "4", features = ["derive"] }
 dirs = "4.0"
 ethers = { workspace = true }
-expanded-pathbuf = "0.1"
+expanded-pathbuf = { workspace = true }
 log = "0.4.19"
 pin-utils = "0.1"
 silius-bundler = { path = "../../crates/bundler" }

bin/silius/src/cli.rs

@@ -1,14 +1,18 @@
 use crate::utils::{parse_address, parse_u256, parse_uopool_mode};
 use clap::Parser;
 use ethers::types::{Address, U256};
+use expanded_pathbuf::ExpandedPathBuf;
 use silius_primitives::UoPoolMode;
 use std::net::SocketAddr;
 
-#[derive(Clone, Debug, Parser, PartialEq)]
+#[derive(Clone, Debug, Parser)]
 pub struct UoPoolServiceOpts {
     #[clap(long, default_value = "127.0.0.1:3001")]
     pub uopool_grpc_listen_address: SocketAddr,
 
+    #[clap(long)]
+    pub datadir: Option<ExpandedPathBuf>,
+
     #[clap(long, value_parser=parse_u256, default_value = "1")]
     pub min_stake: U256,
 

bin/silius/src/create-wallet.rs

@@ -1,9 +1,8 @@
 use anyhow::Result;
 use clap::Parser;
-use dirs::home_dir;
 use ethers::types::U256;
 use expanded_pathbuf::ExpandedPathBuf;
-use silius::utils::parse_u256;
+use silius::utils::{parse_u256, unwrap_path_or_home};
 use silius_primitives::Wallet;
 use tracing::info;
 
@@ -28,14 +27,7 @@ fn main() -> Result<()> {
 
     tracing_subscriber::fmt::init();
 
-    let path = if let Some(output_path) = opt.output_path {
-        output_path
-    } else {
-        home_dir()
-            .map(|h| h.join(".silius"))
-            .ok_or_else(|| anyhow::anyhow!("Get Home directory error"))
-            .map(ExpandedPathBuf)?
-    };
+    let path = unwrap_path_or_home(opt.output_path)?;
 
     if opt.build_fb_wallet == Some(true) {
         let wallet = Wallet::build_random(path, &opt.chain_id, true)?;

bin/silius/src/silius-uopool.rs

@@ -6,7 +6,7 @@ use ethers::{
 };
 use silius::{
     cli::UoPoolServiceOpts,
-    utils::{parse_address, parse_u256},
+    utils::{parse_address, parse_u256, unwrap_path_or_home},
 };
 use silius_grpc::uopool_service_run;
 use silius_primitives::{chain::SUPPORTED_CHAINS, Chain};
@@ -62,10 +62,13 @@ async fn main() -> Result<()> {
         }
     }
 
+    let datadir = unwrap_path_or_home(opt.uopool_opts.datadir)?;
+
     info!("Starting uopool gRPC service...");
 
     uopool_service_run(
         opt.uopool_opts.uopool_grpc_listen_address,
+        datadir,
         opt.entry_points,
         eth_client,
         chain,

bin/silius/src/silius.rs

@@ -7,7 +7,7 @@ use ethers::{
 use expanded_pathbuf::ExpandedPathBuf;
 use silius::{
     cli::{BundlerServiceOpts, RpcServiceOpts, UoPoolServiceOpts},
-    utils::{parse_address, parse_u256, run_until_ctrl_c},
+    utils::{parse_address, parse_u256, run_until_ctrl_c, unwrap_path_or_home},
 };
 use silius_grpc::{
     bundler_client::BundlerClient, bundler_service_run, uo_pool_client::UoPoolClient,
@@ -114,9 +114,12 @@ fn main() -> Result<()> {
 
 
                 if !opt.no_uopool {
+                    let datadir = unwrap_path_or_home(opt.uopool_opts.datadir)?;
+
                     info!("Starting uopool gRPC service...");
                     uopool_service_run(
                         opt.uopool_opts.uopool_grpc_listen_address,
+                        datadir,
                         opt.entry_points.clone(),
                         eth_client,
                         chain,

bin/silius/src/utils.rs

@@ -1,9 +1,23 @@
+use dirs::home_dir;
 use ethers::types::{Address, U256};
+use expanded_pathbuf::ExpandedPathBuf;
 use pin_utils::pin_mut;
 use silius_primitives::UoPoolMode;
 use std::{future::Future, str::FromStr};
 use tracing::info;
 
+/// Unwrap path or returns home directory
+pub fn unwrap_path_or_home(path: Option<ExpandedPathBuf>) -> anyhow::Result<ExpandedPathBuf> {
+    if let Some(path) = path {
+        Ok(path)
+    } else {
+        home_dir()
+            .map(|h| h.join(".silius"))
+            .ok_or_else(|| anyhow::anyhow!("Get Home directory error"))
+            .map(ExpandedPathBuf)
+    }
+}
+
 /// Parses address from string
 pub fn parse_address(s: &str) -> Result<Address, String> {
     Address::from_str(s).map_err(|_| format!("String {s} is not a valid address"))

crates/grpc/Cargo.toml

@@ -16,6 +16,7 @@ arrayref = "0.3"
 async-trait = { workspace = true }
 dashmap = "5.4.0"
 ethers = { workspace = true }
+expanded-pathbuf = { workspace = true }
 parking_lot = { workspace = true }
 prost = "0.11"
 serde_json = { workspace = true }

crates/grpc/src/builder.rs

@@ -1,5 +1,4 @@
-use std::sync::Arc;
-
+use crate::uopool::{GAS_INCREASE_PERC, MAX_UOS_PER_UNSTAKED_SENDER};
 use ethers::{
     providers::Middleware,
     types::{Address, U256},
@@ -13,14 +12,14 @@ use silius_uopool::{
     validate::validator::StandardUserOperationValidator, Mempool, MempoolBox, Reputation,
     ReputationBox, UoPool, VecCh, VecUo,
 };
+use std::fmt::{Debug, Display};
+use std::sync::Arc;
 
-use crate::uopool::{GAS_INCREASE_PERC, MAX_UOS_PER_UNSTAKED_SENDER};
-
-pub struct UoPoolBuilder<M, P, R>
+pub struct UoPoolBuilder<M, P, R, E>
 where
     M: Middleware + Clone + 'static,
-    P: Mempool<UserOperations = VecUo, CodeHashes = VecCh, Error = anyhow::Error> + Send + Sync,
-    R: Reputation<ReputationEntries = Vec<ReputationEntry>, Error = anyhow::Error> + Send + Sync,
+    P: Mempool<UserOperations = VecUo, CodeHashes = VecCh, Error = E> + Send + Sync,
+    R: Reputation<ReputationEntries = Vec<ReputationEntry>, Error = E> + Send + Sync,
 {
     is_unsafe: bool,
     eth_client: Arc<M>,
@@ -31,15 +30,16 @@ where
     min_unstake_delay: U256,
     min_priority_fee_per_gas: U256,
     whitelist: Vec<Address>,
-    mempool: MempoolBox<VecUo, VecCh, P>,
-    reputation: ReputationBox<Vec<ReputationEntry>, R>,
+    mempool: MempoolBox<VecUo, VecCh, P, E>,
+    reputation: ReputationBox<Vec<ReputationEntry>, R, E>,
 }
 
-impl<M, P, R> UoPoolBuilder<M, P, R>
+impl<M, P, R, E> UoPoolBuilder<M, P, R, E>
 where
     M: Middleware + Clone + 'static,
-    P: Mempool<UserOperations = VecUo, CodeHashes = VecCh, Error = anyhow::Error> + Send + Sync,
-    R: Reputation<ReputationEntries = Vec<ReputationEntry>, Error = anyhow::Error> + Send + Sync,
+    P: Mempool<UserOperations = VecUo, CodeHashes = VecCh, Error = E> + Send + Sync,
+    R: Reputation<ReputationEntries = Vec<ReputationEntry>, Error = E> + Send + Sync,
+    E: Debug + Display,
 {
     #[allow(clippy::too_many_arguments)]
     pub fn new(
@@ -55,8 +55,8 @@ where
         mempool: P,
         reputation: R,
     ) -> Self {
-        let mempool = MempoolBox::<VecUo, VecCh, P>::new(mempool);
-        let mut reputation = ReputationBox::<Vec<ReputationEntry>, R>::new(reputation);
+        let mempool = MempoolBox::<VecUo, VecCh, P, E>::new(mempool);
+        let mut reputation = ReputationBox::<Vec<ReputationEntry>, R, E>::new(reputation);
         reputation.init(
             MIN_INCLUSION_RATE_DENOMINATOR,
             THROTTLING_SLACK,
@@ -82,7 +82,7 @@ where
         }
     }
 
-    pub fn uo_pool(&self) -> UoPool<M, StandardUserOperationValidator<M, P, R>, P, R> {
+    pub fn uo_pool(&self) -> UoPool<M, StandardUserOperationValidator<M, P, R, E>, P, R, E> {
         let entry_point = EntryPoint::<M>::new(self.eth_client.clone(), self.entrypoint_addr);
 
         let validator = if self.is_unsafe {
@@ -105,7 +105,7 @@ where
             )
         };
 
-        UoPool::<M, StandardUserOperationValidator<M, P, R>, P, R>::new(
+        UoPool::<M, StandardUserOperationValidator<M, P, R, E>, P, R, E>::new(
             entry_point,
             validator,
             self.mempool.clone(),

crates/grpc/src/uopool.rs

@@ -11,45 +11,50 @@ use ethers::{
     providers::{Http, Middleware, Provider},
     types::{Address, U256},
 };
+use expanded_pathbuf::ExpandedPathBuf;
 use silius_contracts::entry_point::EntryPointErr;
 use silius_primitives::reputation::ReputationEntry;
 use silius_primitives::{uopool::AddError, Chain, UoPoolMode};
 use silius_uopool::{
-    mempool_id, validate::validator::StandardUserOperationValidator, MemoryMempool,
-    MemoryReputation, MempoolId, Reputation, UoPool as UserOperationPool,
+    init_env, DBError, DatabaseMempool, DatabaseReputation, Mempool, VecCh, VecUo, WriteMap,
 };
-use silius_uopool::{Mempool, VecCh, VecUo};
+use silius_uopool::{
+    mempool_id, validate::validator::StandardUserOperationValidator, MempoolId, Reputation,
+    UoPool as UserOperationPool,
+};
+use std::fmt::{Debug, Display};
 use std::{net::SocketAddr, sync::Arc, time::Duration};
 use tonic::{Code, Request, Response, Status};
 use tracing::{info, warn};
 
 pub const MAX_UOS_PER_UNSTAKED_SENDER: usize = 4;
 pub const GAS_INCREASE_PERC: u64 = 10;
 
-type StandardUserPool<M, P, R> =
-    UserOperationPool<M, StandardUserOperationValidator<M, P, R>, P, R>;
+type StandardUserPool<M, P, R, E> =
+    UserOperationPool<M, StandardUserOperationValidator<M, P, R, E>, P, R, E>;
 
-pub struct UoPoolService<M, P, R>
+pub struct UoPoolService<M, P, R, E>
 where
     M: Middleware + Clone + 'static,
-    P: Mempool<UserOperations = VecUo, CodeHashes = VecCh, Error = anyhow::Error> + Send + Sync,
-    R: Reputation<ReputationEntries = Vec<ReputationEntry>, Error = anyhow::Error> + Send + Sync,
+    P: Mempool<UserOperations = VecUo, CodeHashes = VecCh, Error = E> + Send + Sync,
+    R: Reputation<ReputationEntries = Vec<ReputationEntry>, Error = E> + Send + Sync,
 {
-    pub uo_pools: Arc<DashMap<MempoolId, UoPoolBuilder<M, P, R>>>,
+    pub uo_pools: Arc<DashMap<MempoolId, UoPoolBuilder<M, P, R, E>>>,
     pub chain: Chain,
 }
 
-impl<M, P, R> UoPoolService<M, P, R>
+impl<M, P, R, E> UoPoolService<M, P, R, E>
 where
     M: Middleware + Clone + 'static,
-    P: Mempool<UserOperations = VecUo, CodeHashes = VecCh, Error = anyhow::Error> + Send + Sync,
-    R: Reputation<ReputationEntries = Vec<ReputationEntry>, Error = anyhow::Error> + Send + Sync,
+    P: Mempool<UserOperations = VecUo, CodeHashes = VecCh, Error = E> + Send + Sync,
+    R: Reputation<ReputationEntries = Vec<ReputationEntry>, Error = E> + Send + Sync,
+    E: Debug + Display,
 {
-    pub fn new(uo_pools: Arc<DashMap<MempoolId, UoPoolBuilder<M, P, R>>>, chain: Chain) -> Self {
+    pub fn new(uo_pools: Arc<DashMap<MempoolId, UoPoolBuilder<M, P, R, E>>>, chain: Chain) -> Self {
         Self { uo_pools, chain }
     }
 
-    fn get_uo_pool(&self, ep: &Address) -> tonic::Result<StandardUserPool<M, P, R>> {
+    fn get_uo_pool(&self, ep: &Address) -> tonic::Result<StandardUserPool<M, P, R, E>> {
         let m_id = mempool_id(ep, &U256::from(self.chain.id()));
         self.uo_pools
             .get(&m_id)
@@ -62,18 +67,13 @@ where
 }
 
 #[async_trait]
-impl<M, P, R> uo_pool_server::UoPool for UoPoolService<M, P, R>
+impl<M, P, R, E> uo_pool_server::UoPool for UoPoolService<M, P, R, E>
 where
     EntryPointErr: From<<M as Middleware>::Error>,
     M: Middleware + Clone + 'static,
-    P: Mempool<UserOperations = VecUo, CodeHashes = VecCh, Error = anyhow::Error>
-        + Send
-        + Sync
-        + 'static,
-    R: Reputation<ReputationEntries = Vec<ReputationEntry>, Error = anyhow::Error>
-        + Send
-        + Sync
-        + 'static,
+    P: Mempool<UserOperations = VecUo, CodeHashes = VecCh, Error = E> + Send + Sync + 'static,
+    R: Reputation<ReputationEntries = Vec<ReputationEntry>, Error = E> + Send + Sync + 'static,
+    E: Debug + Display + 'static,
 {
     async fn add(&self, req: Request<AddRequest>) -> Result<Response<AddResponse>, Status> {
         let req = req.into_inner();
@@ -347,6 +347,7 @@ where
 #[allow(clippy::too_many_arguments)]
 pub async fn uopool_service_run(
     grpc_listen_address: SocketAddr,
+    datadir: ExpandedPathBuf,
     eps: Vec<Address>,
     eth_client: Arc<Provider<Http>>,
     chain: Chain,
@@ -362,9 +363,18 @@ pub async fn uopool_service_run(
 
         let m_map = Arc::new(DashMap::<
             MempoolId,
-            UoPoolBuilder<Provider<Http>, MemoryMempool, MemoryReputation>,
+            UoPoolBuilder<
+                Provider<Http>,
+                DatabaseMempool<WriteMap>,
+                DatabaseReputation<WriteMap>,
+                DBError,
+            >,
         >::new());
 
+        let env = Arc::new(init_env::<WriteMap>(datadir.join("db")).expect("Init mdbx failed"));
+        env.create_tables()
+            .expect("Create mdbx database tables failed");
+
         for ep in eps {
             let id = mempool_id(&ep, &U256::from(chain.id()));
             let builder = UoPoolBuilder::new(
@@ -377,16 +387,17 @@ pub async fn uopool_service_run(
                 min_unstake_delay,
                 min_priority_fee_per_gas,
                 whitelist.clone(),
-                MemoryMempool::default(),
-                MemoryReputation::default(),
+                DatabaseMempool::new(env.clone()),
+                DatabaseReputation::new(env.clone()),
             );
             m_map.insert(id, builder);
         }
 
         let svc = uo_pool_server::UoPoolServer::new(UoPoolService::<
             Provider<Http>,
-            MemoryMempool,
-            MemoryReputation,
+            DatabaseMempool<WriteMap>,
+            DatabaseReputation<WriteMap>,
+            DBError,
         >::new(m_map.clone(), chain));
 
         tokio::spawn(async move {

crates/primitives/Cargo.toml

@@ -14,7 +14,7 @@ AA (ERC-4337) bundler commonly used types
 anyhow = { workspace = true }
 educe = { version = "0.4", features = ["Debug", "Default"] }
 ethers = { workspace = true }
-expanded-pathbuf = "0.1"
+expanded-pathbuf = { workspace = true }
 lazy_static = "1.4.0"
 rustc-hex = "^2.0.1"
 serde = "1"

crates/uopool/src/database/env.rs

@@ -8,7 +8,7 @@ use reth_db::{
     },
     Error, TableType,
 };
-use std::{fmt::Display, path::PathBuf};
+use std::{fmt::Display, fs, path::PathBuf};
 
 // Code adapted from: https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/implementation/mdbx/mod.rs
 #[derive(Debug)]
@@ -74,6 +74,8 @@ fn default_page_size() -> usize {
 impl<E: EnvironmentKind> Env<E> {
     /// Sets up the database environment
     pub fn open(path: PathBuf) -> anyhow::Result<Self> {
+        fs::create_dir_all(&path)?;
+
         let env = Environment::new()
             .set_max_dbs(TABLES.len())
             .set_geometry(Geometry {