From c6f0aefa6ffd70f3128574235f4fcfa06ab3ec12 Mon Sep 17 00:00:00 2001 From: rajarshimaitra Date: Fri, 30 Dec 2022 17:24:57 +0530 Subject: [PATCH] Add a electrum_sync example --- Cargo.toml | 1 + bdk_electrum_example/.gitignore | 1 + bdk_electrum_example/Cargo.toml | 17 +++ bdk_electrum_example/src/electrum.rs | 213 +++++++++++++++++++++++++++ bdk_electrum_example/src/main.rs | 186 +++++++++++++++++++++++ 5 files changed, 418 insertions(+) create mode 100644 bdk_electrum_example/.gitignore create mode 100644 bdk_electrum_example/Cargo.toml create mode 100644 bdk_electrum_example/src/electrum.rs create mode 100644 bdk_electrum_example/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index 242cf94d..c774f5c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "bdk_chain", "bdk_cli_lib", "bdk_esplora_example", + "bdk_electrum_example", "bdk_file_store", "bdk_tmp_plan", "bdk_coin_select" diff --git a/bdk_electrum_example/.gitignore b/bdk_electrum_example/.gitignore new file mode 100644 index 00000000..ea8c4bf7 --- /dev/null +++ b/bdk_electrum_example/.gitignore @@ -0,0 +1 @@ +/target diff --git a/bdk_electrum_example/Cargo.toml b/bdk_electrum_example/Cargo.toml new file mode 100644 index 00000000..e2c5bb88 --- /dev/null +++ b/bdk_electrum_example/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "bdk_electrum_example" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[dependencies] +# BDK Core +bdk_chain = { path = "../bdk_chain", features = ["serde"] } +bdk_cli = { path = "../bdk_cli_lib"} + +# Electrum +electrum-client = { version = "0.12" } + +# Auxiliary +log = "0.4" +env_logger = "0.7" diff --git a/bdk_electrum_example/src/electrum.rs b/bdk_electrum_example/src/electrum.rs new file mode 100644 index 00000000..f2098bc1 --- /dev/null +++ b/bdk_electrum_example/src/electrum.rs @@ -0,0 +1,213 @@ +use std::{collections::BTreeMap, ops::Deref}; + +use bdk_chain::{ + bitcoin::{BlockHash, Script, Txid}, + sparse_chain::{InsertTxErr, SparseChain}, + BlockId, TxHeight, +}; +use bdk_cli::Broadcast; +use electrum_client::{Client, Config, ElectrumApi}; + +#[derive(Debug)] +pub enum ElectrumError { + Client(electrum_client::Error), + Reorg, +} + +impl core::fmt::Display for ElectrumError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ElectrumError::Client(e) => write!(f, "{}", e), + ElectrumError::Reorg => write!( + f, + "Reorg detected at sync time. Please run the sync call again" + ), + } + } +} + +impl std::error::Error for ElectrumError {} + +impl From for ElectrumError { + fn from(e: electrum_client::Error) -> Self { + Self::Client(e) + } +} + +pub struct ElectrumClient { + client: Client, +} + +impl ElectrumClient { + pub fn new(url: &str) -> Result { + let client = Client::from_config(url, Config::default())?; + Ok(Self { client }) + } +} + +impl Deref for ElectrumClient { + type Target = Client; + fn deref(&self) -> &Self::Target { + &self.client + } +} + +impl Broadcast for ElectrumClient { + type Error = ElectrumError; + fn broadcast(&self, tx: &bdk_chain::bitcoin::Transaction) -> Result<(), Self::Error> { + let _ = self.client.transaction_broadcast(tx)?; + Ok(()) + } +} + +impl ElectrumClient { + /// Fetch latest block height. + pub fn get_tip(&self) -> Result<(u32, BlockHash), ElectrumError> { + // TODO: unsubscribe when added to the client, or is there a better call to use here? + Ok(self + .client + .block_headers_subscribe() + .map(|data| (data.height as u32, data.header.block_hash()))?) + } + + /// Scan for a given list of scripts, and create an initial [`bdk_core::sparse_chain::SparseChain`] update candidate. + /// This will only contain [`Txid`]s in SparseChain, and no actual transaction data. + /// + /// User needs to fetch the required transaction data and create the final [`bdk_core::keychain::KeychainChangeSet`] before applying it. + pub fn spk_txid_scan( + &self, + spks: impl Iterator, + local_chain: &BTreeMap, + batch_size: usize, + ) -> Result { + let mut dummy_keychains = BTreeMap::new(); + dummy_keychains.insert((), spks.enumerate().map(|(i, spk)| (i as u32, spk))); + + Ok(self + .wallet_txid_scan(dummy_keychains, None, local_chain, batch_size)? + .0) + } + + /// Scan for a keychain tracker, and create an initial [`bdk_core::sparse_chain::SparseChain`] update candidate. + /// This will only contain [`Txid`]s in SparseChain, and no actual transaction data. + /// + /// User needs to fetch the required transaction data and create the final [`bdk_core::keychain::KeychainChangeSet`] before applying it. + pub fn wallet_txid_scan( + &self, + scripts: BTreeMap>, + stop_gap: Option, + local_chain: &BTreeMap, + batch_size: usize, + ) -> Result<(SparseChain, BTreeMap), ElectrumError> { + let mut sparse_chain = SparseChain::default(); + + // Check for reorgs. + // In case of reorg, new checkpoints until the last common checkpoint is added to the structure + for (&existing_height, &existing_hash) in local_chain.iter().rev() { + let current_hash = self + .client + .block_header(existing_height as usize)? + .block_hash(); + sparse_chain + .insert_checkpoint(BlockId { + height: existing_height, + hash: current_hash, + }) + .expect("This never errors because we are working with a fresh chain"); + + if current_hash == existing_hash { + break; + } + } + + // Insert the new tip + let (tip_height, tip_hash) = self.get_tip()?; + if sparse_chain + .insert_checkpoint(BlockId { + height: tip_height, + hash: tip_hash, + }) + .is_err() + { + // There has been an reorg since the last call to `block_header` in the loop above at the current tip. + return Err(ElectrumError::Reorg); + } + + let mut keychain_index_update = BTreeMap::new(); + + // Fetch Keychain's last_active_index and all related txids. + // Add them into the SparseChain + for (keychain, mut scripts) in scripts { + let mut last_active_index = 0; + let mut unused_script_count = 0usize; + + loop { + let batch_scripts = (0..batch_size) + .map(|_| scripts.next()) + .filter_map(|item| item) + .collect::>(); + + if batch_scripts.is_empty() { + // We reached the end of the script list + break; + } + + for (history, index) in self + .batch_script_get_history(batch_scripts.iter().map(|(_, script)| script))? + .iter() + .zip(batch_scripts.iter().map(|(index, _)| index)) + { + let txid_list = history + .iter() + .map(|history_result| { + if history_result.height > 0 + && (history_result.height as u32) <= tip_height + { + return ( + history_result.tx_hash, + TxHeight::Confirmed(history_result.height as u32), + ); + } else { + return (history_result.tx_hash, TxHeight::Unconfirmed); + }; + }) + .collect::>(); + + if txid_list.is_empty() { + unused_script_count += 1; + } else { + last_active_index = *index; + unused_script_count = 0; + } + + for (txid, index) in txid_list { + if let Err(err) = sparse_chain.insert_tx(txid, index) { + match err { + InsertTxErr::TxTooHigh => { + unreachable!("We should not encounter this error as we ensured TxHeight <= tip_height"); + } + InsertTxErr::TxMoved => { + /* This means there is a reorg, we will handle this situation below */ + } + } + } + } + } + } + + if unused_script_count >= stop_gap.unwrap_or(usize::MAX) { + break; + } + + keychain_index_update.insert(keychain, last_active_index); + } + + // Check for Reorg during the above sync process + let our_latest = sparse_chain.latest_checkpoint().expect("must exist"); + if our_latest.hash != self.block_header(our_latest.height as usize)?.block_hash() { + return Err(ElectrumError::Reorg); + } + + Ok((sparse_chain, keychain_index_update)) + } +} diff --git a/bdk_electrum_example/src/main.rs b/bdk_electrum_example/src/main.rs new file mode 100644 index 00000000..342a5706 --- /dev/null +++ b/bdk_electrum_example/src/main.rs @@ -0,0 +1,186 @@ +mod electrum; + +use std::fmt::Debug; + +use bdk_chain::{bitcoin::Network, keychain::KeychainChangeSet}; +use electrum::ElectrumClient; + +use bdk_cli::{ + anyhow::{self, anyhow, Context}, + clap::{self, Parser, Subcommand}, +}; +use log::debug; + +use electrum_client::ElectrumApi; + +#[derive(Subcommand, Debug, Clone)] +enum ElectrumCommands { + /// Scans the addresses in the wallet using esplora API. + Scan { + /// When a gap this large has been found for a keychain it will stop. + #[clap(long, default_value = "5")] + stop_gap: usize, + #[clap(flatten)] + scan_option: ScanOption, + }, + /// Scans particular addresses using esplora API + Sync { + /// Scan all the unused addresses + #[clap(long)] + unused: bool, + /// Scan the script addresses that have unspent outputs + #[clap(long)] + unspent: bool, + /// Scan every address that you have derived + #[clap(long)] + all: bool, + #[clap(flatten)] + scan_option: ScanOption, + }, +} + +#[derive(Parser, Debug, Clone, PartialEq)] +pub struct ScanOption { + /// Set batch size for each script_history call to electrum client + #[clap(long, default_value = "25")] + pub batch_size: usize, +} + +fn main() -> anyhow::Result<()> { + let (args, keymap, mut tracker, mut db) = bdk_cli::init::()?; + + let electrum_url = match args.network { + Network::Bitcoin => "ssl://electrum.blockstream.info:50002", + Network::Testnet => "ssl://electrum.blockstream.info:60002", + Network::Regtest => "ssl://localhost:60401", + // TODO: Find a electrum signet endpoint + Network::Signet => return Err(anyhow::anyhow!("Signet nor supported for Electrum")), + }; + + let client = ElectrumClient::new(electrum_url)?; + + let electrum_cmd = match args.command { + bdk_cli::Commands::ChainSpecific(electrum_cmd) => electrum_cmd, + general_command => { + return bdk_cli::handle_commands( + general_command, + client, + &mut tracker, + &mut db, + args.network, + &keymap, + ) + } + }; + + let mut keychain_changeset = KeychainChangeSet::default(); + + let chain_update = match electrum_cmd { + ElectrumCommands::Scan { + stop_gap, + scan_option, + } => { + let scripts = tracker.txout_index.iter_all_script_pubkeys_by_keychain(); + + let (new_sparsechain, keychain_index_update) = client.wallet_txid_scan( + scripts, + Some(stop_gap), + tracker.chain().checkpoints(), + scan_option.batch_size, + )?; + + keychain_changeset.derivation_indices = keychain_index_update; + + new_sparsechain + } + ElectrumCommands::Sync { + mut unused, + mut unspent, + all, + scan_option, + } => { + let txout_index = &tracker.txout_index; + if !(all || unused || unspent) { + unused = true; + unspent = true; + } else if all { + unused = false; + unspent = false + } + let mut spks: Box> = + Box::new(core::iter::empty()); + if unused { + spks = Box::new(spks.chain(txout_index.inner().unused(..).map( + |(index, script)| { + eprintln!("Checking if address at {:?} has been used", index); + script.clone() + }, + ))); + } + + if all { + spks = Box::new(spks.chain(txout_index.script_pubkeys().iter().map( + |(index, script)| { + eprintln!("scanning {:?}", index); + script.clone() + }, + ))); + } + + if unspent { + spks = Box::new(spks.chain(tracker.full_utxos().map(|(_index, ftxout)| { + eprintln!("checking if {} has been spent", ftxout.outpoint); + ftxout.txout.script_pubkey + }))); + } + + let new_sparsechain = client + .spk_txid_scan(spks, tracker.chain().checkpoints(), scan_option.batch_size) + .context("scanning the blockchain")?; + + new_sparsechain + } + }; + + let sparsechain_changeset = match tracker.chain().determine_changeset(&chain_update) { + Ok(cs) => cs, + Err(update_failure) => { + return Err(anyhow!( + "Changeset determination failed : {:?}", + update_failure + )) + } + }; + + let new_txids = tracker + .chain() + .changeset_additions(&sparsechain_changeset) + .collect::>(); + + let new_txs = client.batch_transaction_get(new_txids.iter())?; + + let chaingraph_changeset = match tracker + .chain_graph() + .inflate_changeset(sparsechain_changeset, new_txs) + { + Ok(cs) => cs, + Err(cs_error) => { + return Err(anyhow!( + "Chaingraph changeset creation failed : {:?}", + cs_error + )) + } + }; + + keychain_changeset.chain_graph = chaingraph_changeset; + + db.append_changeset(&keychain_changeset)?; + if let Err(err) = tracker.apply_changeset(keychain_changeset) { + return Err(anyhow!( + "Changeset application failed in tracker : {:?}", + err + )); + } + debug!("sync completed!!"); + Ok(()) +}