From e09f5170948f5f8fd1a86c5ede7501ac22d10f50 Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Wed, 13 May 2020 08:36:30 -0600 Subject: [PATCH] Add solana-tokens (#10011) * Initial commit * Execute transfers * Refactor for testing * Cleanup readme * Rewrite * Cleanup * Cleanup * Cleanup client * Use a Null Client to move prints closer to where messages are sent * Upgrade Solana * Move core functionality into its own module * Handle transaction errors * Merge allocations * Fixes * Cleanup readme * Fix markdown * Add example input * Add integration test - currently fails * Add integration test * Add metrics * Use RpcClient in dry-run, just don't send messages * More metrics * Fix dry run with no keys * Only require one approval if fee-payer is the sender keypair * Fix bugs * Don't create the transaction log if nothing to put into it; otherwise the next innvocation won't add the header * Apply previous transactions to allocations with matching recipients * Bail out of any account already has a balance * Polish * Add new 'balances' command * 9 decimal places * Add missing file * Better dry-run; keypair options now optional * Change field name from 'bid' to 'accepted' Also, tolerate precision change from 2 decimal places to 4 * Write to transaction log immediately * Rename allocations_csv to bids_csv So that we can bypass bids_csv with an allocations CSV file * Upgrade Solana * Remove faucet from integration test * Cleaner integration test Won't work until this lands and is released: https://github.com/solana-labs/solana/pull/9717 * Update README * Add TravicCI script to build and test (#1) * Add distribute-stake command (#2) * Distribute -> DistributeTokens (#3) * Cache cargo deps (#4) * Add docs (#5) * Switch to latest Solana 1.1 release (#7) * distribute -> distribute-tokens (#9) * Switch from CSV to a pickledb database (#8) * Switch from CSV to a pickledb database * Allow PickleDb errors to bubble up * Dedup * Hoist db * Add finalized field to TransactionInfo * Don't allow RPC client to resign transactions * Remove dead code * Use transport::Result * Record unconfirmed transaction * Fix: separate stake account per allocation * Catch transport errors * Panic if we attempt to replay a transaction that hasn't been finalized * Attempt to fix CI PickleDb isn't calling flush() or close() after writing to files. No issue on MacOS, but looks racy in CI. * Revert "Attempt to fix CI" This reverts commit 1632394f636c54402b3578120e8817dd1660e19b. * Poll for signature before returning * Add --sol-for-fees option for stake distributions * Add --allocations-csv option (#14) * Add allocations-csv option * Add tests or GTFO * Apply review feedback * apply feedback * Add read_allocations function * Update arg_parser.rs * Fix balances command (#17) * Fix balances command * Fix readme * Add --force to transfer to non-empty accounts (#18) * Add --no-wait (#16) * Add ThinClient methods to implement --no-wait * Plumb --no-wait through No tests yet * Check transaction status on startup * Easier to test * Wait until transaction is finalized before checking if it failed with an error It's possible that a minority fork thinks it failed. * Add unit tests * Remove dead code and rustfmt * Don't flush database to file if doing a dry-run * Continue when transactions not yet finalized (#20) If those transactions are dropped, the next run will execute them. * Return the number of confirmations (#21) * Add read_allocations() unit-test (#22) Delete the copy-pasted top-level test. Fixes #19 * Add a CSV printer (#23) * Remove all the copypasta (#24) * Move resolve_distribute_stake_args into its own function * Add stake args to token args * Unify option names * Move Command::DistributeStake into DistributeTokens * Remove process_distribute_stake * Only unique signers * Use sender keypair to fund new fee-payer accounts * Unify distribute_tokens and distribute_stake * Rename print-database command to transaction-log (#25) * Send all transactions as quickly as possible, then wait (#26) * Send all transactions as quickly as possible, then wait * Exit when finalized or blockhashes have expired * Don't need blockhash in the CSV output * Better types CSV library was choking on Pubkey as a type. PickleDb doesn't have that problem. * Resend if blockhash has not expired * Attempt to fix CI * Move log to stderr * Add constructor, tuck away client (#30) * Add constructor, tuck away client * Fix unwrap() caught by CI * Fix optional option flagged as required * Bunch of cleanup (#31) * Remove untested --no-wait feature * Make --transactions-db an option, not an arg So that in the future, we can make it optional * Remove more untested features Too many false positives in that santity check. Use --dry-run instead. * Add dry-run mode to ThinClient * Cleaner dry-run * Make key parameters required Just don't use them in --dry-run * Add option to write the transaction log --dry-run doesn't write to the database. Use this option if you want a copy of the transaction log before the final run. * Revert --transaction-log addition Implement #27 first * Fix CI * Update readme * Fix CI in copypasta * Sort transaction log by finalized date (#33) * Make --transaction-db option implicit (#34) * Move db functionality into its own module (#35) * Move db functionality into its own module * Rename tokens module to commands * Version bump * Upgrade Solana * Add solana-tokens to build * Remove Cargo.lock * Remove vscode file * Remove TravisCI build script * Install solana-tokens Co-authored-by: Dan Albert --- Cargo.lock | 56 +++ Cargo.toml | 1 + scripts/cargo-install-all.sh | 2 + tokens/.gitignore | 2 + tokens/Cargo.toml | 34 ++ tokens/README.md | 105 ++++++ tokens/src/arg_parser.rs | 305 ++++++++++++++++ tokens/src/args.rs | 117 ++++++ tokens/src/commands.rs | 688 +++++++++++++++++++++++++++++++++++ tokens/src/db.rs | 351 ++++++++++++++++++ tokens/src/lib.rs | 5 + tokens/src/main.rs | 44 +++ tokens/src/thin_client.rs | 174 +++++++++ tokens/tests/commands.rs | 18 + 14 files changed, 1902 insertions(+) create mode 100644 tokens/.gitignore create mode 100644 tokens/Cargo.toml create mode 100644 tokens/README.md create mode 100644 tokens/src/arg_parser.rs create mode 100644 tokens/src/args.rs create mode 100644 tokens/src/commands.rs create mode 100644 tokens/src/db.rs create mode 100644 tokens/src/lib.rs create mode 100644 tokens/src/main.rs create mode 100644 tokens/src/thin_client.rs create mode 100644 tokens/tests/commands.rs diff --git a/Cargo.lock b/Cargo.lock index b2829294b6e4e5..d5529b423e98eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1358,6 +1358,12 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "half" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177" + [[package]] name = "hash32" version = "0.1.1" @@ -2572,6 +2578,19 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pickledb" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9161694d67f6c5163519d42be942ae36bbdb55f439460144f105bc4f9f7d1d61" +dependencies = [ + "bincode", + "serde", + "serde_cbor", + "serde_json", + "serde_yaml", +] + [[package]] name = "pin-project" version = "0.4.9" @@ -3427,6 +3446,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_cbor" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e18acfa2f90e8b735b2836ab8d538de304cbb6729a7360729ea5a895d15a622" +dependencies = [ + "half", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.110" @@ -4826,6 +4855,33 @@ dependencies = [ "users", ] +[[package]] +name = "solana-tokens" +version = "1.2.0" +dependencies = [ + "chrono", + "clap", + "console 0.10.3", + "csv", + "dirs 2.0.2", + "indexmap", + "indicatif", + "itertools 0.9.0", + "pickledb", + "serde", + "solana-clap-utils", + "solana-cli-config", + "solana-client", + "solana-core", + "solana-remote-wallet", + "solana-runtime", + "solana-sdk", + "solana-stake-program", + "solana-transaction-status", + "tempfile", + "thiserror", +] + [[package]] name = "solana-transaction-status" version = "1.2.0" diff --git a/Cargo.toml b/Cargo.toml index ee3147b4a8e573..61389a8afb95af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,7 @@ members = [ "stake-accounts", "stake-monitor", "sys-tuner", + "tokens", "transaction-status", "upload-perf", "net-utils", diff --git a/scripts/cargo-install-all.sh b/scripts/cargo-install-all.sh index 02fb81f7a40fc3..9ce5115963ce6a 100755 --- a/scripts/cargo-install-all.sh +++ b/scripts/cargo-install-all.sh @@ -67,6 +67,7 @@ if [[ $CI_OS_NAME = windows ]]; then solana-install-init solana-keygen solana-stake-accounts + solana-tokens ) else ./fetch-perf-libs.sh @@ -100,6 +101,7 @@ else solana-stake-accounts solana-stake-monitor solana-sys-tuner + solana-tokens solana-validator solana-watchtower ) diff --git a/tokens/.gitignore b/tokens/.gitignore new file mode 100644 index 00000000000000..e3c0d4bb353841 --- /dev/null +++ b/tokens/.gitignore @@ -0,0 +1,2 @@ +target/ +*.csv diff --git a/tokens/Cargo.toml b/tokens/Cargo.toml new file mode 100644 index 00000000000000..1cbbc4a4c3f85d --- /dev/null +++ b/tokens/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "solana-tokens" +description = "Blockchain, Rebuilt for Scale" +authors = ["Solana Maintainers "] +edition = "2018" +version = "1.2.0" +repository = "https://github.com/solana-labs/solana" +license = "Apache-2.0" +homepage = "https://solana.com/" + +[dependencies] +chrono = { version = "0.4", features = ["serde"] } +clap = "2.33.0" +console = "0.10.3" +csv = "1.1.3" +dirs = "2.0.2" +indexmap = "1.3.2" +indicatif = "0.14.0" +itertools = "0.9.0" +pickledb = "0.4.1" +serde = { version = "1.0", features = ["derive"] } +solana-clap-utils = { path = "../clap-utils", version = "1.2.0" } +solana-cli-config = { path = "../cli-config", version = "1.2.0" } +solana-client = { path = "../client", version = "1.2.0" } +solana-remote-wallet = { path = "../remote-wallet", version = "1.2.0" } +solana-runtime = { path = "../runtime", version = "1.2.0" } +solana-sdk = { path = "../sdk", version = "1.2.0" } +solana-stake-program = { path = "../programs/stake", version = "1.2.0" } +solana-transaction-status = { path = "../transaction-status", version = "1.2.0" } +tempfile = "3.1.0" +thiserror = "1.0" + +[dev-dependencies] +solana-core = { path = "../core", version = "1.2.0" } diff --git a/tokens/README.md b/tokens/README.md new file mode 100644 index 00000000000000..940183dfd71819 --- /dev/null +++ b/tokens/README.md @@ -0,0 +1,105 @@ +# Distribute Solana tokens + +A user may want to make payments to multiple accounts over multiple iterations. +The user will have a spreadsheet listing public keys and token amounts, and +some process for transferring tokens to them, and ensuring that no more than the +expected amount are sent. The command-line tool here automates that process. + +## Distribute tokens + +Send tokens to the recipients in ``. + +Example bids.csv: + +```text +primary_address,bid_amount_dollars +6Vo87BaDhp4v4GHwVDhw5huhxVF8CyxSXYtkUwVHbbPv,6.6 +``` + +```bash +solana-tokens distribute-tokens --from --dollars-per-sol --from-bids --input-csv --fee-payer +``` + +Example transaction log before: + +```text +recipient,amount,signature +6Vo87BaDhp4v4GHwVDhw5huhxVF8CyxSXYtkUwVHbbPv,30,1111111111111111111111111111111111111111111111111111111111111111 +``` + +Send tokens to the recipients in `` if the distribution is +not already recordered in the transaction log. + +```bash +solana-tokens distribute-tokens --from --dollars-per-sol --from-bids --input-csv --fee-payer +``` + +Example output: + +```text +Recipient Amount +6Vo87BaDhp4v4GHwVDhw5huhxVF8CyxSXYtkUwVHbbPv 70 +3ihfUy1n9gaqihM5bJCiTAGLgWc5zo3DqVUS6T736NLM 42 +UKUcTXgbeTYh65RaVV5gSf6xBHevqHvAXMo3e8Q6np8k 43 +``` + + +Example transaction log after: + +```bash +solana-tokens transaction-log --output-path transactions.csv +``` + +```text +recipient,amount,signature +6Vo87BaDhp4v4GHwVDhw5huhxVF8CyxSXYtkUwVHbbPv,30,1111111111111111111111111111111111111111111111111111111111111111 +6Vo87BaDhp4v4GHwVDhw5huhxVF8CyxSXYtkUwVHbbPv,70,1111111111111111111111111111111111111111111111111111111111111111 +3ihfUy1n9gaqihM5bJCiTAGLgWc5zo3DqVUS6T736NLM,42,1111111111111111111111111111111111111111111111111111111111111111 +UKUcTXgbeTYh65RaVV5gSf6xBHevqHvAXMo3e8Q6np8k,43,1111111111111111111111111111111111111111111111111111111111111111 +``` + +### Calculate what tokens should be sent + +List the differences between a list of expected distributions and the record of what +transactions have already been sent. + +```bash +solana-tokens distribute-tokens --dollars-per-sol --dry-run --from-bids --input-csv +``` + +Example bids.csv: + +```text +primary_address,bid_amount_dollars +6Vo87BaDhp4v4GHwVDhw5huhxVF8CyxSXYtkUwVHbbPv,6.6 +6Vo87BaDhp4v4GHwVDhw5huhxVF8CyxSXYtkUwVHbbPv,15.4 +3ihfUy1n9gaqihM5bJCiTAGLgWc5zo3DqVUS6T736NLM,9.24 +UKUcTXgbeTYh65RaVV5gSf6xBHevqHvAXMo3e8Q6np8k,9.46 +``` + +Example output: + +```text +Recipient Amount +6Vo87BaDhp4v4GHwVDhw5huhxVF8CyxSXYtkUwVHbbPv 70 +3ihfUy1n9gaqihM5bJCiTAGLgWc5zo3DqVUS6T736NLM 42 +UKUcTXgbeTYh65RaVV5gSf6xBHevqHvAXMo3e8Q6np8k 43 +``` + +## Distribute stake accounts + +Distributing tokens via stake accounts works similarly to how tokens are distributed. The +big difference is that new stake accounts are split from existing ones. By splitting, +the new accounts inherit any lockup or custodian settings of the original. + +```bash +solana-tokens distribute-stake --stake-account-address \ + --input-csv \ + --stake-authority --withdraw-authority --fee-payer +``` + +Currently, this will subtract 1 SOL from each allocation and store it the +recipient address. That SOL can be used to pay transaction fees on staking +operations such as delegating stake. The rest of the allocation is put in +a stake account. The new stake account address is output in the transaction +log. diff --git a/tokens/src/arg_parser.rs b/tokens/src/arg_parser.rs new file mode 100644 index 00000000000000..96e38e208ed58a --- /dev/null +++ b/tokens/src/arg_parser.rs @@ -0,0 +1,305 @@ +use crate::args::{ + Args, BalancesArgs, Command, DistributeTokensArgs, StakeArgs, TransactionLogArgs, +}; +use clap::{value_t, value_t_or_exit, App, Arg, ArgMatches, SubCommand}; +use solana_clap_utils::input_validators::{is_valid_pubkey, is_valid_signer}; +use solana_cli_config::CONFIG_FILE; +use std::ffi::OsString; +use std::process::exit; + +fn get_matches<'a, I, T>(args: I) -> ArgMatches<'a> +where + I: IntoIterator, + T: Into + Clone, +{ + let default_config_file = CONFIG_FILE.as_ref().unwrap(); + App::new("solana-tokens") + .about("about") + .version("version") + .arg( + Arg::with_name("config_file") + .long("config") + .takes_value(true) + .value_name("FILEPATH") + .default_value(default_config_file) + .help("Config file"), + ) + .arg( + Arg::with_name("url") + .long("url") + .global(true) + .takes_value(true) + .value_name("URL") + .help("RPC entrypoint address. i.e. http://devnet.solana.com"), + ) + .subcommand( + SubCommand::with_name("distribute-tokens") + .about("Distribute tokens") + .arg( + Arg::with_name("campaign_name") + .long("campaign-name") + .takes_value(true) + .value_name("NAME") + .help("Campaign name for storing transaction data"), + ) + .arg( + Arg::with_name("from_bids") + .long("from-bids") + .help("Input CSV contains bids in dollars, not allocations in SOL"), + ) + .arg( + Arg::with_name("input_csv") + .long("input-csv") + .required(true) + .takes_value(true) + .value_name("FILE") + .help("Input CSV file"), + ) + .arg( + Arg::with_name("dollars_per_sol") + .long("dollars-per-sol") + .takes_value(true) + .value_name("NUMBER") + .help("Dollars per SOL, if input CSV contains bids"), + ) + .arg( + Arg::with_name("dry_run") + .long("dry-run") + .help("Do not execute any transfers"), + ) + .arg( + Arg::with_name("sender_keypair") + .long("from") + .required(true) + .takes_value(true) + .value_name("SENDING_KEYPAIR") + .validator(is_valid_signer) + .help("Keypair to fund accounts"), + ) + .arg( + Arg::with_name("fee_payer") + .long("fee-payer") + .required(true) + .takes_value(true) + .value_name("KEYPAIR") + .validator(is_valid_signer) + .help("Fee payer"), + ), + ) + .subcommand( + SubCommand::with_name("distribute-stake") + .about("Distribute stake accounts") + .arg( + Arg::with_name("campaign_name") + .long("campaign-name") + .takes_value(true) + .value_name("NAME") + .help("Campaign name for storing transaction data"), + ) + .arg( + Arg::with_name("input_csv") + .long("input-csv") + .required(true) + .takes_value(true) + .value_name("FILE") + .help("Allocations CSV file"), + ) + .arg( + Arg::with_name("dry_run") + .long("dry-run") + .help("Do not execute any transfers"), + ) + .arg( + Arg::with_name("sender_keypair") + .long("from") + .required(true) + .takes_value(true) + .value_name("SENDING_KEYPAIR") + .validator(is_valid_signer) + .help("Keypair to fund accounts"), + ) + .arg( + Arg::with_name("stake_account_address") + .required(true) + .long("stake-account-address") + .takes_value(true) + .value_name("ACCOUNT_ADDRESS") + .validator(is_valid_pubkey) + .help("Stake Account Address"), + ) + .arg( + Arg::with_name("sol_for_fees") + .default_value("1.0") + .long("sol-for-fees") + .takes_value(true) + .value_name("SOL_AMOUNT") + .help("Amount of SOL to put in system account to pay for fees"), + ) + .arg( + Arg::with_name("stake_authority") + .long("stake-authority") + .required(true) + .takes_value(true) + .value_name("KEYPAIR") + .validator(is_valid_signer) + .help("Stake Authority Keypair"), + ) + .arg( + Arg::with_name("withdraw_authority") + .long("withdraw-authority") + .required(true) + .takes_value(true) + .value_name("KEYPAIR") + .validator(is_valid_signer) + .help("Withdraw Authority Keypair"), + ) + .arg( + Arg::with_name("fee_payer") + .long("fee-payer") + .required(true) + .takes_value(true) + .value_name("KEYPAIR") + .validator(is_valid_signer) + .help("Fee payer"), + ), + ) + .subcommand( + SubCommand::with_name("balances") + .about("Balance of each account") + .arg( + Arg::with_name("input_csv") + .long("input-csv") + .required(true) + .takes_value(true) + .value_name("FILE") + .help("Bids CSV file"), + ) + .arg( + Arg::with_name("from_bids") + .long("from-bids") + .help("Input CSV contains bids in dollars, not allocations in SOL"), + ) + .arg( + Arg::with_name("dollars_per_sol") + .long("dollars-per-sol") + .takes_value(true) + .value_name("NUMBER") + .help("Dollars per SOL"), + ), + ) + .subcommand( + SubCommand::with_name("transaction-log") + .about("Print the database to a CSV file") + .arg( + Arg::with_name("campaign_name") + .long("campaign-name") + .takes_value(true) + .value_name("NAME") + .help("Campaign name for storing transaction data"), + ) + .arg( + Arg::with_name("output_path") + .long("output-path") + .required(true) + .takes_value(true) + .value_name("FILE") + .help("Output file"), + ), + ) + .get_matches_from(args) +} + +fn create_db_path(campaign_name: Option) -> String { + let (prefix, hyphen) = if let Some(name) = campaign_name { + (name, "-") + } else { + ("".to_string(), "") + }; + let path = dirs::home_dir().unwrap(); + let filename = format!("{}{}transactions.db", prefix, hyphen); + path.join(".config") + .join("solana-tokens") + .join(filename) + .to_str() + .unwrap() + .to_string() +} + +fn parse_distribute_tokens_args(matches: &ArgMatches<'_>) -> DistributeTokensArgs { + DistributeTokensArgs { + input_csv: value_t_or_exit!(matches, "input_csv", String), + from_bids: matches.is_present("from_bids"), + transaction_db: create_db_path(value_t!(matches, "campaign_name", String).ok()), + dollars_per_sol: value_t!(matches, "dollars_per_sol", f64).ok(), + dry_run: matches.is_present("dry_run"), + sender_keypair: value_t_or_exit!(matches, "sender_keypair", String), + fee_payer: value_t_or_exit!(matches, "fee_payer", String), + stake_args: None, + } +} + +fn parse_distribute_stake_args(matches: &ArgMatches<'_>) -> DistributeTokensArgs { + let stake_args = StakeArgs { + stake_account_address: value_t_or_exit!(matches, "stake_account_address", String), + sol_for_fees: value_t_or_exit!(matches, "sol_for_fees", f64), + stake_authority: value_t_or_exit!(matches, "stake_authority", String), + withdraw_authority: value_t_or_exit!(matches, "withdraw_authority", String), + }; + DistributeTokensArgs { + input_csv: value_t_or_exit!(matches, "input_csv", String), + from_bids: false, + transaction_db: create_db_path(value_t!(matches, "campaign_name", String).ok()), + dollars_per_sol: None, + dry_run: matches.is_present("dry_run"), + sender_keypair: value_t_or_exit!(matches, "sender_keypair", String), + fee_payer: value_t_or_exit!(matches, "fee_payer", String), + stake_args: Some(stake_args), + } +} + +fn parse_balances_args(matches: &ArgMatches<'_>) -> BalancesArgs { + BalancesArgs { + input_csv: value_t_or_exit!(matches, "input_csv", String), + from_bids: matches.is_present("from_bids"), + dollars_per_sol: value_t!(matches, "dollars_per_sol", f64).ok(), + } +} + +fn parse_transaction_log_args(matches: &ArgMatches<'_>) -> TransactionLogArgs { + TransactionLogArgs { + transaction_db: value_t_or_exit!(matches, "transaction_db", String), + output_path: value_t_or_exit!(matches, "output_path", String), + } +} + +pub fn parse_args(args: I) -> Args +where + I: IntoIterator, + T: Into + Clone, +{ + let matches = get_matches(args); + let config_file = matches.value_of("config_file").unwrap().to_string(); + let url = matches.value_of("url").map(|x| x.to_string()); + + let command = match matches.subcommand() { + ("distribute-tokens", Some(matches)) => { + Command::DistributeTokens(parse_distribute_tokens_args(matches)) + } + ("distribute-stake", Some(matches)) => { + Command::DistributeTokens(parse_distribute_stake_args(matches)) + } + ("balances", Some(matches)) => Command::Balances(parse_balances_args(matches)), + ("transaction-log", Some(matches)) => { + Command::TransactionLog(parse_transaction_log_args(matches)) + } + _ => { + eprintln!("{}", matches.usage()); + exit(1); + } + }; + Args { + config_file, + url, + command, + } +} diff --git a/tokens/src/args.rs b/tokens/src/args.rs new file mode 100644 index 00000000000000..0eed6946388d8e --- /dev/null +++ b/tokens/src/args.rs @@ -0,0 +1,117 @@ +use clap::ArgMatches; +use solana_clap_utils::keypair::{pubkey_from_path, signer_from_path}; +use solana_remote_wallet::remote_wallet::{maybe_wallet_manager, RemoteWalletManager}; +use solana_sdk::{pubkey::Pubkey, signature::Signer}; +use std::{error::Error, sync::Arc}; + +pub struct DistributeTokensArgs { + pub input_csv: String, + pub from_bids: bool, + pub transaction_db: String, + pub dollars_per_sol: Option, + pub dry_run: bool, + pub sender_keypair: K, + pub fee_payer: K, + pub stake_args: Option>, +} + +pub struct StakeArgs { + pub sol_for_fees: f64, + pub stake_account_address: P, + pub stake_authority: K, + pub withdraw_authority: K, +} + +pub struct BalancesArgs { + pub input_csv: String, + pub from_bids: bool, + pub dollars_per_sol: Option, +} + +pub struct TransactionLogArgs { + pub transaction_db: String, + pub output_path: String, +} + +pub enum Command { + DistributeTokens(DistributeTokensArgs), + Balances(BalancesArgs), + TransactionLog(TransactionLogArgs), +} + +pub struct Args { + pub config_file: String, + pub url: Option, + pub command: Command, +} + +pub fn resolve_stake_args( + wallet_manager: &mut Option>, + args: StakeArgs, +) -> Result>, Box> { + let matches = ArgMatches::default(); + let resolved_args = StakeArgs { + stake_account_address: pubkey_from_path( + &matches, + &args.stake_account_address, + "stake account address", + wallet_manager, + ) + .unwrap(), + sol_for_fees: args.sol_for_fees, + stake_authority: signer_from_path( + &matches, + &args.stake_authority, + "stake authority", + wallet_manager, + ) + .unwrap(), + withdraw_authority: signer_from_path( + &matches, + &args.withdraw_authority, + "withdraw authority", + wallet_manager, + ) + .unwrap(), + }; + Ok(resolved_args) +} + +pub fn resolve_command( + command: Command, +) -> Result>, Box> { + match command { + Command::DistributeTokens(args) => { + let mut wallet_manager = maybe_wallet_manager()?; + let matches = ArgMatches::default(); + let resolved_stake_args = args + .stake_args + .map(|args| resolve_stake_args(&mut wallet_manager, args)); + let resolved_args = DistributeTokensArgs { + input_csv: args.input_csv, + from_bids: args.from_bids, + transaction_db: args.transaction_db, + dollars_per_sol: args.dollars_per_sol, + dry_run: args.dry_run, + sender_keypair: signer_from_path( + &matches, + &args.sender_keypair, + "sender", + &mut wallet_manager, + ) + .unwrap(), + fee_payer: signer_from_path( + &matches, + &args.fee_payer, + "fee-payer", + &mut wallet_manager, + ) + .unwrap(), + stake_args: resolved_stake_args.map_or(Ok(None), |r| r.map(Some))?, + }; + Ok(Command::DistributeTokens(resolved_args)) + } + Command::Balances(args) => Ok(Command::Balances(args)), + Command::TransactionLog(args) => Ok(Command::TransactionLog(args)), + } +} diff --git a/tokens/src/commands.rs b/tokens/src/commands.rs new file mode 100644 index 00000000000000..efae68bdeb27ca --- /dev/null +++ b/tokens/src/commands.rs @@ -0,0 +1,688 @@ +use crate::args::{BalancesArgs, DistributeTokensArgs, StakeArgs, TransactionLogArgs}; +use crate::db::{self, TransactionInfo}; +use crate::thin_client::{Client, ThinClient}; +use console::style; +use csv::{ReaderBuilder, Trim}; +use indexmap::IndexMap; +use indicatif::{ProgressBar, ProgressStyle}; +use itertools::Itertools; +use pickledb::PickleDb; +use serde::{Deserialize, Serialize}; +use solana_sdk::{ + message::Message, + native_token::{lamports_to_sol, sol_to_lamports}, + signature::{Signature, Signer}, + system_instruction, + transport::TransportError, +}; +use solana_stake_program::{ + stake_instruction, + stake_state::{Authorized, Lockup, StakeAuthorize}, +}; +use std::{ + cmp::{self}, + io, + thread::sleep, + time::Duration, +}; + +#[derive(Serialize, Deserialize, Debug, Clone)] +struct Bid { + accepted_amount_dollars: f64, + primary_address: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +struct Allocation { + recipient: String, + amount: f64, +} + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("I/O error")] + IoError(#[from] io::Error), + #[error("CSV error")] + CsvError(#[from] csv::Error), + #[error("PickleDb error")] + PickleDbError(#[from] pickledb::error::Error), + #[error("Transport error")] + TransportError(#[from] TransportError), + #[error("Signature not found")] + SignatureNotFound, +} + +fn unique_signers(signers: Vec<&dyn Signer>) -> Vec<&dyn Signer> { + signers.into_iter().unique_by(|s| s.pubkey()).collect_vec() +} + +fn merge_allocations(allocations: &[Allocation]) -> Vec { + let mut allocation_map = IndexMap::new(); + for allocation in allocations { + allocation_map + .entry(&allocation.recipient) + .or_insert(Allocation { + recipient: allocation.recipient.clone(), + amount: 0.0, + }) + .amount += allocation.amount; + } + allocation_map.values().cloned().collect() +} + +fn apply_previous_transactions( + allocations: &mut Vec, + transaction_infos: &[TransactionInfo], +) { + for transaction_info in transaction_infos { + let mut amount = transaction_info.amount; + for allocation in allocations.iter_mut() { + if allocation.recipient != transaction_info.recipient.to_string() { + continue; + } + if allocation.amount >= amount { + allocation.amount -= amount; + break; + } else { + amount -= allocation.amount; + allocation.amount = 0.0; + } + } + } + allocations.retain(|x| x.amount > 0.5); +} + +fn create_allocation(bid: &Bid, dollars_per_sol: f64) -> Allocation { + Allocation { + recipient: bid.primary_address.clone(), + amount: bid.accepted_amount_dollars / dollars_per_sol, + } +} + +fn distribute_tokens( + client: &ThinClient, + db: &mut PickleDb, + allocations: &[Allocation], + args: &DistributeTokensArgs>, +) -> Result<(), Error> { + for allocation in allocations { + let new_stake_account_keypair = Keypair::new(); + let new_stake_account_address = new_stake_account_keypair.pubkey(); + + let mut signers = vec![&*args.fee_payer, &*args.sender_keypair]; + if let Some(stake_args) = &args.stake_args { + signers.push(&*stake_args.stake_authority); + signers.push(&*stake_args.withdraw_authority); + signers.push(&new_stake_account_keypair); + } + let signers = unique_signers(signers); + + println!("{:<44} {:>24.9}", allocation.recipient, allocation.amount); + let instructions = if let Some(stake_args) = &args.stake_args { + let sol_for_fees = stake_args.sol_for_fees; + let sender_pubkey = args.sender_keypair.pubkey(); + let stake_authority = stake_args.stake_authority.pubkey(); + let withdraw_authority = stake_args.withdraw_authority.pubkey(); + + let mut instructions = stake_instruction::split( + &stake_args.stake_account_address, + &stake_authority, + sol_to_lamports(allocation.amount - sol_for_fees), + &new_stake_account_address, + ); + + let recipient = allocation.recipient.parse().unwrap(); + + // Make the recipient the new stake authority + instructions.push(stake_instruction::authorize( + &new_stake_account_address, + &stake_authority, + &recipient, + StakeAuthorize::Staker, + )); + + // Make the recipient the new withdraw authority + instructions.push(stake_instruction::authorize( + &new_stake_account_address, + &withdraw_authority, + &recipient, + StakeAuthorize::Withdrawer, + )); + + instructions.push(system_instruction::transfer( + &sender_pubkey, + &recipient, + sol_to_lamports(sol_for_fees), + )); + + instructions + } else { + let from = args.sender_keypair.pubkey(); + let to = allocation.recipient.parse().unwrap(); + let lamports = sol_to_lamports(allocation.amount); + let instruction = system_instruction::transfer(&from, &to, lamports); + vec![instruction] + }; + + let fee_payer_pubkey = args.fee_payer.pubkey(); + let message = Message::new_with_payer(&instructions, Some(&fee_payer_pubkey)); + match client.send_message(message, &signers) { + Ok(transaction) => { + db::set_transaction_info( + db, + &allocation.recipient.parse().unwrap(), + allocation.amount, + &transaction, + Some(&new_stake_account_address), + false, + )?; + } + Err(e) => { + eprintln!("Error sending tokens to {}: {}", allocation.recipient, e); + } + }; + } + Ok(()) +} + +fn read_allocations( + input_csv: &str, + from_bids: bool, + dollars_per_sol: Option, +) -> Vec { + let rdr = ReaderBuilder::new().trim(Trim::All).from_path(input_csv); + if from_bids { + let bids: Vec = rdr.unwrap().deserialize().map(|bid| bid.unwrap()).collect(); + bids.into_iter() + .map(|bid| create_allocation(&bid, dollars_per_sol.unwrap())) + .collect() + } else { + rdr.unwrap() + .deserialize() + .map(|entry| entry.unwrap()) + .collect() + } +} + +fn new_spinner_progress_bar() -> ProgressBar { + let progress_bar = ProgressBar::new(42); + progress_bar + .set_style(ProgressStyle::default_spinner().template("{spinner:.green} {wide_msg}")); + progress_bar.enable_steady_tick(100); + progress_bar +} + +pub fn process_distribute_tokens( + client: &ThinClient, + args: &DistributeTokensArgs>, +) -> Result, Error> { + let mut allocations: Vec = + read_allocations(&args.input_csv, args.from_bids, args.dollars_per_sol); + + let starting_total_tokens: f64 = allocations.iter().map(|x| x.amount).sum(); + println!( + "{} ◎{}", + style("Total in input_csv:").bold(), + starting_total_tokens, + ); + if let Some(dollars_per_sol) = args.dollars_per_sol { + println!( + "{} ${}", + style("Total in input_csv:").bold(), + starting_total_tokens * dollars_per_sol, + ); + } + + let mut db = db::open_db(&args.transaction_db, args.dry_run)?; + + // Start by finalizing any transactions from the previous run. + let confirmations = finalize_transactions(client, &mut db)?; + + let transaction_infos = db::read_transaction_infos(&db); + apply_previous_transactions(&mut allocations, &transaction_infos); + + if allocations.is_empty() { + eprintln!("No work to do"); + return Ok(confirmations); + } + + println!( + "{}", + style(format!( + "{:<44} {:>24}", + "Recipient", "Expected Balance (◎)" + )) + .bold() + ); + + let distributed_tokens: f64 = transaction_infos.iter().map(|x| x.amount).sum(); + let undistributed_tokens: f64 = allocations.iter().map(|x| x.amount).sum(); + println!("{} ◎{}", style("Distributed:").bold(), distributed_tokens,); + if let Some(dollars_per_sol) = args.dollars_per_sol { + println!( + "{} ${}", + style("Distributed:").bold(), + distributed_tokens * dollars_per_sol, + ); + } + println!( + "{} ◎{}", + style("Undistributed:").bold(), + undistributed_tokens, + ); + if let Some(dollars_per_sol) = args.dollars_per_sol { + println!( + "{} ${}", + style("Undistributed:").bold(), + undistributed_tokens * dollars_per_sol, + ); + } + println!( + "{} ◎{}", + style("Total:").bold(), + distributed_tokens + undistributed_tokens, + ); + if let Some(dollars_per_sol) = args.dollars_per_sol { + println!( + "{} ${}", + style("Total:").bold(), + (distributed_tokens + undistributed_tokens) * dollars_per_sol, + ); + } + + distribute_tokens(client, &mut db, &allocations, args)?; + + let opt_confirmations = finalize_transactions(client, &mut db)?; + Ok(opt_confirmations) +} + +fn finalize_transactions( + client: &ThinClient, + db: &mut PickleDb, +) -> Result, Error> { + let mut opt_confirmations = update_finalized_transactions(client, db)?; + + let progress_bar = new_spinner_progress_bar(); + + while opt_confirmations.is_some() { + if let Some(confirmations) = opt_confirmations { + progress_bar.set_message(&format!( + "[{}/{}] Finalizing transactions", + confirmations, 32, + )); + } + + // Sleep for about 1 slot + sleep(Duration::from_millis(500)); + let opt_conf = update_finalized_transactions(client, db)?; + opt_confirmations = opt_conf; + } + + Ok(opt_confirmations) +} + +// Update the finalized bit on any transactions that are now rooted +// Return the lowest number of confirmations on the unfinalized transactions or None if all are finalized. +fn update_finalized_transactions( + client: &ThinClient, + db: &mut PickleDb, +) -> Result, Error> { + let transaction_infos = db::read_transaction_infos(db); + let unconfirmed_transactions: Vec<_> = transaction_infos + .iter() + .filter_map(|info| { + if info.finalized_date.is_some() { + None + } else { + Some(&info.transaction) + } + }) + .collect(); + let unconfirmed_signatures = unconfirmed_transactions + .iter() + .map(|tx| tx.signatures[0]) + .filter(|sig| *sig != Signature::default()) // Filter out dry-run signatures + .collect_vec(); + let transaction_statuses = client.get_signature_statuses(&unconfirmed_signatures)?; + let recent_blockhashes = client.get_recent_blockhashes()?; + + let mut confirmations = None; + for (transaction, opt_transaction_status) in unconfirmed_transactions + .into_iter() + .zip(transaction_statuses.into_iter()) + { + match db::update_finalized_transaction( + db, + &transaction.signatures[0], + opt_transaction_status, + &transaction.message.recent_blockhash, + &recent_blockhashes, + ) { + Ok(Some(confs)) => { + confirmations = Some(cmp::min(confs, confirmations.unwrap_or(usize::MAX))); + } + result => { + result?; + } + } + } + Ok(confirmations) +} + +pub fn process_balances( + client: &ThinClient, + args: &BalancesArgs, +) -> Result<(), csv::Error> { + let allocations: Vec = + read_allocations(&args.input_csv, args.from_bids, args.dollars_per_sol); + let allocations = merge_allocations(&allocations); + + println!( + "{}", + style(format!( + "{:<44} {:>24} {:>24} {:>24}", + "Recipient", "Expected Balance (◎)", "Actual Balance (◎)", "Difference (◎)" + )) + .bold() + ); + + for allocation in &allocations { + let address = allocation.recipient.parse().unwrap(); + let expected = lamports_to_sol(sol_to_lamports(allocation.amount)); + let actual = lamports_to_sol(client.get_balance(&address).unwrap()); + println!( + "{:<44} {:>24.9} {:>24.9} {:>24.9}", + allocation.recipient, + expected, + actual, + actual - expected + ); + } + + Ok(()) +} + +pub fn process_transaction_log(args: &TransactionLogArgs) -> Result<(), Error> { + let db = db::open_db(&args.transaction_db, true)?; + db::write_transaction_log(&db, &args.output_path)?; + Ok(()) +} + +use solana_sdk::{pubkey::Pubkey, signature::Keypair}; +use tempfile::{tempdir, NamedTempFile}; +pub fn test_process_distribute_tokens_with_client(client: C, sender_keypair: Keypair) { + let thin_client = ThinClient::new(client, false); + let fee_payer = Keypair::new(); + let transaction = thin_client + .transfer(sol_to_lamports(1.0), &sender_keypair, &fee_payer.pubkey()) + .unwrap(); + thin_client + .poll_for_confirmation(&transaction.signatures[0]) + .unwrap(); + + let alice_pubkey = Pubkey::new_rand(); + let allocation = Allocation { + recipient: alice_pubkey.to_string(), + amount: 1000.0, + }; + let allocations_file = NamedTempFile::new().unwrap(); + let input_csv = allocations_file.path().to_str().unwrap().to_string(); + let mut wtr = csv::WriterBuilder::new().from_writer(allocations_file); + wtr.serialize(&allocation).unwrap(); + wtr.flush().unwrap(); + + let dir = tempdir().unwrap(); + let transaction_db = dir + .path() + .join("transactions.db") + .to_str() + .unwrap() + .to_string(); + + let args: DistributeTokensArgs> = DistributeTokensArgs { + sender_keypair: Box::new(sender_keypair), + fee_payer: Box::new(fee_payer), + dry_run: false, + input_csv, + from_bids: false, + transaction_db: transaction_db.clone(), + dollars_per_sol: None, + stake_args: None, + }; + let confirmations = process_distribute_tokens(&thin_client, &args).unwrap(); + assert_eq!(confirmations, None); + + let transaction_infos = + db::read_transaction_infos(&db::open_db(&transaction_db, true).unwrap()); + assert_eq!(transaction_infos.len(), 1); + assert_eq!(transaction_infos[0].recipient, alice_pubkey); + let expected_amount = sol_to_lamports(allocation.amount); + assert_eq!( + sol_to_lamports(transaction_infos[0].amount), + expected_amount + ); + + assert_eq!( + thin_client.get_balance(&alice_pubkey).unwrap(), + expected_amount, + ); + + // Now, run it again, and check there's no double-spend. + process_distribute_tokens(&thin_client, &args).unwrap(); + let transaction_infos = + db::read_transaction_infos(&db::open_db(&transaction_db, true).unwrap()); + assert_eq!(transaction_infos.len(), 1); + assert_eq!(transaction_infos[0].recipient, alice_pubkey); + let expected_amount = sol_to_lamports(allocation.amount); + assert_eq!( + sol_to_lamports(transaction_infos[0].amount), + expected_amount + ); + + assert_eq!( + thin_client.get_balance(&alice_pubkey).unwrap(), + expected_amount, + ); +} + +pub fn test_process_distribute_stake_with_client(client: C, sender_keypair: Keypair) { + let thin_client = ThinClient::new(client, false); + let fee_payer = Keypair::new(); + let transaction = thin_client + .transfer(sol_to_lamports(1.0), &sender_keypair, &fee_payer.pubkey()) + .unwrap(); + thin_client + .poll_for_confirmation(&transaction.signatures[0]) + .unwrap(); + + let stake_account_keypair = Keypair::new(); + let stake_account_address = stake_account_keypair.pubkey(); + let stake_authority = Keypair::new(); + let withdraw_authority = Keypair::new(); + + let authorized = Authorized { + staker: stake_authority.pubkey(), + withdrawer: withdraw_authority.pubkey(), + }; + let lockup = Lockup::default(); + let instructions = stake_instruction::create_account( + &sender_keypair.pubkey(), + &stake_account_address, + &authorized, + &lockup, + sol_to_lamports(3000.0), + ); + let message = Message::new(&instructions); + let signers = [&sender_keypair, &stake_account_keypair]; + thin_client.send_message(message, &signers).unwrap(); + + let alice_pubkey = Pubkey::new_rand(); + let allocation = Allocation { + recipient: alice_pubkey.to_string(), + amount: 1000.0, + }; + let file = NamedTempFile::new().unwrap(); + let input_csv = file.path().to_str().unwrap().to_string(); + let mut wtr = csv::WriterBuilder::new().from_writer(file); + wtr.serialize(&allocation).unwrap(); + wtr.flush().unwrap(); + + let dir = tempdir().unwrap(); + let transaction_db = dir + .path() + .join("transactions.db") + .to_str() + .unwrap() + .to_string(); + + let stake_args: StakeArgs> = StakeArgs { + stake_account_address, + stake_authority: Box::new(stake_authority), + withdraw_authority: Box::new(withdraw_authority), + sol_for_fees: 1.0, + }; + let args: DistributeTokensArgs> = DistributeTokensArgs { + fee_payer: Box::new(fee_payer), + dry_run: false, + input_csv, + transaction_db: transaction_db.clone(), + stake_args: Some(stake_args), + from_bids: false, + sender_keypair: Box::new(sender_keypair), + dollars_per_sol: None, + }; + let confirmations = process_distribute_tokens(&thin_client, &args).unwrap(); + assert_eq!(confirmations, None); + + let transaction_infos = + db::read_transaction_infos(&db::open_db(&transaction_db, true).unwrap()); + assert_eq!(transaction_infos.len(), 1); + assert_eq!(transaction_infos[0].recipient, alice_pubkey); + let expected_amount = sol_to_lamports(allocation.amount); + assert_eq!( + sol_to_lamports(transaction_infos[0].amount), + expected_amount + ); + + assert_eq!( + thin_client.get_balance(&alice_pubkey).unwrap(), + sol_to_lamports(1.0), + ); + let new_stake_account_address = transaction_infos[0].new_stake_account_address.unwrap(); + assert_eq!( + thin_client.get_balance(&new_stake_account_address).unwrap(), + expected_amount - sol_to_lamports(1.0), + ); + + // Now, run it again, and check there's no double-spend. + process_distribute_tokens(&thin_client, &args).unwrap(); + let transaction_infos = + db::read_transaction_infos(&db::open_db(&transaction_db, true).unwrap()); + assert_eq!(transaction_infos.len(), 1); + assert_eq!(transaction_infos[0].recipient, alice_pubkey); + let expected_amount = sol_to_lamports(allocation.amount); + assert_eq!( + sol_to_lamports(transaction_infos[0].amount), + expected_amount + ); + + assert_eq!( + thin_client.get_balance(&alice_pubkey).unwrap(), + sol_to_lamports(1.0), + ); + assert_eq!( + thin_client.get_balance(&new_stake_account_address).unwrap(), + expected_amount - sol_to_lamports(1.0), + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use solana_runtime::{bank::Bank, bank_client::BankClient}; + use solana_sdk::{genesis_config::create_genesis_config, transaction::Transaction}; + + #[test] + fn test_process_distribute_tokens() { + let (genesis_config, sender_keypair) = create_genesis_config(sol_to_lamports(9_000_000.0)); + let bank = Bank::new(&genesis_config); + let bank_client = BankClient::new(bank); + test_process_distribute_tokens_with_client(bank_client, sender_keypair); + } + + #[test] + fn test_process_distribute_stake() { + let (genesis_config, sender_keypair) = create_genesis_config(sol_to_lamports(9_000_000.0)); + let bank = Bank::new(&genesis_config); + let bank_client = BankClient::new(bank); + test_process_distribute_stake_with_client(bank_client, sender_keypair); + } + + #[test] + fn test_read_allocations() { + let alice_pubkey = Pubkey::new_rand(); + let allocation = Allocation { + recipient: alice_pubkey.to_string(), + amount: 42.0, + }; + let file = NamedTempFile::new().unwrap(); + let input_csv = file.path().to_str().unwrap().to_string(); + let mut wtr = csv::WriterBuilder::new().from_writer(file); + wtr.serialize(&allocation).unwrap(); + wtr.flush().unwrap(); + + assert_eq!(read_allocations(&input_csv, false, None), vec![allocation]); + } + + #[test] + fn test_read_allocations_from_bids() { + let alice_pubkey = Pubkey::new_rand(); + let bid = Bid { + primary_address: alice_pubkey.to_string(), + accepted_amount_dollars: 42.0, + }; + let file = NamedTempFile::new().unwrap(); + let input_csv = file.path().to_str().unwrap().to_string(); + let mut wtr = csv::WriterBuilder::new().from_writer(file); + wtr.serialize(&bid).unwrap(); + wtr.flush().unwrap(); + + let allocation = Allocation { + recipient: bid.primary_address, + amount: 84.0, + }; + assert_eq!( + read_allocations(&input_csv, true, Some(0.5)), + vec![allocation] + ); + } + + #[test] + fn test_apply_previous_transactions() { + let alice = Pubkey::new_rand(); + let bob = Pubkey::new_rand(); + let mut allocations = vec![ + Allocation { + recipient: alice.to_string(), + amount: 1.0, + }, + Allocation { + recipient: bob.to_string(), + amount: 1.0, + }, + ]; + let transaction_infos = vec![TransactionInfo { + recipient: bob, + amount: 1.0, + new_stake_account_address: None, + finalized_date: None, + transaction: Transaction::new_unsigned_instructions(&[]), + }]; + apply_previous_transactions(&mut allocations, &transaction_infos); + assert_eq!(allocations.len(), 1); + + // Ensure that we applied the transaction to the allocation with + // a matching recipient address (to bob, not alice). + assert_eq!(allocations[0].recipient, alice.to_string()); + } +} diff --git a/tokens/src/db.rs b/tokens/src/db.rs new file mode 100644 index 00000000000000..66f96a8423bb2c --- /dev/null +++ b/tokens/src/db.rs @@ -0,0 +1,351 @@ +use chrono::prelude::*; +use pickledb::{error::Error, PickleDb, PickleDbDumpPolicy}; +use serde::{Deserialize, Serialize}; +use solana_sdk::{hash::Hash, pubkey::Pubkey, signature::Signature, transaction::Transaction}; +use solana_transaction_status::TransactionStatus; +use std::{cmp::Ordering, fs, io, path::Path}; + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct TransactionInfo { + pub recipient: Pubkey, + pub amount: f64, + pub new_stake_account_address: Option, + pub finalized_date: Option>, + pub transaction: Transaction, +} + +#[derive(Serialize, Deserialize, Debug, Default, PartialEq)] +struct SignedTransactionInfo { + recipient: String, + amount: f64, + new_stake_account_address: String, + finalized_date: Option>, + signature: String, +} + +impl Default for TransactionInfo { + fn default() -> Self { + let mut transaction = Transaction::new_unsigned_instructions(&[]); + transaction.signatures.push(Signature::default()); + Self { + recipient: Pubkey::default(), + amount: 0.0, + new_stake_account_address: None, + finalized_date: None, + transaction, + } + } +} + +pub fn open_db(path: &str, dry_run: bool) -> Result { + let policy = if dry_run { + PickleDbDumpPolicy::NeverDump + } else { + PickleDbDumpPolicy::AutoDump + }; + let path = Path::new(path); + let db = if path.exists() { + PickleDb::load_yaml(path, policy)? + } else { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).unwrap(); + } + PickleDb::new_yaml(path, policy) + }; + Ok(db) +} + +pub fn compare_transaction_infos(a: &TransactionInfo, b: &TransactionInfo) -> Ordering { + let ordering = match (a.finalized_date, b.finalized_date) { + (Some(a), Some(b)) => a.cmp(&b), + (None, Some(_)) => Ordering::Greater, + (Some(_), None) => Ordering::Less, // Future finalized date will be greater + _ => Ordering::Equal, + }; + if ordering == Ordering::Equal { + return a.recipient.to_string().cmp(&b.recipient.to_string()); + } + ordering +} + +pub fn write_transaction_log>(db: &PickleDb, path: &P) -> Result<(), io::Error> { + let mut wtr = csv::WriterBuilder::new().from_path(path).unwrap(); + let mut transaction_infos = read_transaction_infos(db); + transaction_infos.sort_by(compare_transaction_infos); + for info in transaction_infos { + let signed_info = SignedTransactionInfo { + recipient: info.recipient.to_string(), + amount: info.amount, + new_stake_account_address: info + .new_stake_account_address + .map(|x| x.to_string()) + .unwrap_or_else(|| "".to_string()), + finalized_date: info.finalized_date, + signature: info.transaction.signatures[0].to_string(), + }; + wtr.serialize(&signed_info)?; + } + wtr.flush() +} + +pub fn read_transaction_infos(db: &PickleDb) -> Vec { + db.iter() + .map(|kv| kv.get_value::().unwrap()) + .collect() +} + +pub fn set_transaction_info( + db: &mut PickleDb, + recipient: &Pubkey, + amount: f64, + transaction: &Transaction, + new_stake_account_address: Option<&Pubkey>, + finalized: bool, +) -> Result<(), Error> { + let finalized_date = if finalized { Some(Utc::now()) } else { None }; + let transaction_info = TransactionInfo { + recipient: *recipient, + amount, + new_stake_account_address: new_stake_account_address.cloned(), + finalized_date, + transaction: transaction.clone(), + }; + let signature = transaction.signatures[0]; + db.set(&signature.to_string(), &transaction_info)?; + Ok(()) +} + +// Set the finalized bit in the database if the transaction is rooted. +// Remove the TransactionInfo from the database if the transaction failed. +// Return the number of confirmations on the transaction or None if finalized. +pub fn update_finalized_transaction( + db: &mut PickleDb, + signature: &Signature, + opt_transaction_status: Option, + blockhash: &Hash, + recent_blockhashes: &[Hash], +) -> Result, Error> { + if opt_transaction_status.is_none() { + if !recent_blockhashes.contains(blockhash) { + eprintln!("Signature not found {} and blockhash expired", signature); + eprintln!("Discarding transaction record"); + db.rem(&signature.to_string())?; + return Ok(None); + } + + // Return zero to signal the transaction may still be in flight. + return Ok(Some(0)); + } + let transaction_status = opt_transaction_status.unwrap(); + + if let Some(confirmations) = transaction_status.confirmations { + // The transaction was found but is not yet finalized. + return Ok(Some(confirmations)); + } + + if let Err(e) = &transaction_status.status { + // The transaction was finalized, but execution failed. Drop it. + eprintln!( + "Error in transaction with signature {}: {}", + signature, + e.to_string() + ); + eprintln!("Discarding transaction record"); + db.rem(&signature.to_string())?; + return Ok(None); + } + + // Transaction is rooted. Set finalized in the database. + let mut transaction_info = db.get::(&signature.to_string()).unwrap(); + transaction_info.finalized_date = Some(Utc::now()); + db.set(&signature.to_string(), &transaction_info)?; + Ok(None) +} + +#[cfg(test)] +mod tests { + use super::*; + use csv::{ReaderBuilder, Trim}; + use solana_sdk::transaction::TransactionError; + use tempfile::NamedTempFile; + + #[test] + fn test_sort_transaction_infos_finalized_first() { + let info0 = TransactionInfo { + finalized_date: Some(Utc.ymd(2014, 7, 8).and_hms(9, 10, 11)), + ..TransactionInfo::default() + }; + let info1 = TransactionInfo { + finalized_date: Some(Utc.ymd(2014, 7, 8).and_hms(9, 10, 42)), + ..TransactionInfo::default() + }; + let info2 = TransactionInfo::default(); + let info3 = TransactionInfo { + recipient: Pubkey::new_rand(), + ..TransactionInfo::default() + }; + + // Sorted first by date + assert_eq!(compare_transaction_infos(&info0, &info1), Ordering::Less); + + // Finalized transactions should be before unfinalized ones + assert_eq!(compare_transaction_infos(&info1, &info2), Ordering::Less); + + // Then sorted by recipient + assert_eq!(compare_transaction_infos(&info2, &info3), Ordering::Less); + } + + #[test] + fn test_write_transaction_log() { + let mut db = + PickleDb::new_yaml(NamedTempFile::new().unwrap(), PickleDbDumpPolicy::NeverDump); + let signature = Signature::default(); + let transaction_info = TransactionInfo::default(); + db.set(&signature.to_string(), &transaction_info).unwrap(); + + let csv_file = NamedTempFile::new().unwrap(); + write_transaction_log(&db, &csv_file).unwrap(); + + let mut rdr = ReaderBuilder::new().trim(Trim::All).from_reader(csv_file); + let signed_infos: Vec = + rdr.deserialize().map(|entry| entry.unwrap()).collect(); + + let signed_info = SignedTransactionInfo { + recipient: Pubkey::default().to_string(), + signature: Signature::default().to_string(), + ..SignedTransactionInfo::default() + }; + assert_eq!(signed_infos, vec![signed_info]); + } + + #[test] + fn test_update_finalized_transaction_not_landed() { + // Keep waiting for a transaction that hasn't landed yet. + let mut db = + PickleDb::new_yaml(NamedTempFile::new().unwrap(), PickleDbDumpPolicy::NeverDump); + let signature = Signature::default(); + let blockhash = Hash::default(); + let transaction_info = TransactionInfo::default(); + db.set(&signature.to_string(), &transaction_info).unwrap(); + assert!(matches!( + update_finalized_transaction(&mut db, &signature, None, &blockhash, &[blockhash]) + .unwrap(), + Some(0) + )); + + // Unchanged + assert_eq!( + db.get::(&signature.to_string()).unwrap(), + transaction_info + ); + + // Same as before, but now with an expired blockhash + assert_eq!( + update_finalized_transaction(&mut db, &signature, None, &blockhash, &[]).unwrap(), + None + ); + + // Ensure TransactionInfo has been purged. + assert_eq!(db.get::(&signature.to_string()), None); + } + + #[test] + fn test_update_finalized_transaction_confirming() { + // Keep waiting for a transaction that is still being confirmed. + let mut db = + PickleDb::new_yaml(NamedTempFile::new().unwrap(), PickleDbDumpPolicy::NeverDump); + let signature = Signature::default(); + let blockhash = Hash::default(); + let transaction_info = TransactionInfo::default(); + db.set(&signature.to_string(), &transaction_info).unwrap(); + let transaction_status = TransactionStatus { + slot: 0, + confirmations: Some(1), + status: Ok(()), + err: None, + }; + assert_eq!( + update_finalized_transaction( + &mut db, + &signature, + Some(transaction_status), + &blockhash, + &[blockhash] + ) + .unwrap(), + Some(1) + ); + + // Unchanged + assert_eq!( + db.get::(&signature.to_string()).unwrap(), + transaction_info + ); + } + + #[test] + fn test_update_finalized_transaction_failed() { + // Don't wait if the transaction failed to execute. + let mut db = + PickleDb::new_yaml(NamedTempFile::new().unwrap(), PickleDbDumpPolicy::NeverDump); + let signature = Signature::default(); + let blockhash = Hash::default(); + let transaction_info = TransactionInfo::default(); + db.set(&signature.to_string(), &transaction_info).unwrap(); + let status = Err(TransactionError::AccountNotFound); + let transaction_status = TransactionStatus { + slot: 0, + confirmations: None, + status, + err: None, + }; + assert_eq!( + update_finalized_transaction( + &mut db, + &signature, + Some(transaction_status), + &blockhash, + &[blockhash] + ) + .unwrap(), + None + ); + + // Ensure TransactionInfo has been purged. + assert_eq!(db.get::(&signature.to_string()), None); + } + + #[test] + fn test_update_finalized_transaction_finalized() { + // Don't wait once the transaction has been finalized. + let mut db = + PickleDb::new_yaml(NamedTempFile::new().unwrap(), PickleDbDumpPolicy::NeverDump); + let signature = Signature::default(); + let blockhash = Hash::default(); + let transaction_info = TransactionInfo::default(); + db.set(&signature.to_string(), &transaction_info).unwrap(); + let transaction_status = TransactionStatus { + slot: 0, + confirmations: None, + status: Ok(()), + err: None, + }; + assert_eq!( + update_finalized_transaction( + &mut db, + &signature, + Some(transaction_status), + &blockhash, + &[blockhash] + ) + .unwrap(), + None + ); + + assert!(db + .get::(&signature.to_string()) + .unwrap() + .finalized_date + .is_some()); + } +} diff --git a/tokens/src/lib.rs b/tokens/src/lib.rs new file mode 100644 index 00000000000000..f955230c8ceb81 --- /dev/null +++ b/tokens/src/lib.rs @@ -0,0 +1,5 @@ +pub mod arg_parser; +pub mod args; +pub mod commands; +mod db; +pub mod thin_client; diff --git a/tokens/src/main.rs b/tokens/src/main.rs new file mode 100644 index 00000000000000..3ea9db57ddc8c9 --- /dev/null +++ b/tokens/src/main.rs @@ -0,0 +1,44 @@ +use solana_cli_config::Config; +use solana_cli_config::CONFIG_FILE; +use solana_client::rpc_client::RpcClient; +use solana_tokens::{ + arg_parser::parse_args, + args::{resolve_command, Command}, + commands, + thin_client::ThinClient, +}; +use std::env; +use std::error::Error; +use std::path::Path; +use std::process; + +fn main() -> Result<(), Box> { + let command_args = parse_args(env::args_os()); + let config = if Path::new(&command_args.config_file).exists() { + Config::load(&command_args.config_file)? + } else { + let default_config_file = CONFIG_FILE.as_ref().unwrap(); + if command_args.config_file != *default_config_file { + eprintln!("Error: config file not found"); + process::exit(1); + } + Config::default() + }; + let json_rpc_url = command_args.url.unwrap_or(config.json_rpc_url); + let client = RpcClient::new(json_rpc_url); + + match resolve_command(command_args.command)? { + Command::DistributeTokens(args) => { + let thin_client = ThinClient::new(client, args.dry_run); + commands::process_distribute_tokens(&thin_client, &args)?; + } + Command::Balances(args) => { + let thin_client = ThinClient::new(client, false); + commands::process_balances(&thin_client, &args)?; + } + Command::TransactionLog(args) => { + commands::process_transaction_log(&args)?; + } + } + Ok(()) +} diff --git a/tokens/src/thin_client.rs b/tokens/src/thin_client.rs new file mode 100644 index 00000000000000..5b11889f7016ed --- /dev/null +++ b/tokens/src/thin_client.rs @@ -0,0 +1,174 @@ +use solana_client::rpc_client::RpcClient; +use solana_runtime::bank_client::BankClient; +use solana_sdk::{ + account::Account, + client::{AsyncClient, SyncClient}, + fee_calculator::FeeCalculator, + hash::Hash, + message::Message, + pubkey::Pubkey, + signature::{Signature, Signer}, + signers::Signers, + system_instruction, + sysvar::{ + recent_blockhashes::{self, RecentBlockhashes}, + Sysvar, + }, + transaction::Transaction, + transport::{Result, TransportError}, +}; +use solana_transaction_status::TransactionStatus; + +pub trait Client { + fn send_transaction1(&self, transaction: Transaction) -> Result; + fn get_signature_statuses1( + &self, + signatures: &[Signature], + ) -> Result>>; + fn get_balance1(&self, pubkey: &Pubkey) -> Result; + fn get_recent_blockhash1(&self) -> Result<(Hash, FeeCalculator)>; + fn get_account1(&self, pubkey: &Pubkey) -> Result>; +} + +impl Client for RpcClient { + fn send_transaction1(&self, transaction: Transaction) -> Result { + self.send_transaction(&transaction) + .map_err(|e| TransportError::Custom(e.to_string())) + } + + fn get_signature_statuses1( + &self, + signatures: &[Signature], + ) -> Result>> { + self.get_signature_statuses(signatures) + .map(|response| response.value) + .map_err(|e| TransportError::Custom(e.to_string())) + } + + fn get_balance1(&self, pubkey: &Pubkey) -> Result { + self.get_balance(pubkey) + .map_err(|e| TransportError::Custom(e.to_string())) + } + + fn get_recent_blockhash1(&self) -> Result<(Hash, FeeCalculator)> { + self.get_recent_blockhash() + .map_err(|e| TransportError::Custom(e.to_string())) + } + + fn get_account1(&self, pubkey: &Pubkey) -> Result> { + self.get_account(pubkey) + .map(Some) + .map_err(|e| TransportError::Custom(e.to_string())) + } +} + +impl Client for BankClient { + fn send_transaction1(&self, transaction: Transaction) -> Result { + self.async_send_transaction(transaction) + } + + fn get_signature_statuses1( + &self, + signatures: &[Signature], + ) -> Result>> { + signatures + .iter() + .map(|signature| { + self.get_signature_status(signature).map(|opt| { + opt.map(|status| TransactionStatus { + slot: 0, + confirmations: None, + status, + err: None, + }) + }) + }) + .collect() + } + + fn get_balance1(&self, pubkey: &Pubkey) -> Result { + self.get_balance(pubkey) + } + + fn get_recent_blockhash1(&self) -> Result<(Hash, FeeCalculator)> { + self.get_recent_blockhash() + } + + fn get_account1(&self, pubkey: &Pubkey) -> Result> { + self.get_account(pubkey) + } +} + +pub struct ThinClient { + client: C, + dry_run: bool, +} + +impl ThinClient { + pub fn new(client: C, dry_run: bool) -> Self { + Self { client, dry_run } + } + + pub fn send_transaction(&self, transaction: Transaction) -> Result { + if self.dry_run { + return Ok(Signature::default()); + } + self.client.send_transaction1(transaction) + } + + pub fn poll_for_confirmation(&self, signature: &Signature) -> Result<()> { + while self.get_signature_statuses(&[*signature])?[0].is_none() { + std::thread::sleep(std::time::Duration::from_millis(500)); + } + Ok(()) + } + + pub fn get_signature_statuses( + &self, + signatures: &[Signature], + ) -> Result>> { + self.client.get_signature_statuses1(signatures) + } + + pub fn send_message(&self, message: Message, signers: &S) -> Result { + if self.dry_run { + return Ok(Transaction::new_unsigned(message)); + } + let (blockhash, _fee_caluclator) = self.get_recent_blockhash()?; + let transaction = Transaction::new(signers, message, blockhash); + self.send_transaction(transaction.clone())?; + Ok(transaction) + } + + pub fn transfer( + &self, + lamports: u64, + sender_keypair: &S, + to_pubkey: &Pubkey, + ) -> Result { + let create_instruction = + system_instruction::transfer(&sender_keypair.pubkey(), &to_pubkey, lamports); + let message = Message::new(&[create_instruction]); + self.send_message(message, &[sender_keypair]) + } + + pub fn get_recent_blockhash(&self) -> Result<(Hash, FeeCalculator)> { + self.client.get_recent_blockhash1() + } + + pub fn get_balance(&self, pubkey: &Pubkey) -> Result { + self.client.get_balance1(pubkey) + } + + pub fn get_account(&self, pubkey: &Pubkey) -> Result> { + self.client.get_account1(pubkey) + } + + pub fn get_recent_blockhashes(&self) -> Result> { + let opt_blockhashes_account = self.get_account(&recent_blockhashes::id())?; + let blockhashes_account = opt_blockhashes_account.unwrap(); + let recent_blockhashes = RecentBlockhashes::from_account(&blockhashes_account).unwrap(); + let hashes = recent_blockhashes.iter().map(|x| x.blockhash).collect(); + Ok(hashes) + } +} diff --git a/tokens/tests/commands.rs b/tokens/tests/commands.rs new file mode 100644 index 00000000000000..3bdab39c4420dd --- /dev/null +++ b/tokens/tests/commands.rs @@ -0,0 +1,18 @@ +use solana_client::rpc_client::RpcClient; +use solana_core::validator::{TestValidator, TestValidatorOptions}; +use solana_sdk::native_token::sol_to_lamports; +use solana_tokens::commands::test_process_distribute_tokens_with_client; +use std::fs::remove_dir_all; + +#[test] +fn test_process_distribute_with_rpc_client() { + let validator = TestValidator::run_with_options(TestValidatorOptions { + mint_lamports: sol_to_lamports(9_000_000.0), + ..TestValidatorOptions::default() + }); + let rpc_client = RpcClient::new_socket(validator.leader_data.rpc); + test_process_distribute_tokens_with_client(rpc_client, validator.alice); + + validator.server.close().unwrap(); + remove_dir_all(validator.ledger_path).unwrap(); +}