From 4cd8ef001f7b8c05b40831e00a9bb0f14f9d6418 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Thu, 23 Mar 2023 13:34:34 -0500 Subject: [PATCH 1/4] build: use optimizations in packages for dev builds --- Cargo.toml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b930bacb78..4af1002184 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -109,8 +109,16 @@ monitoring_prom = ["prometheus"] slog_json = ["slog-json", "stacks_common/slog_json", "clarity/slog_json"] testing = [] -[profile.dev.package.regex] -opt-level = 2 +# Use a bit more than default optimization for +# dev builds to speed up test execution +[profile.dev] +opt-level = 1 + +# Use release-level optimization for dependencies +# This slows down "first" builds on development environments, +# but won't impact subsequent builds. +[profile.dev.package."*"] +opt-level = 3 [target.'cfg(all(any(target_arch = "x86_64", target_arch = "x86", target_arch = "aarch64"), not(target_env = "msvc")))'.dependencies] sha2 = { version = "0.10", features = ["asm"] } From 23de9d1f299801d29f9ab05d1578b9855e04a699 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Fri, 17 Feb 2023 16:01:53 -0600 Subject: [PATCH 2/4] feat: add a peer-pub-key command for obtaining pubkey from local peer seed --- src/main.rs | 15 +++++++++ stacks-common/src/util/secp256k1.rs | 47 ++++++++++++++++++++++++++++ testnet/stacks-node/src/neon_node.rs | 23 +------------- 3 files changed, 63 insertions(+), 22 deletions(-) diff --git a/src/main.rs b/src/main.rs index fcf8abb7e2..61062a815c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -81,6 +81,8 @@ use blockstack_lib::util::get_epoch_time_ms; use blockstack_lib::util::hash::{hex_bytes, to_hex}; use blockstack_lib::util::log; use blockstack_lib::util::retry::LogReader; +use blockstack_lib::util::secp256k1::Secp256k1PrivateKey; +use blockstack_lib::util::secp256k1::Secp256k1PublicKey; use blockstack_lib::util::sleep_ms; use blockstack_lib::util_lib::strings::UrlString; use blockstack_lib::{ @@ -116,6 +118,19 @@ fn main() { process::exit(0); } + if argv[1] == "peer-pub-key" { + if argv.len() < 3 { + eprintln!("Usage: {} peer-pub-key ", argv[0]); + process::exit(1); + } + + let local_seed = hex_bytes(&argv[2]).expect("Failed to parse hex input local-peer-seed"); + let node_privkey = Secp256k1PrivateKey::from_seed(&local_seed); + let pubkey = Secp256k1PublicKey::from_private(&node_privkey).to_hex(); + println!("{}", pubkey); + process::exit(0); + } + if argv[1] == "decode-bitcoin-header" { if argv.len() < 4 { eprintln!( diff --git a/stacks-common/src/util/secp256k1.rs b/stacks-common/src/util/secp256k1.rs index 42e25c56ef..47a0421297 100644 --- a/stacks-common/src/util/secp256k1.rs +++ b/stacks-common/src/util/secp256k1.rs @@ -37,6 +37,8 @@ use serde::Serialize; use rand::thread_rng; use rand::RngCore; +use super::hash::Sha256Sum; + // per-thread Secp256k1 context thread_local!(static _secp256k1: Secp256k1 = Secp256k1::new()); @@ -268,6 +270,27 @@ impl Secp256k1PrivateKey { } } + /// Create a Secp256k1PrivateKey from seed bytes by repeatedly + /// SHA256 hashing the seed bytes until a private key is found. + /// + /// If `seed` is a valid private key, it will be returned without hashing. + /// The returned private key's compress_public flag will be `true` + pub fn from_seed(seed: &[u8]) -> Secp256k1PrivateKey { + let mut re_hashed_seed = Vec::from(seed); + loop { + if let Ok(mut sk) = Secp256k1PrivateKey::from_slice(&re_hashed_seed[..]) { + // set this to true: LocalPeer will be doing this anyways, + // and that's currently the only way this method is used + sk.set_compress_public(true); + return sk; + } else { + re_hashed_seed = Sha256Sum::from_data(&re_hashed_seed[..]) + .as_bytes() + .to_vec() + } + } + } + pub fn from_hex(hex_string: &str) -> Result { let data = hex_bytes(hex_string).map_err(|_e| "Failed to decode hex private key")?; Secp256k1PrivateKey::from_slice(&data[..]).map_err(|_e| "Invalid private key hex string") @@ -458,6 +481,30 @@ mod tests { assert_eq!(Secp256k1PrivateKey::from_hex(&h_comp), Ok(t1)); } + #[test] + /// Test the behavior of from_seed using hard-coded values from previous existing integration tests + fn sk_from_seed() { + let sk = Secp256k1PrivateKey::from_seed(&[2; 32]); + assert_eq!( + Secp256k1PublicKey::from_private(&sk).to_hex(), + "024d4b6cd1361032ca9bd2aeb9d900aa4d45d9ead80ac9423374c451a7254d0766" + ); + assert_eq!( + sk.to_hex(), + "020202020202020202020202020202020202020202020202020202020202020201" + ); + + let sk = Secp256k1PrivateKey::from_seed(&[0]); + assert_eq!( + Secp256k1PublicKey::from_private(&sk).to_hex(), + "0243311589af63c2adda04fcd7792c038a05c12a4fe40351b3eb1612ff6b2e5a0e" + ); + assert_eq!( + sk.to_hex(), + "6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d01" + ); + } + #[test] fn test_parse_serialize() { let ctx: Secp256k1 = Secp256k1::new(); diff --git a/testnet/stacks-node/src/neon_node.rs b/testnet/stacks-node/src/neon_node.rs index 80c4da6fae..f69b8e94a3 100644 --- a/testnet/stacks-node/src/neon_node.rs +++ b/testnet/stacks-node/src/neon_node.rs @@ -218,7 +218,6 @@ use crate::syncctl::PoxSyncWatchdogComms; use stacks::monitoring; use stacks_common::types::chainstate::StacksBlockId; -use stacks_common::types::chainstate::StacksPrivateKey; use stacks_common::util::vrf::VRFProof; use clarity::vm::ast::ASTRules; @@ -3808,25 +3807,6 @@ impl PeerThread { } impl StacksNode { - /// Create a StacksPrivateKey from a given seed buffer - pub fn make_node_private_key_from_seed(seed: &[u8]) -> StacksPrivateKey { - let node_privkey = { - let mut re_hashed_seed = seed.to_vec(); - let my_private_key = loop { - match Secp256k1PrivateKey::from_slice(&re_hashed_seed[..]) { - Ok(sk) => break sk, - Err(_) => { - re_hashed_seed = Sha256Sum::from_data(&re_hashed_seed[..]) - .as_bytes() - .to_vec() - } - } - }; - my_private_key - }; - node_privkey - } - /// Set up the AST size-precheck height, if configured fn setup_ast_size_precheck(config: &Config, sortdb: &mut SortitionDB) { if let Some(ast_precheck_size_height) = config.burnchain.ast_precheck_size_height { @@ -3898,8 +3878,7 @@ impl StacksNode { "Failed to parse socket: {}", &config.node.p2p_address )); - let node_privkey = - StacksNode::make_node_private_key_from_seed(&config.node.local_peer_seed); + let node_privkey = Secp256k1PrivateKey::from_seed(&config.node.local_peer_seed); let mut peerdb = PeerDB::connect( &config.get_peer_db_file_path(), From 79df94ce1ad8757fa6d6718a5418bde35ded7290 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Tue, 28 Mar 2023 09:52:59 -0500 Subject: [PATCH 3/4] fix test issue with peer-pub-key changes --- testnet/stacks-node/src/tests/epoch_21.rs | 20 +++++++------------ .../src/tests/neon_integrations.rs | 6 ++---- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/testnet/stacks-node/src/tests/epoch_21.rs b/testnet/stacks-node/src/tests/epoch_21.rs index d6e4732955..0c8a41a451 100644 --- a/testnet/stacks-node/src/tests/epoch_21.rs +++ b/testnet/stacks-node/src/tests/epoch_21.rs @@ -7,6 +7,7 @@ use stacks::burnchains::Burnchain; use stacks::chainstate::stacks::db::StacksChainState; use stacks::chainstate::stacks::StacksBlockHeader; use stacks::types::chainstate::StacksAddress; +use stacks::util::secp256k1::Secp256k1PrivateKey; use crate::config::Config; use crate::config::EventKeyType; @@ -62,7 +63,6 @@ use clarity::vm::ClarityVersion; use stacks::core::BURNCHAIN_TX_SEARCH_WINDOW; use crate::burnchains::bitcoin_regtest_controller::UTXO; -use crate::neon_node::StacksNode; use crate::operations::BurnchainOpSigner; use crate::Keychain; @@ -2132,8 +2132,7 @@ fn test_pox_reorgs_three_flaps() { confs.push(conf); } - let node_privkey_1 = - StacksNode::make_node_private_key_from_seed(&confs[0].node.local_peer_seed); + let node_privkey_1 = Secp256k1PrivateKey::from_seed(&confs[0].node.local_peer_seed); for i in 1..num_miners { let chain_id = confs[0].burnchain.chain_id; let peer_version = confs[0].burnchain.peer_version; @@ -2667,8 +2666,7 @@ fn test_pox_reorg_one_flap() { confs.push(conf); } - let node_privkey_1 = - StacksNode::make_node_private_key_from_seed(&confs[0].node.local_peer_seed); + let node_privkey_1 = Secp256k1PrivateKey::from_seed(&confs[0].node.local_peer_seed); for i in 1..num_miners { let chain_id = confs[0].burnchain.chain_id; let peer_version = confs[0].burnchain.peer_version; @@ -3090,8 +3088,7 @@ fn test_pox_reorg_flap_duel() { confs.push(conf); } - let node_privkey_1 = - StacksNode::make_node_private_key_from_seed(&confs[0].node.local_peer_seed); + let node_privkey_1 = Secp256k1PrivateKey::from_seed(&confs[0].node.local_peer_seed); for i in 1..num_miners { let chain_id = confs[0].burnchain.chain_id; let peer_version = confs[0].burnchain.peer_version; @@ -3523,8 +3520,7 @@ fn test_pox_reorg_flap_reward_cycles() { confs.push(conf); } - let node_privkey_1 = - StacksNode::make_node_private_key_from_seed(&confs[0].node.local_peer_seed); + let node_privkey_1 = Secp256k1PrivateKey::from_seed(&confs[0].node.local_peer_seed); for i in 1..num_miners { let chain_id = confs[0].burnchain.chain_id; let peer_version = confs[0].burnchain.peer_version; @@ -3950,8 +3946,7 @@ fn test_pox_missing_five_anchor_blocks() { confs.push(conf); } - let node_privkey_1 = - StacksNode::make_node_private_key_from_seed(&confs[0].node.local_peer_seed); + let node_privkey_1 = Secp256k1PrivateKey::from_seed(&confs[0].node.local_peer_seed); for i in 1..num_miners { let chain_id = confs[0].burnchain.chain_id; let peer_version = confs[0].burnchain.peer_version; @@ -4349,8 +4344,7 @@ fn test_sortition_divergence_pre_21() { confs.push(conf); } - let node_privkey_1 = - StacksNode::make_node_private_key_from_seed(&confs[0].node.local_peer_seed); + let node_privkey_1 = Secp256k1PrivateKey::from_seed(&confs[0].node.local_peer_seed); for i in 1..num_miners { let chain_id = confs[0].burnchain.chain_id; let peer_version = confs[0].burnchain.peer_version; diff --git a/testnet/stacks-node/src/tests/neon_integrations.rs b/testnet/stacks-node/src/tests/neon_integrations.rs index cca910f887..3fc4bc4602 100644 --- a/testnet/stacks-node/src/tests/neon_integrations.rs +++ b/testnet/stacks-node/src/tests/neon_integrations.rs @@ -39,6 +39,7 @@ use stacks::types::chainstate::{ }; use stacks::util::hash::Hash160; use stacks::util::hash::{bytes_to_hex, hex_bytes, to_hex}; +use stacks::util::secp256k1::Secp256k1PrivateKey; use stacks::util::secp256k1::Secp256k1PublicKey; use stacks::util::{get_epoch_time_ms, get_epoch_time_secs, sleep_ms}; use stacks::util_lib::boot::boot_code_id; @@ -77,8 +78,6 @@ use crate::{ use crate::util::hash::{MerkleTree, Sha512Trunc256Sum}; use crate::util::secp256k1::MessageSignature; -use crate::neon_node::StacksNode; - use rand::Rng; use super::bitcoin_regtest::BitcoinCoreController; @@ -10492,8 +10491,7 @@ fn test_competing_miners_build_on_same_chain( confs.push(conf); } - let node_privkey_1 = - StacksNode::make_node_private_key_from_seed(&confs[0].node.local_peer_seed); + let node_privkey_1 = Secp256k1PrivateKey::from_seed(&confs[0].node.local_peer_seed); for i in 1..num_miners { let chain_id = confs[0].burnchain.chain_id; let peer_version = confs[0].burnchain.peer_version; From 136220efb88812ad92f59f79baeaac53b77e5e86 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Thu, 23 Mar 2023 15:38:12 -0500 Subject: [PATCH 4/4] ci: use GH workflow run rather than docker for unit-tests --- .../bitcoin-int-tests/Dockerfile.code-cov | 22 ------------------- .github/workflows/ci.yml | 14 +++++++----- 2 files changed, 9 insertions(+), 27 deletions(-) delete mode 100644 .github/actions/bitcoin-int-tests/Dockerfile.code-cov diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.code-cov b/.github/actions/bitcoin-int-tests/Dockerfile.code-cov deleted file mode 100644 index 209b804731..0000000000 --- a/.github/actions/bitcoin-int-tests/Dockerfile.code-cov +++ /dev/null @@ -1,22 +0,0 @@ -FROM rust:bullseye AS test - -WORKDIR /build - -ENV CARGO_MANIFEST_DIR="$(pwd)" - -RUN rustup component add llvm-tools-preview && \ - cargo install grcov - -ENV RUSTFLAGS="-Cinstrument-coverage" \ - LLVM_PROFILE_FILE="stacks-blockchain-%p-%m.profraw" - -COPY . . - -RUN cargo build --workspace && \ - cargo test --workspace - -# Generate coverage report and upload it to codecov -RUN grcov . --binary-path ./target/debug/ -s . -t lcov --branch --ignore-not-existing --ignore "/*" -o lcov.info - -FROM scratch AS export-stage -COPY --from=test /build/lcov.info / diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1a04a194f..fcdc1ebda1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,17 +40,21 @@ jobs: unit-tests: runs-on: ubuntu-latest steps: + - name: Add code coverage tools + run: | + rustup component add llvm-tools-preview + cargo install grcov - uses: actions/checkout@v2 - name: Run units tests (with coverage) env: - DOCKER_BUILDKIT: 1 - # Remove .dockerignore file so codecov has access to git info + RUSTFLAGS: -Cinstrument-coverage + LLVM_PROFILE_FILE: stacks-blockchain-%p-%m.profraw run: | - rm .dockerignore - docker build -o coverage-output -f ./.github/actions/bitcoin-int-tests/Dockerfile.code-cov . + cargo test --workspace + grcov . --binary-path ./target/debug/ -s . -t lcov --branch --ignore-not-existing --ignore "/*" -o lcov.info - uses: codecov/codecov-action@v2 with: - files: ./coverage-output/lcov.info + files: ./lcov.info name: unit_tests fail_ci_if_error: false