diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.atlas-test b/.github/actions/bitcoin-int-tests/Dockerfile.atlas-test index fc3403fbd2..15931c6491 100644 --- a/.github/actions/bitcoin-int-tests/Dockerfile.atlas-test +++ b/.github/actions/bitcoin-int-tests/Dockerfile.atlas-test @@ -6,10 +6,10 @@ COPY . . RUN cargo test --no-run --workspace -RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-0.20.0/bitcoin-0.20.0-x86_64-linux-gnu.tar.gz -RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz +RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-25.0/bitcoin-25.0-x86_64-linux-gnu.tar.gz +RUN cd / && tar -xvzf bitcoin-25.0-x86_64-linux-gnu.tar.gz -RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/ +RUN ln -s /bitcoin-25.0/bin/bitcoind /bin/ ENV BITCOIND_TEST 1 WORKDIR /src/testnet/stacks-node diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests b/.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests index 2fd43a589e..932cfacb6c 100644 --- a/.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests +++ b/.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests @@ -15,9 +15,9 @@ ENV RUSTFLAGS="-Cinstrument-coverage" \ RUN cargo test --no-run && \ cargo build -RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-0.20.0/bitcoin-0.20.0-x86_64-linux-gnu.tar.gz -RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz +RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-25.0/bitcoin-25.0-x86_64-linux-gnu.tar.gz +RUN cd / && tar -xvzf bitcoin-25.0-x86_64-linux-gnu.tar.gz -RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/ +RUN ln -s /bitcoin-25.0/bin/bitcoind /bin/ ENV BITCOIND_TEST 1 diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.large-genesis b/.github/actions/bitcoin-int-tests/Dockerfile.large-genesis index 1350a6ed86..44c3902318 100644 --- a/.github/actions/bitcoin-int-tests/Dockerfile.large-genesis +++ b/.github/actions/bitcoin-int-tests/Dockerfile.large-genesis @@ -4,10 +4,10 @@ WORKDIR /src COPY . . -RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-0.20.0/bitcoin-0.20.0-x86_64-linux-gnu.tar.gz -RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz +RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-25.0/bitcoin-25.0-x86_64-linux-gnu.tar.gz +RUN cd / && tar -xvzf bitcoin-25.0-x86_64-linux-gnu.tar.gz -RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/ +RUN ln -s /bitcoin-25.0/bin/bitcoind /bin/ RUN rustup component add llvm-tools-preview && \ cargo install grcov diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.net-tests b/.github/actions/bitcoin-int-tests/Dockerfile.net-tests index e7e3f8eaa1..55be5c0db5 100644 --- a/.github/actions/bitcoin-int-tests/Dockerfile.net-tests +++ b/.github/actions/bitcoin-int-tests/Dockerfile.net-tests @@ -4,11 +4,11 @@ WORKDIR /src COPY . . -RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-0.20.0/bitcoin-0.20.0-x86_64-linux-gnu.tar.gz -RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz +RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-25.0/bitcoin-25.0-x86_64-linux-gnu.tar.gz +RUN cd / && tar -xvzf bitcoin-25.0-x86_64-linux-gnu.tar.gz -RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/ -RUN ln -s /bitcoin-0.20.0/bin/bitcoin-cli /bin/ +RUN ln -s /bitcoin-25.0/bin/bitcoind /bin/ +RUN ln -s /bitcoin-25.0/bin/bitcoin-cli /bin/ RUN apt-get update RUN apt-get install -y jq screen net-tools ncat sqlite3 xxd openssl curl diff --git a/.github/workflows/bitcoin-tests.yml b/.github/workflows/bitcoin-tests.yml index 66adcf4f94..a6b088105d 100644 --- a/.github/workflows/bitcoin-tests.yml +++ b/.github/workflows/bitcoin-tests.yml @@ -138,6 +138,8 @@ jobs: - tests::neon_integrations::bad_microblock_pubkey - tests::epoch_24::fix_to_pox_contract - tests::epoch_24::verify_auto_unlock_behavior + - tests::stackerdb::test_stackerdb_load_store + - tests::stackerdb::test_stackerdb_event_observer steps: - name: Checkout the latest code id: git_checkout diff --git a/Cargo.lock b/Cargo.lock index 5bbc1c55e3..e1188ea0b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1482,6 +1482,19 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libstackerdb" +version = "0.0.1" +dependencies = [ + "clarity", + "secp256k1", + "serde", + "serde_derive", + "serde_stacker", + "sha2 0.10.6", + "stacks-common", +] + [[package]] name = "link-cplusplus" version = "1.0.8" @@ -2692,6 +2705,7 @@ dependencies = [ "integer-sqrt", "lazy_static", "libc", + "libstackerdb", "mio 0.6.23", "nix", "percent-encoding", diff --git a/Cargo.toml b/Cargo.toml index 2993d32fa3..377551278e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "pox-locking", "clarity", "stx-genesis", + "libstackerdb", "testnet/stacks-node"] # Use a bit more than default optimization for diff --git a/docs/event-dispatcher.md b/docs/event-dispatcher.md index 17ca78c63e..762814bf1c 100644 --- a/docs/event-dispatcher.md +++ b/docs/event-dispatcher.md @@ -458,3 +458,30 @@ Example: ] } ``` + +### `POST /stackerdb_chunks` + +This payload includes data related to a single mutation to a StackerDB replica +that this node subscribes to. The data will only get sent here if the +corresponding chunk has already been successfully stored. The data includes the +chunk ID, chunk version, smart contract ID, signature, and data hash; the +consumer may request the chunk data itself with a follow-up GET request on the node. + +This endpoint broadcasts events to `AnyEvent` observers, as well as to +`StackerDBChunks` observers. + +Example: + +```json +{ + "contract_id": "STVN97YYA10MY5F6KQJHKNYJNM24C4A1AT39WRW.hello-world", + "modified_slots": [ + { + "slot_id": 0, + "slot_version": 20, + "data_hash": "9d26b8009ab4693d3792791a4ea5e338822b5631af94e567a485b7265ba0f107", + "signature": "015814daf929d8700af344987681f44e913890a12e38550abe8e40f149ef5269f40f4008083a0f2e0ddf65dcd05ecfc151c7ff8a5308ad04c77c0e87b5aeadad31" + } + ] +} +``` diff --git a/libstackerdb/Cargo.toml b/libstackerdb/Cargo.toml new file mode 100644 index 0000000000..53cf128edc --- /dev/null +++ b/libstackerdb/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "libstackerdb" +version = "0.0.1" +authors = [ "Jude Nelson " ] +license = "GPLv3" +homepage = "https://github.com/blockstack/stacks-blockchain" +repository = "https://github.com/blockstack/stacks-blockchain" +description = "Client library for the StackerDB subsystem" +keywords = [ "stacks", "stx", "bitcoin", "crypto", "blockstack", "decentralized", "dapps", "blockchain" ] +readme = "README.md" +resolver = "2" +edition = "2021" + +[lib] +name = "libstackerdb" +path = "./src/libstackerdb.rs" + +[dependencies] +serde = "1" +serde_derive = "1" +serde_stacker = "0.1" +stacks-common = { path = "../stacks-common" } +clarity = { path = "../clarity" } + +[dependencies.secp256k1] +version = "0.24.3" +features = ["serde", "recovery"] + +[target.'cfg(all(any(target_arch = "x86_64", target_arch = "x86", target_arch = "aarch64"), not(target_env = "msvc")))'.dependencies] +sha2 = { version = "0.10", features = ["asm"] } + +[target.'cfg(any(not(any(target_arch = "x86_64", target_arch = "x86", target_arch = "aarch64")), target_env = "msvc"))'.dependencies] +sha2 = { version = "0.10" } diff --git a/libstackerdb/src/libstackerdb.rs b/libstackerdb/src/libstackerdb.rs new file mode 100644 index 0000000000..caba4b0fc8 --- /dev/null +++ b/libstackerdb/src/libstackerdb.rs @@ -0,0 +1,289 @@ +// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +extern crate clarity; +extern crate serde; +extern crate sha2; +extern crate stacks_common; + +use std::error; +use std::fmt; +use std::io::{Read, Write}; + +use sha2::{Digest, Sha512_256}; + +use stacks_common::types::chainstate::{StacksAddress, StacksPrivateKey, StacksPublicKey}; +use stacks_common::types::PrivateKey; +use stacks_common::util::hash::{hex_bytes, to_hex, Hash160, Sha512Trunc256Sum}; + +use stacks_common::codec::read_next; +use stacks_common::codec::read_next_at_most; +use stacks_common::codec::write_next; +use stacks_common::codec::Error as CodecError; +use stacks_common::codec::StacksMessageCodec; + +use stacks_common::util::secp256k1::MessageSignature; + +use serde::{Deserialize, Serialize}; + +use clarity::vm::types::QualifiedContractIdentifier; + +/// maximum chunk size (1 MB) +pub const STACKERDB_MAX_CHUNK_SIZE: u32 = 1024 * 1024; + +#[cfg(test)] +mod tests; + +#[derive(Debug)] +pub enum Error { + /// Error signing a message + SigningError(String), + /// Error verifying a message + VerifyingError(String), +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + Error::SigningError(ref s) => fmt::Display::fmt(s, f), + Error::VerifyingError(ref s) => fmt::Display::fmt(s, f), + } + } +} + +impl error::Error for Error { + fn cause(&self) -> Option<&dyn error::Error> { + match *self { + Error::SigningError(ref _s) => None, + Error::VerifyingError(ref _s) => None, + } + } +} + +/// Slot metadata from the DB. +/// This is derived state from a StackerDBChunkData message. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct SlotMetadata { + /// Slot identifier (unique for each DB instance) + pub slot_id: u32, + /// Slot version (a lamport clock) + pub slot_version: u32, + /// data hash + pub data_hash: Sha512Trunc256Sum, + /// signature over the above + pub signature: MessageSignature, +} + +/// Stacker DB chunk (i.e. as a reply to a chunk request) +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct StackerDBChunkData { + /// slot ID + pub slot_id: u32, + /// slot version (a lamport clock) + pub slot_version: u32, + /// signature from the stacker over (reward cycle consensus hash, slot id, slot version, chunk sha512/256) + pub sig: MessageSignature, + /// the chunk data + #[serde( + serialize_with = "stackerdb_chunk_hex_serialize", + deserialize_with = "stackerdb_chunk_hex_deserialize" + )] + pub data: Vec, +} + +/// StackerDB post chunk acknowledgement +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct StackerDBChunkAckData { + pub accepted: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub metadata: Option, +} + +impl SlotMetadata { + /// Make a new unsigned slot metadata + pub fn new_unsigned( + slot_id: u32, + slot_version: u32, + data_hash: Sha512Trunc256Sum, + ) -> SlotMetadata { + SlotMetadata { + slot_id, + slot_version, + data_hash, + signature: MessageSignature::empty(), + } + } + + /// Get the digest to sign that authenticates this chunk data and metadata + fn auth_digest(&self) -> Sha512Trunc256Sum { + let mut hasher = Sha512_256::new(); + hasher.update(&self.slot_id.to_be_bytes()); + hasher.update(&self.slot_version.to_be_bytes()); + hasher.update(&self.data_hash.0); + Sha512Trunc256Sum::from_hasher(hasher) + } + + /// Sign this slot metadata, committing to slot_id, slot_version, and + /// data_hash. Sets self.signature to the signature. + /// Fails if the underlying crypto library fails + pub fn sign(&mut self, privkey: &StacksPrivateKey) -> Result<(), Error> { + let auth_digest = self.auth_digest(); + let sig = privkey + .sign(&auth_digest.0) + .map_err(|se| Error::SigningError(se.to_string()))?; + + self.signature = sig; + Ok(()) + } + + /// Verify that a given principal signed this chunk metadata. + /// Note that the address version is ignored. + pub fn verify(&self, principal: &StacksAddress) -> Result { + let sigh = self.auth_digest(); + let pubk = StacksPublicKey::recover_to_pubkey(sigh.as_bytes(), &self.signature) + .map_err(|ve| Error::VerifyingError(ve.to_string()))?; + + let pubkh = Hash160::from_node_public_key(&pubk); + Ok(pubkh == principal.bytes) + } +} + +/// Helper methods for StackerDBChunkData messages +impl StackerDBChunkData { + /// Create a new StackerDBChunkData instance. + pub fn new(slot_id: u32, slot_version: u32, data: Vec) -> StackerDBChunkData { + StackerDBChunkData { + slot_id, + slot_version, + sig: MessageSignature::empty(), + data, + } + } + + /// Calculate the hash of the chunk bytes. This is the SHA512/256 hash of the data. + pub fn data_hash(&self) -> Sha512Trunc256Sum { + Sha512Trunc256Sum::from_data(&self.data) + } + + /// Create an owned SlotMetadata describing the metadata of this slot. + pub fn get_slot_metadata(&self) -> SlotMetadata { + SlotMetadata { + slot_id: self.slot_id, + slot_version: self.slot_version, + data_hash: self.data_hash(), + signature: self.sig.clone(), + } + } + + /// Sign this given chunk data message with the given private key. + /// Sets self.signature to the signature. + /// Fails if the underlying signing library fails. + pub fn sign(&mut self, privk: &StacksPrivateKey) -> Result<(), Error> { + let mut md = self.get_slot_metadata(); + md.sign(privk)?; + self.sig = md.signature; + Ok(()) + } + + /// Verify that this chunk was signed by the given + /// public key hash (`addr`). Only fails if the underlying signing library fails. + pub fn verify(&self, addr: &StacksAddress) -> Result { + let md = self.get_slot_metadata(); + md.verify(addr) + } +} + +impl StacksMessageCodec for StackerDBChunkData { + fn consensus_serialize(&self, fd: &mut W) -> Result<(), CodecError> { + write_next(fd, &self.slot_id)?; + write_next(fd, &self.slot_version)?; + write_next(fd, &self.sig)?; + write_next(fd, &self.data)?; + Ok(()) + } + + fn consensus_deserialize(fd: &mut R) -> Result { + let slot_id: u32 = read_next(fd)?; + let slot_version: u32 = read_next(fd)?; + let sig: MessageSignature = read_next(fd)?; + let data: Vec = read_next_at_most(fd, STACKERDB_MAX_CHUNK_SIZE.into())?; + Ok(StackerDBChunkData { + slot_id, + slot_version, + sig, + data, + }) + } +} + +fn stackerdb_chunk_hex_serialize( + chunk: &[u8], + s: S, +) -> Result { + let inst = to_hex(chunk); + s.serialize_str(inst.as_str()) +} + +fn stackerdb_chunk_hex_deserialize<'de, D: serde::Deserializer<'de>>( + d: D, +) -> Result, D::Error> { + let inst_str = String::deserialize(d)?; + hex_bytes(&inst_str).map_err(serde::de::Error::custom) +} + +/// Calculate the GET path for a stacker DB metadata listing +pub fn stackerdb_get_metadata_path(contract_id: QualifiedContractIdentifier) -> String { + format!( + "/v2/stackerdb/{}/{}", + &StacksAddress::from(contract_id.issuer), + &contract_id.name + ) +} + +/// Calculate the GET path for a stacker DB chunk +pub fn stackerdb_get_chunk_path( + contract_id: QualifiedContractIdentifier, + slot_id: u32, + slot_version: Option, +) -> String { + if let Some(version) = slot_version { + format!( + "/v2/stackerdb/{}/{}/{}/{}", + &StacksAddress::from(contract_id.issuer), + &contract_id.name, + slot_id, + version + ) + } else { + format!( + "/v2/stackerdb/{}/{}/{}", + &StacksAddress::from(contract_id.issuer), + &contract_id.name, + slot_id + ) + } +} + +/// Calculate POST path for a stacker DB chunk +pub fn stackerdb_post_chunk_path(contract_id: QualifiedContractIdentifier) -> String { + format!( + "/v2/stackerdb/{}/{}/chunks", + &StacksAddress::from(contract_id.issuer), + &contract_id.name + ) +} diff --git a/stackslib/src/net/stackerdb/tests/bits.rs b/libstackerdb/src/tests/mod.rs similarity index 65% rename from stackslib/src/net/stackerdb/tests/bits.rs rename to libstackerdb/src/tests/mod.rs index 4342162ecd..357e2c8e44 100644 --- a/stackslib/src/net/stackerdb/tests/bits.rs +++ b/libstackerdb/src/tests/mod.rs @@ -14,12 +14,6 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use std::fs; - -use crate::net::stackerdb::SlotMetadata; -use crate::net::StackerDBChunkData; - -use clarity::vm::ContractName; use stacks_common::types::chainstate::StacksAddress; use stacks_common::util::hash::Hash160; @@ -27,8 +21,11 @@ use stacks_common::util::hash::Sha512Trunc256Sum; use stacks_common::util::secp256k1::MessageSignature; use stacks_common::address::{AddressHashMode, C32_ADDRESS_VERSION_MAINNET_SINGLESIG}; -use stacks_common::types::chainstate::{ConsensusHash, StacksPrivateKey, StacksPublicKey}; -use stacks_common::types::PrivateKey; +use stacks_common::types::chainstate::{StacksPrivateKey, StacksPublicKey}; + +use crate::*; + +use clarity::vm::types::QualifiedContractIdentifier; #[test] fn test_stackerdb_slot_metadata_sign_verify() { @@ -76,3 +73,40 @@ fn test_stackerdb_slot_metadata_sign_verify() { bad_slot_metadata.data_hash = Sha512Trunc256Sum([0x20; 32]); assert!(!bad_slot_metadata.verify(&addr).unwrap()); } + +#[test] +fn test_stackerdb_paths() { + let pk = StacksPrivateKey::from_hex( + "4bbe4e7dc879afedf4bf258a7385cf78ccf3a68a77f9cfc624f433d009f812f901", + ) + .unwrap(); + let addr = StacksAddress::from_public_keys( + C32_ADDRESS_VERSION_MAINNET_SINGLESIG, + &AddressHashMode::SerializeP2PKH, + 1, + &vec![StacksPublicKey::from_private(&pk)], + ) + .unwrap(); + + let contract_id = QualifiedContractIdentifier::new(addr.into(), "hello-world".into()); + + assert_eq!( + stackerdb_get_metadata_path(contract_id.clone()), + "/v2/stackerdb/SP1Y0NECNCJ6YDVM7GQ594FF065NN3NT72FASBXB8/hello-world".to_string() + ); + + assert_eq!( + stackerdb_get_chunk_path(contract_id.clone(), 1, Some(2)), + "/v2/stackerdb/SP1Y0NECNCJ6YDVM7GQ594FF065NN3NT72FASBXB8/hello-world/1/2".to_string() + ); + + assert_eq!( + stackerdb_get_chunk_path(contract_id.clone(), 1, None), + "/v2/stackerdb/SP1Y0NECNCJ6YDVM7GQ594FF065NN3NT72FASBXB8/hello-world/1".to_string() + ); + + assert_eq!( + stackerdb_post_chunk_path(contract_id.clone()), + "/v2/stackerdb/SP1Y0NECNCJ6YDVM7GQ594FF065NN3NT72FASBXB8/hello-world/chunks".to_string() + ); +} diff --git a/stackslib/Cargo.toml b/stackslib/Cargo.toml index 5225a609b0..129ce3e8c0 100644 --- a/stackslib/Cargo.toml +++ b/stackslib/Cargo.toml @@ -53,6 +53,7 @@ libc = "0.2.82" clarity = { path = "../clarity" } stacks-common = { path = "../stacks-common" } pox-locking = { path = "../pox-locking" } +libstackerdb = { path = "../libstackerdb" } siphasher = "0.3.7" [target.'cfg(unix)'.dependencies] diff --git a/stackslib/src/chainstate/stacks/db/blocks.rs b/stackslib/src/chainstate/stacks/db/blocks.rs index bfcd34eecc..06cf1c3d91 100644 --- a/stackslib/src/chainstate/stacks/db/blocks.rs +++ b/stackslib/src/chainstate/stacks/db/blocks.rs @@ -60,6 +60,7 @@ use crate::core::mempool::MAXIMUM_MEMPOOL_TX_CHAINING; use crate::core::*; use crate::cost_estimates::EstimatorError; use crate::net::relay::Relayer; +use crate::net::stream::{BlockStreamData, HeaderStreamData, MicroblockStreamData, Streamer}; use crate::net::BlocksInvData; use crate::net::Error as net_error; use crate::net::ExtendedStacksHeader; @@ -448,309 +449,6 @@ impl StagingMicroblock { } } -impl MicroblockStreamData { - fn stream_count(&mut self, fd: &mut W, count: u64) -> Result { - let mut num_written = 0; - while self.num_items_ptr < self.num_items_buf.len() && num_written < count { - // stream length prefix - test_debug!( - "Length prefix: try to send {:?} (ptr={})", - &self.num_items_buf[self.num_items_ptr..], - self.num_items_ptr - ); - let num_sent = match fd.write(&self.num_items_buf[self.num_items_ptr..]) { - Ok(0) => { - // done (disconnected) - test_debug!("Length prefix: wrote 0 bytes",); - return Ok(num_written); - } - Ok(n) => { - self.num_items_ptr += n; - n as u64 - } - Err(e) => { - if e.kind() == io::ErrorKind::Interrupted { - // EINTR; try again - continue; - } else if e.kind() == io::ErrorKind::WouldBlock - || (cfg!(windows) && e.kind() == io::ErrorKind::TimedOut) - { - // blocked - return Ok(num_written); - } else { - return Err(Error::WriteError(e)); - } - } - }; - num_written += num_sent; - test_debug!( - "Length prefix: sent {} bytes ({} total)", - num_sent, - num_written - ); - } - Ok(num_written) - } -} - -impl StreamCursor { - pub fn new_block(index_block_hash: StacksBlockId) -> StreamCursor { - StreamCursor::Block(BlockStreamData { - index_block_hash: index_block_hash, - offset: 0, - total_bytes: 0, - }) - } - - pub fn new_microblock_confirmed( - chainstate: &StacksChainState, - tail_index_microblock_hash: StacksBlockId, - ) -> Result { - // look up parent - let mblock_info = StacksChainState::load_staging_microblock_info_indexed( - &chainstate.db(), - &tail_index_microblock_hash, - )? - .ok_or(Error::NoSuchBlockError)?; - - let parent_index_block_hash = StacksBlockHeader::make_index_block_hash( - &mblock_info.consensus_hash, - &mblock_info.anchored_block_hash, - ); - - // need to send out the consensus_serialize()'ed array length before sending microblocks. - // this is exactly what seq tells us, though. - let num_items_buf = ((mblock_info.sequence as u32) + 1).to_be_bytes(); - - Ok(StreamCursor::Microblocks(MicroblockStreamData { - index_block_hash: StacksBlockId([0u8; 32]), - rowid: None, - offset: 0, - total_bytes: 0, - microblock_hash: mblock_info.microblock_hash, - parent_index_block_hash: parent_index_block_hash, - seq: mblock_info.sequence, - unconfirmed: false, - num_items_buf: num_items_buf, - num_items_ptr: 0, - })) - } - - pub fn new_microblock_unconfirmed( - chainstate: &StacksChainState, - anchored_index_block_hash: StacksBlockId, - seq: u16, - ) -> Result { - let mblock_info = StacksChainState::load_next_descendant_microblock( - &chainstate.db(), - &anchored_index_block_hash, - seq, - )? - .ok_or(Error::NoSuchBlockError)?; - - Ok(StreamCursor::Microblocks(MicroblockStreamData { - index_block_hash: anchored_index_block_hash.clone(), - rowid: None, - offset: 0, - total_bytes: 0, - microblock_hash: mblock_info.block_hash(), - parent_index_block_hash: anchored_index_block_hash, - seq: seq, - unconfirmed: true, - num_items_buf: [0u8; 4], - num_items_ptr: 4, // stops us from trying to send a length prefix - })) - } - - pub fn new_headers( - chainstate: &StacksChainState, - tip: &StacksBlockId, - num_headers_requested: u32, - ) -> Result { - let header_info = StacksChainState::load_staging_block_info(chainstate.db(), tip)? - .ok_or(Error::NoSuchBlockError)?; - - let num_headers = if header_info.height < (num_headers_requested as u64) { - header_info.height as u32 - } else { - num_headers_requested - }; - - test_debug!("Request for {} headers from {}", num_headers, tip); - - Ok(StreamCursor::Headers(HeaderStreamData { - index_block_hash: tip.clone(), - offset: 0, - total_bytes: 0, - num_headers: num_headers, - header_bytes: None, - end_of_stream: false, - corked: false, - })) - } - - pub fn new_tx_stream( - tx_query: MemPoolSyncData, - max_txs: u64, - height: u64, - page_id_opt: Option, - ) -> StreamCursor { - let last_randomized_txid = page_id_opt.unwrap_or_else(|| { - let random_bytes = rand::thread_rng().gen::<[u8; 32]>(); - Txid(random_bytes) - }); - - StreamCursor::MempoolTxs(TxStreamData { - tx_query, - last_randomized_txid: last_randomized_txid, - tx_buf: vec![], - tx_buf_ptr: 0, - num_txs: 0, - max_txs: max_txs, - height: height, - corked: false, - }) - } - - fn stream_one_byte(fd: &mut W, b: u8) -> Result { - loop { - match fd.write(&[b]) { - Ok(0) => { - // done (disconnected) - return Ok(0); - } - Ok(n) => { - return Ok(n as u64); - } - Err(e) => { - if e.kind() == io::ErrorKind::Interrupted { - // EINTR; try again - continue; - } else if e.kind() == io::ErrorKind::WouldBlock - || (cfg!(windows) && e.kind() == io::ErrorKind::TimedOut) - { - // blocked - return Ok(0); - } else { - return Err(Error::WriteError(e)); - } - } - } - } - } - - pub fn get_offset(&self) -> u64 { - match self { - StreamCursor::Block(ref stream) => stream.offset(), - StreamCursor::Microblocks(ref stream) => stream.offset(), - StreamCursor::Headers(ref stream) => stream.offset(), - // no-op for mempool txs - StreamCursor::MempoolTxs(..) => 0, - } - } - - pub fn add_more_bytes(&mut self, nw: u64) { - match self { - StreamCursor::Block(ref mut stream) => stream.add_bytes(nw), - StreamCursor::Microblocks(ref mut stream) => stream.add_bytes(nw), - StreamCursor::Headers(ref mut stream) => stream.add_bytes(nw), - // no-op fo mempool txs - StreamCursor::MempoolTxs(..) => (), - } - } - - pub fn stream_to( - &mut self, - mempool: &MemPoolDB, - chainstate: &mut StacksChainState, - fd: &mut W, - count: u64, - ) -> Result { - match self { - StreamCursor::Microblocks(ref mut stream) => { - let mut num_written = 0; - if !stream.unconfirmed { - // Confirmed microblocks are represented as a consensus-encoded vector of - // microblocks, in reverse sequence order. - // Write 4-byte length prefix first - num_written += stream.stream_count(fd, count)?; - StacksChainState::stream_microblocks_confirmed(&chainstate, fd, stream, count) - .and_then(|bytes_sent| Ok(bytes_sent + num_written)) - } else { - StacksChainState::stream_microblocks_unconfirmed(&chainstate, fd, stream, count) - .and_then(|bytes_sent| Ok(bytes_sent + num_written)) - } - } - StreamCursor::MempoolTxs(ref mut tx_stream) => mempool.stream_txs(fd, tx_stream, count), - StreamCursor::Headers(ref mut stream) => { - let mut num_written = 0; - if stream.total_bytes == 0 { - test_debug!("Opening header stream"); - let byte_written = StreamCursor::stream_one_byte(fd, '[' as u8)?; - num_written += byte_written; - stream.total_bytes += byte_written; - } - if stream.total_bytes > 0 { - let mut sent = chainstate.stream_headers(fd, stream, count)?; - - if stream.end_of_stream && !stream.corked { - // end of stream; cork it - test_debug!("Corking header stream"); - let byte_written = StreamCursor::stream_one_byte(fd, ']' as u8)?; - if byte_written > 0 { - sent += byte_written; - stream.total_bytes += byte_written; - stream.corked = true; - } - } - num_written += sent; - } - Ok(num_written) - } - StreamCursor::Block(ref mut stream) => chainstate.stream_block(fd, stream, count), - } - } -} - -impl Streamer for StreamCursor { - fn offset(&self) -> u64 { - self.get_offset() - } - fn add_bytes(&mut self, nw: u64) { - self.add_more_bytes(nw) - } -} - -impl Streamer for HeaderStreamData { - fn offset(&self) -> u64 { - self.offset - } - fn add_bytes(&mut self, nw: u64) { - self.offset += nw; - self.total_bytes += nw; - } -} - -impl Streamer for BlockStreamData { - fn offset(&self) -> u64 { - self.offset - } - fn add_bytes(&mut self, nw: u64) { - self.offset += nw; - self.total_bytes += nw; - } -} - -impl Streamer for MicroblockStreamData { - fn offset(&self) -> u64 { - self.offset - } - fn add_bytes(&mut self, nw: u64) { - self.offset += nw; - self.total_bytes += nw; - } -} - impl StacksChainState { fn get_index_block_pathbuf(blocks_dir: &str, index_block_hash: &StacksBlockId) -> PathBuf { let block_hash_bytes = index_block_hash.as_bytes(); @@ -3291,7 +2989,14 @@ impl StacksChainState { /// Stream a single header's data from disk /// If this method returns 0, it's because we're EOF on the header and should begin the next. - fn stream_one_header( + /// + /// The data streamed to `fd` is meant to be part of a JSON array. The header data will be + /// encoded as JSON, and a `,` will be written after it if there are more headers to follow. + /// The caller is responsible for writing `[` before writing headers, and writing `]` after all + /// headers have been written. + /// + /// Returns the number of bytes written + pub fn stream_one_header( blocks_conn: &DBConn, block_path: &str, fd: &mut W, @@ -3374,9 +3079,10 @@ impl StacksChainState { } /// Stream multiple headers from disk, moving in reverse order from the chain tip back. + /// The format will be a JSON array. /// Returns total number of bytes written (will be equal to the number of bytes read). /// Returns 0 if we run out of headers - fn stream_headers( + pub fn stream_headers( &self, fd: &mut W, stream: &mut HeaderStreamData, @@ -3472,7 +3178,7 @@ impl StacksChainState { /// Stream a single microblock's data from the staging database. /// If this method returns 0, it's because we're EOF on the blob. - fn stream_one_microblock( + pub fn stream_one_microblock( blocks_conn: &DBConn, fd: &mut W, stream: &mut MicroblockStreamData, @@ -3530,7 +3236,7 @@ impl StacksChainState { /// Stream multiple microblocks from staging, moving in reverse order from the stream tail to the stream head. /// Returns total number of bytes written (will be equal to the number of bytes read). /// Returns 0 if we run out of microblocks in the staging db - fn stream_microblocks_confirmed( + pub fn stream_microblocks_confirmed( chainstate: &StacksChainState, fd: &mut W, stream: &mut MicroblockStreamData, @@ -3597,7 +3303,7 @@ impl StacksChainState { } /// Stream block data from the chunk store. - fn stream_data_from_chunk_store( + pub fn stream_data_from_chunk_store( blocks_path: &str, fd: &mut W, stream: &mut BlockStreamData, @@ -10452,128 +10158,7 @@ pub mod test { } } - fn stream_one_header_to_vec( - blocks_conn: &DBConn, - blocks_path: &str, - stream: &mut StreamCursor, - count: u64, - ) -> Result, chainstate_error> { - if let StreamCursor::Headers(ref mut stream) = stream { - let mut bytes = vec![]; - StacksChainState::stream_one_header(blocks_conn, blocks_path, &mut bytes, stream, count) - .map(|nr| { - assert_eq!(bytes.len(), nr as usize); - - // truncate trailing ',' if it exists - let len = bytes.len(); - if len > 0 { - if bytes[len - 1] == ',' as u8 { - let _ = bytes.pop(); - } - } - bytes - }) - } else { - panic!("not a header stream"); - } - } - - fn stream_one_staging_microblock_to_vec( - blocks_conn: &DBConn, - stream: &mut StreamCursor, - count: u64, - ) -> Result, chainstate_error> { - if let StreamCursor::Microblocks(ref mut stream) = stream { - let mut bytes = vec![]; - StacksChainState::stream_one_microblock(blocks_conn, &mut bytes, stream, count).map( - |nr| { - assert_eq!(bytes.len(), nr as usize); - bytes - }, - ) - } else { - panic!("not a microblock stream"); - } - } - - fn stream_chunk_to_vec( - blocks_path: &str, - stream: &mut StreamCursor, - count: u64, - ) -> Result, chainstate_error> { - if let StreamCursor::Block(ref mut stream) = stream { - let mut bytes = vec![]; - StacksChainState::stream_data_from_chunk_store(blocks_path, &mut bytes, stream, count) - .map(|nr| { - assert_eq!(bytes.len(), nr as usize); - bytes - }) - } else { - panic!("not a block stream"); - } - } - - fn stream_headers_to_vec( - chainstate: &mut StacksChainState, - stream: &mut StreamCursor, - count: u64, - ) -> Result, chainstate_error> { - let mempool = MemPoolDB::open_test( - chainstate.mainnet, - chainstate.chain_id, - &chainstate.root_path, - ) - .unwrap(); - let mut bytes = vec![]; - stream - .stream_to(&mempool, chainstate, &mut bytes, count) - .map(|nr| { - assert_eq!(bytes.len(), nr as usize); - bytes - }) - } - - fn stream_unconfirmed_microblocks_to_vec( - chainstate: &mut StacksChainState, - stream: &mut StreamCursor, - count: u64, - ) -> Result, chainstate_error> { - let mempool = MemPoolDB::open_test( - chainstate.mainnet, - chainstate.chain_id, - &chainstate.root_path, - ) - .unwrap(); - let mut bytes = vec![]; - stream - .stream_to(&mempool, chainstate, &mut bytes, count) - .map(|nr| { - assert_eq!(bytes.len(), nr as usize); - bytes - }) - } - - fn stream_confirmed_microblocks_to_vec( - chainstate: &mut StacksChainState, - stream: &mut StreamCursor, - count: u64, - ) -> Result, chainstate_error> { - let mempool = MemPoolDB::open_test( - chainstate.mainnet, - chainstate.chain_id, - &chainstate.root_path, - ) - .unwrap(); - let mut bytes = vec![]; - stream - .stream_to(&mempool, chainstate, &mut bytes, count) - .map(|nr| { - assert_eq!(bytes.len(), nr as usize); - bytes - }) - } - - fn decode_microblock_stream(mblock_bytes: &Vec) -> Vec { + pub fn decode_microblock_stream(mblock_bytes: &Vec) -> Vec { // decode stream let mut mblock_ptr = mblock_bytes.as_slice(); let mut mblocks = vec![]; @@ -10601,569 +10186,6 @@ pub mod test { mblocks } - #[test] - fn stacks_db_stream_blocks() { - let mut chainstate = instantiate_chainstate(false, 0x80000000, function_name!()); - let privk = StacksPrivateKey::from_hex( - "eb05c83546fdd2c79f10f5ad5434a90dd28f7e3acb7c092157aa1bc3656b012c01", - ) - .unwrap(); - - let block = make_16k_block(&privk); - - let consensus_hash = ConsensusHash([2u8; 20]); - let parent_consensus_hash = ConsensusHash([1u8; 20]); - let index_block_header = - StacksBlockHeader::make_index_block_hash(&consensus_hash, &block.block_hash()); - - // can't stream a non-existant block - let mut stream = StreamCursor::new_block(index_block_header.clone()); - assert!(stream_chunk_to_vec(&chainstate.blocks_path, &mut stream, 123).is_err()); - - // stream unmodified - let stream_2 = StreamCursor::new_block(index_block_header.clone()); - assert_eq!(stream, stream_2); - - // store block to staging - store_staging_block( - &mut chainstate, - &consensus_hash, - &block, - &parent_consensus_hash, - 1, - 2, - ); - - // stream it back - let mut all_block_bytes = vec![]; - loop { - let mut next_bytes = - stream_chunk_to_vec(&chainstate.blocks_path, &mut stream, 16).unwrap(); - if next_bytes.len() == 0 { - break; - } - test_debug!( - "Got {} more bytes from staging; add to {} total", - next_bytes.len(), - all_block_bytes.len() - ); - all_block_bytes.append(&mut next_bytes); - } - - // should decode back into the block - let staging_block = StacksBlock::consensus_deserialize(&mut &all_block_bytes[..]).unwrap(); - assert_eq!(staging_block, block); - - // accept it - set_block_processed(&mut chainstate, &consensus_hash, &block.block_hash(), true); - - // can still stream it - let mut stream = StreamCursor::new_block(index_block_header.clone()); - - // stream from chunk store - let mut all_block_bytes = vec![]; - loop { - let mut next_bytes = - stream_chunk_to_vec(&chainstate.blocks_path, &mut stream, 16).unwrap(); - if next_bytes.len() == 0 { - break; - } - test_debug!( - "Got {} more bytes from chunkstore; add to {} total", - next_bytes.len(), - all_block_bytes.len() - ); - all_block_bytes.append(&mut next_bytes); - } - - // should decode back into the block - let staging_block = StacksBlock::consensus_deserialize(&mut &all_block_bytes[..]).unwrap(); - assert_eq!(staging_block, block); - } - - #[test] - fn stacks_db_stream_headers() { - let mut chainstate = instantiate_chainstate(false, 0x80000000, function_name!()); - let privk = StacksPrivateKey::from_hex( - "eb05c83546fdd2c79f10f5ad5434a90dd28f7e3acb7c092157aa1bc3656b012c01", - ) - .unwrap(); - - let mut blocks: Vec = vec![]; - let mut blocks_index_hashes: Vec = vec![]; - - // make a linear stream - for i in 0..32 { - let mut block = make_empty_coinbase_block(&privk); - - if i == 0 { - block.header.total_work.work = 1; - block.header.total_work.burn = 1; - } - if i > 0 { - block.header.parent_block = blocks.get(i - 1).unwrap().block_hash(); - block.header.total_work.work = - blocks.get(i - 1).unwrap().header.total_work.work + 1; - block.header.total_work.burn = - blocks.get(i - 1).unwrap().header.total_work.burn + 1; - } - - let consensus_hash = ConsensusHash([((i + 1) as u8); 20]); - let parent_consensus_hash = ConsensusHash([(i as u8); 20]); - - store_staging_block( - &mut chainstate, - &consensus_hash, - &block, - &parent_consensus_hash, - i as u64, - i as u64, - ); - - blocks_index_hashes.push(StacksBlockHeader::make_index_block_hash( - &consensus_hash, - &block.block_hash(), - )); - blocks.push(block); - } - - let mut blocks_fork = blocks[0..16].to_vec(); - let mut blocks_fork_index_hashes = blocks_index_hashes[0..16].to_vec(); - - // make a stream that branches off - for i in 16..32 { - let mut block = make_empty_coinbase_block(&privk); - - if i == 16 { - block.header.parent_block = blocks.get(i - 1).unwrap().block_hash(); - block.header.total_work.work = - blocks.get(i - 1).unwrap().header.total_work.work + 1; - block.header.total_work.burn = - blocks.get(i - 1).unwrap().header.total_work.burn + 2; - } else { - block.header.parent_block = blocks_fork.get(i - 1).unwrap().block_hash(); - block.header.total_work.work = - blocks_fork.get(i - 1).unwrap().header.total_work.work + 1; - block.header.total_work.burn = - blocks_fork.get(i - 1).unwrap().header.total_work.burn + 2; - } - - let consensus_hash = ConsensusHash([((i + 1) as u8) | 0x80; 20]); - let parent_consensus_hash = if i == 16 { - ConsensusHash([(i as u8); 20]) - } else { - ConsensusHash([(i as u8) | 0x80; 20]) - }; - - store_staging_block( - &mut chainstate, - &consensus_hash, - &block, - &parent_consensus_hash, - i as u64, - i as u64, - ); - - blocks_fork_index_hashes.push(StacksBlockHeader::make_index_block_hash( - &consensus_hash, - &block.block_hash(), - )); - blocks_fork.push(block); - } - - // can't stream a non-existant header - assert!(StreamCursor::new_headers(&chainstate, &StacksBlockId([0x11; 32]), 1).is_err()); - - // stream back individual headers - for i in 0..blocks.len() { - let mut stream = - StreamCursor::new_headers(&chainstate, &blocks_index_hashes[i], 1).unwrap(); - let mut next_header_bytes = vec![]; - loop { - // torture test - let mut next_bytes = stream_one_header_to_vec( - &chainstate.db(), - &chainstate.blocks_path, - &mut stream, - 25, - ) - .unwrap(); - if next_bytes.len() == 0 { - break; - } - next_header_bytes.append(&mut next_bytes); - } - test_debug!("Got {} total bytes", next_header_bytes.len()); - let header: ExtendedStacksHeader = - serde_json::from_reader(&mut &next_header_bytes[..]).unwrap(); - - assert_eq!(header.consensus_hash, ConsensusHash([(i + 1) as u8; 20])); - assert_eq!(header.header, blocks[i].header); - - if i > 0 { - assert_eq!(header.parent_block_id, blocks_index_hashes[i - 1]); - } - } - - // stream back a run of headers - let block_expected_headers: Vec = - blocks.iter().rev().map(|blk| blk.header.clone()).collect(); - - let block_expected_index_hashes: Vec = blocks_index_hashes - .iter() - .rev() - .map(|idx| idx.clone()) - .collect(); - - let block_fork_expected_headers: Vec = blocks_fork - .iter() - .rev() - .map(|blk| blk.header.clone()) - .collect(); - - let block_fork_expected_index_hashes: Vec = blocks_fork_index_hashes - .iter() - .rev() - .map(|idx| idx.clone()) - .collect(); - - // get them all -- ask for more than there is - let mut stream = - StreamCursor::new_headers(&chainstate, blocks_index_hashes.last().unwrap(), 4096) - .unwrap(); - let header_bytes = - stream_headers_to_vec(&mut chainstate, &mut stream, 1024 * 1024).unwrap(); - - eprintln!( - "headers: {}", - String::from_utf8(header_bytes.clone()).unwrap() - ); - let headers: Vec = - serde_json::from_reader(&mut &header_bytes[..]).unwrap(); - - assert_eq!(headers.len(), block_expected_headers.len()); - for ((i, h), eh) in headers - .iter() - .enumerate() - .zip(block_expected_headers.iter()) - { - assert_eq!(h.header, *eh); - assert_eq!(h.consensus_hash, ConsensusHash([(32 - i) as u8; 20])); - if i + 1 < block_expected_index_hashes.len() { - assert_eq!(h.parent_block_id, block_expected_index_hashes[i + 1]); - } - } - - let mut stream = - StreamCursor::new_headers(&chainstate, blocks_fork_index_hashes.last().unwrap(), 4096) - .unwrap(); - let header_bytes = - stream_headers_to_vec(&mut chainstate, &mut stream, 1024 * 1024).unwrap(); - let fork_headers: Vec = - serde_json::from_reader(&mut &header_bytes[..]).unwrap(); - - assert_eq!(fork_headers.len(), block_fork_expected_headers.len()); - for ((i, h), eh) in fork_headers - .iter() - .enumerate() - .zip(block_fork_expected_headers.iter()) - { - let consensus_hash = if i >= 16 { - ConsensusHash([((32 - i) as u8); 20]) - } else { - ConsensusHash([((32 - i) as u8) | 0x80; 20]) - }; - - assert_eq!(h.header, *eh); - assert_eq!(h.consensus_hash, consensus_hash); - if i + 1 < block_fork_expected_index_hashes.len() { - assert_eq!(h.parent_block_id, block_fork_expected_index_hashes[i + 1]); - } - } - - assert_eq!(fork_headers[16..32], headers[16..32]); - - // ask for only a few - let mut stream = - StreamCursor::new_headers(&chainstate, blocks_index_hashes.last().unwrap(), 10) - .unwrap(); - let mut header_bytes = vec![]; - loop { - // torture test - let mut next_bytes = stream_headers_to_vec(&mut chainstate, &mut stream, 17).unwrap(); - if next_bytes.len() == 0 { - break; - } - header_bytes.append(&mut next_bytes); - } - - eprintln!( - "header bytes: {}", - String::from_utf8(header_bytes.clone()).unwrap() - ); - - let headers: Vec = - serde_json::from_reader(&mut &header_bytes[..]).unwrap(); - - assert_eq!(headers.len(), 10); - for (i, hdr) in headers.iter().enumerate() { - assert_eq!(hdr.header, block_expected_headers[i]); - assert_eq!(hdr.parent_block_id, block_expected_index_hashes[i + 1]); - } - - // ask for only a few - let mut stream = - StreamCursor::new_headers(&chainstate, blocks_fork_index_hashes.last().unwrap(), 10) - .unwrap(); - let mut header_bytes = vec![]; - loop { - // torture test - let mut next_bytes = stream_headers_to_vec(&mut chainstate, &mut stream, 17).unwrap(); - if next_bytes.len() == 0 { - break; - } - header_bytes.append(&mut next_bytes); - } - let headers: Vec = - serde_json::from_reader(&mut &header_bytes[..]).unwrap(); - - assert_eq!(headers.len(), 10); - for (i, hdr) in headers.iter().enumerate() { - assert_eq!(hdr.header, block_fork_expected_headers[i]); - assert_eq!(hdr.parent_block_id, block_fork_expected_index_hashes[i + 1]); - } - } - - #[test] - fn stacks_db_stream_staging_microblocks() { - let mut chainstate = instantiate_chainstate(false, 0x80000000, function_name!()); - let privk = StacksPrivateKey::from_hex( - "eb05c83546fdd2c79f10f5ad5434a90dd28f7e3acb7c092157aa1bc3656b012c01", - ) - .unwrap(); - - let block = make_empty_coinbase_block(&privk); - let mut mblocks = make_sample_microblock_stream(&privk, &block.block_hash()); - mblocks.truncate(15); - - let consensus_hash = ConsensusHash([2u8; 20]); - let parent_consensus_hash = ConsensusHash([1u8; 20]); - let index_block_header = - StacksBlockHeader::make_index_block_hash(&consensus_hash, &block.block_hash()); - - // can't stream a non-existant microblock - if let Err(super::Error::NoSuchBlockError) = - StreamCursor::new_microblock_confirmed(&chainstate, index_block_header.clone()) - { - } else { - panic!("Opened nonexistant microblock"); - } - - if let Err(super::Error::NoSuchBlockError) = - StreamCursor::new_microblock_unconfirmed(&chainstate, index_block_header.clone(), 0) - { - } else { - panic!("Opened nonexistant microblock"); - } - - // store microblocks to staging and stream them back - for (i, mblock) in mblocks.iter().enumerate() { - store_staging_microblock( - &mut chainstate, - &consensus_hash, - &block.block_hash(), - mblock, - ); - - // read back all the data we have so far, block-by-block - let mut staging_mblocks = vec![]; - for j in 0..(i + 1) { - let mut next_mblock_bytes = vec![]; - let mut stream = StreamCursor::new_microblock_unconfirmed( - &chainstate, - index_block_header.clone(), - j as u16, - ) - .unwrap(); - loop { - let mut next_bytes = - stream_one_staging_microblock_to_vec(&chainstate.db(), &mut stream, 4096) - .unwrap(); - if next_bytes.len() == 0 { - break; - } - test_debug!( - "Got {} more bytes from staging; add to {} total", - next_bytes.len(), - next_mblock_bytes.len() - ); - next_mblock_bytes.append(&mut next_bytes); - } - test_debug!("Got {} total bytes", next_mblock_bytes.len()); - - // should deserialize to a microblock - let staging_mblock = - StacksMicroblock::consensus_deserialize(&mut &next_mblock_bytes[..]).unwrap(); - staging_mblocks.push(staging_mblock); - } - - assert_eq!(staging_mblocks.len(), mblocks[0..(i + 1)].len()); - for j in 0..(i + 1) { - test_debug!("check {}", j); - assert_eq!(staging_mblocks[j], mblocks[j]) - } - - // can also read partial stream in one shot, from any seq - for k in 0..(i + 1) { - test_debug!("start at seq {}", k); - let mut staging_mblock_bytes = vec![]; - let mut stream = StreamCursor::new_microblock_unconfirmed( - &chainstate, - index_block_header.clone(), - k as u16, - ) - .unwrap(); - loop { - let mut next_bytes = - stream_unconfirmed_microblocks_to_vec(&mut chainstate, &mut stream, 4096) - .unwrap(); - if next_bytes.len() == 0 { - break; - } - test_debug!( - "Got {} more bytes from staging; add to {} total", - next_bytes.len(), - staging_mblock_bytes.len() - ); - staging_mblock_bytes.append(&mut next_bytes); - } - - test_debug!("Got {} total bytes", staging_mblock_bytes.len()); - - // decode stream - let staging_mblocks = decode_microblock_stream(&staging_mblock_bytes); - - assert_eq!(staging_mblocks.len(), mblocks[k..(i + 1)].len()); - for j in 0..staging_mblocks.len() { - test_debug!("check {}", j); - assert_eq!(staging_mblocks[j], mblocks[k + j]) - } - } - } - } - - #[test] - fn stacks_db_stream_confirmed_microblocks() { - let mut chainstate = instantiate_chainstate(false, 0x80000000, function_name!()); - let privk = StacksPrivateKey::from_hex( - "eb05c83546fdd2c79f10f5ad5434a90dd28f7e3acb7c092157aa1bc3656b012c01", - ) - .unwrap(); - - let block = make_empty_coinbase_block(&privk); - let mut mblocks = make_sample_microblock_stream(&privk, &block.block_hash()); - mblocks.truncate(5); - - let mut child_block = make_empty_coinbase_block(&privk); - child_block.header.parent_block = block.block_hash(); - child_block.header.parent_microblock = mblocks.last().as_ref().unwrap().block_hash(); - child_block.header.parent_microblock_sequence = - mblocks.last().as_ref().unwrap().header.sequence; - - let consensus_hash = ConsensusHash([2u8; 20]); - let parent_consensus_hash = ConsensusHash([1u8; 20]); - let child_consensus_hash = ConsensusHash([3u8; 20]); - - let index_block_header = - StacksBlockHeader::make_index_block_hash(&consensus_hash, &block.block_hash()); - - // store microblocks to staging - for (i, mblock) in mblocks.iter().enumerate() { - store_staging_microblock( - &mut chainstate, - &consensus_hash, - &block.block_hash(), - mblock, - ); - } - - // store block to staging - store_staging_block( - &mut chainstate, - &consensus_hash, - &block, - &parent_consensus_hash, - 1, - 2, - ); - - // store child block to staging - store_staging_block( - &mut chainstate, - &child_consensus_hash, - &child_block, - &consensus_hash, - 1, - 2, - ); - - // accept it - set_block_processed(&mut chainstate, &consensus_hash, &block.block_hash(), true); - set_block_processed( - &mut chainstate, - &child_consensus_hash, - &child_block.block_hash(), - true, - ); - - for i in 0..mblocks.len() { - // set different parts of this stream as confirmed - set_microblocks_processed( - &mut chainstate, - &child_consensus_hash, - &child_block.block_hash(), - &mblocks[i].block_hash(), - ); - - // verify that we can stream everything - let microblock_index_header = - StacksBlockHeader::make_index_block_hash(&consensus_hash, &mblocks[i].block_hash()); - let mut stream = StreamCursor::new_microblock_confirmed( - &chainstate, - microblock_index_header.clone(), - ) - .unwrap(); - - let mut confirmed_mblock_bytes = vec![]; - loop { - let mut next_bytes = - stream_confirmed_microblocks_to_vec(&mut chainstate, &mut stream, 16).unwrap(); - if next_bytes.len() == 0 { - break; - } - test_debug!( - "Got {} more bytes from staging; add to {} total", - next_bytes.len(), - confirmed_mblock_bytes.len() - ); - confirmed_mblock_bytes.append(&mut next_bytes); - } - - // decode stream (should be length-prefixed) - let mut confirmed_mblocks = - Vec::::consensus_deserialize(&mut &confirmed_mblock_bytes[..]) - .unwrap(); - - confirmed_mblocks.reverse(); - - assert_eq!(confirmed_mblocks.len(), mblocks[0..(i + 1)].len()); - for j in 0..(i + 1) { - test_debug!("check {}", j); - assert_eq!(confirmed_mblocks[j], mblocks[j]) - } - } - } - #[test] fn stacks_db_get_blocks_inventory() { let mut chainstate = instantiate_chainstate(false, 0x80000000, function_name!()); diff --git a/stackslib/src/chainstate/stacks/db/mod.rs b/stackslib/src/chainstate/stacks/db/mod.rs index cf5a8ffb5f..22ee6fc560 100644 --- a/stackslib/src/chainstate/stacks/db/mod.rs +++ b/stackslib/src/chainstate/stacks/db/mod.rs @@ -524,92 +524,6 @@ impl<'a> DerefMut for ChainstateTx<'a> { } } -/// Interface for streaming data -pub trait Streamer { - fn offset(&self) -> u64; - fn add_bytes(&mut self, nw: u64); -} - -/// Opaque structure for streaming block, microblock, and header data from disk -#[derive(Debug, PartialEq, Clone)] -pub enum StreamCursor { - Block(BlockStreamData), - Microblocks(MicroblockStreamData), - Headers(HeaderStreamData), - MempoolTxs(TxStreamData), -} - -#[derive(Debug, PartialEq, Clone)] -pub struct BlockStreamData { - /// index block hash of the block to download - index_block_hash: StacksBlockId, - /// offset into whatever is being read (the blob, or the file in the chunk store) - offset: u64, - /// total number of bytes read. - total_bytes: u64, -} - -#[derive(Debug, PartialEq, Clone)] -pub struct MicroblockStreamData { - /// index block hash of the block to download - index_block_hash: StacksBlockId, - /// microblock blob row id - rowid: Option, - /// offset into whatever is being read (the blob, or the file in the chunk store) - offset: u64, - /// total number of bytes read. - total_bytes: u64, - - /// length prefix - num_items_buf: [u8; 4], - num_items_ptr: usize, - - /// microblock pointer - microblock_hash: BlockHeaderHash, - parent_index_block_hash: StacksBlockId, - - /// unconfirmed state - seq: u16, - unconfirmed: bool, -} - -#[derive(Debug, PartialEq, Clone)] -pub struct HeaderStreamData { - /// index block hash of the block to download - index_block_hash: StacksBlockId, - /// offset into whatever is being read (the blob, or the file in the chunk store) - offset: u64, - /// total number of bytes read. - total_bytes: u64, - /// number of headers requested - num_headers: u32, - - /// header buffer data - header_bytes: Option>, - end_of_stream: bool, - corked: bool, -} - -#[derive(Debug, PartialEq, Clone)] -pub struct TxStreamData { - /// Mempool sync data requested - pub tx_query: MemPoolSyncData, - /// last txid loaded - pub last_randomized_txid: Txid, - /// serialized transaction buffer that's being sent - pub tx_buf: Vec, - pub tx_buf_ptr: usize, - /// number of transactions visited in the DB so far - pub num_txs: u64, - /// maximum we can visit in the query - pub max_txs: u64, - /// height of the chain at time of query - pub height: u64, - /// Are we done sending transactions, and are now in the process of sending the trailing page - /// ID? - pub corked: bool, -} - pub const CHAINSTATE_VERSION: &'static str = "3"; const CHAINSTATE_INITIAL_SCHEMA: &'static [&'static str] = &[ diff --git a/stackslib/src/core/mempool.rs b/stackslib/src/core/mempool.rs index 27e8417e0e..c6b155f1a4 100644 --- a/stackslib/src/core/mempool.rs +++ b/stackslib/src/core/mempool.rs @@ -41,8 +41,8 @@ use crate::burnchains::Txid; use crate::chainstate::burn::db::sortdb::SortitionDB; use crate::chainstate::burn::ConsensusHash; use crate::chainstate::stacks::{ - db::blocks::MemPoolRejection, db::ClarityTx, db::StacksChainState, db::TxStreamData, - index::Error as MarfError, Error as ChainstateError, StacksTransaction, + db::blocks::MemPoolRejection, db::ClarityTx, db::StacksChainState, index::Error as MarfError, + Error as ChainstateError, StacksTransaction, }; use crate::chainstate::stacks::{StacksMicroblock, TransactionPayload}; use crate::core::ExecutionCost; @@ -50,6 +50,7 @@ use crate::core::StacksEpochId; use crate::core::FIRST_BURNCHAIN_CONSENSUS_HASH; use crate::core::FIRST_STACKS_BLOCK_HASH; use crate::monitoring::increment_stx_mempool_gc; +use crate::net::stream::TxStreamData; use crate::util_lib::db::query_int; use crate::util_lib::db::query_row_columns; use crate::util_lib::db::query_rows; diff --git a/stackslib/src/core/tests/mod.rs b/stackslib/src/core/tests/mod.rs index 3533ce2ad8..ea713481cd 100644 --- a/stackslib/src/core/tests/mod.rs +++ b/stackslib/src/core/tests/mod.rs @@ -25,7 +25,6 @@ use crate::chainstate::burn::ConsensusHash; use crate::chainstate::stacks::db::test::chainstate_path; use crate::chainstate::stacks::db::test::instantiate_chainstate; use crate::chainstate::stacks::db::test::instantiate_chainstate_with_balances; -use crate::chainstate::stacks::db::StreamCursor; use crate::chainstate::stacks::events::StacksTransactionReceipt; use crate::chainstate::stacks::miner::TransactionResult; use crate::chainstate::stacks::test::codec_all_transactions; @@ -46,6 +45,7 @@ use crate::core::mempool::TxTag; use crate::core::mempool::{BLOOM_COUNTER_DEPTH, BLOOM_COUNTER_ERROR_RATE, MAX_BLOOM_COUNTER_TXS}; use crate::core::FIRST_BURNCHAIN_CONSENSUS_HASH; use crate::core::FIRST_STACKS_BLOCK_HASH; +use crate::net::stream::StreamCursor; use crate::net::Error as NetError; use crate::net::HttpResponseType; use crate::net::MemPoolSyncData; diff --git a/stackslib/src/lib.rs b/stackslib/src/lib.rs index acceccf36d..ba2a6b3fe8 100644 --- a/stackslib/src/lib.rs +++ b/stackslib/src/lib.rs @@ -95,6 +95,8 @@ pub mod net; #[macro_use] pub extern crate clarity; +pub extern crate libstackerdb; + pub use clarity::vm; #[macro_use] diff --git a/stackslib/src/main.rs b/stackslib/src/main.rs index d734524870..bbeb297e74 100644 --- a/stackslib/src/main.rs +++ b/stackslib/src/main.rs @@ -21,6 +21,7 @@ #![allow(non_upper_case_globals)] extern crate blockstack_lib; +extern crate libstackerdb; extern crate rusqlite; #[macro_use] extern crate stacks_common; @@ -100,6 +101,8 @@ use std::collections::HashSet; use std::fs::{File, OpenOptions}; use std::io::BufReader; +use libstackerdb::StackerDBChunkData; + fn main() { let mut argv: Vec = env::args().collect(); if argv.len() < 2 { @@ -1074,6 +1077,35 @@ simulating a miner. process::exit(0); } + if argv[1] == "post-stackerdb" { + if argv.len() < 4 { + eprintln!( + "Usage: {} post-stackerdb slot_id slot_version privkey data", + &argv[0] + ); + process::exit(1); + } + let slot_id: u32 = argv[2].parse().unwrap(); + let slot_version: u32 = argv[3].parse().unwrap(); + let privkey: String = argv[4].clone(); + let data: String = argv[5].clone(); + + let buf = if data == "-" { + let mut buffer = vec![]; + io::stdin().read_to_end(&mut buffer).unwrap(); + buffer + } else { + data.as_bytes().to_vec() + }; + + let mut chunk = StackerDBChunkData::new(slot_id, slot_version, buf); + let privk = StacksPrivateKey::from_hex(&privkey).unwrap(); + chunk.sign(&privk).unwrap(); + + println!("{}", &serde_json::to_string(&chunk).unwrap()); + process::exit(0); + } + if argv[1] == "replay-chainstate" { if argv.len() < 7 { eprintln!("Usage: {} OLD_CHAINSTATE_PATH OLD_SORTITION_DB_PATH OLD_BURNCHAIN_DB_PATH NEW_CHAINSTATE_PATH NEW_BURNCHAIN_DB_PATH", &argv[0]); diff --git a/stackslib/src/net/asn.rs b/stackslib/src/net/asn.rs index 9a8347463e..9a7612a292 100644 --- a/stackslib/src/net/asn.rs +++ b/stackslib/src/net/asn.rs @@ -1,5 +1,5 @@ // Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation -// Copyright (C) 2020 Stacks Open Internet Foundation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by diff --git a/stackslib/src/net/chat.rs b/stackslib/src/net/chat.rs index bac191ef76..721b58d456 100644 --- a/stackslib/src/net/chat.rs +++ b/stackslib/src/net/chat.rs @@ -1,5 +1,5 @@ // Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation -// Copyright (C) 2020 Stacks Open Internet Foundation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -49,7 +49,6 @@ use crate::net::neighbors::MAX_NEIGHBOR_BLOCK_DELAY; use crate::net::p2p::PeerNetwork; use crate::net::relay::*; use crate::net::stackerdb::StackerDBs; -use crate::net::ContractId; use crate::net::Error as net_error; use crate::net::GetBlocksInv; use crate::net::GetPoxInv; @@ -72,6 +71,8 @@ use crate::core::StacksEpoch; use crate::types::chainstate::PoxId; use crate::types::StacksPublicKeyBuffer; +use clarity::vm::types::QualifiedContractIdentifier; + // did we or did we not successfully send a message? #[derive(Debug, Clone)] pub struct NeighborHealthPoint { @@ -380,7 +381,7 @@ pub struct ConversationP2P { pub stats: NeighborStats, /// which stacker DBs this peer replicates - pub db_smart_contracts: Vec, + pub db_smart_contracts: Vec, /// outbound replies pub reply_handles: VecDeque, @@ -699,7 +700,7 @@ impl ConversationP2P { } /// Does this remote neighbor support a particular StackerDB? - pub fn replicates_stackerdb(&self, db: &ContractId) -> bool { + pub fn replicates_stackerdb(&self, db: &QualifiedContractIdentifier) -> bool { for cid in self.db_smart_contracts.iter() { if cid == db { return true; @@ -1181,6 +1182,11 @@ impl ConversationP2P { self.db_smart_contracts.clear(); } + /// Getter for stacker DB contracts + pub fn get_stackerdb_contract_ids(&self) -> &[QualifiedContractIdentifier] { + &self.db_smart_contracts + } + /// Handle an inbound NAT-punch request -- just tell the peer what we think their IP/port are. /// No authentication from the peer is necessary. fn handle_natpunch_request(&self, chain_view: &BurnchainView, nonce: u32) -> StacksMessage { @@ -2062,7 +2068,7 @@ impl ConversationP2P { } /// Validate a pushed stackerdb chunk. - /// Update bandwidth accounting, but forward the stackerdb chunk along. + /// Update bandwidth accounting, but forward the stackerdb chunk along if we can accept it. /// Possibly return a reply handle for a NACK if we throttle the remote sender fn validate_stackerdb_push( &mut self, @@ -2096,6 +2102,7 @@ impl ConversationP2P { .reply_nack(local_peer, chain_view, preamble, NackErrorCodes::Throttled) .and_then(|handle| Ok(Some(handle))); } + Ok(None) } @@ -2778,7 +2785,9 @@ mod test { data_url.clone(), &asn4_entries, Some(&initial_neighbors), - &vec![ContractId::parse("SP000000000000000000002Q6VF78.sbtc").unwrap()], + &vec![ + QualifiedContractIdentifier::parse("SP000000000000000000002Q6VF78.sbtc").unwrap(), + ], ) .unwrap(); let sortdb = SortitionDB::connect( @@ -2964,7 +2973,7 @@ mod test { burnchain.clone(), chain_view.clone(), ConnectionOptions::default(), - vec![], + HashMap::new(), StacksEpoch::unit_test_pre_2_05(0), ); network @@ -3087,7 +3096,10 @@ mod test { burnchain.network_id, local_peer_1.data_url, local_peer_1.port, - &[ContractId::parse("SP000000000000000000002Q6VF78.sbtc").unwrap()], + &[ + QualifiedContractIdentifier::parse("SP000000000000000000002Q6VF78.sbtc") + .unwrap(), + ], ) .unwrap(); @@ -3097,7 +3109,10 @@ mod test { burnchain.network_id, local_peer_2.data_url, local_peer_2.port, - &[ContractId::parse("SP000000000000000000002Q6VF78.sbtc").unwrap()], + &[ + QualifiedContractIdentifier::parse("SP000000000000000000002Q6VF78.sbtc") + .unwrap(), + ], ) .unwrap(); @@ -3106,12 +3121,18 @@ mod test { assert_eq!( local_peer_1.stacker_dbs, - vec![ContractId::parse("SP000000000000000000002Q6VF78.sbtc").unwrap()] + vec![ + QualifiedContractIdentifier::parse("SP000000000000000000002Q6VF78.sbtc") + .unwrap() + ] ); assert_eq!( local_peer_2.stacker_dbs, - vec![ContractId::parse("SP000000000000000000002Q6VF78.sbtc").unwrap()] + vec![ + QualifiedContractIdentifier::parse("SP000000000000000000002Q6VF78.sbtc") + .unwrap() + ] ); let mut convo_1 = ConversationP2P::new( @@ -3208,8 +3229,10 @@ mod test { // remote peer always replies with its supported smart contracts assert_eq!( db_data.smart_contracts, - vec![ContractId::parse("SP000000000000000000002Q6VF78.sbtc") - .unwrap()] + vec![QualifiedContractIdentifier::parse( + "SP000000000000000000002Q6VF78.sbtc" + ) + .unwrap()] ); // peers learn each others' smart contract DBs @@ -3219,7 +3242,10 @@ mod test { ); assert_eq!(convo_1.db_smart_contracts.len(), 1); assert!(convo_1.replicates_stackerdb( - &ContractId::parse("SP000000000000000000002Q6VF78.sbtc").unwrap() + &QualifiedContractIdentifier::parse( + "SP000000000000000000002Q6VF78.sbtc" + ) + .unwrap() )); } else { assert_eq!(db_data.rc_consensus_hash, chain_view_2.rc_consensus_hash); @@ -3231,7 +3257,10 @@ mod test { ); assert_eq!(convo_1.db_smart_contracts.len(), 0); assert!(!convo_1.replicates_stackerdb( - &ContractId::parse("SP000000000000000000002Q6VF78.sbtc").unwrap() + &QualifiedContractIdentifier::parse( + "SP000000000000000000002Q6VF78.sbtc" + ) + .unwrap() )); } } @@ -4940,9 +4969,10 @@ mod test { StackerDBHandshakeData { rc_consensus_hash: chain_view.rc_consensus_hash.clone(), // placeholder sbtc address for now - smart_contracts: vec![ - ContractId::parse("SP000000000000000000002Q6VF78.sbtc").unwrap() - ], + smart_contracts: vec![QualifiedContractIdentifier::parse( + "SP000000000000000000002Q6VF78.sbtc", + ) + .unwrap()], }, ); diff --git a/stackslib/src/net/codec.rs b/stackslib/src/net/codec.rs index 1a59f52407..32f304e4d6 100644 --- a/stackslib/src/net/codec.rs +++ b/stackslib/src/net/codec.rs @@ -1,5 +1,5 @@ // Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation -// Copyright (C) 2020 Stacks Open Internet Foundation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -37,7 +37,6 @@ use crate::chainstate::stacks::StacksTransaction; use crate::chainstate::stacks::MAX_BLOCK_LEN; use crate::core::PEER_VERSION_TESTNET; use crate::net::db::LocalPeer; -use crate::net::stackerdb::STACKERDB_MAX_CHUNK_SIZE; use crate::net::Error as net_error; use crate::net::*; use stacks_common::codec::{read_next_at_most, read_next_exact, MAX_MESSAGE_LEN}; @@ -753,31 +752,30 @@ impl StacksMessageCodec for MemPoolSyncData { } } -/// We can't implement StacksMessageCodec directly for T: ContractIdExtension, so -/// we have to resort to these crude methods. -fn contract_id_consensus_serialize( +fn contract_id_consensus_serialize( fd: &mut W, - cid: &T, + cid: &QualifiedContractIdentifier, ) -> Result<(), codec_error> { - let addr = cid.address(); - let name = cid.name(); - write_next(fd, &addr.version)?; - write_next(fd, &addr.bytes.0)?; - write_next(fd, &name)?; + let addr = &cid.issuer; + let name = &cid.name; + write_next(fd, &addr.0)?; + write_next(fd, &addr.1)?; + write_next(fd, name)?; Ok(()) } -fn contract_id_consensus_deserialize( +fn contract_id_consensus_deserialize( fd: &mut R, -) -> Result { +) -> Result { let version: u8 = read_next(fd)?; let bytes: [u8; 20] = read_next(fd)?; let name: ContractName = read_next(fd)?; - let qn = T::from_parts( + let qn = QualifiedContractIdentifier::new( StacksAddress { version, bytes: Hash160(bytes), - }, + } + .into(), name, ); Ok(qn) @@ -803,7 +801,7 @@ impl StacksMessageCodec for StackerDBHandshakeData { let len_u8: u8 = read_next(fd)?; let mut smart_contracts = Vec::with_capacity(len_u8 as usize); for _ in 0..len_u8 { - let cid: ContractId = contract_id_consensus_deserialize(fd)?; + let cid: QualifiedContractIdentifier = contract_id_consensus_deserialize(fd)?; smart_contracts.push(cid); } Ok(StackerDBHandshakeData { @@ -821,7 +819,7 @@ impl StacksMessageCodec for StackerDBGetChunkInvData { } fn consensus_deserialize(fd: &mut R) -> Result { - let contract_id: ContractId = contract_id_consensus_deserialize(fd)?; + let contract_id: QualifiedContractIdentifier = contract_id_consensus_deserialize(fd)?; let rc_consensus_hash: ConsensusHash = read_next(fd)?; Ok(StackerDBGetChunkInvData { contract_id, @@ -860,7 +858,7 @@ impl StacksMessageCodec for StackerDBGetChunkData { } fn consensus_deserialize(fd: &mut R) -> Result { - let contract_id: ContractId = contract_id_consensus_deserialize(fd)?; + let contract_id: QualifiedContractIdentifier = contract_id_consensus_deserialize(fd)?; let rc_consensus_hash: ConsensusHash = read_next(fd)?; let slot_id: u32 = read_next(fd)?; let slot_version: u32 = read_next(fd)?; @@ -873,29 +871,6 @@ impl StacksMessageCodec for StackerDBGetChunkData { } } -impl StacksMessageCodec for StackerDBChunkData { - fn consensus_serialize(&self, fd: &mut W) -> Result<(), codec_error> { - write_next(fd, &self.slot_id)?; - write_next(fd, &self.slot_version)?; - write_next(fd, &self.sig)?; - write_next(fd, &self.data)?; - Ok(()) - } - - fn consensus_deserialize(fd: &mut R) -> Result { - let slot_id: u32 = read_next(fd)?; - let slot_version: u32 = read_next(fd)?; - let sig: MessageSignature = read_next(fd)?; - let data: Vec = read_next_at_most(fd, STACKERDB_MAX_CHUNK_SIZE.into())?; - Ok(StackerDBChunkData { - slot_id, - slot_version, - sig, - data, - }) - } -} - impl StacksMessageCodec for StackerDBPushChunkData { fn consensus_serialize(&self, fd: &mut W) -> Result<(), codec_error> { contract_id_consensus_serialize(fd, &self.contract_id)?; @@ -905,7 +880,7 @@ impl StacksMessageCodec for StackerDBPushChunkData { } fn consensus_deserialize(fd: &mut R) -> Result { - let contract_id: ContractId = contract_id_consensus_deserialize(fd)?; + let contract_id: QualifiedContractIdentifier = contract_id_consensus_deserialize(fd)?; let rc_consensus_hash: ConsensusHash = read_next(fd)?; let chunk_data: StackerDBChunkData = read_next(fd)?; Ok(StackerDBPushChunkData { @@ -2216,8 +2191,10 @@ pub mod test { let data = StackerDBHandshakeData { rc_consensus_hash: ConsensusHash([0x01; 20]), smart_contracts: vec![ - ContractId::parse("SP8QPP8TVXYAXS1VFSERG978A6WKBF59NSYJQEMN.foo").unwrap(), - ContractId::parse("SP28D54YKFCMRKXBR6BR0E4BPN57S62RSM4XEVPRP.bar").unwrap(), + QualifiedContractIdentifier::parse("SP8QPP8TVXYAXS1VFSERG978A6WKBF59NSYJQEMN.foo") + .unwrap(), + QualifiedContractIdentifier::parse("SP28D54YKFCMRKXBR6BR0E4BPN57S62RSM4XEVPRP.bar") + .unwrap(), ], }; let bytes = vec![ @@ -2241,7 +2218,10 @@ pub mod test { #[test] fn codec_StackerDBGetChunkInvData() { let data = StackerDBGetChunkInvData { - contract_id: ContractId::parse("SP8QPP8TVXYAXS1VFSERG978A6WKBF59NSYJQEMN.foo").unwrap(), + contract_id: QualifiedContractIdentifier::parse( + "SP8QPP8TVXYAXS1VFSERG978A6WKBF59NSYJQEMN.foo", + ) + .unwrap(), rc_consensus_hash: ConsensusHash([0x01; 20]), }; @@ -2281,7 +2261,10 @@ pub mod test { #[test] fn codec_StackerDBGetChunkData() { let data = StackerDBGetChunkData { - contract_id: ContractId::parse("SP8QPP8TVXYAXS1VFSERG978A6WKBF59NSYJQEMN.foo").unwrap(), + contract_id: QualifiedContractIdentifier::parse( + "SP8QPP8TVXYAXS1VFSERG978A6WKBF59NSYJQEMN.foo", + ) + .unwrap(), rc_consensus_hash: ConsensusHash([0x01; 20]), slot_id: 2, slot_version: 3, @@ -2341,7 +2324,10 @@ pub mod test { }; let push_data = StackerDBPushChunkData { - contract_id: ContractId::parse("SP8QPP8TVXYAXS1VFSERG978A6WKBF59NSYJQEMN.foo").unwrap(), + contract_id: QualifiedContractIdentifier::parse( + "SP8QPP8TVXYAXS1VFSERG978A6WKBF59NSYJQEMN.foo", + ) + .unwrap(), rc_consensus_hash: ConsensusHash([0x01; 20]), chunk_data: data, }; @@ -2508,11 +2494,11 @@ pub mod test { }, StackerDBHandshakeData { rc_consensus_hash: ConsensusHash([0x01; 20]), - smart_contracts: vec![ContractId::parse("SP8QPP8TVXYAXS1VFSERG978A6WKBF59NSYJQEMN.foo").unwrap(), ContractId::parse("SP28D54YKFCMRKXBR6BR0E4BPN57S62RSM4XEVPRP.bar").unwrap()] + smart_contracts: vec![QualifiedContractIdentifier::parse("SP8QPP8TVXYAXS1VFSERG978A6WKBF59NSYJQEMN.foo").unwrap(), QualifiedContractIdentifier::parse("SP28D54YKFCMRKXBR6BR0E4BPN57S62RSM4XEVPRP.bar").unwrap()] } ), StacksMessageType::StackerDBGetChunkInv(StackerDBGetChunkInvData { - contract_id: ContractId::parse("SP8QPP8TVXYAXS1VFSERG978A6WKBF59NSYJQEMN.foo").unwrap(), + contract_id: QualifiedContractIdentifier::parse("SP8QPP8TVXYAXS1VFSERG978A6WKBF59NSYJQEMN.foo").unwrap(), rc_consensus_hash: ConsensusHash([0x01; 20]), }), StacksMessageType::StackerDBChunkInv(StackerDBChunkInvData { @@ -2520,7 +2506,7 @@ pub mod test { num_outbound_replicas: 4, }), StacksMessageType::StackerDBGetChunk(StackerDBGetChunkData { - contract_id: ContractId::parse("SP8QPP8TVXYAXS1VFSERG978A6WKBF59NSYJQEMN.foo").unwrap(), + contract_id: QualifiedContractIdentifier::parse("SP8QPP8TVXYAXS1VFSERG978A6WKBF59NSYJQEMN.foo").unwrap(), rc_consensus_hash: ConsensusHash([0x01; 20]), slot_id: 2, slot_version: 3 @@ -2532,7 +2518,7 @@ pub mod test { data: vec![0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff] }), StacksMessageType::StackerDBPushChunk(StackerDBPushChunkData { - contract_id: ContractId::parse("SP8QPP8TVXYAXS1VFSERG978A6WKBF59NSYJQEMN.foo").unwrap(), + contract_id: QualifiedContractIdentifier::parse("SP8QPP8TVXYAXS1VFSERG978A6WKBF59NSYJQEMN.foo").unwrap(), rc_consensus_hash: ConsensusHash([0x01; 20]), chunk_data: StackerDBChunkData { slot_id: 2, diff --git a/stackslib/src/net/connection.rs b/stackslib/src/net/connection.rs index f637fd7c52..7ecffa9575 100644 --- a/stackslib/src/net/connection.rs +++ b/stackslib/src/net/connection.rs @@ -1,5 +1,5 @@ // Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation -// Copyright (C) 2020 Stacks Open Internet Foundation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by diff --git a/stackslib/src/net/db.rs b/stackslib/src/net/db.rs index 692db831da..90bbf9c969 100644 --- a/stackslib/src/net/db.rs +++ b/stackslib/src/net/db.rs @@ -1,5 +1,5 @@ // Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation -// Copyright (C) 2020 Stacks Open Internet Foundation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -28,6 +28,7 @@ use std::convert::From; use std::convert::TryFrom; use std::fs; +use clarity::vm::types::QualifiedContractIdentifier; use clarity::vm::types::StacksAddressExtensions; use clarity::vm::types::StandardPrincipalData; @@ -58,7 +59,6 @@ use rand::Rng; use rand::RngCore; use crate::net::asn::ASEntry4; -use crate::net::ContractId; use crate::net::Neighbor; use crate::net::NeighborAddress; use crate::net::NeighborKey; @@ -106,10 +106,11 @@ impl FromColumn for PeerAddress { } } -impl FromRow for ContractId { - fn from_row<'a>(row: &'a Row) -> Result { +impl FromRow for QualifiedContractIdentifier { + fn from_row<'a>(row: &'a Row) -> Result { let cid_str: String = row.get_unwrap("smart_contract_id"); - let cid = ContractId::parse(&cid_str).map_err(|_e| db_error::ParseError)?; + let cid = + QualifiedContractIdentifier::parse(&cid_str).map_err(|_e| db_error::ParseError)?; Ok(cid) } @@ -127,7 +128,7 @@ pub struct LocalPeer { pub port: u16, pub services: u16, pub data_url: UrlString, - pub stacker_dbs: Vec, + pub stacker_dbs: Vec, // filled in and curated at runtime pub public_ip_address: Option<(PeerAddress, u16)>, @@ -170,7 +171,7 @@ impl LocalPeer { privkey: Option, key_expire: u64, data_url: UrlString, - stacker_dbs: Vec, + stacker_dbs: Vec, ) -> LocalPeer { let mut pkey = privkey.unwrap_or(Secp256k1PrivateKey::new()); pkey.set_compress_public(true); @@ -246,11 +247,12 @@ impl FromRow for LocalPeer { nonce_buf.copy_from_slice(&nonce_bytes[0..32]); let data_url = UrlString::try_from(data_url_str).map_err(|_e| db_error::ParseError)?; - let stacker_dbs: Vec = if let Some(stackerdbs_json) = stackerdbs_json { - serde_json::from_str(&stackerdbs_json).map_err(|_| db_error::ParseError)? - } else { - vec![] - }; + let stacker_dbs: Vec = + if let Some(stackerdbs_json) = stackerdbs_json { + serde_json::from_str(&stackerdbs_json).map_err(|_| db_error::ParseError)? + } else { + vec![] + }; Ok(LocalPeer { network_id: network_id, @@ -435,7 +437,7 @@ impl PeerDB { p2p_port: u16, asn4_entries: &[ASEntry4], initial_neighbors: &[Neighbor], - stacker_dbs: &[ContractId], + stacker_dbs: &[QualifiedContractIdentifier], ) -> Result<(), db_error> { let localpeer = LocalPeer::new( network_id, @@ -571,13 +573,13 @@ impl PeerDB { parent_network_id: u32, data_url: UrlString, p2p_port: u16, - stacker_dbs: &[ContractId], + stacker_dbs: &[QualifiedContractIdentifier], ) -> Result<(), db_error> { let local_peer_args: &[&dyn ToSql] = &[ &p2p_port, &data_url.as_str(), &serde_json::to_string(stacker_dbs) - .expect("FATAL: unable to serialize Vec"), + .expect("FATAL: unable to serialize Vec"), &network_id, &parent_network_id, ]; @@ -635,7 +637,7 @@ impl PeerDB { data_url: UrlString, asn4_recs: &[ASEntry4], initial_neighbors: Option<&[Neighbor]>, - stacker_dbs: &[ContractId], + stacker_dbs: &[QualifiedContractIdentifier], ) -> Result { let mut create_flag = false; let open_flags = if fs::metadata(path).is_err() { @@ -1052,7 +1054,7 @@ impl PeerDB { pub fn insert_or_replace_stacker_dbs( tx: &Transaction, slot: u32, - smart_contracts: &[ContractId], + smart_contracts: &[QualifiedContractIdentifier], ) -> Result<(), db_error> { for cid in smart_contracts { test_debug!("Add Stacker DB contract to slot {}: {}", slot, cid); @@ -1336,7 +1338,7 @@ impl PeerDB { fn get_stacker_dbs_by_slot( conn: &Connection, used_slot: u32, - ) -> Result, db_error> { + ) -> Result, db_error> { let mut db_set = HashSet::new(); let qry = "SELECT smart_contract_id FROM stackerdb_peers WHERE peer_slot = ?1"; let dbs = query_rows(conn, qry, &[&used_slot])?; @@ -1350,7 +1352,7 @@ impl PeerDB { /// Get the slots for all peers that replicate a particular stacker DB fn get_stacker_db_slots( conn: &Connection, - smart_contract: &ContractId, + smart_contract: &QualifiedContractIdentifier, ) -> Result, db_error> { let qry = "SELECT peer_slot FROM stackerdb_peers WHERE smart_contract_id = ?1"; let args: &[&dyn ToSql] = &[&smart_contract.to_string()]; @@ -1361,7 +1363,7 @@ impl PeerDB { fn static_get_peer_stacker_dbs( conn: &Connection, neighbor: &Neighbor, - ) -> Result, db_error> { + ) -> Result, db_error> { let used_slot_opt = PeerDB::find_peer_slot( conn, neighbor.addr.network_id, @@ -1376,7 +1378,10 @@ impl PeerDB { } /// Get a peer's advertized stacker DBs by their IDs. - pub fn get_peer_stacker_dbs(&self, neighbor: &Neighbor) -> Result, db_error> { + pub fn get_peer_stacker_dbs( + &self, + neighbor: &Neighbor, + ) -> Result, db_error> { PeerDB::static_get_peer_stacker_dbs(&self.conn, neighbor) } @@ -1387,7 +1392,7 @@ impl PeerDB { pub fn update_peer_stacker_dbs( tx: &Transaction, neighbor: &Neighbor, - dbs: &[ContractId], + dbs: &[QualifiedContractIdentifier], ) -> Result<(), db_error> { let slot = if let Some(slot) = PeerDB::find_peer_slot( tx, @@ -1402,7 +1407,8 @@ impl PeerDB { let cur_dbs_set: HashSet<_> = PeerDB::static_get_peer_stacker_dbs(tx, neighbor)? .into_iter() .collect(); - let new_dbs_set: HashSet = dbs.iter().map(|cid| cid.clone()).collect(); + let new_dbs_set: HashSet = + dbs.iter().map(|cid| cid.clone()).collect(); let to_insert: Vec<_> = new_dbs_set.difference(&cur_dbs_set).collect(); let to_delete: Vec<_> = cur_dbs_set.difference(&new_dbs_set).collect(); @@ -1432,7 +1438,7 @@ impl PeerDB { pub fn try_insert_peer( tx: &Transaction, neighbor: &Neighbor, - stacker_dbs: &[ContractId], + stacker_dbs: &[QualifiedContractIdentifier], ) -> Result { let present = PeerDB::has_peer( tx, @@ -1801,7 +1807,7 @@ impl PeerDB { pub fn find_stacker_db_replicas( conn: &DBConn, network_id: u32, - smart_contract: &ContractId, + smart_contract: &QualifiedContractIdentifier, max_count: usize, ) -> Result, db_error> { if max_count == 0 { @@ -1863,8 +1869,14 @@ mod test { assert_eq!(local_peer.stacker_dbs, vec![]); let mut stackerdbs = vec![ - ContractId::new(StandardPrincipalData(0x01, [0x02; 20]), "db-1".into()), - ContractId::new(StandardPrincipalData(0x02, [0x03; 20]), "db-2".into()), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x01, [0x02; 20]), + "db-1".into(), + ), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x02, [0x03; 20]), + "db-2".into(), + ), ]; stackerdbs.sort(); @@ -2042,8 +2054,14 @@ mod test { // basic storage and retrieval let mut stackerdbs = vec![ - ContractId::new(StandardPrincipalData(0x01, [0x02; 20]), "db-1".into()), - ContractId::new(StandardPrincipalData(0x02, [0x03; 20]), "db-2".into()), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x01, [0x02; 20]), + "db-1".into(), + ), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x02, [0x03; 20]), + "db-2".into(), + ), ]; stackerdbs.sort(); @@ -2066,8 +2084,14 @@ mod test { // adding DBs to the same slot just grows the total list let mut new_stackerdbs = vec![ - ContractId::new(StandardPrincipalData(0x03, [0x04; 20]), "db-3".into()), - ContractId::new(StandardPrincipalData(0x04, [0x05; 20]), "db-5".into()), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x03, [0x04; 20]), + "db-3".into(), + ), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x04, [0x05; 20]), + "db-5".into(), + ), ]; new_stackerdbs.sort(); @@ -2271,8 +2295,14 @@ mod test { .unwrap(); let mut stackerdbs = vec![ - ContractId::new(StandardPrincipalData(0x01, [0x02; 20]), "db-1".into()), - ContractId::new(StandardPrincipalData(0x02, [0x03; 20]), "db-2".into()), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x01, [0x02; 20]), + "db-1".into(), + ), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x02, [0x03; 20]), + "db-2".into(), + ), ]; stackerdbs.sort(); @@ -2302,8 +2332,14 @@ mod test { // insert new stacker DBs -- keep one the same, and add a different one let mut changed_stackerdbs = vec![ - ContractId::new(StandardPrincipalData(0x01, [0x02; 20]), "db-1".into()), - ContractId::new(StandardPrincipalData(0x03, [0x04; 20]), "db-3".into()), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x01, [0x02; 20]), + "db-1".into(), + ), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x03, [0x04; 20]), + "db-3".into(), + ), ]; changed_stackerdbs.sort(); @@ -2336,8 +2372,14 @@ mod test { // add back stacker DBs let mut new_stackerdbs = vec![ - ContractId::new(StandardPrincipalData(0x04, [0x05; 20]), "db-4".into()), - ContractId::new(StandardPrincipalData(0x05, [0x06; 20]), "db-5".into()), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x04, [0x05; 20]), + "db-4".into(), + ), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x05, [0x06; 20]), + "db-5".into(), + ), ]; new_stackerdbs.sort(); @@ -2358,8 +2400,14 @@ mod test { // Do it twice -- it should be idempotent for _ in 0..2 { let mut replace_stackerdbs = vec![ - ContractId::new(StandardPrincipalData(0x06, [0x07; 20]), "db-6".into()), - ContractId::new(StandardPrincipalData(0x07, [0x08; 20]), "db-7".into()), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x06, [0x07; 20]), + "db-6".into(), + ), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x07, [0x08; 20]), + "db-7".into(), + ), ]; replace_stackerdbs.sort(); @@ -2448,8 +2496,14 @@ mod test { .unwrap(); let mut stackerdbs = vec![ - ContractId::new(StandardPrincipalData(0x01, [0x02; 20]), "db-1".into()), - ContractId::new(StandardPrincipalData(0x02, [0x03; 20]), "db-2".into()), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x01, [0x02; 20]), + "db-1".into(), + ), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x02, [0x03; 20]), + "db-2".into(), + ), ]; stackerdbs.sort(); @@ -2481,8 +2535,14 @@ mod test { // insert new stacker DBs -- keep one the same, and add a different one let mut changed_stackerdbs = vec![ - ContractId::new(StandardPrincipalData(0x01, [0x02; 20]), "db-1".into()), - ContractId::new(StandardPrincipalData(0x03, [0x04; 20]), "db-3".into()), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x01, [0x02; 20]), + "db-1".into(), + ), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x03, [0x04; 20]), + "db-3".into(), + ), ]; changed_stackerdbs.sort(); @@ -2536,8 +2596,14 @@ mod test { assert_eq!(replicas.len(), 0); let mut replace_stackerdbs = vec![ - ContractId::new(StandardPrincipalData(0x06, [0x07; 20]), "db-6".into()), - ContractId::new(StandardPrincipalData(0x07, [0x08; 20]), "db-7".into()), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x06, [0x07; 20]), + "db-6".into(), + ), + QualifiedContractIdentifier::new( + StandardPrincipalData(0x07, [0x08; 20]), + "db-7".into(), + ), ]; replace_stackerdbs.sort(); diff --git a/stackslib/src/net/dns.rs b/stackslib/src/net/dns.rs index 63f7527a60..f342cf12ef 100644 --- a/stackslib/src/net/dns.rs +++ b/stackslib/src/net/dns.rs @@ -1,21 +1,18 @@ -/* - copyright: (c) 2013-2020 by Blockstack PBC, a public benefit corporation. - - This file is part of Blockstack. - - Blockstack is free software. You may redistribute or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License or - (at your option) any later version. - - Blockstack is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY, including without the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with Blockstack. If not, see . -*/ +// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . use std::sync::mpsc::sync_channel; use std::sync::mpsc::Receiver; diff --git a/stackslib/src/net/download.rs b/stackslib/src/net/download.rs index a94286892b..1695b11f0b 100644 --- a/stackslib/src/net/download.rs +++ b/stackslib/src/net/download.rs @@ -1,21 +1,18 @@ -/* - copyright: (c) 2013-2020 by Blockstack PBC, a public benefit corporation. - - This file is part of Blockstack. - - Blockstack is free software. You may redistribute or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License or - (at your option) any later version. - - Blockstack is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY, including without the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with Blockstack. If not, see . -*/ +// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . use std::collections::HashMap; use std::collections::HashSet; diff --git a/stackslib/src/net/http.rs b/stackslib/src/net/http.rs index 116813cd63..7d48a80063 100644 --- a/stackslib/src/net/http.rs +++ b/stackslib/src/net/http.rs @@ -1,21 +1,18 @@ -/* - copyright: (c) 2013-2020 by Blockstack PBC, a public benefit corporation. - - This file is part of Blockstack. - - Blockstack is free software. You may redistribute or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License or - (at your option) any later version. - - Blockstack is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY, including without the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with Blockstack. If not, see . -*/ +// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . use std::collections::{HashMap, HashSet}; use std::convert::TryFrom; @@ -37,6 +34,8 @@ use serde_json; use time; use url::{form_urlencoded, Url}; +use libstackerdb::STACKERDB_MAX_CHUNK_SIZE; + use crate::burnchains::{Address, Txid}; use crate::chainstate::burn::ConsensusHash; use crate::chainstate::stacks::{ @@ -62,6 +61,7 @@ use crate::net::NeighborAddress; use crate::net::PeerAddress; use crate::net::PeerHost; use crate::net::ProtocolFamily; +use crate::net::StackerDBChunkData; use crate::net::StacksHttpMessage; use crate::net::StacksHttpPreamble; use crate::net::UnconfirmedTransactionResponse; @@ -79,7 +79,7 @@ use clarity::vm::{ representations::{ CONTRACT_NAME_REGEX_STRING, PRINCIPAL_DATA_REGEX_STRING, STANDARD_PRINCIPAL_REGEX_STRING, }, - types::{PrincipalData, BOUND_VALUE_SERIALIZATION_HEX}, + types::{PrincipalData, QualifiedContractIdentifier, BOUND_VALUE_SERIALIZATION_HEX}, ClarityName, ContractName, Value, }; use stacks_common::util::hash::hex_bytes; @@ -163,6 +163,26 @@ lazy_static! { Regex::new(r#"^/v2/attachments/([0-9a-f]{40})$"#).unwrap(); static ref PATH_POST_MEMPOOL_QUERY: Regex = Regex::new(r#"^/v2/mempool/query$"#).unwrap(); + static ref PATH_GET_STACKERDB_METADATA: Regex = + Regex::new(&format!( + r#"^/v2/stackerdb/(?P
{})/(?P{})$"#, + *STANDARD_PRINCIPAL_REGEX_STRING, *CONTRACT_NAME_REGEX_STRING + )).unwrap(); + static ref PATH_GET_STACKERDB_CHUNK: Regex = + Regex::new(&format!( + r#"^/v2/stackerdb/(?P
{})/(?P{})/(?P[0-9]+)$"#, + *STANDARD_PRINCIPAL_REGEX_STRING, *CONTRACT_NAME_REGEX_STRING + )).unwrap(); + static ref PATH_GET_STACKERDB_VERSIONED_CHUNK: Regex = + Regex::new(&format!( + r#"^/v2/stackerdb/(?P
{})/(?P{})/(?P[0-9]+)/(?P[0-9]+)$"#, + *STANDARD_PRINCIPAL_REGEX_STRING, *CONTRACT_NAME_REGEX_STRING + )).unwrap(); + static ref PATH_POST_STACKERDB_CHUNK: Regex = + Regex::new(&format!( + r#"/v2/stackerdb/(?P
{})/(?P{})/chunks$"#, + *STANDARD_PRINCIPAL_REGEX_STRING, *CONTRACT_NAME_REGEX_STRING + )).unwrap(); static ref PATH_OPTIONS_WILDCARD: Regex = Regex::new("^/v2/.{0,4096}$").unwrap(); } @@ -1612,6 +1632,26 @@ impl HttpRequestType { &PATH_POST_MEMPOOL_QUERY, &HttpRequestType::parse_post_mempool_query, ), + ( + "GET", + &PATH_GET_STACKERDB_METADATA, + &HttpRequestType::parse_get_stackerdb_metadata, + ), + ( + "GET", + &PATH_GET_STACKERDB_CHUNK, + &HttpRequestType::parse_get_stackerdb_chunk, + ), + ( + "GET", + &PATH_GET_STACKERDB_VERSIONED_CHUNK, + &HttpRequestType::parse_get_stackerdb_versioned_chunk, + ), + ( + "POST", + &PATH_POST_STACKERDB_CHUNK, + &HttpRequestType::parse_post_stackerdb_chunk, + ), ]; // use url::Url to parse path and query string @@ -2425,7 +2465,8 @@ impl HttpRequestType { preamble: &HttpRequestPreamble, fd: &mut R, ) -> Result { - let body: PostTransactionRequestBody = serde_json::from_reader(fd) + let mut bound_fd = BoundReader::from_reader(fd, preamble.get_content_length() as u64); + let body: PostTransactionRequestBody = serde_json::from_reader(&mut bound_fd) .map_err(|_e| net_error::DeserializeError("Failed to parse body".into()))?; let tx = { @@ -2710,6 +2751,155 @@ impl HttpRequestType { )) } + fn parse_get_stackerdb_metadata( + _protocol: &mut StacksHttp, + preamble: &HttpRequestPreamble, + regex: &Captures, + _query: Option<&str>, + _fd: &mut R, + ) -> Result { + if preamble.get_content_length() != 0 { + return Err(net_error::DeserializeError( + "Invalid Http request: expected 0-length body".to_string(), + )); + } + + HttpRequestType::parse_get_contract_arguments(preamble, regex).map( + |(preamble, addr, name)| { + let contract_id = QualifiedContractIdentifier::new(addr.into(), name); + HttpRequestType::GetStackerDBMetadata(preamble, contract_id) + }, + ) + } + + fn parse_get_stackerdb_chunk( + _protocol: &mut StacksHttp, + preamble: &HttpRequestPreamble, + regex: &Captures, + _query: Option<&str>, + _fd: &mut R, + ) -> Result { + if preamble.get_content_length() != 0 { + return Err(net_error::DeserializeError( + "Invalid Http request: expected 0-length body".to_string(), + )); + } + + let slot_id: u32 = regex + .name("slot_id") + .ok_or(net_error::DeserializeError( + "Failed to match slot ID".to_string(), + ))? + .as_str() + .parse() + .map_err(|_| net_error::DeserializeError("Failed to decode slot ID".to_string()))?; + + HttpRequestType::parse_get_contract_arguments(preamble, regex).map( + |(preamble, addr, name)| { + let contract_id = QualifiedContractIdentifier::new(addr.into(), name); + HttpRequestType::GetStackerDBChunk(preamble, contract_id, slot_id, None) + }, + ) + } + + fn parse_get_stackerdb_versioned_chunk( + _protocol: &mut StacksHttp, + preamble: &HttpRequestPreamble, + regex: &Captures, + _query: Option<&str>, + _fd: &mut R, + ) -> Result { + if preamble.get_content_length() != 0 { + return Err(net_error::DeserializeError( + "Invalid Http request: expected 0-length body".to_string(), + )); + } + + let slot_id: u32 = regex + .name("slot_id") + .ok_or(net_error::DeserializeError( + "Failed to match slot ID".to_string(), + ))? + .as_str() + .parse() + .map_err(|_| net_error::DeserializeError("Failed to decode slot ID".to_string()))?; + + let version: u32 = regex + .name("slot_version") + .ok_or(net_error::DeserializeError( + "Failed to match slot version".to_string(), + ))? + .as_str() + .parse() + .map_err(|_| { + net_error::DeserializeError("Failed to decode slot version".to_string()) + })?; + + HttpRequestType::parse_get_contract_arguments(preamble, regex).map( + |(preamble, addr, name)| { + let contract_id = QualifiedContractIdentifier::new(addr.into(), name); + HttpRequestType::GetStackerDBChunk(preamble, contract_id, slot_id, Some(version)) + }, + ) + } + + fn parse_post_stackerdb_chunk( + _protocol: &mut StacksHttp, + preamble: &HttpRequestPreamble, + regex: &Captures, + _query: Option<&str>, + fd: &mut R, + ) -> Result { + if preamble.get_content_length() == 0 { + return Err(net_error::DeserializeError( + "Invalid Http request: expected non-zero-length body for PostStackerDBChunk" + .to_string(), + )); + } + + if preamble.get_content_length() > MAX_PAYLOAD_LEN { + return Err(net_error::DeserializeError( + "Invalid Http request: PostStackerDBChunk body is too big".to_string(), + )); + } + + // content-type must be given, and must be application/json + match preamble.content_type { + None => { + return Err(net_error::DeserializeError( + "Missing Content-Type for stackerdb chunk".to_string(), + )); + } + Some(ref c) => { + if *c != HttpContentType::JSON { + return Err(net_error::DeserializeError( + "Wrong Content-Type for stackerdb; expected application/json".to_string(), + )); + } + } + }; + + let contract_addr = StacksAddress::from_string(®ex["address"]).ok_or_else(|| { + net_error::DeserializeError("Failed to parse contract address".into()) + })?; + let contract_name = ContractName::try_from(regex["contract"].to_string()) + .map_err(|_e| net_error::DeserializeError("Failed to parse contract name".into()))?; + + let contract_id = QualifiedContractIdentifier::new(contract_addr.into(), contract_name); + + let mut bound_fd = BoundReader::from_reader(fd, preamble.get_content_length() as u64); + let chunk_data: StackerDBChunkData = + serde_json::from_reader(&mut bound_fd).map_err(|_e| { + net_error::DeserializeError("Failed to parse StackerDB chunk body".into()) + })?; + + Ok(HttpRequestType::PostStackerDBChunk( + HttpRequestMetadata::from_preamble(preamble), + contract_id, + chunk_data, + )) + } + fn parse_options_preflight( _protocol: &mut StacksHttp, preamble: &HttpRequestPreamble, @@ -2751,6 +2941,9 @@ impl HttpRequestType { HttpRequestType::GetAttachment(ref md, ..) => md, HttpRequestType::MemPoolQuery(ref md, ..) => md, HttpRequestType::FeeRateEstimate(ref md, _, _) => md, + HttpRequestType::GetStackerDBMetadata(ref md, ..) => md, + HttpRequestType::GetStackerDBChunk(ref md, ..) => md, + HttpRequestType::PostStackerDBChunk(ref md, ..) => md, HttpRequestType::ClientError(ref md, ..) => md, } } @@ -2783,6 +2976,9 @@ impl HttpRequestType { HttpRequestType::GetAttachment(ref mut md, ..) => md, HttpRequestType::MemPoolQuery(ref mut md, ..) => md, HttpRequestType::FeeRateEstimate(ref mut md, _, _) => md, + HttpRequestType::GetStackerDBMetadata(ref mut md, ..) => md, + HttpRequestType::GetStackerDBChunk(ref mut md, ..) => md, + HttpRequestType::PostStackerDBChunk(ref mut md, ..) => md, HttpRequestType::ClientError(ref mut md, ..) => md, } } @@ -2965,6 +3161,36 @@ impl HttpRequestType { } None => "/v2/mempool/query".to_string(), }, + HttpRequestType::GetStackerDBMetadata(_, contract_id) => format!( + "/v2/stackerdb/{}/{}", + StacksAddress::from(contract_id.issuer.clone()), + &contract_id.name + ), + HttpRequestType::GetStackerDBChunk(_, contract_id, slot_id, slot_version_opt) => { + if let Some(version) = slot_version_opt { + format!( + "/v2/stackerdb/{}/{}/{}/{}", + StacksAddress::from(contract_id.issuer.clone()), + &contract_id.name, + slot_id, + version + ) + } else { + format!( + "/v2/stackerdb/{}/{}/{}", + StacksAddress::from(contract_id.issuer.clone()), + &contract_id.name, + slot_id + ) + } + } + HttpRequestType::PostStackerDBChunk(_, contract_id, ..) => { + format!( + "/v2/stackerdb/{}/{}/chunks", + StacksAddress::from(contract_id.issuer.clone()), + &contract_id.name + ) + } HttpRequestType::FeeRateEstimate(_, _, _) => self.get_path().to_string(), HttpRequestType::ClientError(_md, e) => match e { ClientError::NotFound(path) => path.to_string(), @@ -3008,6 +3234,13 @@ impl HttpRequestType { HttpRequestType::GetIsTraitImplemented(..) => "/v2/traits/:principal/:contract_name", HttpRequestType::MemPoolQuery(..) => "/v2/mempool/query", HttpRequestType::FeeRateEstimate(_, _, _) => "/v2/fees/transaction", + HttpRequestType::GetStackerDBMetadata(..) => "/v2/stackerdb/:principal/:contract_name", + HttpRequestType::GetStackerDBChunk(..) => { + "/v2/stackerdb/:principal/:contract_name/:slot_id(/:slot_version)?" + } + HttpRequestType::PostStackerDBChunk(..) => { + "/v2/stackerdb/:principal/:contract_name/chunks" + } HttpRequestType::OptionsPreflight(..) | HttpRequestType::ClientError(..) => "/", } } @@ -3181,6 +3414,28 @@ impl HttpRequestType { fd.write_all(&request_body_bytes) .map_err(net_error::WriteError)?; } + HttpRequestType::PostStackerDBChunk(md, _, request) => { + let mut request_body_bytes = vec![]; + serde_json::to_writer(&mut request_body_bytes, request).map_err(|e| { + net_error::SerializeError(format!( + "Failed to serialize StackerDB POST chunk to JSON: {:?}", + &e + )) + })?; + HttpRequestPreamble::new_serialized( + fd, + &md.version, + "POST", + &self.request_path(), + &md.peer, + md.keep_alive, + Some(request_body_bytes.len() as u32), + Some(&HttpContentType::JSON), + |fd| stacks_height_headers(fd, md), + )?; + fd.write_all(&request_body_bytes) + .map_err(net_error::WriteError)?; + } other_type => { let md = other_type.metadata(); let request_path = other_type.request_path(); @@ -3268,6 +3523,7 @@ impl HttpResponseType { Ok(resp) } + /// Parse a SIP-003 bytestream. The first 4 bytes are a big-endian length prefix fn parse_bytestream( preamble: &HttpResponsePreamble, fd: &mut R, @@ -3361,17 +3617,18 @@ impl HttpResponseType { }) } - fn parse_text( + fn parse_raw_bytes( preamble: &HttpResponsePreamble, fd: &mut R, len_hint: Option, max_len: u64, + expected_content_type: HttpContentType, ) -> Result, net_error> { - // content-type has to be text/plain - if preamble.content_type != HttpContentType::Text { - return Err(net_error::DeserializeError( - "Invalid content-type: expected text/plain".to_string(), - )); + if preamble.content_type != expected_content_type { + return Err(net_error::DeserializeError(format!( + "Invalid content-type: expected {}", + expected_content_type + ))); } let buf = if preamble.is_chunked() && len_hint.is_none() { let mut chunked_fd = HttpChunkedTransferReader::from_reader(fd, max_len); @@ -3402,6 +3659,24 @@ impl HttpResponseType { Ok(buf) } + fn parse_text( + preamble: &HttpResponsePreamble, + fd: &mut R, + len_hint: Option, + max_len: u64, + ) -> Result, net_error> { + Self::parse_raw_bytes(preamble, fd, len_hint, max_len, HttpContentType::Text) + } + + fn parse_bytes( + preamble: &HttpResponsePreamble, + fd: &mut R, + len_hint: Option, + max_len: u64, + ) -> Result, net_error> { + Self::parse_raw_bytes(preamble, fd, len_hint, max_len, HttpContentType::Bytes) + } + // len_hint is given by the StacksHttp protocol implementation pub fn parse( protocol: &mut StacksHttp, @@ -3491,6 +3766,22 @@ impl HttpResponseType { &PATH_POST_MEMPOOL_QUERY, &HttpResponseType::parse_post_mempool_query, ), + ( + &PATH_GET_STACKERDB_METADATA, + &HttpResponseType::parse_get_stackerdb_metadata, + ), + ( + &PATH_GET_STACKERDB_CHUNK, + &HttpResponseType::parse_get_stackerdb_chunk, + ), + ( + &PATH_GET_STACKERDB_VERSIONED_CHUNK, + &HttpResponseType::parse_get_stackerdb_chunk, + ), + ( + &PATH_POST_STACKERDB_CHUNK, + &HttpResponseType::parse_stackerdb_chunk_response, + ), ]; // use url::Url to parse path and query string @@ -4041,6 +4332,51 @@ impl HttpResponseType { )) } + fn parse_get_stackerdb_metadata( + _protocol: &mut StacksHttp, + request_version: HttpVersion, + preamble: &HttpResponsePreamble, + fd: &mut R, + len_hint: Option, + ) -> Result { + let slot_metadata = + HttpResponseType::parse_json(preamble, fd, len_hint, MAX_MESSAGE_LEN as u64)?; + Ok(HttpResponseType::StackerDBMetadata( + HttpResponseMetadata::from_preamble(request_version, preamble), + slot_metadata, + )) + } + + fn parse_get_stackerdb_chunk( + _protocol: &mut StacksHttp, + request_version: HttpVersion, + preamble: &HttpResponsePreamble, + fd: &mut R, + len_hint: Option, + ) -> Result { + let chunk = + HttpResponseType::parse_bytes(preamble, fd, len_hint, STACKERDB_MAX_CHUNK_SIZE as u64)?; + Ok(HttpResponseType::StackerDBChunk( + HttpResponseMetadata::from_preamble(request_version, preamble), + chunk, + )) + } + + fn parse_stackerdb_chunk_response( + _protocol: &mut StacksHttp, + request_version: HttpVersion, + preamble: &HttpResponsePreamble, + fd: &mut R, + len_hint: Option, + ) -> Result { + let slot_ack = + HttpResponseType::parse_json(preamble, fd, len_hint, MAX_MESSAGE_LEN as u64)?; + Ok(HttpResponseType::StackerDBChunkAck( + HttpResponseMetadata::from_preamble(request_version, preamble), + slot_ack, + )) + } + fn error_reason(code: u16) -> &'static str { match code { 400 => "Bad Request", @@ -4105,6 +4441,9 @@ impl HttpResponseType { HttpResponseType::MemPoolTxs(ref md, ..) => md, HttpResponseType::OptionsPreflight(ref md) => md, HttpResponseType::TransactionFeeEstimation(ref md, _) => md, + HttpResponseType::StackerDBMetadata(ref md, ..) => md, + HttpResponseType::StackerDBChunk(ref md, ..) => md, + HttpResponseType::StackerDBChunkAck(ref md, ..) => md, // errors HttpResponseType::BadRequestJSON(ref md, _) => md, HttpResponseType::BadRequest(ref md, _) => md, @@ -4406,6 +4745,42 @@ impl HttpResponseType { None => HttpResponseType::send_bytestream(protocol, md, fd, txs), }?; } + HttpResponseType::StackerDBMetadata(ref md, ref slot_metadata) => { + HttpResponsePreamble::new_serialized( + fd, + 200, + "OK", + md.content_length.clone(), + &HttpContentType::JSON, + md.request_id, + |ref mut fd| keep_alive_headers(fd, md), + )?; + HttpResponseType::send_json(protocol, md, fd, slot_metadata)?; + } + HttpResponseType::StackerDBChunk(ref md, ref chunk, ..) => { + HttpResponsePreamble::new_serialized( + fd, + 200, + "OK", + md.content_length.clone(), + &HttpContentType::Bytes, + md.request_id, + |ref mut fd| keep_alive_headers(fd, md), + )?; + HttpResponseType::send_text(protocol, md, fd, chunk)?; + } + HttpResponseType::StackerDBChunkAck(ref md, ref ack_data) => { + HttpResponsePreamble::new_serialized( + fd, + 200, + "OK", + md.content_length.clone(), + &HttpContentType::JSON, + md.request_id, + |ref mut fd| keep_alive_headers(fd, md), + )?; + HttpResponseType::send_json(protocol, md, fd, ack_data)?; + } HttpResponseType::OptionsPreflight(ref md) => { HttpResponsePreamble::new_serialized( fd, @@ -4525,6 +4900,9 @@ impl MessageSequence for StacksHttpMessage { HttpRequestType::GetAttachmentsInv(..) => "HTTP(GetAttachmentsInv)", HttpRequestType::MemPoolQuery(..) => "HTTP(MemPoolQuery)", HttpRequestType::OptionsPreflight(..) => "HTTP(OptionsPreflight)", + HttpRequestType::GetStackerDBMetadata(..) => "HTTP(GetStackerDBMetadata)", + HttpRequestType::GetStackerDBChunk(..) => "HTTP(GetStackerDBChunk)", + HttpRequestType::PostStackerDBChunk(..) => "HTTP(PostStackerDBChunk)", HttpRequestType::ClientError(..) => "HTTP(ClientError)", HttpRequestType::FeeRateEstimate(_, _, _) => "HTTP(FeeRateEstimate)", }, @@ -4555,6 +4933,9 @@ impl MessageSequence for StacksHttpMessage { HttpResponseType::UnconfirmedTransaction(_, _) => "HTTP(UnconfirmedTransaction)", HttpResponseType::MemPoolTxStream(..) => "HTTP(MemPoolTxStream)", HttpResponseType::MemPoolTxs(..) => "HTTP(MemPoolTxs)", + HttpResponseType::StackerDBMetadata(..) => "HTTP(StackerDBMetadata)", + HttpResponseType::StackerDBChunk(..) => "HTTP(StackerDBChunk)", + HttpResponseType::StackerDBChunkAck(..) => "HTTP(StackerDBChunkAck)", HttpResponseType::OptionsPreflight(_) => "HTTP(OptionsPreflight)", HttpResponseType::BadRequestJSON(..) | HttpResponseType::BadRequest(..) => { "HTTP(400)" @@ -6216,6 +6597,7 @@ mod test { ) .unwrap(), authenticated: true, + stackerdbs: Some(vec![]), }, RPCNeighbor { network_id: 3, @@ -6230,6 +6612,7 @@ mod test { ) .unwrap(), authenticated: false, + stackerdbs: Some(vec![]), }, ], inbound: vec![], diff --git a/stackslib/src/net/inv.rs b/stackslib/src/net/inv.rs index 6c13178873..1a1919f289 100644 --- a/stackslib/src/net/inv.rs +++ b/stackslib/src/net/inv.rs @@ -1,21 +1,18 @@ -/* - copyright: (c) 2013-2020 by Blockstack PBC, a public benefit corporation. - - This file is part of Blockstack. - - Blockstack is free software. You may redistribute or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License or - (at your option) any later version. - - Blockstack is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY, including without the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with Blockstack. If not, see . -*/ +// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . use std::cmp; use std::collections::BTreeMap; diff --git a/stackslib/src/net/mod.rs b/stackslib/src/net/mod.rs index e7f5afd77a..7f1528e74b 100644 --- a/stackslib/src/net/mod.rs +++ b/stackslib/src/net/mod.rs @@ -1,5 +1,5 @@ // Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation -// Copyright (C) 2020 Stacks Open Internet Foundation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -104,6 +104,10 @@ pub use self::http::StacksHttp; use crate::core::StacksEpoch; +use libstackerdb::{ + Error as libstackerdb_error, SlotMetadata, StackerDBChunkAckData, StackerDBChunkData, +}; + /// Implements `ASEntry4` object, which is used in db.rs to store the AS number of an IP address. pub mod asn; /// Implements the Atlas network. This network uses the infrastructure created in `src/net` to @@ -140,12 +144,16 @@ pub mod relay; pub mod rpc; pub mod server; pub mod stackerdb; +pub mod stream; use crate::net::stackerdb::StackerDBConfig; use crate::net::stackerdb::StackerDBSync; use crate::net::stackerdb::StackerDBSyncResult; use crate::net::stackerdb::StackerDBs; +pub use crate::net::neighbors::{NeighborComms, PeerNetworkComms}; +pub use crate::net::stream::StreamCursor; + #[cfg(test)] pub mod tests; @@ -260,27 +268,42 @@ pub enum Error { /// burnchain error BurnchainError(burnchain_error), /// chunk is stale - StaleChunk(u32, u32), + StaleChunk { + supplied_version: u32, + latest_version: u32, + }, /// no such slot - NoSuchSlot(ContractId, u32), + NoSuchSlot(QualifiedContractIdentifier, u32), /// no such DB - NoSuchStackerDB(ContractId), + NoSuchStackerDB(QualifiedContractIdentifier), /// stacker DB exists - StackerDBExists(ContractId), + StackerDBExists(QualifiedContractIdentifier), /// slot signer is wrong BadSlotSigner(StacksAddress, u32), /// too many writes to a slot - TooManySlotWrites(u32, u32), + TooManySlotWrites { + supplied_version: u32, + max_writes: u32, + }, /// too frequent writes to a slot TooFrequentSlotWrites(u64), /// Invalid control smart contract for a Stacker DB - InvalidStackerDBContract(ContractId), + InvalidStackerDBContract(QualifiedContractIdentifier, String), /// state machine step took too long StepTimeout, /// stacker DB chunk is too big StackerDBChunkTooBig(usize), } +impl From for Error { + fn from(e: libstackerdb_error) -> Self { + match e { + libstackerdb_error::SigningError(s) => Error::SigningError(s), + libstackerdb_error::VerifyingError(s) => Error::VerifyingError(s), + } + } +} + impl From for Error { fn from(e: codec_error) -> Self { match e { @@ -380,8 +403,15 @@ impl fmt::Display for Error { Error::Transient(ref s) => write!(f, "Transient network error: {}", s), Error::ExpectedEndOfStream => write!(f, "Expected end-of-stream"), Error::BurnchainError(ref e) => fmt::Display::fmt(e, f), - Error::StaleChunk(ref current, ref given) => { - write!(f, "Stale DB chunk (cur={},given={})", current, given) + Error::StaleChunk { + supplied_version, + latest_version, + } => { + write!( + f, + "Stale DB chunk (supplied={},latest={})", + supplied_version, latest_version + ) } Error::NoSuchSlot(ref addr, ref slot_id) => { write!(f, "No such DB slot ({},{})", addr, slot_id) @@ -395,17 +425,24 @@ impl fmt::Display for Error { Error::BadSlotSigner(ref addr, ref slot_id) => { write!(f, "Bad DB slot signer ({},{})", addr, slot_id) } - Error::TooManySlotWrites(ref max, ref given) => { - write!(f, "Too many slot writes (max={},given={})", max, given) + Error::TooManySlotWrites { + supplied_version, + max_writes, + } => { + write!( + f, + "Too many slot writes (max={},given={})", + max_writes, supplied_version + ) } Error::TooFrequentSlotWrites(ref deadline) => { write!(f, "Too frequent slot writes (deadline={})", deadline) } - Error::InvalidStackerDBContract(ref contract_id) => { + Error::InvalidStackerDBContract(ref contract_id, ref reason) => { write!( f, - "Invalid StackerDB control smart contract {}", - contract_id + "Invalid StackerDB control smart contract {}: {}", + contract_id, reason ) } Error::StepTimeout => write!(f, "State-machine step took too long"), @@ -473,12 +510,12 @@ impl error::Error for Error { Error::Transient(ref _s) => None, Error::ExpectedEndOfStream => None, Error::BurnchainError(ref e) => Some(e), - Error::StaleChunk(..) => None, + Error::StaleChunk { .. } => None, Error::NoSuchSlot(..) => None, Error::NoSuchStackerDB(..) => None, Error::StackerDBExists(..) => None, Error::BadSlotSigner(..) => None, - Error::TooManySlotWrites(..) => None, + Error::TooManySlotWrites { .. } => None, Error::TooFrequentSlotWrites(..) => None, Error::InvalidStackerDBContract(..) => None, Error::StepTimeout => None, @@ -988,41 +1025,6 @@ pub enum MemPoolSyncData { TxTags([u8; 32], Vec), } -/// Make QualifiedContractIdentifier usable to the networking code -pub trait ContractIdExtension { - fn from_parts(addr: StacksAddress, name: ContractName) -> Self; - fn address(&self) -> StacksAddress; - fn name(&self) -> ContractName; - fn from_str(txt: &str) -> Option - where - Self: Sized; -} - -/// short-hand type alias -pub type ContractId = QualifiedContractIdentifier; - -impl ContractIdExtension for ContractId { - fn from_parts(addr: StacksAddress, name: ContractName) -> ContractId { - let id_addr = StandardPrincipalData(addr.version, addr.bytes.0); - ContractId::new(id_addr, name) - } - - fn address(&self) -> StacksAddress { - StacksAddress { - version: self.issuer.0, - bytes: Hash160(self.issuer.1.clone()), - } - } - - fn name(&self) -> ContractName { - self.name.clone() - } - - fn from_str(txt: &str) -> Option { - ContractId::parse(txt).ok() - } -} - /// Inform the remote peer of (a page of) the list of stacker DB contracts this node supports #[derive(Debug, Clone, PartialEq)] pub struct StackerDBHandshakeData { @@ -1030,14 +1032,14 @@ pub struct StackerDBHandshakeData { pub rc_consensus_hash: ConsensusHash, /// list of smart contracts that we index. /// there can be as many as 256 entries. - pub smart_contracts: Vec, + pub smart_contracts: Vec, } /// Request for a chunk inventory #[derive(Debug, Clone, PartialEq)] pub struct StackerDBGetChunkInvData { /// smart contract being used to determine chunk quantity and order - pub contract_id: ContractId, + pub contract_id: QualifiedContractIdentifier, /// consensus hash of the sortition that started this reward cycle pub rc_consensus_hash: ConsensusHash, } @@ -1056,7 +1058,7 @@ pub struct StackerDBChunkInvData { #[derive(Debug, Clone, PartialEq)] pub struct StackerDBGetChunkData { /// smart contract being used to determine slot quantity and order - pub contract_id: ContractId, + pub contract_id: QualifiedContractIdentifier, /// consensus hash of the sortition that started this reward cycle pub rc_consensus_hash: ConsensusHash, /// slot ID @@ -1065,24 +1067,11 @@ pub struct StackerDBGetChunkData { pub slot_version: u32, } -/// Stacker DB chunk reply to a StackerDBGetChunkData -#[derive(Debug, Clone, PartialEq)] -pub struct StackerDBChunkData { - /// slot ID - pub slot_id: u32, - /// slot version (a lamport clock) - pub slot_version: u32, - /// signature from the stacker over (reward cycle consensus hash, slot id, slot version, chunk sha512/256) - pub sig: MessageSignature, - /// the chunk data - pub data: Vec, -} - /// Stacker DB chunk push #[derive(Debug, Clone, PartialEq)] pub struct StackerDBPushChunkData { /// smart contract being used to determine chunk quantity and order - pub contract_id: ContractId, + pub contract_id: QualifiedContractIdentifier, /// consensus hash of the sortition that started this reward cycle pub rc_consensus_hash: ConsensusHash, /// the pushed chunk @@ -1278,6 +1267,9 @@ pub struct RPCPeerInfoData { #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub last_pox_anchor: Option, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub stackerdbs: Option>, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -1629,10 +1621,17 @@ pub struct RPCNeighbor { pub port: u16, pub public_key_hash: Hash160, pub authenticated: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub stackerdbs: Option>, } impl RPCNeighbor { - pub fn from_neighbor_key_and_pubkh(nk: NeighborKey, pkh: Hash160, auth: bool) -> RPCNeighbor { + pub fn from_neighbor_key_and_pubkh( + nk: NeighborKey, + pkh: Hash160, + auth: bool, + stackerdbs: Vec, + ) -> RPCNeighbor { RPCNeighbor { network_id: nk.network_id, peer_version: nk.peer_version, @@ -1640,6 +1639,7 @@ impl RPCNeighbor { port: nk.port, public_key_hash: pkh, authenticated: auth, + stackerdbs: Some(stackerdbs), } } } @@ -1731,6 +1731,19 @@ pub enum HttpRequestType { TipRequest, ), MemPoolQuery(HttpRequestMetadata, MemPoolSyncData, Option), + /// StackerDB HTTP queries + GetStackerDBMetadata(HttpRequestMetadata, QualifiedContractIdentifier), + GetStackerDBChunk( + HttpRequestMetadata, + QualifiedContractIdentifier, + u32, + Option, + ), + PostStackerDBChunk( + HttpRequestMetadata, + QualifiedContractIdentifier, + StackerDBChunkData, + ), /// catch-all for any errors we should surface from parsing ClientError(HttpRequestMetadata, ClientError), } @@ -1849,6 +1862,9 @@ pub enum HttpResponseType { MemPoolTxs(HttpResponseMetadata, Option, Vec), OptionsPreflight(HttpResponseMetadata), TransactionFeeEstimation(HttpResponseMetadata, RPCFeeEstimateResponse), + StackerDBMetadata(HttpResponseMetadata, Vec), + StackerDBChunk(HttpResponseMetadata, Vec), + StackerDBChunkAck(HttpResponseMetadata, StackerDBChunkAckData), // peer-given error responses BadRequest(HttpResponseMetadata, String), BadRequestJSON(HttpResponseMetadata, serde_json::Value), @@ -2170,6 +2186,7 @@ pub struct NetworkResult { pub uploaded_transactions: Vec, // transactions sent to us by the http server pub uploaded_blocks: Vec, // blocks sent to us via the http server pub uploaded_microblocks: Vec, // microblocks sent to us by the http server + pub uploaded_stackerdb_chunks: Vec, // chunks we received from the HTTP server pub attachments: Vec<(AttachmentInstance, Attachment)>, pub synced_transactions: Vec, // transactions we downloaded via a mempool sync pub stacker_db_sync_results: Vec, // chunks for stacker DBs we downloaded @@ -2178,7 +2195,7 @@ pub struct NetworkResult { pub num_download_passes: u64, pub burn_height: u64, pub rc_consensus_hash: ConsensusHash, - pub stacker_db_configs: HashMap, + pub stacker_db_configs: HashMap, } impl NetworkResult { @@ -2188,7 +2205,7 @@ impl NetworkResult { num_download_passes: u64, burn_height: u64, rc_consensus_hash: ConsensusHash, - stacker_db_configs: HashMap, + stacker_db_configs: HashMap, ) -> NetworkResult { NetworkResult { unhandled_messages: HashMap::new(), @@ -2201,6 +2218,7 @@ impl NetworkResult { uploaded_transactions: vec![], uploaded_blocks: vec![], uploaded_microblocks: vec![], + uploaded_stackerdb_chunks: vec![], attachments: vec![], synced_transactions: vec![], stacker_db_sync_results: vec![], @@ -2233,6 +2251,14 @@ impl NetworkResult { self.attachments.len() > 0 } + pub fn has_stackerdb_chunks(&self) -> bool { + self.stacker_db_sync_results + .iter() + .fold(0, |acc, x| acc + x.chunks_to_store.len()) + > 0 + || self.uploaded_stackerdb_chunks.len() > 0 + } + pub fn transactions(&self) -> Vec { self.pushed_transactions .values() @@ -2247,6 +2273,7 @@ impl NetworkResult { || self.has_microblocks() || self.has_transactions() || self.has_attachments() + || self.has_stackerdb_chunks() } pub fn consume_unsolicited( @@ -2308,6 +2335,9 @@ impl NetworkResult { StacksMessageType::Microblocks(mblock_data) => { self.uploaded_microblocks.push(mblock_data); } + StacksMessageType::StackerDBPushChunk(chunk_data) => { + self.uploaded_stackerdb_chunks.push(chunk_data); + } _ => { // drop warn!("Dropping unknown HTTP message"); @@ -2701,7 +2731,7 @@ pub mod test { /// on cycle numbers bounded (inclusive) by the supplied u64s pub check_pox_invariants: Option<(u64, u64)>, /// Which stacker DBs will this peer replicate? - pub stacker_dbs: Vec, + pub stacker_dbs: Vec, /// Stacker DB configurations for each stacker_dbs entry above, if different from /// StackerDBConfig::noop() pub stacker_db_configs: Vec>, @@ -2831,7 +2861,9 @@ pub mod test { ) } - pub fn get_stacker_db_configs(&self) -> HashMap { + pub fn get_stacker_db_configs( + &self, + ) -> HashMap { let mut ret = HashMap::new(); for (contract_id, config_opt) in self.stacker_dbs.iter().zip(self.stacker_db_configs.iter()) @@ -2844,6 +2876,15 @@ pub mod test { } ret } + + pub fn add_stacker_db( + &mut self, + contract_id: QualifiedContractIdentifier, + config: StackerDBConfig, + ) { + self.stacker_dbs.push(contract_id); + self.stacker_db_configs.push(Some(config)); + } } pub fn dns_thread_start(max_inflight: u64) -> (DNSClient, thread::JoinHandle<()>) { @@ -2912,11 +2953,12 @@ pub mod test { fn init_stacker_dbs( root_path: &str, peerdb: &PeerDB, - stacker_dbs: &[ContractId], + stacker_dbs: &[QualifiedContractIdentifier], stacker_db_configs: &[Option], - ) -> Vec> { + ) -> HashMap)> + { let stackerdb_path = format!("{}/stacker_db.sqlite", root_path); - let mut stacker_db_syncs = vec![]; + let mut stacker_db_syncs = HashMap::new(); let local_peer = PeerDB::get_local_peer(peerdb.conn()).unwrap(); for (i, contract_id) in stacker_dbs.iter().enumerate() { let mut db_config = if let Some(config_opt) = stacker_db_configs.get(i) { @@ -2949,7 +2991,8 @@ pub mod test { stacker_dbs, ) .expect(&format!("FATAL: could not open '{}'", stackerdb_path)); - stacker_db_syncs.push(stacker_db_sync); + + stacker_db_syncs.insert(contract_id.clone(), (db_config.clone(), stacker_db_sync)); } stacker_db_syncs } diff --git a/stackslib/src/net/neighbors/neighbor.rs b/stackslib/src/net/neighbors/neighbor.rs index 2defb6f06f..733e0edd8c 100644 --- a/stackslib/src/net/neighbors/neighbor.rs +++ b/stackslib/src/net/neighbors/neighbor.rs @@ -14,9 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use crate::net::{ - db::PeerDB, ContractId, Error as net_error, Neighbor, NeighborAddress, NeighborKey, -}; +use crate::net::{db::PeerDB, Error as net_error, Neighbor, NeighborAddress, NeighborKey}; use crate::util_lib::db::{DBConn, DBTx}; @@ -34,6 +32,8 @@ use rand::thread_rng; use stacks_common::util::get_epoch_time_secs; use stacks_common::util::log; +use clarity::vm::types::QualifiedContractIdentifier; + /// Walk-specific helper functions for neighbors impl Neighbor { pub fn empty(key: &NeighborKey, pubk: &Secp256k1PublicKey, expire_block: u64) -> Neighbor { @@ -57,7 +57,7 @@ impl Neighbor { pub fn save_update<'a>( &mut self, tx: &DBTx<'a>, - stacker_dbs: Option<&[ContractId]>, + stacker_dbs: Option<&[QualifiedContractIdentifier]>, ) -> Result<(), net_error> { self.last_contact_time = get_epoch_time_secs(); PeerDB::update_peer(tx, &self).map_err(net_error::DBError)?; @@ -74,7 +74,7 @@ impl Neighbor { pub fn save<'a>( &mut self, tx: &DBTx<'a>, - stacker_dbs: Option<&[ContractId]>, + stacker_dbs: Option<&[QualifiedContractIdentifier]>, ) -> Result { self.last_contact_time = get_epoch_time_secs(); PeerDB::try_insert_peer(tx, &self, stacker_dbs.unwrap_or(&[])).map_err(net_error::DBError) diff --git a/stackslib/src/net/p2p.rs b/stackslib/src/net/p2p.rs index f1c72202c4..218a840607 100644 --- a/stackslib/src/net/p2p.rs +++ b/stackslib/src/net/p2p.rs @@ -1,5 +1,5 @@ // Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation -// Copyright (C) 2020 Stacks Open Internet Foundation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -75,7 +75,7 @@ use crate::net::relay::*; use crate::net::relay::*; use crate::net::rpc::RPCHandlerArgs; use crate::net::server::*; -use crate::net::stackerdb::{StackerDBConfig, StackerDBSync, StackerDBs}; +use crate::net::stackerdb::{StackerDBConfig, StackerDBSync, StackerDBTx, StackerDBs}; use crate::net::Error as net_error; use crate::net::Neighbor; use crate::net::NeighborKey; @@ -95,6 +95,7 @@ use crate::chainstate::stacks::StacksBlockHeader; use crate::types::chainstate::{PoxId, SortitionId}; use clarity::vm::ast::ASTRules; +use clarity::vm::types::QualifiedContractIdentifier; /// inter-thread request to send a p2p message from another thread in this program. #[derive(Debug)] @@ -318,9 +319,10 @@ pub struct PeerNetwork { pub attachments_downloader: Option, // peer stacker DB state machines - pub stacker_db_syncs: Option>>, + pub stacker_db_syncs: + Option>>, // configuration state for stacker DBs (loaded at runtime from smart contracts) - pub stacker_db_configs: HashMap, + pub stacker_db_configs: HashMap, // handle to all stacker DB state pub stackerdbs: StackerDBs, @@ -387,7 +389,10 @@ impl PeerNetwork { burnchain: Burnchain, chain_view: BurnchainView, connection_opts: ConnectionOptions, - stacker_db_syncs: Vec>, + stacker_db_syncs: HashMap< + QualifiedContractIdentifier, + (StackerDBConfig, StackerDBSync), + >, epochs: Vec, ) -> PeerNetwork { let http = HttpPeer::new(connection_opts.clone(), 0); @@ -406,9 +411,11 @@ impl PeerNetwork { let first_burn_header_hash = burnchain.first_block_hash.clone(); let first_burn_header_ts = burnchain.first_block_timestamp; + let mut stacker_db_configs = HashMap::new(); let mut stacker_db_sync_map = HashMap::new(); - for stacker_db_sync in stacker_db_syncs.into_iter() { - stacker_db_sync_map.insert(stacker_db_sync.smart_contract_id.clone(), stacker_db_sync); + for (contract_id, (stacker_db_config, stacker_db_sync)) in stacker_db_syncs.into_iter() { + stacker_db_configs.insert(contract_id.clone(), stacker_db_config); + stacker_db_sync_map.insert(contract_id.clone(), stacker_db_sync); } let mut network = PeerNetwork { @@ -473,7 +480,7 @@ impl PeerNetwork { attachments_downloader: None, stacker_db_syncs: Some(stacker_db_sync_map), - stacker_db_configs: HashMap::new(), + stacker_db_configs: stacker_db_configs, stackerdbs: stackerdbs, mempool_state: MempoolSyncState::PickOutboundPeer, @@ -639,6 +646,20 @@ impl PeerNetwork { &self.stackerdbs } + /// Get StackerDBs transaction + pub fn stackerdbs_tx_begin<'a>( + &'a mut self, + stackerdb_contract_id: &QualifiedContractIdentifier, + ) -> Result, net_error> { + if let Some(config) = self.stacker_db_configs.get(stackerdb_contract_id) { + return self + .stackerdbs + .tx_begin(config.clone()) + .map_err(net_error::from); + } + Err(net_error::NoSuchStackerDB(stackerdb_contract_id.clone())) + } + /// Get a ref to the walk pingbacks -- pub fn get_walk_pingbacks(&self) -> &HashMap { &self.walk_pingbacks @@ -671,7 +692,10 @@ impl PeerNetwork { /// Count up the number of outbound StackerDB replicas we talk to, /// given the contract ID that controls it. - pub fn count_outbound_stackerdb_replicas(&self, contract_id: &ContractId) -> usize { + pub fn count_outbound_stackerdb_replicas( + &self, + contract_id: &QualifiedContractIdentifier, + ) -> usize { let mut count = 0; for (_, convo) in self.peers.iter() { if !convo.is_outbound() { @@ -5150,20 +5174,65 @@ impl PeerNetwork { } /// Set the stacker DB configs - pub fn set_stacker_db_configs(&mut self, configs: HashMap) { + pub fn set_stacker_db_configs( + &mut self, + configs: HashMap, + ) { self.stacker_db_configs = configs; } /// Obtain a copy of the stacker DB configs - pub fn get_stacker_db_configs_owned(&self) -> HashMap { + pub fn get_stacker_db_configs_owned( + &self, + ) -> HashMap { self.stacker_db_configs.clone() } /// Obtain a ref to the stacker DB configs - pub fn get_stacker_db_configs(&self) -> &HashMap { + pub fn get_stacker_db_configs(&self) -> &HashMap { &self.stacker_db_configs } + /// Create or reconfigure a StackerDB. + /// Fails only if the underlying DB fails + fn create_or_reconfigure_stackerdb( + &mut self, + stackerdb_contract_id: &QualifiedContractIdentifier, + new_config: &StackerDBConfig, + ) -> Result<(), db_error> { + debug!("Reconfiguring StackerDB {}...", stackerdb_contract_id); + let tx = self.stackerdbs.tx_begin(new_config.clone())?; + match tx.reconfigure_stackerdb(stackerdb_contract_id, &new_config.signers) { + Ok(..) => {} + Err(net_error::NoSuchStackerDB(..)) => { + // need to create it first + info!( + "Creating local replica of StackerDB {}", + stackerdb_contract_id + ); + test_debug!( + "Creating local replica of StackerDB {} with config {:?}", + stackerdb_contract_id, + &new_config + ); + if let Err(e) = tx.create_stackerdb(stackerdb_contract_id, &new_config.signers) { + warn!( + "Failed to create StackerDB replica {}: {:?}", + stackerdb_contract_id, &e + ); + } + } + Err(e) => { + warn!( + "Failed to reconfigure StackerDB replica {}: {:?}", + stackerdb_contract_id, &e + ); + } + } + tx.commit()?; + Ok(()) + } + /// Refresh view of burnchain, if needed. /// If the burnchain view changes, then take the following additional steps: /// * hint to the inventory sync state-machine to restart, since we potentially have a new @@ -5175,7 +5244,7 @@ impl PeerNetwork { &mut self, indexer: &B, sortdb: &SortitionDB, - chainstate: &StacksChainState, + chainstate: &mut StacksChainState, ibd: bool, ) -> Result>, net_error> { // update burnchain snapshot if we need to (careful -- it's expensive) @@ -5259,6 +5328,39 @@ impl PeerNetwork { self.last_anchor_block_txid = ih .get_last_selected_anchor_block_txid()? .unwrap_or(Txid([0x00; 32])); + + // refresh stackerdb configs + let mut new_stackerdb_configs = HashMap::new(); + let stacker_db_configs = mem::replace(&mut self.stacker_db_configs, HashMap::new()); + for (stackerdb_contract_id, stackerdb_config) in stacker_db_configs.into_iter() { + let new_config = match StackerDBConfig::from_smart_contract( + chainstate, + sortdb, + &stackerdb_contract_id, + ) { + Ok(config) => config, + Err(e) => { + warn!( + "Failed to load StackerDB config for {}: {:?}", + &stackerdb_contract_id, &e + ); + StackerDBConfig::noop() + } + }; + if new_config != stackerdb_config && new_config.signers.len() > 0 { + if let Err(e) = + self.create_or_reconfigure_stackerdb(&stackerdb_contract_id, &new_config) + { + warn!( + "Failed to create or reconfigure StackerDB {}: DB error {:?}", + &stackerdb_contract_id, &e + ); + } + } + new_stackerdb_configs.insert(stackerdb_contract_id.clone(), new_config); + } + + self.stacker_db_configs = new_stackerdb_configs; } if sn.canonical_stacks_tip_hash != self.burnchain_tip.canonical_stacks_tip_hash @@ -5803,7 +5905,7 @@ mod test { burnchain, burnchain_view, conn_opts, - vec![], + HashMap::new(), StacksEpoch::unit_test_pre_2_05(0), ); p2p diff --git a/stackslib/src/net/poll.rs b/stackslib/src/net/poll.rs index abbae359fb..c534f24133 100644 --- a/stackslib/src/net/poll.rs +++ b/stackslib/src/net/poll.rs @@ -1,5 +1,5 @@ // Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation -// Copyright (C) 2020 Stacks Open Internet Foundation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by diff --git a/stackslib/src/net/prune.rs b/stackslib/src/net/prune.rs index c31f95d13c..214bfe2582 100644 --- a/stackslib/src/net/prune.rs +++ b/stackslib/src/net/prune.rs @@ -1,5 +1,5 @@ // Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation -// Copyright (C) 2020 Stacks Open Internet Foundation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by diff --git a/stackslib/src/net/relay.rs b/stackslib/src/net/relay.rs index f7903b2a5e..65c8f8b21f 100644 --- a/stackslib/src/net/relay.rs +++ b/stackslib/src/net/relay.rs @@ -1,27 +1,25 @@ -/* - copyright: (c) 2013-2020 by Blockstack PBC, a public benefit corporation. - - This file is part of Blockstack. - - Blockstack is free software. You may redistribute or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License or - (at your option) any later version. - - Blockstack is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY, including without the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with Blockstack. If not, see . -*/ +// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . use std::cmp; use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; use std::collections::VecDeque; +use std::mem; use rand::prelude::*; use rand::thread_rng; @@ -47,7 +45,9 @@ use crate::net::http::*; use crate::net::p2p::*; use crate::net::poll::*; use crate::net::rpc::*; -use crate::net::stackerdb::{StackerDBConfig, StackerDBSyncResult, StackerDBs}; +use crate::net::stackerdb::{ + StackerDBConfig, StackerDBEventDispatcher, StackerDBSyncResult, StackerDBs, +}; use crate::net::Error as net_error; use crate::net::*; use crate::types::chainstate::StacksBlockId; @@ -106,6 +106,39 @@ pub struct ProcessedNetReceipts { pub num_new_unconfirmed_microblocks: u64, } +/// A trait for implementing both mempool event observer methods and stackerdb methods. +/// This is required for event observers to fully report on newly-relayed data. +pub trait RelayEventDispatcher: + MemPoolEventDispatcher + + StackerDBEventDispatcher + + AsMemPoolEventDispatcher + + AsStackerDBEventDispatcher +{ +} +impl RelayEventDispatcher for T {} + +/// Trait for upcasting to MemPoolEventDispatcher +pub trait AsMemPoolEventDispatcher { + fn as_mempool_event_dispatcher(&self) -> &dyn MemPoolEventDispatcher; +} + +/// Trait for upcasting to StackerDBEventDispatcher +pub trait AsStackerDBEventDispatcher { + fn as_stackerdb_event_dispatcher(&self) -> &dyn StackerDBEventDispatcher; +} + +impl AsMemPoolEventDispatcher for T { + fn as_mempool_event_dispatcher(&self) -> &dyn MemPoolEventDispatcher { + self + } +} + +impl AsStackerDBEventDispatcher for T { + fn as_stackerdb_event_dispatcher(&self) -> &dyn StackerDBEventDispatcher { + self + } +} + /// Private trait for keeping track of messages that can be relayed, so we can identify the peers /// who frequently send us duplicates. pub trait RelayPayload { @@ -1704,30 +1737,58 @@ impl Relayer { } } + /// Process HTTP-uploaded stackerdb chunks. + /// They're already stored by the RPC handler, so just forward events for them. + pub fn process_uploaded_stackerdb_chunks( + uploaded_chunks: Vec, + event_observer: Option<&dyn StackerDBEventDispatcher>, + ) { + if let Some(observer) = event_observer { + let mut all_events: HashMap> = + HashMap::new(); + for chunk in uploaded_chunks.into_iter() { + debug!("Got uploaded StackerDB chunk"; "stackerdb_contract_id" => &format!("{}", &chunk.contract_id), "slot_id" => chunk.chunk_data.slot_id, "slot_version" => chunk.chunk_data.slot_version); + if let Some(events) = all_events.get_mut(&chunk.contract_id) { + events.push(chunk.chunk_data); + } else { + all_events.insert(chunk.contract_id.clone(), vec![chunk.chunk_data]); + } + } + for (contract_id, new_chunks) in all_events.into_iter() { + observer.new_stackerdb_chunks(contract_id, new_chunks); + } + } + } + /// Process newly-arrived chunks obtained from a peer stackerdb replica. pub fn process_stacker_db_chunks( stackerdbs: &mut StackerDBs, - stackerdb_configs: &HashMap, - sync_results: &[StackerDBSyncResult], + stackerdb_configs: &HashMap, + sync_results: Vec, + event_observer: Option<&dyn StackerDBEventDispatcher>, ) -> Result<(), Error> { // sort stacker results by contract, so as to minimize the number of transactions. - let mut sync_results_map: HashMap<&ContractId, Vec<&StackerDBSyncResult>> = HashMap::new(); - for sync_result in sync_results { - let sc = &sync_result.contract_id; - if let Some(result_list) = sync_results_map.get_mut(sc) { + let mut sync_results_map: HashMap> = + HashMap::new(); + for sync_result in sync_results.into_iter() { + let sc = sync_result.contract_id.clone(); + if let Some(result_list) = sync_results_map.get_mut(&sc) { result_list.push(sync_result); } else { sync_results_map.insert(sc, vec![sync_result]); } } - for (sc, sync_results) in sync_results_map.iter() { - if let Some(config) = stackerdb_configs.get(sc) { + let mut all_events: HashMap> = + HashMap::new(); + + for (sc, sync_results) in sync_results_map.into_iter() { + if let Some(config) = stackerdb_configs.get(&sc) { let tx = stackerdbs.tx_begin(config.clone())?; - for sync_result in sync_results { - for chunk in sync_result.chunks_to_store.iter() { + for sync_result in sync_results.into_iter() { + for chunk in sync_result.chunks_to_store.into_iter() { let md = chunk.get_slot_metadata(); - if let Err(e) = tx.try_replace_chunk(sc, &md, &chunk.data) { + if let Err(e) = tx.try_replace_chunk(&sc, &md, &chunk.data) { warn!( "Failed to store chunk for StackerDB"; "stackerdb_contract_id" => &format!("{}", &sync_result.contract_id), @@ -1739,6 +1800,12 @@ impl Relayer { } else { debug!("Stored chunk"; "stackerdb_contract_id" => &format!("{}", &sync_result.contract_id), "slot_id" => md.slot_id, "slot_version" => md.slot_version); } + + if let Some(event_list) = all_events.get_mut(&sync_result.contract_id) { + event_list.push(chunk); + } else { + all_events.insert(sync_result.contract_id.clone(), vec![chunk]); + } } } tx.commit()?; @@ -1747,6 +1814,11 @@ impl Relayer { } } + if let Some(observer) = event_observer.as_ref() { + for (contract_id, new_chunks) in all_events.into_iter() { + observer.new_stackerdb_chunks(contract_id, new_chunks); + } + } Ok(()) } @@ -1754,8 +1826,9 @@ impl Relayer { /// extract all StackerDBPushChunk messages from `unhandled_messages` pub fn process_pushed_stacker_db_chunks( stackerdbs: &mut StackerDBs, - stackerdb_configs: &HashMap, + stackerdb_configs: &HashMap, unhandled_messages: &mut HashMap>, + event_observer: Option<&dyn StackerDBEventDispatcher>, ) -> Result<(), Error> { // synthesize StackerDBSyncResults from each chunk let mut sync_results = vec![]; @@ -1771,7 +1844,12 @@ impl Relayer { }); } - Relayer::process_stacker_db_chunks(stackerdbs, stackerdb_configs, &sync_results) + Relayer::process_stacker_db_chunks( + stackerdbs, + stackerdb_configs, + sync_results, + event_observer, + ) } /// Given a network result, consume and store all data. @@ -1793,7 +1871,7 @@ impl Relayer { mempool: &mut MemPoolDB, ibd: bool, coord_comms: Option<&CoordinatorChannels>, - event_observer: Option<&dyn MemPoolEventDispatcher>, + event_observer: Option<&dyn RelayEventDispatcher>, ) -> Result { let mut num_new_blocks = 0; let mut num_new_confirmed_microblocks = 0; @@ -1893,7 +1971,7 @@ impl Relayer { sortdb, chainstate, mempool, - event_observer, + event_observer.map(|obs| obs.as_mempool_event_dispatcher()), )?; if new_txs.len() > 0 { @@ -1922,11 +2000,18 @@ impl Relayer { processed_unconfirmed_state = Relayer::refresh_unconfirmed(chainstate, sortdb); } + // push events for HTTP-uploaded stacker DB chunks + Relayer::process_uploaded_stackerdb_chunks( + mem::replace(&mut network_result.uploaded_stackerdb_chunks, vec![]), + event_observer.map(|obs| obs.as_stackerdb_event_dispatcher()), + ); + // store downloaded stacker DB chunks Relayer::process_stacker_db_chunks( &mut self.stacker_dbs, &network_result.stacker_db_configs, - &network_result.stacker_db_sync_results, + mem::replace(&mut network_result.stacker_db_sync_results, vec![]), + event_observer.map(|obs| obs.as_stackerdb_event_dispatcher()), )?; // store pushed stacker DB chunks @@ -1934,6 +2019,7 @@ impl Relayer { &mut self.stacker_dbs, &network_result.stacker_db_configs, &mut network_result.unhandled_messages, + event_observer.map(|obs| obs.as_stackerdb_event_dispatcher()), )?; let receipts = ProcessedNetReceipts { diff --git a/stackslib/src/net/rpc.rs b/stackslib/src/net/rpc.rs index eaa537a84b..6677085267 100644 --- a/stackslib/src/net/rpc.rs +++ b/stackslib/src/net/rpc.rs @@ -1,21 +1,18 @@ -/* - copyright: (c) 2013-2020 by Blockstack PBC, a public benefit corporation. - - This file is part of Blockstack. - - Blockstack is free software. You may redistribute or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License or - (at your option) any later version. - - Blockstack is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY, including without the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with Blockstack. If not, see . -*/ +// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . use std::collections::HashMap; use std::collections::HashSet; @@ -38,9 +35,7 @@ use crate::burnchains::*; use crate::chainstate::burn::db::sortdb::SortitionDB; use crate::chainstate::burn::ConsensusHash; use crate::chainstate::stacks::db::blocks::CheckError; -use crate::chainstate::stacks::db::{ - blocks::MINIMUM_TX_FEE_RATE_PER_BYTE, StacksChainState, StreamCursor, -}; +use crate::chainstate::stacks::db::{blocks::MINIMUM_TX_FEE_RATE_PER_BYTE, StacksChainState}; use crate::chainstate::stacks::Error as chain_error; use crate::chainstate::stacks::*; use crate::clarity_vm::clarity::ClarityConnection; @@ -59,6 +54,8 @@ use crate::net::http::*; use crate::net::p2p::PeerMap; use crate::net::p2p::PeerNetwork; use crate::net::relay::Relayer; +use crate::net::stackerdb::StackerDBTx; +use crate::net::stackerdb::StackerDBs; use crate::net::BlocksDatum; use crate::net::Error as net_error; use crate::net::HttpRequestMetadata; @@ -74,9 +71,11 @@ use crate::net::PeerHost; use crate::net::ProtocolFamily; use crate::net::RPCFeeEstimate; use crate::net::RPCFeeEstimateResponse; +use crate::net::StackerDBPushChunkData; use crate::net::StacksHttp; use crate::net::StacksHttpMessage; use crate::net::StacksMessageType; +use crate::net::StreamCursor; use crate::net::UnconfirmedTransactionResponse; use crate::net::UnconfirmedTransactionStatus; use crate::net::UrlString; @@ -114,6 +113,7 @@ use clarity::vm::{ types::{PrincipalData, QualifiedContractIdentifier, StandardPrincipalData}, ClarityName, ContractName, SymbolicExpression, Value, }; +use libstackerdb::{StackerDBChunkAckData, StackerDBChunkData}; use stacks_common::util::get_epoch_time_secs; use stacks_common::util::hash::Hash160; use stacks_common::util::hash::{hex_bytes, to_hex}; @@ -124,6 +124,7 @@ use crate::clarity_vm::database::marf::MarfedKV; use stacks_common::types::chainstate::BlockHeaderHash; use stacks_common::types::chainstate::{BurnchainHeaderHash, StacksAddress, StacksBlockId}; use stacks_common::types::StacksPublicKeyBuffer; +use stacks_common::util::secp256k1::MessageSignature; use crate::clarity_vm::clarity::Error as clarity_error; @@ -242,6 +243,7 @@ impl RPCPeerInfoData { let public_key = StacksPublicKey::from_private(&network.local_peer.private_key); let public_key_buf = StacksPublicKeyBuffer::from_public_key(&public_key); let public_key_hash = Hash160::from_node_public_key(&public_key); + let stackerdb_contract_ids = network.get_local_peer().stacker_dbs.clone(); RPCPeerInfoData { peer_version: network.burnchain.peer_version, @@ -274,6 +276,12 @@ impl RPCPeerInfoData { anchor_block_hash: network.last_anchor_block_hash.clone(), anchor_block_txid: network.last_anchor_block_txid.clone(), }), + stackerdbs: Some( + stackerdb_contract_ids + .into_iter() + .map(|cid| format!("{}", cid)) + .collect(), + ), } } } @@ -556,10 +564,12 @@ impl RPCNeighborsInfo { let bootstrap = bootstrap_nodes .into_iter() .map(|n| { + let stackerdb_contract_ids = peerdb.get_peer_stacker_dbs(&n).unwrap_or(vec![]); RPCNeighbor::from_neighbor_key_and_pubkh( n.addr.clone(), Hash160::from_node_public_key(&n.public_key), true, + stackerdb_contract_ids, ) }) .collect(); @@ -578,10 +588,12 @@ impl RPCNeighborsInfo { let sample: Vec = neighbor_sample .into_iter() .map(|n| { + let stackerdb_contract_ids = peerdb.get_peer_stacker_dbs(&n).unwrap_or(vec![]); RPCNeighbor::from_neighbor_key_and_pubkh( n.addr.clone(), Hash160::from_node_public_key(&n.public_key), true, + stackerdb_contract_ids, ) }) .collect(); @@ -596,12 +608,14 @@ impl RPCNeighborsInfo { nk, naddr.public_key_hash, convo.is_authenticated(), + convo.get_stackerdb_contract_ids().to_vec(), )); } else { inbound.push(RPCNeighbor::from_neighbor_key_and_pubkh( nk, naddr.public_key_hash, convo.is_authenticated(), + convo.get_stackerdb_contract_ids().to_vec(), )); } } @@ -2450,6 +2464,175 @@ impl ConversationHttp { response.send(http, fd).and_then(|_| Ok(stream)) } + /// Handle a request for the vector of stacker DB slot metadata + fn handle_get_stackerdb_metadata( + http: &mut StacksHttp, + fd: &mut W, + req: &HttpRequestType, + stackerdbs: &StackerDBs, + stackerdb_contract_id: &QualifiedContractIdentifier, + canonical_stacks_tip_height: u64, + ) -> Result<(), net_error> { + let response_metadata = + HttpResponseMetadata::from_http_request_type(req, Some(canonical_stacks_tip_height)); + + let response = if let Ok(slots) = stackerdbs.get_db_slot_metadata(stackerdb_contract_id) { + HttpResponseType::StackerDBMetadata(response_metadata, slots) + } else { + HttpResponseType::NotFound(response_metadata, "No such StackerDB contract".into()) + }; + + return response.send(http, fd).map(|_| ()); + } + + /// Handle a request for a stacker DB chunk, optionally with a given version + fn handle_get_stackerdb_chunk( + http: &mut StacksHttp, + fd: &mut W, + req: &HttpRequestType, + stackerdbs: &StackerDBs, + stackerdb_contract_id: &QualifiedContractIdentifier, + slot_id: u32, + slot_version: Option, + canonical_stacks_tip_height: u64, + ) -> Result<(), net_error> { + let response_metadata = + HttpResponseMetadata::from_http_request_type(req, Some(canonical_stacks_tip_height)); + + let chunk_res = if let Some(version) = slot_version.as_ref() { + stackerdbs + .get_chunk(stackerdb_contract_id, slot_id, *version) + .map(|chunk_data| chunk_data.map(|chunk_data| chunk_data.data)) + } else { + stackerdbs.get_latest_chunk(stackerdb_contract_id, slot_id) + }; + + let response = match chunk_res { + Ok(Some(chunk)) => HttpResponseType::StackerDBChunk(response_metadata, chunk), + Ok(None) | Err(net_error::NoSuchStackerDB(..)) => { + // not found + HttpResponseType::NotFound(response_metadata, "No such StackerDB chunk".into()) + } + Err(e) => { + // some other error + error!("Failed to load StackerDB chunk"; + "smart_contract_id" => stackerdb_contract_id.to_string(), + "slot_id" => slot_id, + "slot_version" => slot_version, + "error" => format!("{:?}", &e) + ); + HttpResponseType::ServerError( + response_metadata, + format!("Failed to load StackerDB chunk"), + ) + } + }; + + return response.send(http, fd).map(|_| ()); + } + + /// Handle a post for a new StackerDB chunk. + /// If we accept it, then forward it to the relayer as well + /// so an event can be generated for it. + fn handle_post_stackerdb_chunk( + http: &mut StacksHttp, + fd: &mut W, + req: &HttpRequestType, + tx: &mut StackerDBTx, + stackerdb_contract_id: &QualifiedContractIdentifier, + stackerdb_chunk: &StackerDBChunkData, + canonical_stacks_tip_height: u64, + ) -> Result, net_error> { + let response_metadata = + HttpResponseMetadata::from_http_request_type(req, Some(canonical_stacks_tip_height)); + + if let Err(_e) = tx.get_stackerdb_id(stackerdb_contract_id) { + // shouldn't be necessary (this is checked against the peer network's configured DBs), + // but you never know. + let resp = HttpResponseType::NotFound(response_metadata, "No such StackerDB".into()); + return resp.send(http, fd).and_then(|_| Ok(None)); + } + if let Err(_e) = tx.try_replace_chunk( + stackerdb_contract_id, + &stackerdb_chunk.get_slot_metadata(), + &stackerdb_chunk.data, + ) { + let slot_metadata_opt = + match tx.get_slot_metadata(stackerdb_contract_id, stackerdb_chunk.slot_id) { + Ok(slot_opt) => slot_opt, + Err(e) => { + // some other error + error!("Failed to load replaced StackerDB chunk metadata"; + "smart_contract_id" => stackerdb_contract_id.to_string(), + "error" => format!("{:?}", &e) + ); + let resp = HttpResponseType::ServerError( + response_metadata, + format!("Failed to load StackerDB chunk"), + ); + return resp.send(http, fd).and_then(|_| Ok(None)); + } + }; + + let (reason, slot_metadata_opt) = if let Some(slot_metadata) = slot_metadata_opt { + ( + format!("Data for this slot and version already exist"), + Some(slot_metadata), + ) + } else { + ( + format!( + "{:?}", + net_error::NoSuchSlot( + stackerdb_contract_id.clone(), + stackerdb_chunk.slot_id + ) + ), + None, + ) + }; + + let ack = StackerDBChunkAckData { + accepted: false, + reason: Some(reason), + metadata: slot_metadata_opt, + }; + let resp = HttpResponseType::StackerDBChunkAck(response_metadata, ack); + return resp.send(http, fd).and_then(|_| Ok(None)); + } + + let slot_metadata = if let Some(md) = + tx.get_slot_metadata(stackerdb_contract_id, stackerdb_chunk.slot_id)? + { + md + } else { + // shouldn't be reachable + let resp = HttpResponseType::ServerError( + response_metadata, + format!("Failed to load slot metadata after storing chunk"), + ); + return resp.send(http, fd).and_then(|_| Ok(None)); + }; + + // success! + let ack = StackerDBChunkAckData { + accepted: true, + reason: None, + metadata: Some(slot_metadata), + }; + + let resp = HttpResponseType::StackerDBChunkAck(response_metadata, ack); + return resp.send(http, fd).and_then(|_| { + Ok(Some(StacksMessageType::StackerDBPushChunk( + StackerDBPushChunkData { + contract_id: stackerdb_contract_id.clone(), + rc_consensus_hash: ConsensusHash([0u8; 20]), // unused, + chunk_data: stackerdb_chunk.clone(), + }, + ))) + }); + } + /// Handle an external HTTP request. /// Some requests, such as those for blocks, will create new reply streams. This method adds /// those new streams into the `reply_streams` set. @@ -3022,6 +3205,62 @@ impl ConversationHttp { } None } + HttpRequestType::GetStackerDBMetadata(ref _md, ref stackerdb_contract_id) => { + ConversationHttp::handle_get_stackerdb_metadata( + &mut self.connection.protocol, + &mut reply, + &req, + network.get_stackerdbs(), + stackerdb_contract_id, + network.burnchain_tip.canonical_stacks_tip_height, + )?; + None + } + HttpRequestType::GetStackerDBChunk( + ref _md, + ref stackerdb_contract_id, + ref slot_id, + ref slot_version_opt, + ) => { + ConversationHttp::handle_get_stackerdb_chunk( + &mut self.connection.protocol, + &mut reply, + &req, + network.get_stackerdbs(), + stackerdb_contract_id, + *slot_id, + slot_version_opt.clone(), + network.burnchain_tip.canonical_stacks_tip_height, + )?; + None + } + HttpRequestType::PostStackerDBChunk( + ref _md, + ref stackerdb_contract_id, + ref chunk_data, + ) => { + let tip_height = network.burnchain_tip.canonical_stacks_tip_height; + if let Ok(mut tx) = network.stackerdbs_tx_begin(stackerdb_contract_id) { + ret = ConversationHttp::handle_post_stackerdb_chunk( + &mut self.connection.protocol, + &mut reply, + &req, + &mut tx, + stackerdb_contract_id, + chunk_data, + tip_height, + )?; + tx.commit()?; + } else { + let response_metadata = + HttpResponseMetadata::from_http_request_type(&req, Some(tip_height)); + let resp = + HttpResponseType::NotFound(response_metadata, "No such StackerDB".into()); + resp.send(&mut self.connection.protocol, &mut reply) + .map(|_| ())?; + }; + None + } HttpRequestType::ClientError(ref _md, ref err) => { let response_metadata = HttpResponseMetadata::from_http_request_type( &req, @@ -3704,6 +3943,53 @@ impl ConversationHttp { page_id_opt, ) } + + /// Make a request for a stackerDB's metadata + pub fn new_get_stackerdb_metadata( + &self, + stackerdb_contract_id: QualifiedContractIdentifier, + ) -> HttpRequestType { + HttpRequestType::GetStackerDBMetadata( + HttpRequestMetadata::from_host(self.peer_host.clone(), None), + stackerdb_contract_id, + ) + } + + /// Make a request for a stackerDB's chunk + pub fn new_get_stackerdb_chunk( + &self, + stackerdb_contract_id: QualifiedContractIdentifier, + slot_id: u32, + slot_version: Option, + ) -> HttpRequestType { + HttpRequestType::GetStackerDBChunk( + HttpRequestMetadata::from_host(self.peer_host.clone(), None), + stackerdb_contract_id, + slot_id, + slot_version, + ) + } + + /// Make a new post for a stackerDB's chunk + pub fn new_post_stackerdb_chunk( + &self, + stackerdb_contract_id: QualifiedContractIdentifier, + slot_id: u32, + slot_version: u32, + sig: MessageSignature, + data: Vec, + ) -> HttpRequestType { + HttpRequestType::PostStackerDBChunk( + HttpRequestMetadata::from_host(self.peer_host.clone(), None), + stackerdb_contract_id, + StackerDBChunkData { + slot_id, + slot_version, + sig, + data, + }, + ) + } } #[cfg(test)] @@ -3719,19 +4005,21 @@ mod test { use crate::chainstate::burn::ConsensusHash; use crate::chainstate::stacks::db::blocks::test::*; use crate::chainstate::stacks::db::StacksChainState; - use crate::chainstate::stacks::db::StreamCursor; use crate::chainstate::stacks::miner::*; use crate::chainstate::stacks::test::*; use crate::chainstate::stacks::Error as chain_error; use crate::chainstate::stacks::*; use crate::net::codec::*; use crate::net::http::*; + use crate::net::stream::*; use crate::net::test::*; use crate::net::*; use clarity::vm::types::*; + use libstackerdb::SlotMetadata; + use libstackerdb::STACKERDB_MAX_CHUNK_SIZE; use stacks_common::address::*; use stacks_common::util::get_epoch_time_secs; - use stacks_common::util::hash::hex_bytes; + use stacks_common::util::hash::{hex_bytes, Sha512Trunc256Sum}; use stacks_common::util::pipe::*; use crate::chainstate::stacks::C32_ADDRESS_VERSION_TESTNET_SINGLESIG; @@ -3755,7 +4043,29 @@ mod test { (var-set bar 1) (ok 1))) (begin - (map-set unit-map { account: 'ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R } { units: 123 }))"; + (map-set unit-map { account: 'ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R } { units: 123 })) + + ;; stacker DB + (define-read-only (stackerdb-get-signer-slots) + (ok (list + { + signer: 'ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R, + num-slots: u3 + } + { + signer: 'STVN97YYA10MY5F6KQJHKNYJNM24C4A1AT39WRW, + num-slots: u3 + }))) + + (define-read-only (stackerdb-get-config) + (ok { + chunk-size: u4096, + write-freq: u0, + max-writes: u4096, + max-neighbors: u32, + hint-replicas: (list ) + })) + "; const TEST_CONTRACT_UNCONFIRMED: &'static str = "(define-read-only (ro-test) (ok 1))"; @@ -3836,12 +4146,6 @@ mod test { &ConversationHttp, ) -> bool, { - let mut peer_1_config = TestPeerConfig::new(test_name, peer_1_p2p, peer_1_http); - let mut peer_2_config = TestPeerConfig::new(test_name, peer_2_p2p, peer_2_http); - - let peer_1_indexer = BitcoinIndexer::new_unit_test(&peer_1_config.burnchain.working_dir); - let peer_2_indexer = BitcoinIndexer::new_unit_test(&peer_2_config.burnchain.working_dir); - // ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R let privk1 = StacksPrivateKey::from_hex( "9f1f85a512a96a244e4c0d762788500687feb97481639572e3bffbd6860e6ab001", @@ -3872,6 +4176,22 @@ mod test { ) .unwrap(); + let mut peer_1_config = TestPeerConfig::new(test_name, peer_1_p2p, peer_1_http); + let mut peer_2_config = TestPeerConfig::new(test_name, peer_2_p2p, peer_2_http); + + // stacker DBs get initialized thru reconfiguration when the above block gets processed + peer_1_config.add_stacker_db( + QualifiedContractIdentifier::new(addr1.clone().into(), "hello-world".into()), + StackerDBConfig::noop(), + ); + peer_2_config.add_stacker_db( + QualifiedContractIdentifier::new(addr1.clone().into(), "hello-world".into()), + StackerDBConfig::noop(), + ); + + let peer_1_indexer = BitcoinIndexer::new_unit_test(&peer_1_config.burnchain.working_dir); + let peer_2_indexer = BitcoinIndexer::new_unit_test(&peer_2_config.burnchain.working_dir); + peer_1_config.initial_balances = vec![ (addr1.to_account_principal(), 1000000000), (addr2.to_account_principal(), 1000000000), @@ -4163,13 +4483,13 @@ mod test { peer_2.mempool.replace(mempool); let peer_1_sortdb = peer_1.sortdb.take().unwrap(); - let peer_1_stacks_node = peer_1.stacks_node.take().unwrap(); + let mut peer_1_stacks_node = peer_1.stacks_node.take().unwrap(); let _ = peer_1 .network .refresh_burnchain_view( &peer_1_indexer, &peer_1_sortdb, - &peer_1_stacks_node.chainstate, + &mut peer_1_stacks_node.chainstate, false, ) .unwrap(); @@ -4177,13 +4497,13 @@ mod test { peer_1.stacks_node = Some(peer_1_stacks_node); let peer_2_sortdb = peer_2.sortdb.take().unwrap(); - let peer_2_stacks_node = peer_2.stacks_node.take().unwrap(); + let mut peer_2_stacks_node = peer_2.stacks_node.take().unwrap(); let _ = peer_2 .network .refresh_burnchain_view( &peer_2_indexer, &peer_2_sortdb, - &peer_2_stacks_node.chainstate, + &mut peer_2_stacks_node.chainstate, false, ) .unwrap(); @@ -4263,7 +4583,7 @@ mod test { .refresh_burnchain_view( &peer_2_indexer, &peer_2_sortdb, - &peer_2_stacks_node.chainstate, + &mut peer_2_stacks_node.chainstate, false, ) .unwrap(); @@ -4316,7 +4636,7 @@ mod test { .refresh_burnchain_view( &peer_1_indexer, &peer_1_sortdb, - &peer_1_stacks_node.chainstate, + &mut peer_1_stacks_node.chainstate, false, ) .unwrap(); @@ -4416,6 +4736,7 @@ mod test { .unwrap() )) ); + assert!(peer_data.stackerdbs.is_some()); true } _ => { @@ -4572,6 +4893,18 @@ mod test { neighbor_info.bootstrap[0].port, peer_client.config.server_port ); // we see ourselves as the bootstrap + for n in neighbor_info.sample.iter() { + assert!(n.stackerdbs.is_some()); + } + for n in neighbor_info.bootstrap.iter() { + assert!(n.stackerdbs.is_some()); + } + for n in neighbor_info.inbound.iter() { + assert!(n.stackerdbs.is_some()); + } + for n in neighbor_info.outbound.iter() { + assert!(n.stackerdbs.is_some()); + } true } _ => { @@ -6671,6 +7004,590 @@ mod test { ); } + #[test] + #[ignore] + fn test_rpc_get_stackerdb_metadata() { + test_rpc( + function_name!(), + 40817, + 40818, + 50815, + 50816, + false, + |ref mut peer_client, + ref mut convo_client, + ref mut peer_server, + ref mut convo_server| { + convo_client.new_get_stackerdb_metadata( + QualifiedContractIdentifier::parse( + "ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world", + ) + .unwrap(), + ) + }, + |ref http_request, + ref http_response, + ref mut peer_client, + ref mut peer_server, + ref convo_client, + ref convo_server| { + let req_md = http_request.metadata().clone(); + println!("{:?}", http_response); + match http_response { + HttpResponseType::StackerDBMetadata(_, metadata) => { + // config was updated + assert_eq!(metadata.len(), 6); + for (i, slot) in metadata.iter().enumerate() { + assert_eq!(slot.slot_id, i as u32); + assert_eq!(slot.slot_version, 0); + assert_eq!(slot.data_hash, Sha512Trunc256Sum([0u8; 32])); + assert_eq!(slot.signature, MessageSignature::empty()); + } + true + } + _ => false, + } + }, + ) + } + + #[test] + #[ignore] + fn test_rpc_get_stackerdb_versioned_chunk() { + test_rpc( + function_name!(), + 40819, + 40820, + 50817, + 50818, + false, + |ref mut peer_client, + ref mut convo_client, + ref mut peer_server, + ref mut convo_server| { + debug!("Set up peer stackerDB"); + // insert a value in slot 0 + let contract_id = QualifiedContractIdentifier::parse( + "ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world", + ) + .unwrap(); + let privk1 = StacksPrivateKey::from_hex( + "9f1f85a512a96a244e4c0d762788500687feb97481639572e3bffbd6860e6ab001", + ) + .unwrap(); + + let data = "hello world".as_bytes(); + let data_hash = Sha512Trunc256Sum::from_data(data); + let mut slot_metadata = SlotMetadata::new_unsigned(0, 1, data_hash); + slot_metadata.sign(&privk1).unwrap(); + + let tx = peer_server + .network + .stackerdbs + .tx_begin(StackerDBConfig::noop()) + .unwrap(); + tx.try_replace_chunk(&contract_id, &slot_metadata, "hello world".as_bytes()) + .unwrap(); + tx.commit().unwrap(); + + // now go ask for it + convo_client.new_get_stackerdb_chunk(contract_id, 0, Some(1)) + }, + |ref http_request, + ref http_response, + ref mut peer_client, + ref mut peer_server, + ref convo_client, + ref convo_server| { + let req_md = http_request.metadata().clone(); + println!("{:?}", http_response); + match http_response { + HttpResponseType::StackerDBChunk(_, chunk_data) => { + assert_eq!(chunk_data, "hello world".as_bytes()); + true + } + _ => false, + } + }, + ) + } + + #[test] + #[ignore] + fn test_rpc_get_stackerdb_latest_chunk() { + test_rpc( + function_name!(), + 40821, + 40822, + 50819, + 50820, + false, + |ref mut peer_client, + ref mut convo_client, + ref mut peer_server, + ref mut convo_server| { + debug!("Set up peer stackerDB"); + // insert a value in slot 0 + let contract_id = QualifiedContractIdentifier::parse( + "ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world", + ) + .unwrap(); + let privk1 = StacksPrivateKey::from_hex( + "9f1f85a512a96a244e4c0d762788500687feb97481639572e3bffbd6860e6ab001", + ) + .unwrap(); + + let data = "hello world".as_bytes(); + let data_hash = Sha512Trunc256Sum::from_data(data); + let mut slot_metadata = SlotMetadata::new_unsigned(0, 1, data_hash); + slot_metadata.sign(&privk1).unwrap(); + + let tx = peer_server + .network + .stackerdbs + .tx_begin(StackerDBConfig::noop()) + .unwrap(); + tx.try_replace_chunk(&contract_id, &slot_metadata, "hello world".as_bytes()) + .unwrap(); + tx.commit().unwrap(); + + // now go ask for it + convo_client.new_get_stackerdb_chunk(contract_id, 0, None) + }, + |ref http_request, + ref http_response, + ref mut peer_client, + ref mut peer_server, + ref convo_client, + ref convo_server| { + let req_md = http_request.metadata().clone(); + println!("{:?}", http_response); + match http_response { + HttpResponseType::StackerDBChunk(_, chunk_data) => { + assert_eq!(chunk_data, "hello world".as_bytes()); + true + } + _ => false, + } + }, + ) + } + + #[test] + #[ignore] + fn test_rpc_get_stackerdb_nonexistant_chunk() { + test_rpc( + function_name!(), + 40821, + 40822, + 50819, + 50820, + false, + |ref mut peer_client, + ref mut convo_client, + ref mut peer_server, + ref mut convo_server| { + debug!("Set up peer stackerDB"); + // insert a value in slot 0 + let contract_id = QualifiedContractIdentifier::parse( + "ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world", + ) + .unwrap(); + let privk1 = StacksPrivateKey::from_hex( + "9f1f85a512a96a244e4c0d762788500687feb97481639572e3bffbd6860e6ab001", + ) + .unwrap(); + + let data = "hello world".as_bytes(); + let data_hash = Sha512Trunc256Sum::from_data(data); + let mut slot_metadata = SlotMetadata::new_unsigned(0, 1, data_hash); + slot_metadata.sign(&privk1).unwrap(); + + let tx = peer_server + .network + .stackerdbs + .tx_begin(StackerDBConfig::noop()) + .unwrap(); + tx.try_replace_chunk(&contract_id, &slot_metadata, "hello world".as_bytes()) + .unwrap(); + tx.commit().unwrap(); + + // now go ask for it + convo_client.new_get_stackerdb_chunk(contract_id, 0, Some(2)) + }, + |ref http_request, + ref http_response, + ref mut peer_client, + ref mut peer_server, + ref convo_client, + ref convo_server| { + let req_md = http_request.metadata().clone(); + println!("{:?}", http_response); + match http_response { + HttpResponseType::NotFound(..) => true, + _ => false, + } + }, + ) + } + + #[test] + #[ignore] + fn test_rpc_get_stackerdb_nonexistant_db() { + test_rpc( + function_name!(), + 40823, + 40824, + 50821, + 50822, + false, + |ref mut peer_client, + ref mut convo_client, + ref mut peer_server, + ref mut convo_server| { + debug!("Set up peer stackerDB"); + // insert a value in slot 0 + let contract_id = QualifiedContractIdentifier::parse( + "ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world", + ) + .unwrap(); + let privk1 = StacksPrivateKey::from_hex( + "9f1f85a512a96a244e4c0d762788500687feb97481639572e3bffbd6860e6ab001", + ) + .unwrap(); + + let data = "hello world".as_bytes(); + let data_hash = Sha512Trunc256Sum::from_data(data); + let mut slot_metadata = SlotMetadata::new_unsigned(0, 1, data_hash); + slot_metadata.sign(&privk1).unwrap(); + + let tx = peer_server + .network + .stackerdbs + .tx_begin(StackerDBConfig::noop()) + .unwrap(); + tx.try_replace_chunk(&contract_id, &slot_metadata, "hello world".as_bytes()) + .unwrap(); + tx.commit().unwrap(); + + // now go ask for it, but from the wrong contract + let contract_id = QualifiedContractIdentifier::parse( + "ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.nope", + ) + .unwrap(); + convo_client.new_get_stackerdb_chunk(contract_id, 0, None) + }, + |ref http_request, + ref http_response, + ref mut peer_client, + ref mut peer_server, + ref convo_client, + ref convo_server| { + let req_md = http_request.metadata().clone(); + println!("{:?}", http_response); + match http_response { + HttpResponseType::NotFound(..) => true, + _ => false, + } + }, + ) + } + + #[test] + #[ignore] + fn test_rpc_post_stackerdb_chunk() { + test_rpc( + function_name!(), + 40823, + 40824, + 50821, + 50822, + false, + |ref mut peer_client, + ref mut convo_client, + ref mut peer_server, + ref mut convo_server| { + // insert a value in slot 0 + let contract_id = QualifiedContractIdentifier::parse( + "ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world", + ) + .unwrap(); + let privk1 = StacksPrivateKey::from_hex( + "9f1f85a512a96a244e4c0d762788500687feb97481639572e3bffbd6860e6ab001", + ) + .unwrap(); + + let data = "hello world".as_bytes(); + let data_hash = Sha512Trunc256Sum::from_data(data); + let mut slot_metadata = SlotMetadata::new_unsigned(0, 1, data_hash); + slot_metadata.sign(&privk1).unwrap(); + + convo_client.new_post_stackerdb_chunk( + contract_id, + slot_metadata.slot_id, + slot_metadata.slot_version, + slot_metadata.signature, + data.to_vec(), + ) + }, + |ref http_request, + ref http_response, + ref mut peer_client, + ref mut peer_server, + ref convo_client, + ref convo_server| { + let req_md = http_request.metadata().clone(); + println!("{:?}", http_response); + match http_response { + HttpResponseType::StackerDBChunkAck(_, ack) => { + assert!(ack.accepted); + assert!(ack.metadata.is_some()); + + let md = ack.metadata.clone().unwrap(); + + assert_eq!(md.slot_id, 0); + assert_eq!(md.slot_version, 1); + + let data = "hello world".as_bytes(); + let data_hash = Sha512Trunc256Sum::from_data(data); + assert_eq!(md.data_hash, data_hash); + + // server actually has it + let contract_id = QualifiedContractIdentifier::parse( + "ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world", + ) + .unwrap(); + assert_eq!( + peer_server + .network + .stackerdbs + .get_latest_chunk(&contract_id, 0) + .unwrap() + .unwrap(), + "hello world".as_bytes() + ); + true + } + _ => false, + } + }, + ) + } + + #[test] + #[ignore] + fn test_rpc_post_stale_stackerdb_chunk() { + test_rpc( + function_name!(), + 40825, + 40826, + 50823, + 50824, + false, + |ref mut peer_client, + ref mut convo_client, + ref mut peer_server, + ref mut convo_server| { + // insert a value in slot 0 + let contract_id = QualifiedContractIdentifier::parse( + "ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world", + ) + .unwrap(); + let privk1 = StacksPrivateKey::from_hex( + "9f1f85a512a96a244e4c0d762788500687feb97481639572e3bffbd6860e6ab001", + ) + .unwrap(); + + let data = "hello world".as_bytes(); + let data_hash = Sha512Trunc256Sum::from_data(data); + let mut slot_metadata = SlotMetadata::new_unsigned(0, 1, data_hash); + slot_metadata.sign(&privk1).unwrap(); + + let tx = peer_server + .network + .stackerdbs + .tx_begin(StackerDBConfig::noop()) + .unwrap(); + tx.try_replace_chunk(&contract_id, &slot_metadata, "hello world".as_bytes()) + .unwrap(); + tx.commit().unwrap(); + + // conflicting data + let conflict_data = "conflict".as_bytes(); + let conflict_data_hash = Sha512Trunc256Sum::from_data(conflict_data); + let mut conflict_slot_metadata = + SlotMetadata::new_unsigned(0, 1, conflict_data_hash); + conflict_slot_metadata.sign(&privk1).unwrap(); + + convo_client.new_post_stackerdb_chunk( + contract_id, + conflict_slot_metadata.slot_id, + conflict_slot_metadata.slot_version, + conflict_slot_metadata.signature, + data.to_vec(), + ) + }, + |ref http_request, + ref http_response, + ref mut peer_client, + ref mut peer_server, + ref convo_client, + ref convo_server| { + let req_md = http_request.metadata().clone(); + println!("{:?}", http_response); + match http_response { + HttpResponseType::StackerDBChunkAck(_, ack) => { + assert!(!ack.accepted); + assert!(ack.reason.is_some()); + assert!(ack.metadata.is_some()); + + let md = ack.metadata.clone().unwrap(); + + assert_eq!(md.slot_id, 0); + assert_eq!(md.slot_version, 1); + + let data = "hello world".as_bytes(); + let data_hash = Sha512Trunc256Sum::from_data(data); + assert_eq!(md.data_hash, data_hash); + + // server actually has it + let contract_id = QualifiedContractIdentifier::parse( + "ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world", + ) + .unwrap(); + assert_eq!( + peer_server + .network + .stackerdbs + .get_latest_chunk(&contract_id, 0) + .unwrap() + .unwrap(), + "hello world".as_bytes() + ); + true + } + _ => false, + } + }, + ) + } + + #[test] + #[ignore] + fn test_rpc_post_nonexistant_stackerdb_chunk() { + test_rpc( + function_name!(), + 40827, + 40828, + 50825, + 50826, + false, + |ref mut peer_client, + ref mut convo_client, + ref mut peer_server, + ref mut convo_server| { + // insert a value in slot 0 + let contract_id = QualifiedContractIdentifier::parse( + "ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world", + ) + .unwrap(); + let privk1 = StacksPrivateKey::from_hex( + "9f1f85a512a96a244e4c0d762788500687feb97481639572e3bffbd6860e6ab001", + ) + .unwrap(); + + let data = "hello world".as_bytes(); + let data_hash = Sha512Trunc256Sum::from_data(data); + let mut slot_metadata = SlotMetadata::new_unsigned(0, 1, data_hash); + slot_metadata.sign(&privk1).unwrap(); + + // ... but for the wrong DB + let contract_id = QualifiedContractIdentifier::parse( + "ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.nope", + ) + .unwrap(); + convo_client.new_post_stackerdb_chunk( + contract_id, + slot_metadata.slot_id, + slot_metadata.slot_version, + slot_metadata.signature, + data.to_vec(), + ) + }, + |ref http_request, + ref http_response, + ref mut peer_client, + ref mut peer_server, + ref convo_client, + ref convo_server| { + let req_md = http_request.metadata().clone(); + println!("{:?}", http_response); + match http_response { + HttpResponseType::NotFound(..) => true, + _ => false, + } + }, + ) + } + + #[test] + #[ignore] + fn test_rpc_post_overflow_stackerdb_chunk() { + test_rpc( + function_name!(), + 40829, + 40830, + 50827, + 50828, + false, + |ref mut peer_client, + ref mut convo_client, + ref mut peer_server, + ref mut convo_server| { + // insert a value in slot 0 + let contract_id = QualifiedContractIdentifier::parse( + "ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R.hello-world", + ) + .unwrap(); + let privk1 = StacksPrivateKey::from_hex( + "9f1f85a512a96a244e4c0d762788500687feb97481639572e3bffbd6860e6ab001", + ) + .unwrap(); + + let data = "hello world".as_bytes(); + let data_hash = Sha512Trunc256Sum::from_data(data); + + // invalid slot! + let mut slot_metadata = SlotMetadata::new_unsigned(100000, 1, data_hash); + slot_metadata.sign(&privk1).unwrap(); + + convo_client.new_post_stackerdb_chunk( + contract_id, + slot_metadata.slot_id, + slot_metadata.slot_version, + slot_metadata.signature, + data.to_vec(), + ) + }, + |ref http_request, + ref http_response, + ref mut peer_client, + ref mut peer_server, + ref convo_client, + ref convo_server| { + match http_response { + HttpResponseType::StackerDBChunkAck(_, ack) => { + assert!(!ack.accepted); + assert!(ack.reason.is_some()); + assert!(ack.metadata.is_none()); + true + } + _ => false, + } + }, + ) + } + #[test] fn test_getinfo_compat() { let old_getinfo_json = r#"{"peer_version":402653189,"pox_consensus":"b712eb731b613eebae814a8f416c5c15bc8391ec","burn_block_height":727631,"stable_pox_consensus":"53b5ed79842080500d7d83daa36aa1069dedf983","stable_burn_block_height":727624,"server_version":"stacks-node 0.0.1 (feat/faster-inv-generation:68f33190a, release build, linux [x86_64])","network_id":1,"parent_network_id":3652501241,"stacks_tip_height":52537,"stacks_tip":"b3183f2ac588e12319ff0fde78f97e62c92a218d87828c35710c29aaf7adbedc","stacks_tip_consensus_hash":"b712eb731b613eebae814a8f416c5c15bc8391ec","genesis_chainstate_hash":"74237aa39aa50a83de11a4f53e9d3bb7d43461d1de9873f402e5453ae60bc59b","unanchored_tip":"e76f68d607480e9984b4062b2691fb60a88423177898f5780b40ace17ae8982a","unanchored_seq":0,"exit_at_block_height":null}"#; diff --git a/stackslib/src/net/server.rs b/stackslib/src/net/server.rs index 6aea1396c7..466a33d0b0 100644 --- a/stackslib/src/net/server.rs +++ b/stackslib/src/net/server.rs @@ -1,21 +1,18 @@ -/* - copyright: (c) 2013-2020 by Blockstack PBC, a public benefit corporation. - - This file is part of Blockstack. - - Blockstack is free software. You may redistribute or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License or - (at your option) any later version. - - Blockstack is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY, including without the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with Blockstack. If not, see . -*/ +// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . use std::io::Error as io_error; use std::io::ErrorKind; diff --git a/stackslib/src/net/stackerdb/bits.rs b/stackslib/src/net/stackerdb/bits.rs deleted file mode 100644 index 54709d8cdc..0000000000 --- a/stackslib/src/net/stackerdb/bits.rs +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation -// Copyright (C) 2020-2023 Stacks Open Internet Foundation -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -/// This module contains methods for interacting with the data contained within the messages -use crate::net::{ - Error as net_error, StackerDBChunkData, StackerDBChunkInvData, StackerDBGetChunkData, - StackerDBGetChunkInvData, -}; - -use sha2::{Digest, Sha512_256}; - -use stacks_common::util::hash::{Hash160, Sha512Trunc256Sum}; - -use stacks_common::types::chainstate::{ConsensusHash, StacksAddress, StacksPrivateKey}; - -use stacks_common::types::PrivateKey; - -use crate::net::stackerdb::SlotMetadata; - -use crate::chainstate::stacks::StacksPublicKey; - -use stacks_common::util::secp256k1::MessageSignature; - -impl SlotMetadata { - /// Get the digest to sign that authenticates this chunk data and metadata - fn auth_digest(&self) -> Sha512Trunc256Sum { - let mut hasher = Sha512_256::new(); - hasher.update(&self.slot_id.to_be_bytes()); - hasher.update(&self.slot_version.to_be_bytes()); - hasher.update(&self.data_hash.0); - Sha512Trunc256Sum::from_hasher(hasher) - } - - /// Sign this slot metadata, committing to slot_id, slot_version, and - /// data_hash. Sets self.signature to the signature. - /// Fails if the underlying crypto library fails - pub fn sign(&mut self, privkey: &StacksPrivateKey) -> Result<(), net_error> { - let auth_digest = self.auth_digest(); - let sig = privkey - .sign(&auth_digest.0) - .map_err(|se| net_error::SigningError(se.to_string()))?; - - self.signature = sig; - Ok(()) - } - - /// Verify that a given principal signed this chunk metadata. - /// Note that the address version is ignored. - pub fn verify(&self, principal: &StacksAddress) -> Result { - let sigh = self.auth_digest(); - let pubk = StacksPublicKey::recover_to_pubkey(sigh.as_bytes(), &self.signature) - .map_err(|ve| net_error::VerifyingError(ve.to_string()))?; - - let pubkh = Hash160::from_node_public_key(&pubk); - Ok(pubkh == principal.bytes) - } -} - -/// Helper methods for StackerDBChunkData messages -impl StackerDBChunkData { - /// Create a new StackerDBChunkData instance. - pub fn new(slot_id: u32, slot_version: u32, data: Vec) -> StackerDBChunkData { - StackerDBChunkData { - slot_id, - slot_version, - sig: MessageSignature::empty(), - data, - } - } - - /// Calculate the hash of the chunk bytes. This is the SHA512/256 hash of the data. - pub fn data_hash(&self) -> Sha512Trunc256Sum { - Sha512Trunc256Sum::from_data(&self.data) - } - - /// Create an owned SlotMetadata describing the metadata of this slot. - pub fn get_slot_metadata(&self) -> SlotMetadata { - SlotMetadata { - slot_id: self.slot_id, - slot_version: self.slot_version, - data_hash: self.data_hash(), - signature: self.sig.clone(), - } - } - - /// Sign this given chunk data message with the given private key. - /// Sets self.signature to the signature. - /// Fails if the underlying signing library fails. - pub fn sign(&mut self, privk: &StacksPrivateKey) -> Result<(), net_error> { - let mut md = self.get_slot_metadata(); - md.sign(privk)?; - self.sig = md.signature; - Ok(()) - } - - /// Verify that this chunk was signed by the given - /// public key hash (`addr`). Only fails if the underlying signing library fails. - pub fn verify(&self, addr: &StacksAddress) -> Result { - let md = self.get_slot_metadata(); - md.verify(addr) - } -} diff --git a/stackslib/src/net/stackerdb/config.rs b/stackslib/src/net/stackerdb/config.rs index 68ef8a895f..885d92c19c 100644 --- a/stackslib/src/net/stackerdb/config.rs +++ b/stackslib/src/net/stackerdb/config.rs @@ -41,7 +41,6 @@ use crate::net::stackerdb::{ StackerDBConfig, StackerDBs, STACKERDB_INV_MAX, STACKERDB_MAX_CHUNK_SIZE, }; -use crate::net::ContractId; use crate::net::Error as net_error; use crate::net::NeighborAddress; use crate::net::PeerAddress; @@ -54,6 +53,7 @@ use crate::clarity_vm::clarity::{ClarityReadOnlyConnection, Error as clarity_err use clarity::vm::analysis::ContractAnalysis; use clarity::vm::clarity::ClarityConnection; use clarity::vm::database::BurnStateDB; +use clarity::vm::types::QualifiedContractIdentifier; use clarity::vm::types::StandardPrincipalData; use clarity::vm::types::{ BufferLength, FixedFunction, FunctionType, ListTypeData, PrincipalData, SequenceSubtype, @@ -79,7 +79,7 @@ lazy_static! { ]) .expect("FATAL: failed to construct signer list type") .into(), - MAX_HINT_REPLICAS + STACKERDB_INV_MAX ) .expect("FATAL: could not construct signer list type") .into(), @@ -120,45 +120,48 @@ lazy_static! { } impl StackerDBConfig { - /// Check that a smart contract is consistent with being a StackerDB controller - fn is_contract_valid(epoch: &StacksEpochId, analysis: ContractAnalysis) -> bool { + /// Check that a smart contract is consistent with being a StackerDB controller. + /// Returns Ok(..) if the contract is valid + /// Returns Err(reason) if the contract is invalid. A human-readable reason will be given. + fn is_contract_valid(epoch: &StacksEpochId, analysis: ContractAnalysis) -> Result<(), String> { for (name, func_return_type) in REQUIRED_FUNCTIONS.iter() { let func = if let Some(f) = analysis.read_only_function_types.get(name) { f } else if let Some(f) = analysis.public_function_types.get(name) { f } else { - warn!("Contract is missing function '{}'", name); - return false; + let reason = format!("Contract is missing function '{}'", name); + return Err(reason); }; match func { FunctionType::Fixed(FixedFunction { args, returns }) => { if args.len() != 0 { - return false; + let reason = format!("Contract function '{}' has an invalid signature: it must take zero arguments", name); + return Err(reason); } if !func_return_type .admits_type(epoch, &returns) .unwrap_or(false) { - warn!("Contract function '{}' has an invalid return type: expected {:?}, got {:?}", name, func_return_type, returns); - return false; + let reason = format!("Contract function '{}' has an invalid return type: expected {:?}, got {:?}", name, func_return_type, returns); + return Err(reason); } } _ => { - warn!("Contract function '{}' is not a fixed function", name); - return false; + let reason = format!("Contract function '{}' is not a fixed function", name); + return Err(reason); } } } - true + Ok(()) } /// Evaluate the contract to get its signer slots fn eval_signer_slots( chainstate: &mut StacksChainState, burn_dbconn: &dyn BurnStateDB, - contract_id: &ContractId, + contract_id: &QualifiedContractIdentifier, tip: &StacksBlockId, ) -> Result, net_error> { let value = chainstate.eval_read_only( @@ -172,11 +175,15 @@ impl StackerDBConfig { let slot_list = match result { Err(err_val) => { let err_code = err_val.expect_u128(); - warn!( + let reason = format!( "Contract {} failed to run `stackerdb-get-signer-slots`: error u{}", contract_id, &err_code ); - return Err(net_error::InvalidStackerDBContract(contract_id.clone())); + warn!("{}", &reason); + return Err(net_error::InvalidStackerDBContract( + contract_id.clone(), + reason, + )); } Ok(ok_val) => ok_val.expect_list(), }; @@ -197,11 +204,15 @@ impl StackerDBConfig { .expect_u128(); if num_slots_uint > (STACKERDB_INV_MAX as u128) { - warn!( + let reason = format!( "Contract {} stipulated more than maximum number of slots for one signer ({})", contract_id, STACKERDB_INV_MAX ); - return Err(net_error::InvalidStackerDBContract(contract_id.clone())); + warn!("{}", &reason); + return Err(net_error::InvalidStackerDBContract( + contract_id.clone(), + reason, + )); } let num_slots = num_slots_uint as u32; total_num_slots = @@ -213,18 +224,26 @@ impl StackerDBConfig { )))?; if total_num_slots > STACKERDB_INV_MAX.into() { - warn!( + let reason = format!( "Contract {} stipulated more than the maximum number of slots", contract_id ); - return Err(net_error::InvalidStackerDBContract(contract_id.clone())); + warn!("{}", &reason); + return Err(net_error::InvalidStackerDBContract( + contract_id.clone(), + reason, + )); } // standard principals only let addr = match signer_principal { PrincipalData::Contract(..) => { - warn!("Contract {} stipulated a contract principal as a writer, which is not supported", contract_id); - return Err(net_error::InvalidStackerDBContract(contract_id.clone())); + let reason = format!("Contract {} stipulated a contract principal as a writer, which is not supported", contract_id); + warn!("{}", &reason); + return Err(net_error::InvalidStackerDBContract( + contract_id.clone(), + reason, + )); } PrincipalData::Standard(StandardPrincipalData(version, bytes)) => StacksAddress { version, @@ -241,7 +260,7 @@ impl StackerDBConfig { fn eval_config( chainstate: &mut StacksChainState, burn_dbconn: &dyn BurnStateDB, - contract_id: &ContractId, + contract_id: &QualifiedContractIdentifier, tip: &StacksBlockId, signers: Vec<(StacksAddress, u32)>, ) -> Result { @@ -252,11 +271,15 @@ impl StackerDBConfig { let config_tuple = match result { Err(err_val) => { let err_code = err_val.expect_u128(); - warn!( + let reason = format!( "Contract {} failed to run `stackerdb-get-config`: err u{}", contract_id, &err_code ); - return Err(net_error::InvalidStackerDBContract(contract_id.clone())); + warn!("{}", &reason); + return Err(net_error::InvalidStackerDBContract( + contract_id.clone(), + reason, + )); } Ok(ok_val) => ok_val.expect_tuple(), }; @@ -266,12 +289,17 @@ impl StackerDBConfig { .expect("FATAL: missing 'chunk-size'") .clone() .expect_u128(); + if chunk_size > STACKERDB_MAX_CHUNK_SIZE as u128 { - warn!( + let reason = format!( "Contract {} stipulates a chunk size beyond STACKERDB_MAX_CHUNK_SIZE", contract_id ); - return Err(net_error::InvalidStackerDBContract(contract_id.clone())); + warn!("{}", &reason); + return Err(net_error::InvalidStackerDBContract( + contract_id.clone(), + reason, + )); } let write_freq = config_tuple @@ -280,11 +308,15 @@ impl StackerDBConfig { .clone() .expect_u128(); if write_freq > u64::MAX as u128 { - warn!( + let reason = format!( "Contract {} stipulates a write frequency beyond u64::MAX", contract_id ); - return Err(net_error::InvalidStackerDBContract(contract_id.clone())); + warn!("{}", &reason); + return Err(net_error::InvalidStackerDBContract( + contract_id.clone(), + reason, + )); } let max_writes = config_tuple @@ -293,11 +325,15 @@ impl StackerDBConfig { .clone() .expect_u128(); if max_writes > u32::MAX as u128 { - warn!( + let reason = format!( "Contract {} stipulates a max-write bound beyond u32::MAX", contract_id ); - return Err(net_error::InvalidStackerDBContract(contract_id.clone())); + warn!("{}", &reason); + return Err(net_error::InvalidStackerDBContract( + contract_id.clone(), + reason, + )); } let max_neighbors = config_tuple @@ -306,11 +342,15 @@ impl StackerDBConfig { .clone() .expect_u128(); if max_neighbors > usize::MAX as u128 { - warn!( + let reason = format!( "Contract {} stipulates a maximum number of neighbors beyond usize::MAX", contract_id ); - return Err(net_error::InvalidStackerDBContract(contract_id.clone())); + warn!("{}", &reason); + return Err(net_error::InvalidStackerDBContract( + contract_id.clone(), + reason, + )); } let hint_replicas_list = config_tuple @@ -342,28 +382,40 @@ impl StackerDBConfig { for byte_val in addr_byte_list.into_iter() { let byte = byte_val.expect_u128(); if byte > (u8::MAX as u128) { - warn!( + let reason = format!( "Contract {} stipulates an addr byte above u8::MAX", contract_id ); - return Err(net_error::InvalidStackerDBContract(contract_id.clone())); + warn!("{}", &reason); + return Err(net_error::InvalidStackerDBContract( + contract_id.clone(), + reason, + )); } addr_bytes.push(byte as u8); } if addr_bytes.len() != 16 { - warn!( + let reason = format!( "Contract {} did not stipulate a full 16-octet IP address", contract_id ); - return Err(net_error::InvalidStackerDBContract(contract_id.clone())); + warn!("{}", &reason); + return Err(net_error::InvalidStackerDBContract( + contract_id.clone(), + reason, + )); } if port < 1024 || port > ((u16::MAX - 1) as u128) { - warn!( + let reason = format!( "Contract {} stipulates a port lower than 1024 or above u16::MAX - 1", contract_id ); - return Err(net_error::InvalidStackerDBContract(contract_id.clone())); + warn!("{}", &reason); + return Err(net_error::InvalidStackerDBContract( + contract_id.clone(), + reason, + )); } let mut pubkey_hash_slice = [0u8; 20]; @@ -393,7 +445,7 @@ impl StackerDBConfig { pub fn from_smart_contract( chainstate: &mut StacksChainState, sortition_db: &SortitionDB, - contract_id: &ContractId, + contract_id: &QualifiedContractIdentifier, ) -> Result { let chain_tip = chainstate .get_stacks_chain_tip(sortition_db)? @@ -423,12 +475,18 @@ impl StackerDBConfig { .ok_or(net_error::NoSuchStackerDB(contract_id.clone()))?; // contract must be consistent with StackerDB control interface - if !Self::is_contract_valid(&cur_epoch.epoch_id, analysis) { - debug!( - "Contract {} does not conform to StackerDB trait", - contract_id + if let Err(invalid_reason) = + Self::is_contract_valid(&cur_epoch.epoch_id, analysis) + { + let reason = format!( + "Contract {} does not conform to StackerDB trait: {}", + contract_id, invalid_reason ); - return Err(net_error::InvalidStackerDBContract(contract_id.clone())); + warn!("{}", &reason); + return Err(net_error::InvalidStackerDBContract( + contract_id.clone(), + reason, + )); } Ok(()) @@ -436,11 +494,15 @@ impl StackerDBConfig { })?; if res.is_none() { - warn!( + let reason = format!( "Could not evaluate contract {} at {}", contract_id, &chain_tip_hash ); - return Err(net_error::InvalidStackerDBContract(contract_id.clone())); + warn!("{}", &reason); + return Err(net_error::InvalidStackerDBContract( + contract_id.clone(), + reason, + )); } else if let Some(Err(e)) = res { warn!( "Could not use contract {} for StackerDB: {:?}", diff --git a/stackslib/src/net/stackerdb/db.rs b/stackslib/src/net/stackerdb/db.rs index 69811fdc33..4b4232f676 100644 --- a/stackslib/src/net/stackerdb/db.rs +++ b/stackslib/src/net/stackerdb/db.rs @@ -21,13 +21,13 @@ use std::fs; use std::io; use std::path::Path; +use libstackerdb::SlotMetadata; +use libstackerdb::STACKERDB_MAX_CHUNK_SIZE; + use crate::chainstate::stacks::address::PoxAddress; -use crate::net::stackerdb::{ - SlotMetadata, StackerDBConfig, StackerDBTx, StackerDBs, STACKERDB_INV_MAX, - STACKERDB_MAX_CHUNK_SIZE, -}; +use crate::net::stackerdb::{StackerDBConfig, StackerDBTx, StackerDBs, STACKERDB_INV_MAX}; use crate::net::Error as net_error; -use crate::net::{ContractId, StackerDBChunkData, StackerDBHandshakeData}; +use crate::net::{StackerDBChunkData, StackerDBHandshakeData}; use rusqlite::{ types::ToSql, Connection, OpenFlags, OptionalExtension, Row, Transaction, NO_PARAMS, @@ -44,6 +44,7 @@ use stacks_common::util::get_epoch_time_secs; use stacks_common::util::hash::Sha512Trunc256Sum; use stacks_common::util::secp256k1::MessageSignature; +use clarity::vm::types::QualifiedContractIdentifier; use clarity::vm::ContractName; const STACKER_DB_SCHEMA: &'static [&'static str] = &[ @@ -52,8 +53,10 @@ const STACKER_DB_SCHEMA: &'static [&'static str] = &[ "#, r#" CREATE TABLE databases( - -- smart contract for this stackerdb + -- internal numeric identifier for this stackerdb's smart contract identifier + -- (so we don't have to copy it into each chunk row) stackerdb_id INTEGER NOT NULL, + -- smart contract ID for this stackerdb smart_contract_id TEXT UNIQUE NOT NULL, PRIMARY KEY(stackerdb_id) ); @@ -76,7 +79,7 @@ const STACKER_DB_SCHEMA: &'static [&'static str] = &[ -- the following is NOT covered by the signature -- address of the creator of this chunk - signer STRING NOT NULL, + signer TEXT NOT NULL, -- the chunk data itself data BLOB NOT NULL, -- UNIX timestamp when the chunk was written. @@ -94,6 +97,7 @@ const STACKER_DB_SCHEMA: &'static [&'static str] = &[ pub const NO_VERSION: i64 = 0; /// Private struct for loading the data we need to validate an incoming chunk +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] pub struct SlotValidation { pub signer: StacksAddress, pub version: u32, @@ -101,7 +105,7 @@ pub struct SlotValidation { } impl FromRow for SlotMetadata { - fn from_row<'a>(row: &'a Row) -> Result { + fn from_row(row: &Row) -> Result { let slot_id: u32 = row.get_unwrap("slot_id"); let slot_version: u32 = row.get_unwrap("version"); let data_hash_str: String = row.get_unwrap("data_hash"); @@ -121,7 +125,7 @@ impl FromRow for SlotMetadata { } impl FromRow for SlotValidation { - fn from_row<'a>(row: &'a Row) -> Result { + fn from_row(row: &Row) -> Result { let signer = StacksAddress::from_column(row, "signer")?; let version: u32 = row.get_unwrap("version"); let write_time_i64: i64 = row.get_unwrap("write_time"); @@ -139,7 +143,7 @@ impl FromRow for SlotValidation { } impl FromRow for StackerDBChunkData { - fn from_row<'a>(row: &'a Row) -> Result { + fn from_row(row: &Row) -> Result { let slot_id: u32 = row.get_unwrap("slot_id"); let slot_version: u32 = row.get_unwrap("version"); let data: Vec = row.get_unwrap("data"); @@ -157,7 +161,10 @@ impl FromRow for StackerDBChunkData { /// Get the local numeric ID of a stacker DB. /// Returns Err(NoSuchStackerDB(..)) if it doesn't exist -fn inner_get_stackerdb_id(conn: &DBConn, smart_contract: &ContractId) -> Result { +fn inner_get_stackerdb_id( + conn: &DBConn, + smart_contract: &QualifiedContractIdentifier, +) -> Result { let sql = "SELECT rowid FROM databases WHERE smart_contract_id = ?1"; let args: &[&dyn ToSql] = &[&smart_contract.to_string()]; Ok(query_row(conn, sql, args)?.ok_or(net_error::NoSuchStackerDB(smart_contract.clone()))?) @@ -168,7 +175,7 @@ fn inner_get_stackerdb_id(conn: &DBConn, smart_contract: &ContractId) -> Result< /// Inner method body for related methods in both the DB instance and the transaction instance. fn inner_get_slot_metadata( conn: &DBConn, - smart_contract: &ContractId, + smart_contract: &QualifiedContractIdentifier, slot_id: u32, ) -> Result, net_error> { let stackerdb_id = inner_get_stackerdb_id(conn, smart_contract)?; @@ -182,7 +189,7 @@ fn inner_get_slot_metadata( /// Inner method body for related methods in both the DB instance and the transaction instance. fn inner_get_slot_validation( conn: &DBConn, - smart_contract: &ContractId, + smart_contract: &QualifiedContractIdentifier, slot_id: u32, ) -> Result, net_error> { let stackerdb_id = inner_get_stackerdb_id(conn, smart_contract)?; @@ -193,8 +200,8 @@ fn inner_get_slot_validation( } impl<'a> StackerDBTx<'a> { - pub fn commit(self) -> Result<(), net_error> { - self.sql_tx.commit().map_err(net_error::from) + pub fn commit(self) -> Result<(), db_error> { + self.sql_tx.commit().map_err(db_error::from) } pub fn conn(&self) -> &DBConn { @@ -203,7 +210,10 @@ impl<'a> StackerDBTx<'a> { /// Delete a stacker DB table and its contents. /// Idempotent. - pub fn delete_stackerdb(&self, smart_contract_id: &ContractId) -> Result<(), net_error> { + pub fn delete_stackerdb( + &self, + smart_contract_id: &QualifiedContractIdentifier, + ) -> Result<(), net_error> { let qry = "DELETE FROM databases WHERE smart_contract_id = ?1"; let args: &[&dyn ToSql] = &[&smart_contract_id.to_string()]; let mut stmt = self.sql_tx.prepare(qry)?; @@ -212,13 +222,18 @@ impl<'a> StackerDBTx<'a> { } /// List all stacker DB smart contracts we have available - pub fn get_stackerdb_contract_ids(&self) -> Result, net_error> { + pub fn get_stackerdb_contract_ids( + &self, + ) -> Result, net_error> { let sql = "SELECT smart_contract_id FROM databases ORDER BY smart_contract_id"; query_rows(&self.conn(), sql, NO_PARAMS).map_err(|e| e.into()) } /// Get the Stacker DB ID for a smart contract - pub fn get_stackerdb_id(&self, smart_contract: &ContractId) -> Result { + pub fn get_stackerdb_id( + &self, + smart_contract: &QualifiedContractIdentifier, + ) -> Result { inner_get_stackerdb_id(&self.conn(), smart_contract) } @@ -227,7 +242,7 @@ impl<'a> StackerDBTx<'a> { /// (and thus the key used to authenticate them) pub fn create_stackerdb( &self, - smart_contract: &ContractId, + smart_contract: &QualifiedContractIdentifier, slots: &[(StacksAddress, u32)], ) -> Result<(), net_error> { if slots.len() > (STACKERDB_INV_MAX as usize) { @@ -250,6 +265,7 @@ impl<'a> StackerDBTx<'a> { let mut slot_id = 0u32; for (principal, slot_count) in slots.iter() { + test_debug!("Create StackerDB slots: ({}, {})", &principal, slot_count); for _ in 0..*slot_count { let args: &[&dyn ToSql] = &[ &stackerdb_id, @@ -273,7 +289,10 @@ impl<'a> StackerDBTx<'a> { /// Clear a database's slots and its data. /// Idempotent. /// Fails if the DB doesn't exist - pub fn clear_stackerdb_slots(&self, smart_contract: &ContractId) -> Result<(), net_error> { + pub fn clear_stackerdb_slots( + &self, + smart_contract: &QualifiedContractIdentifier, + ) -> Result<(), net_error> { let stackerdb_id = self.get_stackerdb_id(smart_contract)?; let qry = "DELETE FROM chunks WHERE stackerdb_id = ?1"; let args: &[&dyn ToSql] = &[&stackerdb_id]; @@ -288,7 +307,7 @@ impl<'a> StackerDBTx<'a> { /// If the address for a slot changes, then its data will be dropped. pub fn reconfigure_stackerdb( &self, - smart_contract: &ContractId, + smart_contract: &QualifiedContractIdentifier, slots: &[(StacksAddress, u32)], ) -> Result<(), net_error> { let stackerdb_id = self.get_stackerdb_id(smart_contract)?; @@ -336,7 +355,7 @@ impl<'a> StackerDBTx<'a> { /// Get the slot metadata pub fn get_slot_metadata( &self, - smart_contract: &ContractId, + smart_contract: &QualifiedContractIdentifier, slot_id: u32, ) -> Result, net_error> { inner_get_slot_metadata(self.conn(), smart_contract, slot_id) @@ -345,7 +364,7 @@ impl<'a> StackerDBTx<'a> { /// Get a chunk's validation data pub fn get_slot_validation( &self, - smart_contract: &ContractId, + smart_contract: &QualifiedContractIdentifier, slot_id: u32, ) -> Result, net_error> { inner_get_slot_validation(self.conn(), smart_contract, slot_id) @@ -356,7 +375,7 @@ impl<'a> StackerDBTx<'a> { /// there. These will not be checked. fn insert_chunk( &self, - smart_contract: &ContractId, + smart_contract: &QualifiedContractIdentifier, slot_desc: &SlotMetadata, chunk: &[u8], ) -> Result<(), net_error> { @@ -382,7 +401,7 @@ impl<'a> StackerDBTx<'a> { /// Otherwise, this errors out with Error::StaleChunk pub fn try_replace_chunk( &self, - smart_contract: &ContractId, + smart_contract: &QualifiedContractIdentifier, slot_desc: &SlotMetadata, chunk: &[u8], ) -> Result<(), net_error> { @@ -404,16 +423,16 @@ impl<'a> StackerDBTx<'a> { )); } if slot_desc.slot_version <= slot_validation.version { - return Err(net_error::StaleChunk( - slot_validation.version, - slot_desc.slot_version, - )); + return Err(net_error::StaleChunk { + latest_version: slot_validation.version, + supplied_version: slot_desc.slot_version, + }); } if slot_desc.slot_version > self.config.max_writes { - return Err(net_error::TooManySlotWrites( - self.config.max_writes, - slot_validation.version, - )); + return Err(net_error::TooManySlotWrites { + max_writes: self.config.max_writes, + supplied_version: slot_validation.version, + }); } self.insert_chunk(smart_contract, slot_desc, chunk) } @@ -425,33 +444,28 @@ impl StackerDBs { let mut create_flag = false; let open_flags = if path != ":memory:" { - match fs::metadata(path) { - Err(e) => { - if e.kind() == io::ErrorKind::NotFound { - // need to create - if readwrite { - create_flag = true; - let ppath = Path::new(path); - let pparent_path = ppath - .parent() - .expect(&format!("BUG: no parent of '{}'", path)); - fs::create_dir_all(&pparent_path).map_err(|e| db_error::IOError(e))?; - - OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE - } else { - return Err(db_error::NoDBError.into()); - } - } else { - return Err(db_error::IOError(e).into()); - } + if let Err(e) = fs::metadata(path) { + if e.kind() != io::ErrorKind::NotFound { + return Err(db_error::IOError(e).into()); } - Ok(_md) => { - // can just open - if readwrite { - OpenFlags::SQLITE_OPEN_READ_WRITE - } else { - OpenFlags::SQLITE_OPEN_READ_ONLY - } + if !readwrite { + return Err(db_error::NoDBError.into()); + } + + create_flag = true; + let ppath = Path::new(path); + let pparent_path = ppath + .parent() + .expect(&format!("BUG: no parent of '{}'", path)); + fs::create_dir_all(&pparent_path).map_err(|e| db_error::IOError(e))?; + + OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE + } else { + // can just open + if readwrite { + OpenFlags::SQLITE_OPEN_READ_WRITE + } else { + OpenFlags::SQLITE_OPEN_READ_ONLY } } } else { @@ -497,18 +511,23 @@ impl StackerDBs { pub fn tx_begin<'a>( &'a mut self, config: StackerDBConfig, - ) -> Result, net_error> { + ) -> Result, db_error> { let sql_tx = tx_begin_immediate(&mut self.conn)?; Ok(StackerDBTx { sql_tx, config }) } /// Get the Stacker DB ID for a smart contract - pub fn get_stackerdb_id(&self, smart_contract: &ContractId) -> Result { + pub fn get_stackerdb_id( + &self, + smart_contract: &QualifiedContractIdentifier, + ) -> Result { inner_get_stackerdb_id(&self.conn, smart_contract) } /// List all stacker DB smart contracts we have available - pub fn get_stackerdb_contract_ids(&self) -> Result, net_error> { + pub fn get_stackerdb_contract_ids( + &self, + ) -> Result, net_error> { let sql = "SELECT smart_contract_id FROM databases ORDER BY smart_contract_id"; query_rows(&self.conn, sql, NO_PARAMS).map_err(|e| e.into()) } @@ -519,7 +538,7 @@ impl StackerDBs { /// Returns Err(..) if the DB doesn't exist of some other DB error happens pub fn get_slot_signer( &self, - smart_contract: &ContractId, + smart_contract: &QualifiedContractIdentifier, slot_id: u32, ) -> Result, net_error> { let stackerdb_id = self.get_stackerdb_id(smart_contract)?; @@ -531,23 +550,38 @@ impl StackerDBs { /// Get the slot metadata pub fn get_slot_metadata( &self, - smart_contract: &ContractId, + smart_contract: &QualifiedContractIdentifier, slot_id: u32, ) -> Result, net_error> { inner_get_slot_metadata(&self.conn, smart_contract, slot_id) } + /// Get the slot metadata for the whole DB + /// (used for RPC) + pub fn get_db_slot_metadata( + &self, + smart_contract: &QualifiedContractIdentifier, + ) -> Result, net_error> { + let stackerdb_id = inner_get_stackerdb_id(&self.conn, smart_contract)?; + let sql = "SELECT slot_id,version,data_hash,signature FROM chunks WHERE stackerdb_id = ?1 ORDER BY slot_id ASC"; + let args: &[&dyn ToSql] = &[&stackerdb_id]; + query_rows(&self.conn, &sql, args).map_err(|e| e.into()) + } + /// Get a slot's validation data pub fn get_slot_validation( &self, - smart_contract: &ContractId, + smart_contract: &QualifiedContractIdentifier, slot_id: u32, ) -> Result, net_error> { inner_get_slot_validation(&self.conn, smart_contract, slot_id) } /// Get the list of slot ID versions for a given DB instance at a given reward cycle - pub fn get_slot_versions(&self, smart_contract: &ContractId) -> Result, net_error> { + pub fn get_slot_versions( + &self, + smart_contract: &QualifiedContractIdentifier, + ) -> Result, net_error> { let stackerdb_id = self.get_stackerdb_id(smart_contract)?; let sql = "SELECT version FROM chunks WHERE stackerdb_id = ?1 ORDER BY slot_id"; let args: &[&dyn ToSql] = &[&stackerdb_id]; @@ -557,7 +591,7 @@ impl StackerDBs { /// Get the list of slot write timestamps for a given DB instance at a given reward cycle pub fn get_slot_write_timestamps( &self, - smart_contract: &ContractId, + smart_contract: &QualifiedContractIdentifier, ) -> Result, net_error> { let stackerdb_id = self.get_stackerdb_id(smart_contract)?; let sql = "SELECT write_time FROM chunks WHERE stackerdb_id = ?1 ORDER BY slot_id"; @@ -571,33 +605,24 @@ impl StackerDBs { /// Returns Err(..) if the DB does not exist, or some other DB error occurs pub fn get_latest_chunk( &self, - smart_contract: &ContractId, + smart_contract: &QualifiedContractIdentifier, slot_id: u32, ) -> Result>, net_error> { let stackerdb_id = self.get_stackerdb_id(smart_contract)?; let qry = "SELECT data FROM chunks WHERE stackerdb_id = ?1 AND slot_id = ?2"; let args: &[&dyn ToSql] = &[&stackerdb_id, &slot_id]; - let mut stmt = self - .conn - .prepare(&qry) - .map_err(|e| net_error::DBError(e.into()))?; - - let mut rows = stmt.query(args).map_err(|e| net_error::DBError(e.into()))?; - - let mut data = None; - while let Some(row) = rows.next().map_err(|e| net_error::DBError(e.into()))? { - data = Some(row.get_unwrap(0)); - break; - } - Ok(data) + self.conn + .query_row(qry, args, |row| row.get(0)) + .optional() + .map_err(|e| e.into()) } /// Get a versioned chunk out of this database. If the version is not present, then None will /// be returned. pub fn get_chunk( &self, - smart_contract: &ContractId, + smart_contract: &QualifiedContractIdentifier, slot_id: u32, slot_version: u32, ) -> Result, net_error> { diff --git a/stackslib/src/net/stackerdb/mod.rs b/stackslib/src/net/stackerdb/mod.rs index ba33727712..3862544676 100644 --- a/stackslib/src/net/stackerdb/mod.rs +++ b/stackslib/src/net/stackerdb/mod.rs @@ -114,12 +114,10 @@ #[cfg(test)] pub mod tests; -pub mod bits; pub mod config; pub mod db; pub mod sync; -use crate::net::ContractId; use crate::net::Error as net_error; use crate::net::NackData; use crate::net::NackErrorCodes; @@ -147,16 +145,17 @@ use crate::net::p2p::PeerNetwork; use stacks_common::types::chainstate::StacksAddress; use stacks_common::util::get_epoch_time_secs; +use libstackerdb::{SlotMetadata, STACKERDB_MAX_CHUNK_SIZE}; + +use clarity::vm::types::QualifiedContractIdentifier; + /// maximum chunk inventory size pub const STACKERDB_INV_MAX: u32 = 4096; -/// maximum chunk size (1 MB) -pub const STACKERDB_MAX_CHUNK_SIZE: u32 = 1024 * 1024; - /// Final result of synchronizing state with a remote set of DB replicas pub struct StackerDBSyncResult { /// which contract this is a replica for - pub contract_id: ContractId, + pub contract_id: QualifiedContractIdentifier, /// slot inventory for this replica pub chunk_invs: HashMap, /// list of data to store @@ -217,20 +216,6 @@ pub struct StackerDBTx<'a> { config: StackerDBConfig, } -/// Slot metadata from the DB. -/// This is derived state from a StackerDBChunkData message. -#[derive(Clone, Debug, PartialEq)] -pub struct SlotMetadata { - /// Slot identifier (unique for each DB instance) - pub slot_id: u32, - /// Slot version (a lamport clock) - pub slot_version: u32, - /// data hash - pub data_hash: Sha512Trunc256Sum, - /// signature over the above - pub signature: MessageSignature, -} - /// Possible states a DB sync state-machine can be in #[derive(Debug)] pub enum StackerDBSyncState { @@ -248,7 +233,7 @@ pub struct StackerDBSync { /// what state are we in? state: StackerDBSyncState, /// which contract this is a replica for - pub smart_contract_id: ContractId, + pub smart_contract_id: QualifiedContractIdentifier, /// number of chunks in this DB pub num_slots: usize, /// how frequently we accept chunk writes, in seconds @@ -287,6 +272,9 @@ pub struct StackerDBSync { pub total_pushed: u64, /// last time the state-transition function ran to completion last_run_ts: u64, + /// whether or not we should immediately re-fetch chunks because we learned about new chunks + /// from our peers when they replied to our chunk-pushes with new inventory state + need_resync: bool, } impl StackerDBSyncResult { @@ -303,6 +291,16 @@ impl StackerDBSyncResult { } } +/// Event dispatcher trait for pushing out new chunk arrival info +pub trait StackerDBEventDispatcher { + /// A set of one or more chunks has been obtained by this replica + fn new_stackerdb_chunks( + &self, + contract_id: QualifiedContractIdentifier, + chunk_info: Vec, + ); +} + impl PeerNetwork { /// Run all stacker DB sync state-machines. /// Return a list of sync results on success, to be incorporated into the NetworkResult. @@ -352,7 +350,10 @@ impl PeerNetwork { } /// Create a StackerDBChunksInv, or a Nack if the requested DB isn't replicated here - pub fn make_StackerDBChunksInv_or_Nack(&self, contract_id: &ContractId) -> StacksMessageType { + pub fn make_StackerDBChunksInv_or_Nack( + &self, + contract_id: &QualifiedContractIdentifier, + ) -> StacksMessageType { let slot_versions = match self.stackerdbs.get_slot_versions(contract_id) { Ok(versions) => versions, Err(e) => { @@ -380,7 +381,7 @@ impl PeerNetwork { /// Returns Err(..) on DB error pub fn validate_received_chunk( &self, - smart_contract_id: &ContractId, + smart_contract_id: &QualifiedContractIdentifier, config: &StackerDBConfig, data: &StackerDBChunkData, expected_versions: &[u32], @@ -443,7 +444,12 @@ impl PeerNetwork { /// the inventory vector is updated with this chunk's data. /// /// Note that this can happen *during* a StackerDB sync's execution, so be very careful about - /// modifying a state machine's contents! + /// modifying a state machine's contents! The only modification possible here is to wakeup + /// the state machine in case it's asleep (i.e. blocked on waiting for the next sync round). + /// + /// The write frequency is not checked for this chunk. This is because the `ConversationP2P` on + /// which this chunk arrived will have already bandwidth-throttled the remote peer, and because + /// messages can be arbitrarily delayed (and bunched up) by the network anyway. /// /// Return Ok(true) if we should store the chunk /// Return Ok(false) if we should drop it. diff --git a/stackslib/src/net/stackerdb/sync.rs b/stackslib/src/net/stackerdb/sync.rs index 7f57dcd23f..2c429b61d2 100644 --- a/stackslib/src/net/stackerdb/sync.rs +++ b/stackslib/src/net/stackerdb/sync.rs @@ -33,13 +33,14 @@ use crate::net::connection::ReplyHandleP2P; use crate::net::p2p::PeerNetwork; use crate::net::Error as net_error; use crate::net::{ - ContractId, NackData, Neighbor, NeighborAddress, NeighborKey, StackerDBChunkData, - StackerDBChunkInvData, StackerDBGetChunkData, StackerDBGetChunkInvData, StackerDBPushChunkData, - StacksMessageType, + NackData, Neighbor, NeighborAddress, NeighborKey, StackerDBChunkData, StackerDBChunkInvData, + StackerDBGetChunkData, StackerDBGetChunkInvData, StackerDBPushChunkData, StacksMessageType, }; use crate::net::neighbors::NeighborComms; +use clarity::vm::types::QualifiedContractIdentifier; + use rand::prelude::SliceRandom; use rand::thread_rng; use rand::Rng; @@ -51,7 +52,7 @@ const MAX_DB_NEIGHBORS: usize = 32; impl StackerDBSync { /// TODO: replace `stackerdbs` with a type parameter pub fn new( - smart_contract: ContractId, + smart_contract: QualifiedContractIdentifier, config: &StackerDBConfig, comms: NC, stackerdbs: StackerDBs, @@ -78,6 +79,7 @@ impl StackerDBSync { total_stored: 0, total_pushed: 0, last_run_ts: 0, + need_resync: false, }; dbsync.reset(None, config)?; Ok(dbsync) @@ -159,6 +161,8 @@ impl StackerDBSync { self.num_slots = config.num_slots() as usize; self.write_freq = config.write_freq; + self.need_resync = false; + Ok(result) } @@ -183,9 +187,15 @@ impl StackerDBSync { pub fn make_chunk_request_schedule( &self, network: &PeerNetwork, + local_slot_versions_opt: Option>, ) -> Result)>, net_error> { let rc_consensus_hash = network.get_chain_view().rc_consensus_hash.clone(); - let local_slot_versions = self.stackerdbs.get_slot_versions(&self.smart_contract_id)?; + let local_slot_versions = if let Some(local_slot_versions) = local_slot_versions_opt { + local_slot_versions + } else { + self.stackerdbs.get_slot_versions(&self.smart_contract_id)? + }; + let local_write_timestamps = self .stackerdbs .get_slot_write_timestamps(&self.smart_contract_id)?; @@ -422,12 +432,42 @@ impl StackerDBSync { /// Update bookkeeping about which chunks we have pushed. /// Stores the new chunk inventory to RAM. + /// Returns true if the inventory changed (indicating that we need to resync) + /// Returns false otherwise pub fn add_pushed_chunk( &mut self, + network: &PeerNetwork, naddr: NeighborAddress, new_inv: StackerDBChunkInvData, slot_id: u32, - ) { + ) -> bool { + // safety (should already be checked) -- don't accept if the size is wrong + if new_inv.slot_versions.len() != self.num_slots { + return false; + } + + let need_resync = if let Some(old_inv) = self.chunk_invs.get(&naddr) { + let mut resync = false; + for (old_slot_id, old_version) in old_inv.slot_versions.iter().enumerate() { + if *old_version < new_inv.slot_versions[old_slot_id] { + // remote peer indicated that it has a newer version of this chunk. + test_debug!( + "{:?}: peer {:?} has a newer version of slot {} ({} < {})", + network.get_local_peer(), + &naddr, + old_slot_id, + old_version, + new_inv.slot_versions[old_slot_id] + ); + resync = true; + break; + } + } + resync + } else { + false + }; + self.chunk_invs.insert(naddr.clone(), new_inv); self.chunk_push_priorities @@ -440,6 +480,7 @@ impl StackerDBSync { } self.total_pushed += 1; + need_resync } /// Ask inbound neighbors who replicate this DB for their chunk inventories. @@ -704,6 +745,7 @@ impl StackerDBSync { StacksMessageType::StackerDBChunkInv(data) => { if data.slot_versions.len() != self.num_slots { info!("{:?}: Received malformed StackerDBChunkInv from {:?}: expected {} chunks, got {}", network.get_local_peer(), &naddr, self.num_slots, data.slot_versions.len()); + self.comms.add_broken(network, &naddr); continue; } data @@ -736,7 +778,7 @@ impl StackerDBSync { } // got everything. Calculate download priority - let priorities = self.make_chunk_request_schedule(&network)?; + let priorities = self.make_chunk_request_schedule(&network, None)?; let expected_versions = self.stackerdbs.get_slot_versions(&self.smart_contract_id)?; self.chunk_fetch_priorities = priorities; @@ -955,10 +997,12 @@ impl StackerDBSync { Ok(self.chunk_push_priorities.len() == 0) } - /// Collect push-chunk replies from neighbors - /// Returns Ok(true) if all inflight messages have been received (or dealt with) - /// Returns Ok(false) otherwise - pub fn pushchunks_try_finish(&mut self, network: &mut PeerNetwork) -> Result { + /// Collect push-chunk replies from neighbors. + /// If a remote neighbor replies with a chunk-inv for a pushed chunk which contains newer data + /// than we have, then set `self.need_resync` to true. + /// Returns true if all inflight messages have been received (or dealt with) + /// Returns false otherwise + pub fn pushchunks_try_finish(&mut self, network: &mut PeerNetwork) -> bool { for (naddr, message) in self.comms.collect_replies(network).into_iter() { let new_chunk_inv = match message.payload { StacksMessageType::StackerDBChunkInv(data) => data, @@ -978,6 +1022,13 @@ impl StackerDBSync { } }; + // must be well-formed + if new_chunk_inv.slot_versions.len() != self.num_slots { + info!("{:?}: Received malformed StackerDBChunkInv from {:?}: expected {} chunks, got {}", network.get_local_peer(), &naddr, self.num_slots, new_chunk_inv.slot_versions.len()); + self.comms.add_broken(network, &naddr); + continue; + } + // update bookkeeping test_debug!( "{:?}: pushchunks_try_finish: Received StackerDBChunkInv from {:?}", @@ -986,11 +1037,33 @@ impl StackerDBSync { ); if let Some((slot_id, _)) = self.chunk_push_receipts.get(&naddr) { - self.add_pushed_chunk(naddr, new_chunk_inv, *slot_id); + self.need_resync = self.need_resync + || self.add_pushed_chunk(network, naddr, new_chunk_inv, *slot_id); } } - Ok(self.comms.count_inflight() == 0) + self.comms.count_inflight() == 0 + } + + /// Recalculate the download schedule based on chunkinvs received on push + pub fn recalculate_chunk_request_schedule( + &mut self, + network: &PeerNetwork, + ) -> Result<(), net_error> { + // figure out the new expected versions + let mut expected_versions = vec![0u32; self.num_slots as usize]; + for (_, chunk_inv) in self.chunk_invs.iter() { + for (slot_id, slot_version) in chunk_inv.slot_versions.iter().enumerate() { + expected_versions[slot_id] = (*slot_version).max(expected_versions[slot_id]); + } + } + + let priorities = + self.make_chunk_request_schedule(&network, Some(expected_versions.clone()))?; + + self.chunk_fetch_priorities = priorities; + self.expected_versions = expected_versions; + Ok(()) } /// Forcibly wake up the state machine if it is throttled @@ -1071,10 +1144,21 @@ impl StackerDBSync { } StackerDBSyncState::PushChunks => { let pushes_finished = self.pushchunks_begin(network)?; - let inflight_finished = self.pushchunks_try_finish(network)?; + let inflight_finished = self.pushchunks_try_finish(network); let done = pushes_finished && inflight_finished; if done { - self.state = StackerDBSyncState::Finished; + if self.need_resync + && !network.get_connection_opts().disable_stackerdb_get_chunks + { + // someone pushed newer chunk data to us, and getting chunks is + // enabled, so immediately go request them + self.recalculate_chunk_request_schedule(network)?; + self.state = StackerDBSyncState::GetChunks; + } else { + // done syncing + self.state = StackerDBSyncState::Finished; + } + self.need_resync = false; blocked = false; } } diff --git a/stackslib/src/net/stackerdb/tests/config.rs b/stackslib/src/net/stackerdb/tests/config.rs index 125414d1ab..1c63745b37 100644 --- a/stackslib/src/net/stackerdb/tests/config.rs +++ b/stackslib/src/net/stackerdb/tests/config.rs @@ -28,8 +28,6 @@ use crate::chainstate::stacks::TransactionPayload; use crate::chainstate::stacks::TransactionVersion; use crate::net::test::TestEventObserver; -use crate::net::ContractId; -use crate::net::ContractIdExtension; use crate::net::Error as net_error; use crate::net::NeighborAddress; use crate::net::PeerAddress; @@ -46,6 +44,8 @@ use stacks_common::types::chainstate::StacksPublicKey; use stacks_common::types::StacksEpoch; use stacks_common::util::hash::Hash160; +use clarity::vm::types::QualifiedContractIdentifier; + fn make_smart_contract( name: &str, code_body: &str, @@ -490,14 +490,15 @@ fn test_valid_and_invalid_stackerdb_configs() { peer.tenure_with_txs(&txs, &mut coinbase_nonce); for (i, (code, result)) in testcases.iter().enumerate() { - let contract_id = ContractId::from_parts( + let contract_id = QualifiedContractIdentifier::new( StacksAddress::from_public_keys( 26, &AddressHashMode::SerializeP2PKH, 1, &vec![StacksPublicKey::from_private(&contract_owner)], ) - .unwrap(), + .unwrap() + .into(), ContractName::try_from(format!("test-{}", i)).unwrap(), ); peer.with_db_state(|sortdb, chainstate, _, _| { diff --git a/stackslib/src/net/stackerdb/tests/db.rs b/stackslib/src/net/stackerdb/tests/db.rs index 1ac896ffcd..273274e83b 100644 --- a/stackslib/src/net/stackerdb/tests/db.rs +++ b/stackslib/src/net/stackerdb/tests/db.rs @@ -17,10 +17,9 @@ use std::fs; use std::path::Path; -use crate::net::stackerdb::{db::SlotValidation, SlotMetadata, StackerDBConfig, StackerDBs}; +use crate::net::stackerdb::{db::SlotValidation, StackerDBConfig, StackerDBs}; +use libstackerdb::SlotMetadata; -use crate::net::ContractId; -use crate::net::ContractIdExtension; use crate::net::Error as net_error; use crate::net::StackerDBChunkData; @@ -37,6 +36,8 @@ use stacks_common::address::{ }; use stacks_common::types::chainstate::{StacksPrivateKey, StacksPublicKey}; +use clarity::vm::types::QualifiedContractIdentifier; + fn setup_test_path(path: &str) { let dirname = Path::new(path).parent().unwrap().to_str().unwrap(); if fs::metadata(&dirname).is_err() { @@ -75,11 +76,12 @@ fn test_stackerdb_create_list_delete() { // databases with one chunk tx.create_stackerdb( - &ContractId::from_parts( + &QualifiedContractIdentifier::new( StacksAddress { version: 0x01, bytes: Hash160([0x01; 20]), - }, + } + .into(), ContractName::try_from("db1").unwrap(), ), &[( @@ -92,11 +94,12 @@ fn test_stackerdb_create_list_delete() { ) .unwrap(); tx.create_stackerdb( - &ContractId::from_parts( + &QualifiedContractIdentifier::new( StacksAddress { version: 0x02, bytes: Hash160([0x02; 20]), - }, + } + .into(), ContractName::try_from("db2").unwrap(), ), &[( @@ -109,11 +112,12 @@ fn test_stackerdb_create_list_delete() { ) .unwrap(); tx.create_stackerdb( - &ContractId::from_parts( + &QualifiedContractIdentifier::new( StacksAddress { version: 0x03, bytes: Hash160([0x03; 20]), - }, + } + .into(), ContractName::try_from("db3").unwrap(), ), &[( @@ -134,25 +138,28 @@ fn test_stackerdb_create_list_delete() { assert_eq!( dbs, vec![ - ContractId::from_parts( + QualifiedContractIdentifier::new( StacksAddress { version: 0x01, bytes: Hash160([0x01; 20]) - }, + } + .into(), ContractName::try_from("db1").unwrap() ), - ContractId::from_parts( + QualifiedContractIdentifier::new( StacksAddress { version: 0x02, bytes: Hash160([0x02; 20]) - }, + } + .into(), ContractName::try_from("db2").unwrap() ), - ContractId::from_parts( + QualifiedContractIdentifier::new( StacksAddress { version: 0x03, bytes: Hash160([0x03; 20]) - }, + } + .into(), ContractName::try_from("db3").unwrap() ), ] @@ -162,11 +169,12 @@ fn test_stackerdb_create_list_delete() { let tx = db.tx_begin(StackerDBConfig::noop()).unwrap(); if let net_error::StackerDBExists(..) = tx .create_stackerdb( - &ContractId::from_parts( + &QualifiedContractIdentifier::new( StacksAddress { version: 0x01, bytes: Hash160([0x01; 20]), - }, + } + .into(), ContractName::try_from("db1").unwrap(), ), &[], @@ -184,25 +192,28 @@ fn test_stackerdb_create_list_delete() { assert_eq!( dbs, vec![ - ContractId::from_parts( + QualifiedContractIdentifier::new( StacksAddress { version: 0x01, bytes: Hash160([0x01; 20]) - }, + } + .into(), ContractName::try_from("db1").unwrap() ), - ContractId::from_parts( + QualifiedContractIdentifier::new( StacksAddress { version: 0x02, bytes: Hash160([0x02; 20]) - }, + } + .into(), ContractName::try_from("db2").unwrap() ), - ContractId::from_parts( + QualifiedContractIdentifier::new( StacksAddress { version: 0x03, bytes: Hash160([0x03; 20]) - }, + } + .into(), ContractName::try_from("db3").unwrap() ), ] @@ -215,11 +226,12 @@ fn test_stackerdb_create_list_delete() { // remove a db let tx = db.tx_begin(StackerDBConfig::noop()).unwrap(); - tx.delete_stackerdb(&ContractId::from_parts( + tx.delete_stackerdb(&QualifiedContractIdentifier::new( StacksAddress { version: 0x01, bytes: Hash160([0x01; 20]), - }, + } + .into(), ContractName::try_from("db1").unwrap(), )) .unwrap(); @@ -231,18 +243,20 @@ fn test_stackerdb_create_list_delete() { assert_eq!( dbs, vec![ - ContractId::from_parts( + QualifiedContractIdentifier::new( StacksAddress { version: 0x02, bytes: Hash160([0x02; 20]) - }, + } + .into(), ContractName::try_from("db2").unwrap() ), - ContractId::from_parts( + QualifiedContractIdentifier::new( StacksAddress { version: 0x03, bytes: Hash160([0x03; 20]) - }, + } + .into(), ContractName::try_from("db3").unwrap() ), ] @@ -255,11 +269,12 @@ fn test_stackerdb_create_list_delete() { // deletion is idempotent let tx = db.tx_begin(StackerDBConfig::noop()).unwrap(); - tx.delete_stackerdb(&ContractId::from_parts( + tx.delete_stackerdb(&QualifiedContractIdentifier::new( StacksAddress { version: 0x01, bytes: Hash160([0x01; 20]), - }, + } + .into(), ContractName::try_from("db1").unwrap(), )) .unwrap(); @@ -271,18 +286,20 @@ fn test_stackerdb_create_list_delete() { assert_eq!( dbs, vec![ - ContractId::from_parts( + QualifiedContractIdentifier::new( StacksAddress { version: 0x02, bytes: Hash160([0x02; 20]) - }, + } + .into(), ContractName::try_from("db2").unwrap() ), - ContractId::from_parts( + QualifiedContractIdentifier::new( StacksAddress { version: 0x03, bytes: Hash160([0x03; 20]) - }, + } + .into(), ContractName::try_from("db3").unwrap() ), ] @@ -299,11 +316,12 @@ fn test_stackerdb_prepare_clear_slots() { let path = "/tmp/test_stackerdb_prepare_clear_slots.sqlite"; setup_test_path(path); - let sc = ContractId::from_parts( + let sc = QualifiedContractIdentifier::new( StacksAddress { version: 0x01, bytes: Hash160([0x01; 20]), - }, + } + .into(), ContractName::try_from("db1").unwrap(), ); @@ -409,11 +427,12 @@ fn test_stackerdb_insert_query_chunks() { let path = "/tmp/test_stackerdb_insert_query_chunks.sqlite"; setup_test_path(path); - let sc = ContractId::from_parts( + let sc = QualifiedContractIdentifier::new( StacksAddress { version: 0x01, bytes: Hash160([0x01; 20]), - }, + } + .into(), ContractName::try_from("db1").unwrap(), ); @@ -477,11 +496,13 @@ fn test_stackerdb_insert_query_chunks() { assert_eq!(slot_metadata.signature, chunk_data.sig); // should fail -- stale version - if let Err(net_error::StaleChunk(db_version, given_version)) = - tx.try_replace_chunk(&sc, &chunk_data.get_slot_metadata(), &chunk_data.data) + if let Err(net_error::StaleChunk { + supplied_version, + latest_version, + }) = tx.try_replace_chunk(&sc, &chunk_data.get_slot_metadata(), &chunk_data.data) { - assert_eq!(db_version, 1); - assert_eq!(given_version, 1); + assert_eq!(supplied_version, 1); + assert_eq!(latest_version, 1); } else { panic!("Did not get StaleChunk"); } @@ -489,11 +510,13 @@ fn test_stackerdb_insert_query_chunks() { // should fail -- too many writes version chunk_data.slot_version = db_config.max_writes + 1; chunk_data.sign(&pk).unwrap(); - if let Err(net_error::TooManySlotWrites(db_max, cur_version)) = - tx.try_replace_chunk(&sc, &chunk_data.get_slot_metadata(), &chunk_data.data) + if let Err(net_error::TooManySlotWrites { + supplied_version, + max_writes, + }) = tx.try_replace_chunk(&sc, &chunk_data.get_slot_metadata(), &chunk_data.data) { - assert_eq!(db_max, db_config.max_writes); - assert_eq!(cur_version, 1); + assert_eq!(max_writes, db_config.max_writes); + assert_eq!(supplied_version, 1); } else { panic!("Did not get TooManySlotWrites"); } @@ -559,11 +582,12 @@ fn test_reconfigure_stackerdb() { let path = "/tmp/test_stackerdb_reconfigure.sqlite"; setup_test_path(path); - let sc = ContractId::from_parts( + let sc = QualifiedContractIdentifier::new( StacksAddress { version: 0x01, bytes: Hash160([0x01; 20]), - }, + } + .into(), ContractName::try_from("db1").unwrap(), ); diff --git a/stackslib/src/net/stackerdb/tests/mod.rs b/stackslib/src/net/stackerdb/tests/mod.rs index cfcfed7f4f..0838342100 100644 --- a/stackslib/src/net/stackerdb/tests/mod.rs +++ b/stackslib/src/net/stackerdb/tests/mod.rs @@ -14,7 +14,6 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -pub mod bits; pub mod config; pub mod db; pub mod sync; diff --git a/stackslib/src/net/stackerdb/tests/sync.rs b/stackslib/src/net/stackerdb/tests/sync.rs index e77fbcd9a2..87fd47f19e 100644 --- a/stackslib/src/net/stackerdb/tests/sync.rs +++ b/stackslib/src/net/stackerdb/tests/sync.rs @@ -16,7 +16,8 @@ use std::fs; -use crate::net::stackerdb::{db::SlotValidation, SlotMetadata, StackerDBConfig, StackerDBs}; +use crate::net::stackerdb::{db::SlotValidation, StackerDBConfig, StackerDBs}; +use libstackerdb::SlotMetadata; use crate::net::Error as net_error; use crate::net::StackerDBChunkData; @@ -43,12 +44,11 @@ use rand::RngCore; use crate::net::relay::Relayer; use crate::net::test::TestPeer; use crate::net::test::TestPeerConfig; -use crate::net::ContractIdExtension; - -use crate::net::ContractId; use crate::util_lib::test::with_timeout; +use clarity::vm::types::QualifiedContractIdentifier; + const BASE_PORT: u16 = 33000; // Minimum chunk size for FROST is 97 + T * 33, where T = 3000 @@ -84,7 +84,9 @@ fn add_stackerdb(config: &mut TestPeerConfig, stackerdb_config: Option. + +use std::io; +use std::io::{Read, Write}; + +use stacks_common::types::chainstate::BlockHeaderHash; +use stacks_common::types::chainstate::StacksBlockId; + +use crate::burnchains::Txid; +use crate::chainstate::stacks::{StacksBlock, StacksBlockHeader, StacksMicroblock}; + +use crate::chainstate::stacks::db::StacksChainState; +use crate::chainstate::stacks::Error as ChainstateError; + +use crate::core::mempool::MemPoolDB; + +use crate::net::MemPoolSyncData; + +use rand::thread_rng; +use rand::Rng; + +/// Interface for streaming data +pub trait Streamer { + /// Return the offset into the stream at which this Streamer points. This value is equivalent + /// to returning the number of bytes streamed out so far. + fn offset(&self) -> u64; + /// Update the stream's offset pointer by `nw` bytes, so the implementation can keep track of + /// how much data has been sent so far. + fn add_bytes(&mut self, nw: u64); +} + +/// Opaque structure for streaming block, microblock, and header data from disk +#[derive(Debug, PartialEq, Clone)] +pub enum StreamCursor { + Block(BlockStreamData), + Microblocks(MicroblockStreamData), + Headers(HeaderStreamData), + MempoolTxs(TxStreamData), +} + +#[derive(Debug, PartialEq, Clone)] +pub struct BlockStreamData { + /// index block hash of the block to download + pub index_block_hash: StacksBlockId, + /// offset into whatever is being read (the blob, or the file in the chunk store) + pub offset: u64, + /// total number of bytes read. + pub total_bytes: u64, +} + +#[derive(Debug, PartialEq, Clone)] +pub struct MicroblockStreamData { + /// index block hash of the block to download + pub index_block_hash: StacksBlockId, + /// microblock blob row id + pub rowid: Option, + /// offset into whatever is being read (the blob, or the file in the chunk store) + pub offset: u64, + /// total number of bytes read. + pub total_bytes: u64, + + /// length prefix + pub num_items_buf: [u8; 4], + pub num_items_ptr: usize, + + /// microblock pointer + pub microblock_hash: BlockHeaderHash, + pub parent_index_block_hash: StacksBlockId, + + /// unconfirmed state + pub seq: u16, + pub unconfirmed: bool, +} + +#[derive(Debug, PartialEq, Clone)] +pub struct HeaderStreamData { + /// index block hash of the block to download + pub index_block_hash: StacksBlockId, + /// offset into whatever is being read (the blob, or the file in the chunk store) + pub offset: u64, + /// total number of bytes read. + pub total_bytes: u64, + /// number of headers requested + pub num_headers: u32, + + /// header buffer data + pub header_bytes: Option>, + pub end_of_stream: bool, + pub corked: bool, +} + +#[derive(Debug, PartialEq, Clone)] +pub struct TxStreamData { + /// Mempool sync data requested + pub tx_query: MemPoolSyncData, + /// last txid loaded + pub last_randomized_txid: Txid, + /// serialized transaction buffer that's being sent + pub tx_buf: Vec, + pub tx_buf_ptr: usize, + /// number of transactions visited in the DB so far + pub num_txs: u64, + /// maximum we can visit in the query + pub max_txs: u64, + /// height of the chain at time of query + pub height: u64, + /// Are we done sending transactions, and are now in the process of sending the trailing page + /// ID? + pub corked: bool, +} + +impl MicroblockStreamData { + /// Stream the number of microblocks, as a SIP-003-encoded 4-byte big-endian integer. + /// Returns the number of bytes written to `fd` on success + /// Returns chainstate errors otherwise. + fn stream_count(&mut self, fd: &mut W, count: u64) -> Result { + let mut num_written = 0; + while self.num_items_ptr < self.num_items_buf.len() && num_written < count { + // stream length prefix + test_debug!( + "Length prefix: try to send {:?} (ptr={})", + &self.num_items_buf[self.num_items_ptr..], + self.num_items_ptr + ); + let num_sent = match fd.write(&self.num_items_buf[self.num_items_ptr..]) { + Ok(0) => { + // done (disconnected) + test_debug!("Length prefix: wrote 0 bytes",); + return Ok(num_written); + } + Ok(n) => { + self.num_items_ptr += n; + n as u64 + } + Err(e) => { + if e.kind() == io::ErrorKind::Interrupted { + // EINTR; try again + continue; + } else if e.kind() == io::ErrorKind::WouldBlock + || (cfg!(windows) && e.kind() == io::ErrorKind::TimedOut) + { + // blocked + return Ok(num_written); + } else { + return Err(ChainstateError::WriteError(e)); + } + } + }; + num_written += num_sent; + test_debug!( + "Length prefix: sent {} bytes ({} total)", + num_sent, + num_written + ); + } + Ok(num_written) + } +} + +impl StreamCursor { + /// Create a new stream cursor for a Stacks block + pub fn new_block(index_block_hash: StacksBlockId) -> StreamCursor { + StreamCursor::Block(BlockStreamData { + index_block_hash: index_block_hash, + offset: 0, + total_bytes: 0, + }) + } + + /// Create a new stream cursor for a Stacks microblock stream that has been confirmed. + /// Returns an error if the identified microblock stream does not exist. + pub fn new_microblock_confirmed( + chainstate: &StacksChainState, + tail_index_microblock_hash: StacksBlockId, + ) -> Result { + // look up parent + let mblock_info = StacksChainState::load_staging_microblock_info_indexed( + &chainstate.db(), + &tail_index_microblock_hash, + )? + .ok_or(ChainstateError::NoSuchBlockError)?; + + let parent_index_block_hash = StacksBlockHeader::make_index_block_hash( + &mblock_info.consensus_hash, + &mblock_info.anchored_block_hash, + ); + + // need to send out the consensus_serialize()'ed array length before sending microblocks. + // this is exactly what seq tells us, though. + let num_items_buf = ((mblock_info.sequence as u32) + 1).to_be_bytes(); + + Ok(StreamCursor::Microblocks(MicroblockStreamData { + index_block_hash: StacksBlockId([0u8; 32]), + rowid: None, + offset: 0, + total_bytes: 0, + microblock_hash: mblock_info.microblock_hash, + parent_index_block_hash: parent_index_block_hash, + seq: mblock_info.sequence, + unconfirmed: false, + num_items_buf: num_items_buf, + num_items_ptr: 0, + })) + } + + /// Create a new stream cursor for a Stacks microblock stream that is unconfirmed. + /// Returns an error if the parent Stacks block does not exist, or if the sequence number is + /// too far ahead of the unconfirmed stream's tail. + pub fn new_microblock_unconfirmed( + chainstate: &StacksChainState, + anchored_index_block_hash: StacksBlockId, + seq: u16, + ) -> Result { + let mblock_info = StacksChainState::load_next_descendant_microblock( + &chainstate.db(), + &anchored_index_block_hash, + seq, + )? + .ok_or(ChainstateError::NoSuchBlockError)?; + + Ok(StreamCursor::Microblocks(MicroblockStreamData { + index_block_hash: anchored_index_block_hash.clone(), + rowid: None, + offset: 0, + total_bytes: 0, + microblock_hash: mblock_info.block_hash(), + parent_index_block_hash: anchored_index_block_hash, + seq: seq, + unconfirmed: true, + num_items_buf: [0u8; 4], + num_items_ptr: 4, // stops us from trying to send a length prefix + })) + } + + pub fn new_headers( + chainstate: &StacksChainState, + tip: &StacksBlockId, + num_headers_requested: u32, + ) -> Result { + let header_info = StacksChainState::load_staging_block_info(chainstate.db(), tip)? + .ok_or(ChainstateError::NoSuchBlockError)?; + + let num_headers = if header_info.height < (num_headers_requested as u64) { + header_info.height as u32 + } else { + num_headers_requested + }; + + test_debug!("Request for {} headers from {}", num_headers, tip); + + Ok(StreamCursor::Headers(HeaderStreamData { + index_block_hash: tip.clone(), + offset: 0, + total_bytes: 0, + num_headers: num_headers, + header_bytes: None, + end_of_stream: false, + corked: false, + })) + } + + /// Create a new stream cursor for mempool transactions + pub fn new_tx_stream( + tx_query: MemPoolSyncData, + max_txs: u64, + height: u64, + page_id_opt: Option, + ) -> StreamCursor { + let last_randomized_txid = page_id_opt.unwrap_or_else(|| { + let random_bytes = thread_rng().gen::<[u8; 32]>(); + Txid(random_bytes) + }); + + StreamCursor::MempoolTxs(TxStreamData { + tx_query, + last_randomized_txid: last_randomized_txid, + tx_buf: vec![], + tx_buf_ptr: 0, + num_txs: 0, + max_txs: max_txs, + height: height, + corked: false, + }) + } + + /// Write a single byte to the given `fd`. + /// Non-blocking -- masks EINTR by returning 0. + fn stream_one_byte(fd: &mut W, b: u8) -> Result { + loop { + match fd.write(&[b]) { + Ok(0) => { + // done (disconnected) + return Ok(0); + } + Ok(n) => { + return Ok(n as u64); + } + Err(e) => { + if e.kind() == io::ErrorKind::Interrupted { + // EINTR; try again + continue; + } else if e.kind() == io::ErrorKind::WouldBlock + || (cfg!(windows) && e.kind() == io::ErrorKind::TimedOut) + { + // blocked + return Ok(0); + } else { + return Err(ChainstateError::WriteError(e)); + } + } + } + } + } + + /// Get the offset into the stream at which the cursor points + pub fn get_offset(&self) -> u64 { + match self { + StreamCursor::Block(ref stream) => stream.offset(), + StreamCursor::Microblocks(ref stream) => stream.offset(), + StreamCursor::Headers(ref stream) => stream.offset(), + // no-op for mempool txs + StreamCursor::MempoolTxs(..) => 0, + } + } + + /// Update the cursor's offset by nw + pub fn add_more_bytes(&mut self, nw: u64) { + match self { + StreamCursor::Block(ref mut stream) => stream.add_bytes(nw), + StreamCursor::Microblocks(ref mut stream) => stream.add_bytes(nw), + StreamCursor::Headers(ref mut stream) => stream.add_bytes(nw), + // no-op fo mempool txs + StreamCursor::MempoolTxs(..) => (), + } + } + + /// Stream chainstate data into the given `fd`. + /// Depending on what StreamCursor variant we are, the data may come from the chainstate or + /// mempool. + /// Returns the number of bytes streamed on success. + /// Return an error on I/O errors, or if this cursor does not represent chainstate data. + pub fn stream_to( + &mut self, + mempool: &MemPoolDB, + chainstate: &mut StacksChainState, + fd: &mut W, + count: u64, + ) -> Result { + match self { + StreamCursor::Microblocks(ref mut stream) => { + let mut num_written = 0; + if !stream.unconfirmed { + // Confirmed microblocks are represented as a consensus-encoded vector of + // microblocks, in reverse sequence order. + // Write 4-byte length prefix first + num_written += stream.stream_count(fd, count)?; + StacksChainState::stream_microblocks_confirmed(&chainstate, fd, stream, count) + .and_then(|bytes_sent| Ok(bytes_sent + num_written)) + } else { + StacksChainState::stream_microblocks_unconfirmed(&chainstate, fd, stream, count) + .and_then(|bytes_sent| Ok(bytes_sent + num_written)) + } + } + StreamCursor::MempoolTxs(ref mut tx_stream) => mempool.stream_txs(fd, tx_stream, count), + StreamCursor::Headers(ref mut stream) => { + // headers are a JSON array. Start by writing '[', then write each header, and + // then write ']' + let mut num_written = 0; + if stream.total_bytes == 0 { + test_debug!("Opening header stream"); + let byte_written = StreamCursor::stream_one_byte(fd, '[' as u8)?; + num_written += byte_written; + stream.total_bytes += byte_written; + } + if stream.total_bytes > 0 { + let mut sent = chainstate.stream_headers(fd, stream, count)?; + + if stream.end_of_stream && !stream.corked { + // end of stream; cork it + test_debug!("Corking header stream"); + let byte_written = StreamCursor::stream_one_byte(fd, ']' as u8)?; + if byte_written > 0 { + sent += byte_written; + stream.total_bytes += byte_written; + stream.corked = true; + } + } + num_written += sent; + } + Ok(num_written) + } + StreamCursor::Block(ref mut stream) => chainstate.stream_block(fd, stream, count), + } + } +} + +impl Streamer for StreamCursor { + fn offset(&self) -> u64 { + self.get_offset() + } + fn add_bytes(&mut self, nw: u64) { + self.add_more_bytes(nw) + } +} + +impl Streamer for HeaderStreamData { + fn offset(&self) -> u64 { + self.offset + } + fn add_bytes(&mut self, nw: u64) { + self.offset += nw; + self.total_bytes += nw; + } +} + +impl Streamer for BlockStreamData { + fn offset(&self) -> u64 { + self.offset + } + fn add_bytes(&mut self, nw: u64) { + self.offset += nw; + self.total_bytes += nw; + } +} + +impl Streamer for MicroblockStreamData { + fn offset(&self) -> u64 { + self.offset + } + fn add_bytes(&mut self, nw: u64) { + self.offset += nw; + self.total_bytes += nw; + } +} diff --git a/stackslib/src/net/tests/mod.rs b/stackslib/src/net/tests/mod.rs index b99a2d193b..a5cf528989 100644 --- a/stackslib/src/net/tests/mod.rs +++ b/stackslib/src/net/tests/mod.rs @@ -15,3 +15,4 @@ // along with this program. If not, see . pub mod neighbors; +pub mod stream; diff --git a/stackslib/src/net/tests/neighbors.rs b/stackslib/src/net/tests/neighbors.rs index 60012648bb..149988ff5c 100644 --- a/stackslib/src/net/tests/neighbors.rs +++ b/stackslib/src/net/tests/neighbors.rs @@ -1732,14 +1732,14 @@ fn test_step_walk_2_neighbors_different_networks() { }) } -fn stacker_db_id(i: usize) -> ContractId { - ContractId::new( +fn stacker_db_id(i: usize) -> QualifiedContractIdentifier { + QualifiedContractIdentifier::new( StandardPrincipalData(0x01, [i as u8; 20]), format!("db-{}", i).as_str().into(), ) } -fn make_stacker_db_ids(i: usize) -> Vec { +fn make_stacker_db_ids(i: usize) -> Vec { let mut dbs = vec![]; for j in 0..i { dbs.push(stacker_db_id(j)); diff --git a/stackslib/src/net/tests/stream.rs b/stackslib/src/net/tests/stream.rs new file mode 100644 index 0000000000..0edf612c97 --- /dev/null +++ b/stackslib/src/net/tests/stream.rs @@ -0,0 +1,707 @@ +// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use stacks_common::codec::StacksMessageCodec; +use stacks_common::types::chainstate::ConsensusHash; +use stacks_common::types::chainstate::StacksBlockId; +use stacks_common::types::chainstate::StacksPrivateKey; + +use crate::net::ExtendedStacksHeader; +use crate::net::StreamCursor; + +use crate::util_lib::db::DBConn; + +use crate::chainstate::stacks::db::StacksChainState; +use crate::chainstate::stacks::Error as chainstate_error; +use crate::chainstate::stacks::StacksBlock; +use crate::chainstate::stacks::StacksBlockHeader; +use crate::chainstate::stacks::StacksMicroblock; + +use crate::chainstate::stacks::db::blocks::test::*; +use crate::chainstate::stacks::db::test::instantiate_chainstate; + +use crate::core::MemPoolDB; + +fn stream_one_header_to_vec( + blocks_conn: &DBConn, + blocks_path: &str, + stream: &mut StreamCursor, + count: u64, +) -> Result, chainstate_error> { + if let StreamCursor::Headers(ref mut stream) = stream { + let mut bytes = vec![]; + StacksChainState::stream_one_header(blocks_conn, blocks_path, &mut bytes, stream, count) + .map(|nr| { + assert_eq!(bytes.len(), nr as usize); + + // truncate trailing ',' if it exists + let len = bytes.len(); + if len > 0 { + if bytes[len - 1] == ',' as u8 { + let _ = bytes.pop(); + } + } + bytes + }) + } else { + panic!("not a header stream"); + } +} + +fn stream_one_staging_microblock_to_vec( + blocks_conn: &DBConn, + stream: &mut StreamCursor, + count: u64, +) -> Result, chainstate_error> { + if let StreamCursor::Microblocks(ref mut stream) = stream { + let mut bytes = vec![]; + StacksChainState::stream_one_microblock(blocks_conn, &mut bytes, stream, count).map(|nr| { + assert_eq!(bytes.len(), nr as usize); + bytes + }) + } else { + panic!("not a microblock stream"); + } +} + +fn stream_chunk_to_vec( + blocks_path: &str, + stream: &mut StreamCursor, + count: u64, +) -> Result, chainstate_error> { + if let StreamCursor::Block(ref mut stream) = stream { + let mut bytes = vec![]; + StacksChainState::stream_data_from_chunk_store(blocks_path, &mut bytes, stream, count).map( + |nr| { + assert_eq!(bytes.len(), nr as usize); + bytes + }, + ) + } else { + panic!("not a block stream"); + } +} + +fn stream_headers_to_vec( + chainstate: &mut StacksChainState, + stream: &mut StreamCursor, + count: u64, +) -> Result, chainstate_error> { + let mempool = MemPoolDB::open_test( + chainstate.mainnet, + chainstate.chain_id, + &chainstate.root_path, + ) + .unwrap(); + let mut bytes = vec![]; + stream + .stream_to(&mempool, chainstate, &mut bytes, count) + .map(|nr| { + assert_eq!(bytes.len(), nr as usize); + bytes + }) +} + +fn stream_unconfirmed_microblocks_to_vec( + chainstate: &mut StacksChainState, + stream: &mut StreamCursor, + count: u64, +) -> Result, chainstate_error> { + let mempool = MemPoolDB::open_test( + chainstate.mainnet, + chainstate.chain_id, + &chainstate.root_path, + ) + .unwrap(); + let mut bytes = vec![]; + stream + .stream_to(&mempool, chainstate, &mut bytes, count) + .map(|nr| { + assert_eq!(bytes.len(), nr as usize); + bytes + }) +} + +fn stream_confirmed_microblocks_to_vec( + chainstate: &mut StacksChainState, + stream: &mut StreamCursor, + count: u64, +) -> Result, chainstate_error> { + let mempool = MemPoolDB::open_test( + chainstate.mainnet, + chainstate.chain_id, + &chainstate.root_path, + ) + .unwrap(); + let mut bytes = vec![]; + stream + .stream_to(&mempool, chainstate, &mut bytes, count) + .map(|nr| { + assert_eq!(bytes.len(), nr as usize); + bytes + }) +} + +#[test] +fn stacks_db_stream_blocks() { + let mut chainstate = instantiate_chainstate(false, 0x80000000, function_name!()); + let privk = StacksPrivateKey::from_hex( + "eb05c83546fdd2c79f10f5ad5434a90dd28f7e3acb7c092157aa1bc3656b012c01", + ) + .unwrap(); + + let block = make_16k_block(&privk); + + let consensus_hash = ConsensusHash([2u8; 20]); + let parent_consensus_hash = ConsensusHash([1u8; 20]); + let index_block_header = + StacksBlockHeader::make_index_block_hash(&consensus_hash, &block.block_hash()); + + // can't stream a non-existant block + let mut stream = StreamCursor::new_block(index_block_header.clone()); + assert!(stream_chunk_to_vec(&chainstate.blocks_path, &mut stream, 123).is_err()); + + // stream unmodified + let stream_2 = StreamCursor::new_block(index_block_header.clone()); + assert_eq!(stream, stream_2); + + // store block to staging + store_staging_block( + &mut chainstate, + &consensus_hash, + &block, + &parent_consensus_hash, + 1, + 2, + ); + + // stream it back + let mut all_block_bytes = vec![]; + loop { + let mut next_bytes = stream_chunk_to_vec(&chainstate.blocks_path, &mut stream, 16).unwrap(); + if next_bytes.len() == 0 { + break; + } + test_debug!( + "Got {} more bytes from staging; add to {} total", + next_bytes.len(), + all_block_bytes.len() + ); + all_block_bytes.append(&mut next_bytes); + } + + // should decode back into the block + let staging_block = StacksBlock::consensus_deserialize(&mut &all_block_bytes[..]).unwrap(); + assert_eq!(staging_block, block); + + // accept it + set_block_processed(&mut chainstate, &consensus_hash, &block.block_hash(), true); + + // can still stream it + let mut stream = StreamCursor::new_block(index_block_header.clone()); + + // stream from chunk store + let mut all_block_bytes = vec![]; + loop { + let mut next_bytes = stream_chunk_to_vec(&chainstate.blocks_path, &mut stream, 16).unwrap(); + if next_bytes.len() == 0 { + break; + } + test_debug!( + "Got {} more bytes from chunkstore; add to {} total", + next_bytes.len(), + all_block_bytes.len() + ); + all_block_bytes.append(&mut next_bytes); + } + + // should decode back into the block + let staging_block = StacksBlock::consensus_deserialize(&mut &all_block_bytes[..]).unwrap(); + assert_eq!(staging_block, block); +} + +#[test] +fn stacks_db_stream_headers() { + let mut chainstate = instantiate_chainstate(false, 0x80000000, function_name!()); + let privk = StacksPrivateKey::from_hex( + "eb05c83546fdd2c79f10f5ad5434a90dd28f7e3acb7c092157aa1bc3656b012c01", + ) + .unwrap(); + + let mut blocks: Vec = vec![]; + let mut blocks_index_hashes: Vec = vec![]; + + // make a linear stream + for i in 0..32 { + let mut block = make_empty_coinbase_block(&privk); + + if i == 0 { + block.header.total_work.work = 1; + block.header.total_work.burn = 1; + } + if i > 0 { + block.header.parent_block = blocks.get(i - 1).unwrap().block_hash(); + block.header.total_work.work = blocks.get(i - 1).unwrap().header.total_work.work + 1; + block.header.total_work.burn = blocks.get(i - 1).unwrap().header.total_work.burn + 1; + } + + let consensus_hash = ConsensusHash([((i + 1) as u8); 20]); + let parent_consensus_hash = ConsensusHash([(i as u8); 20]); + + store_staging_block( + &mut chainstate, + &consensus_hash, + &block, + &parent_consensus_hash, + i as u64, + i as u64, + ); + + blocks_index_hashes.push(StacksBlockHeader::make_index_block_hash( + &consensus_hash, + &block.block_hash(), + )); + blocks.push(block); + } + + let mut blocks_fork = blocks[0..16].to_vec(); + let mut blocks_fork_index_hashes = blocks_index_hashes[0..16].to_vec(); + + // make a stream that branches off + for i in 16..32 { + let mut block = make_empty_coinbase_block(&privk); + + if i == 16 { + block.header.parent_block = blocks.get(i - 1).unwrap().block_hash(); + block.header.total_work.work = blocks.get(i - 1).unwrap().header.total_work.work + 1; + block.header.total_work.burn = blocks.get(i - 1).unwrap().header.total_work.burn + 2; + } else { + block.header.parent_block = blocks_fork.get(i - 1).unwrap().block_hash(); + block.header.total_work.work = + blocks_fork.get(i - 1).unwrap().header.total_work.work + 1; + block.header.total_work.burn = + blocks_fork.get(i - 1).unwrap().header.total_work.burn + 2; + } + + let consensus_hash = ConsensusHash([((i + 1) as u8) | 0x80; 20]); + let parent_consensus_hash = if i == 16 { + ConsensusHash([(i as u8); 20]) + } else { + ConsensusHash([(i as u8) | 0x80; 20]) + }; + + store_staging_block( + &mut chainstate, + &consensus_hash, + &block, + &parent_consensus_hash, + i as u64, + i as u64, + ); + + blocks_fork_index_hashes.push(StacksBlockHeader::make_index_block_hash( + &consensus_hash, + &block.block_hash(), + )); + blocks_fork.push(block); + } + + // can't stream a non-existant header + assert!(StreamCursor::new_headers(&chainstate, &StacksBlockId([0x11; 32]), 1).is_err()); + + // stream back individual headers + for i in 0..blocks.len() { + let mut stream = + StreamCursor::new_headers(&chainstate, &blocks_index_hashes[i], 1).unwrap(); + let mut next_header_bytes = vec![]; + loop { + // torture test + let mut next_bytes = stream_one_header_to_vec( + &chainstate.db(), + &chainstate.blocks_path, + &mut stream, + 25, + ) + .unwrap(); + if next_bytes.len() == 0 { + break; + } + next_header_bytes.append(&mut next_bytes); + } + test_debug!("Got {} total bytes", next_header_bytes.len()); + let header: ExtendedStacksHeader = + serde_json::from_reader(&mut &next_header_bytes[..]).unwrap(); + + assert_eq!(header.consensus_hash, ConsensusHash([(i + 1) as u8; 20])); + assert_eq!(header.header, blocks[i].header); + + if i > 0 { + assert_eq!(header.parent_block_id, blocks_index_hashes[i - 1]); + } + } + + // stream back a run of headers + let block_expected_headers: Vec = + blocks.iter().rev().map(|blk| blk.header.clone()).collect(); + + let block_expected_index_hashes: Vec = blocks_index_hashes + .iter() + .rev() + .map(|idx| idx.clone()) + .collect(); + + let block_fork_expected_headers: Vec = blocks_fork + .iter() + .rev() + .map(|blk| blk.header.clone()) + .collect(); + + let block_fork_expected_index_hashes: Vec = blocks_fork_index_hashes + .iter() + .rev() + .map(|idx| idx.clone()) + .collect(); + + // get them all -- ask for more than there is + let mut stream = + StreamCursor::new_headers(&chainstate, blocks_index_hashes.last().unwrap(), 4096).unwrap(); + let header_bytes = stream_headers_to_vec(&mut chainstate, &mut stream, 1024 * 1024).unwrap(); + + eprintln!( + "headers: {}", + String::from_utf8(header_bytes.clone()).unwrap() + ); + let headers: Vec = + serde_json::from_reader(&mut &header_bytes[..]).unwrap(); + + assert_eq!(headers.len(), block_expected_headers.len()); + for ((i, h), eh) in headers + .iter() + .enumerate() + .zip(block_expected_headers.iter()) + { + assert_eq!(h.header, *eh); + assert_eq!(h.consensus_hash, ConsensusHash([(32 - i) as u8; 20])); + if i + 1 < block_expected_index_hashes.len() { + assert_eq!(h.parent_block_id, block_expected_index_hashes[i + 1]); + } + } + + let mut stream = + StreamCursor::new_headers(&chainstate, blocks_fork_index_hashes.last().unwrap(), 4096) + .unwrap(); + let header_bytes = stream_headers_to_vec(&mut chainstate, &mut stream, 1024 * 1024).unwrap(); + let fork_headers: Vec = + serde_json::from_reader(&mut &header_bytes[..]).unwrap(); + + assert_eq!(fork_headers.len(), block_fork_expected_headers.len()); + for ((i, h), eh) in fork_headers + .iter() + .enumerate() + .zip(block_fork_expected_headers.iter()) + { + let consensus_hash = if i >= 16 { + ConsensusHash([((32 - i) as u8); 20]) + } else { + ConsensusHash([((32 - i) as u8) | 0x80; 20]) + }; + + assert_eq!(h.header, *eh); + assert_eq!(h.consensus_hash, consensus_hash); + if i + 1 < block_fork_expected_index_hashes.len() { + assert_eq!(h.parent_block_id, block_fork_expected_index_hashes[i + 1]); + } + } + + assert_eq!(fork_headers[16..32], headers[16..32]); + + // ask for only a few + let mut stream = + StreamCursor::new_headers(&chainstate, blocks_index_hashes.last().unwrap(), 10).unwrap(); + let mut header_bytes = vec![]; + loop { + // torture test + let mut next_bytes = stream_headers_to_vec(&mut chainstate, &mut stream, 17).unwrap(); + if next_bytes.len() == 0 { + break; + } + header_bytes.append(&mut next_bytes); + } + + eprintln!( + "header bytes: {}", + String::from_utf8(header_bytes.clone()).unwrap() + ); + + let headers: Vec = + serde_json::from_reader(&mut &header_bytes[..]).unwrap(); + + assert_eq!(headers.len(), 10); + for (i, hdr) in headers.iter().enumerate() { + assert_eq!(hdr.header, block_expected_headers[i]); + assert_eq!(hdr.parent_block_id, block_expected_index_hashes[i + 1]); + } + + // ask for only a few + let mut stream = + StreamCursor::new_headers(&chainstate, blocks_fork_index_hashes.last().unwrap(), 10) + .unwrap(); + let mut header_bytes = vec![]; + loop { + // torture test + let mut next_bytes = stream_headers_to_vec(&mut chainstate, &mut stream, 17).unwrap(); + if next_bytes.len() == 0 { + break; + } + header_bytes.append(&mut next_bytes); + } + let headers: Vec = + serde_json::from_reader(&mut &header_bytes[..]).unwrap(); + + assert_eq!(headers.len(), 10); + for (i, hdr) in headers.iter().enumerate() { + assert_eq!(hdr.header, block_fork_expected_headers[i]); + assert_eq!(hdr.parent_block_id, block_fork_expected_index_hashes[i + 1]); + } +} + +#[test] +fn stacks_db_stream_staging_microblocks() { + let mut chainstate = instantiate_chainstate(false, 0x80000000, function_name!()); + let privk = StacksPrivateKey::from_hex( + "eb05c83546fdd2c79f10f5ad5434a90dd28f7e3acb7c092157aa1bc3656b012c01", + ) + .unwrap(); + + let block = make_empty_coinbase_block(&privk); + let mut mblocks = make_sample_microblock_stream(&privk, &block.block_hash()); + mblocks.truncate(15); + + let consensus_hash = ConsensusHash([2u8; 20]); + let parent_consensus_hash = ConsensusHash([1u8; 20]); + let index_block_header = + StacksBlockHeader::make_index_block_hash(&consensus_hash, &block.block_hash()); + + // can't stream a non-existant microblock + if let Err(chainstate_error::NoSuchBlockError) = + StreamCursor::new_microblock_confirmed(&chainstate, index_block_header.clone()) + { + } else { + panic!("Opened nonexistant microblock"); + } + + if let Err(chainstate_error::NoSuchBlockError) = + StreamCursor::new_microblock_unconfirmed(&chainstate, index_block_header.clone(), 0) + { + } else { + panic!("Opened nonexistant microblock"); + } + + // store microblocks to staging and stream them back + for (i, mblock) in mblocks.iter().enumerate() { + store_staging_microblock( + &mut chainstate, + &consensus_hash, + &block.block_hash(), + mblock, + ); + + // read back all the data we have so far, block-by-block + let mut staging_mblocks = vec![]; + for j in 0..(i + 1) { + let mut next_mblock_bytes = vec![]; + let mut stream = StreamCursor::new_microblock_unconfirmed( + &chainstate, + index_block_header.clone(), + j as u16, + ) + .unwrap(); + loop { + let mut next_bytes = + stream_one_staging_microblock_to_vec(&chainstate.db(), &mut stream, 4096) + .unwrap(); + if next_bytes.len() == 0 { + break; + } + test_debug!( + "Got {} more bytes from staging; add to {} total", + next_bytes.len(), + next_mblock_bytes.len() + ); + next_mblock_bytes.append(&mut next_bytes); + } + test_debug!("Got {} total bytes", next_mblock_bytes.len()); + + // should deserialize to a microblock + let staging_mblock = + StacksMicroblock::consensus_deserialize(&mut &next_mblock_bytes[..]).unwrap(); + staging_mblocks.push(staging_mblock); + } + + assert_eq!(staging_mblocks.len(), mblocks[0..(i + 1)].len()); + for j in 0..(i + 1) { + test_debug!("check {}", j); + assert_eq!(staging_mblocks[j], mblocks[j]) + } + + // can also read partial stream in one shot, from any seq + for k in 0..(i + 1) { + test_debug!("start at seq {}", k); + let mut staging_mblock_bytes = vec![]; + let mut stream = StreamCursor::new_microblock_unconfirmed( + &chainstate, + index_block_header.clone(), + k as u16, + ) + .unwrap(); + loop { + let mut next_bytes = + stream_unconfirmed_microblocks_to_vec(&mut chainstate, &mut stream, 4096) + .unwrap(); + if next_bytes.len() == 0 { + break; + } + test_debug!( + "Got {} more bytes from staging; add to {} total", + next_bytes.len(), + staging_mblock_bytes.len() + ); + staging_mblock_bytes.append(&mut next_bytes); + } + + test_debug!("Got {} total bytes", staging_mblock_bytes.len()); + + // decode stream + let staging_mblocks = decode_microblock_stream(&staging_mblock_bytes); + + assert_eq!(staging_mblocks.len(), mblocks[k..(i + 1)].len()); + for j in 0..staging_mblocks.len() { + test_debug!("check {}", j); + assert_eq!(staging_mblocks[j], mblocks[k + j]) + } + } + } +} + +#[test] +fn stacks_db_stream_confirmed_microblocks() { + let mut chainstate = instantiate_chainstate(false, 0x80000000, function_name!()); + let privk = StacksPrivateKey::from_hex( + "eb05c83546fdd2c79f10f5ad5434a90dd28f7e3acb7c092157aa1bc3656b012c01", + ) + .unwrap(); + + let block = make_empty_coinbase_block(&privk); + let mut mblocks = make_sample_microblock_stream(&privk, &block.block_hash()); + mblocks.truncate(5); + + let mut child_block = make_empty_coinbase_block(&privk); + child_block.header.parent_block = block.block_hash(); + child_block.header.parent_microblock = mblocks.last().as_ref().unwrap().block_hash(); + child_block.header.parent_microblock_sequence = + mblocks.last().as_ref().unwrap().header.sequence; + + let consensus_hash = ConsensusHash([2u8; 20]); + let parent_consensus_hash = ConsensusHash([1u8; 20]); + let child_consensus_hash = ConsensusHash([3u8; 20]); + + let index_block_header = + StacksBlockHeader::make_index_block_hash(&consensus_hash, &block.block_hash()); + + // store microblocks to staging + for (i, mblock) in mblocks.iter().enumerate() { + store_staging_microblock( + &mut chainstate, + &consensus_hash, + &block.block_hash(), + mblock, + ); + } + + // store block to staging + store_staging_block( + &mut chainstate, + &consensus_hash, + &block, + &parent_consensus_hash, + 1, + 2, + ); + + // store child block to staging + store_staging_block( + &mut chainstate, + &child_consensus_hash, + &child_block, + &consensus_hash, + 1, + 2, + ); + + // accept it + set_block_processed(&mut chainstate, &consensus_hash, &block.block_hash(), true); + set_block_processed( + &mut chainstate, + &child_consensus_hash, + &child_block.block_hash(), + true, + ); + + for i in 0..mblocks.len() { + // set different parts of this stream as confirmed + set_microblocks_processed( + &mut chainstate, + &child_consensus_hash, + &child_block.block_hash(), + &mblocks[i].block_hash(), + ); + + // verify that we can stream everything + let microblock_index_header = + StacksBlockHeader::make_index_block_hash(&consensus_hash, &mblocks[i].block_hash()); + let mut stream = + StreamCursor::new_microblock_confirmed(&chainstate, microblock_index_header.clone()) + .unwrap(); + + let mut confirmed_mblock_bytes = vec![]; + loop { + let mut next_bytes = + stream_confirmed_microblocks_to_vec(&mut chainstate, &mut stream, 16).unwrap(); + if next_bytes.len() == 0 { + break; + } + test_debug!( + "Got {} more bytes from staging; add to {} total", + next_bytes.len(), + confirmed_mblock_bytes.len() + ); + confirmed_mblock_bytes.append(&mut next_bytes); + } + + // decode stream (should be length-prefixed) + let mut confirmed_mblocks = + Vec::::consensus_deserialize(&mut &confirmed_mblock_bytes[..]) + .unwrap(); + + confirmed_mblocks.reverse(); + + assert_eq!(confirmed_mblocks.len(), mblocks[0..(i + 1)].len()); + for j in 0..(i + 1) { + test_debug!("check {}", j); + assert_eq!(confirmed_mblocks[j], mblocks[j]) + } + } +} diff --git a/testnet/stacks-node/src/burnchains/bitcoin_regtest_controller.rs b/testnet/stacks-node/src/burnchains/bitcoin_regtest_controller.rs index 4f14fd27f0..cd19661c13 100644 --- a/testnet/stacks-node/src/burnchains/bitcoin_regtest_controller.rs +++ b/testnet/stacks-node/src/burnchains/bitcoin_regtest_controller.rs @@ -2339,9 +2339,6 @@ impl BitcoinRPCRequest { } pub fn import_public_key(config: &Config, public_key: &Secp256k1PublicKey) -> RPCResult<()> { - let rescan = true; - let label = ""; - let pkh = Hash160::from_data(&public_key.to_bytes()) .to_bytes() .to_vec(); @@ -2368,9 +2365,30 @@ impl BitcoinRPCRequest { addr2str(&address), public_key.to_hex() ); + let payload = BitcoinRPCRequest { - method: "importaddress".to_string(), - params: vec![addr2str(&address).into(), label.into(), rescan.into()], + method: "getdescriptorinfo".to_string(), + params: vec![format!("addr({})", &addr2str(&address)).into()], + id: "stacks".to_string(), + jsonrpc: "2.0".to_string(), + }; + + let result = BitcoinRPCRequest::send(&config, payload)?; + let checksum = result + .get(&"result".to_string()) + .and_then(|res| res.as_object()) + .and_then(|obj| obj.get("checksum")) + .and_then(|checksum_val| checksum_val.as_str()) + .ok_or(RPCError::Bitcoind(format!( + "Did not receive an object with `checksum` from `getdescriptorinfo \"{}\"`", + &addr2str(&address) + )))?; + + let payload = BitcoinRPCRequest { + method: "importdescriptors".to_string(), + params: vec![ + json!([{ "desc": format!("addr({})#{}", &addr2str(&address), &checksum), "timestamp": 0, "internal": true }]), + ], id: "stacks".to_string(), jsonrpc: "2.0".to_string(), }; @@ -2422,7 +2440,7 @@ impl BitcoinRPCRequest { pub fn create_wallet(config: &Config, wallet_name: &str) -> RPCResult<()> { let payload = BitcoinRPCRequest { method: "createwallet".to_string(), - params: vec![wallet_name.into()], + params: vec![wallet_name.into(), true.into()], id: "stacks".to_string(), jsonrpc: "2.0".to_string(), }; @@ -2440,6 +2458,7 @@ impl BitcoinRPCRequest { return Err(RPCError::Network(format!("RPC Error: {}", err))); } }; + request.append_header("Content-Type", "application/json"); request.set_body(body); diff --git a/testnet/stacks-node/src/config.rs b/testnet/stacks-node/src/config.rs index 364df90b55..bfc617c22a 100644 --- a/testnet/stacks-node/src/config.rs +++ b/testnet/stacks-node/src/config.rs @@ -708,6 +708,14 @@ impl Config { chain_liveness_poll_time_secs: node .chain_liveness_poll_time_secs .unwrap_or(default_node_config.chain_liveness_poll_time_secs), + stacker_dbs: node + .stacker_dbs + .unwrap_or(vec![]) + .iter() + .filter_map(|contract_id| { + QualifiedContractIdentifier::parse(contract_id).ok() + }) + .collect(), }; (node_config, node.bootstrap_node, node.deny_nodes) } @@ -1521,6 +1529,8 @@ pub struct NodeConfig { /// At most, how often should the chain-liveness thread /// wake up the chains-coordinator. Defaults to 300s (5 min). pub chain_liveness_poll_time_secs: u64, + /// stacker DBs we replicate + pub stacker_dbs: Vec, } #[derive(Clone, Debug)] @@ -1799,6 +1809,7 @@ impl NodeConfig { require_affirmed_anchor_blocks: true, fault_injection_hide_blocks: false, chain_liveness_poll_time_secs: 300, + stacker_dbs: vec![], } } @@ -2005,6 +2016,8 @@ pub struct NodeConfigFile { /// At most, how often should the chain-liveness thread /// wake up the chains-coordinator. Defaults to 300s (5 min). pub chain_liveness_poll_time_secs: Option, + /// Stacker DBs we replicate + pub stacker_dbs: Option>, } #[derive(Clone, Deserialize, Default, Debug)] @@ -2083,6 +2096,7 @@ pub enum EventKeyType { BurnchainBlocks, MinedBlocks, MinedMicroblocks, + StackerDBChunks, } impl EventKeyType { @@ -2107,6 +2121,10 @@ impl EventKeyType { return Some(EventKeyType::Microblocks); } + if raw_key == "stackerdb" { + return Some(EventKeyType::StackerDBChunks); + } + let comps: Vec<_> = raw_key.split("::").collect(); if comps.len() == 1 { let split: Vec<_> = comps[0].split(".").collect(); diff --git a/testnet/stacks-node/src/event_dispatcher.rs b/testnet/stacks-node/src/event_dispatcher.rs index 0c49b7f2fa..42cb385390 100644 --- a/testnet/stacks-node/src/event_dispatcher.rs +++ b/testnet/stacks-node/src/event_dispatcher.rs @@ -20,7 +20,8 @@ use stacks::chainstate::stacks::{ }; use stacks::chainstate::stacks::{StacksBlock, StacksMicroblock}; use stacks::codec::StacksMessageCodec; -use stacks::core::mempool::{MemPoolDropReason, MemPoolEventDispatcher}; +use stacks::core::mempool::MemPoolDropReason; +use stacks::core::mempool::MemPoolEventDispatcher; use stacks::net::atlas::{Attachment, AttachmentInstance}; use stacks::types::chainstate::{BlockHeaderHash, BurnchainHeaderHash, StacksBlockId}; use stacks::util::hash::bytes_to_hex; @@ -36,6 +37,10 @@ use stacks::chainstate::stacks::db::unconfirmed::ProcessedUnconfirmedState; use stacks::chainstate::stacks::miner::TransactionEvent; use stacks::chainstate::stacks::TransactionPayload; +use stacks::net::stackerdb::StackerDBEventDispatcher; + +use stacks::libstackerdb::StackerDBChunkData; + #[derive(Debug, Clone)] struct EventObserver { endpoint: String, @@ -60,6 +65,7 @@ pub const PATH_MEMPOOL_TX_SUBMIT: &str = "new_mempool_tx"; pub const PATH_MEMPOOL_TX_DROP: &str = "drop_mempool_tx"; pub const PATH_MINED_BLOCK: &str = "mined_block"; pub const PATH_MINED_MICROBLOCK: &str = "mined_microblock"; +pub const PATH_STACKERDB_CHUNKS: &str = "stackerdb_chunks"; pub const PATH_BURN_BLOCK_SUBMIT: &str = "new_burn_block"; pub const PATH_BLOCK_PROCESSED: &str = "new_block"; pub const PATH_ATTACHMENT_PROCESSED: &str = "attachments/new"; @@ -84,6 +90,13 @@ pub struct MinedMicroblockEvent { pub anchor_block: BlockHeaderHash, } +/// Event structure for newly-arrived StackerDB data +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct StackerDBChunksEvent { + pub contract_id: QualifiedContractIdentifier, + pub modified_slots: Vec, +} + impl EventObserver { pub fn send_payload(&self, payload: &serde_json::Value, path: &str) { let body = match serde_json::to_vec(&payload) { @@ -336,6 +349,10 @@ impl EventObserver { self.send_payload(payload, PATH_MINED_MICROBLOCK); } + fn send_stackerdb_chunks(&self, payload: &serde_json::Value) { + self.send_payload(payload, PATH_STACKERDB_CHUNKS); + } + fn send_new_burn_block(&self, payload: &serde_json::Value) { self.send_payload(payload, PATH_BURN_BLOCK_SUBMIT); } @@ -411,6 +428,7 @@ pub struct EventDispatcher { any_event_observers_lookup: HashSet, miner_observers_lookup: HashSet, mined_microblocks_observers_lookup: HashSet, + stackerdb_observers_lookup: HashSet, } impl MemPoolEventDispatcher for EventDispatcher { @@ -455,6 +473,17 @@ impl MemPoolEventDispatcher for EventDispatcher { } } +impl StackerDBEventDispatcher for EventDispatcher { + /// Relay new StackerDB chunks + fn new_stackerdb_chunks( + &self, + contract_id: QualifiedContractIdentifier, + chunks: Vec, + ) { + self.process_new_stackerdb_chunks(contract_id, chunks); + } +} + impl BlockEventDispatcher for EventDispatcher { fn announce_block( &self, @@ -520,6 +549,7 @@ impl EventDispatcher { microblock_observers_lookup: HashSet::new(), miner_observers_lookup: HashSet::new(), mined_microblocks_observers_lookup: HashSet::new(), + stackerdb_observers_lookup: HashSet::new(), } } @@ -880,6 +910,36 @@ impl EventDispatcher { } } + /// Forward newly-accepted StackerDB chunk metadata to downstream `stackerdb` observers. + /// Infallible. + pub fn process_new_stackerdb_chunks( + &self, + contract_id: QualifiedContractIdentifier, + new_chunks: Vec, + ) { + let interested_observers: Vec<_> = self + .registered_observers + .iter() + .enumerate() + .filter(|(obs_id, _observer)| { + self.stackerdb_observers_lookup.contains(&(*obs_id as u16)) + }) + .collect(); + if interested_observers.len() < 1 { + return; + } + + let payload = serde_json::to_value(StackerDBChunksEvent { + contract_id, + modified_slots: new_chunks, + }) + .expect("FATAL: failed to serialize StackerDBChunksEvent to JSON"); + + for (_, observer) in interested_observers.iter() { + observer.send_stackerdb_chunks(&payload); + } + } + pub fn process_dropped_mempool_txs(&self, txs: Vec, reason: MemPoolDropReason) { // lazily assemble payload only if we have observers let interested_observers: Vec<_> = self @@ -999,6 +1059,9 @@ impl EventDispatcher { self.mined_microblocks_observers_lookup .insert(observer_index); } + EventKeyType::StackerDBChunks => { + self.stackerdb_observers_lookup.insert(observer_index); + } } } diff --git a/testnet/stacks-node/src/neon_node.rs b/testnet/stacks-node/src/neon_node.rs index 05990409ee..dc7ddcfe20 100644 --- a/testnet/stacks-node/src/neon_node.rs +++ b/testnet/stacks-node/src/neon_node.rs @@ -193,8 +193,8 @@ use stacks::net::{ p2p::PeerNetwork, relay::Relayer, rpc::RPCHandlerArgs, - stackerdb::StackerDBs, - Error as NetError, NetworkResult, PeerAddress, ServiceFlags, + stackerdb::{StackerDBConfig, StackerDBSync, StackerDBs}, + Error as NetError, NetworkResult, PeerAddress, PeerNetworkComms, ServiceFlags, }; use stacks::types::chainstate::{ BlockHeaderHash, BurnchainHeaderHash, SortitionId, StacksAddress, VRFSeed, @@ -225,6 +225,7 @@ use stacks_common::util::vrf::VRFProof; use clarity::vm::ast::ASTRules; use clarity::vm::types::PrincipalData; +use clarity::vm::types::QualifiedContractIdentifier; pub const RELAYER_MAX_BUFFER: usize = 100; const VRF_MOCK_MINER_KEY: u64 = 1; @@ -3894,7 +3895,11 @@ impl StacksNode { /// * bootstrap nodes /// Returns the instantiated PeerDB /// Panics on failure. - fn setup_peer_db(config: &Config, burnchain: &Burnchain) -> PeerDB { + fn setup_peer_db( + config: &Config, + burnchain: &Burnchain, + stackerdb_contract_ids: &[QualifiedContractIdentifier], + ) -> PeerDB { let data_url = UrlString::try_from(format!("{}", &config.node.data_url)).unwrap(); let initial_neighbors = config.node.bootstrap_node.clone(); if initial_neighbors.len() > 0 { @@ -3928,7 +3933,7 @@ impl StacksNode { data_url, &[], Some(&initial_neighbors), - &[], + stackerdb_contract_ids, ) .map_err(|e| { eprintln!( @@ -4015,13 +4020,87 @@ impl StacksNode { .unwrap() }; - let peerdb = Self::setup_peer_db(config, &burnchain); - let atlasdb = AtlasDB::connect(atlas_config.clone(), &config.get_atlas_db_file_path(), true).unwrap(); let stackerdbs = StackerDBs::connect(&config.get_stacker_db_file_path(), true).unwrap(); + let mut chainstate = + open_chainstate_with_faults(config).expect("FATAL: could not open chainstate DB"); + + let mut stackerdb_machines = HashMap::new(); + for stackerdb_contract_id in config.node.stacker_dbs.iter() { + // attempt to load the config + let (instantiate, stacker_db_config) = match StackerDBConfig::from_smart_contract( + &mut chainstate, + &sortdb, + stackerdb_contract_id, + ) { + Ok(c) => (true, c), + Err(e) => { + warn!( + "Failed to load StackerDB config for {}: {:?}", + stackerdb_contract_id, &e + ); + (false, StackerDBConfig::noop()) + } + }; + let mut stackerdbs = + StackerDBs::connect(&config.get_stacker_db_file_path(), true).unwrap(); + + if instantiate { + match stackerdbs.get_stackerdb_id(stackerdb_contract_id) { + Ok(..) => { + // reconfigure + let tx = stackerdbs.tx_begin(stacker_db_config.clone()).unwrap(); + tx.reconfigure_stackerdb(stackerdb_contract_id, &stacker_db_config.signers) + .expect(&format!( + "FATAL: failed to reconfigure StackerDB replica {}", + stackerdb_contract_id + )); + tx.commit().unwrap(); + } + Err(NetError::NoSuchStackerDB(..)) => { + // instantiate replica + let tx = stackerdbs.tx_begin(stacker_db_config.clone()).unwrap(); + tx.create_stackerdb(stackerdb_contract_id, &stacker_db_config.signers) + .expect(&format!( + "FATAL: failed to instantiate StackerDB replica {}", + stackerdb_contract_id + )); + tx.commit().unwrap(); + } + Err(e) => { + panic!("FATAL: failed to query StackerDB state: {:?}", &e); + } + } + } + let stacker_db_sync = match StackerDBSync::new( + stackerdb_contract_id.clone(), + &stacker_db_config, + PeerNetworkComms::new(), + stackerdbs, + ) { + Ok(s) => s, + Err(e) => { + warn!( + "Failed to instantiate StackerDB sync machine for {}: {:?}", + stackerdb_contract_id, &e + ); + continue; + } + }; + + stackerdb_machines.insert( + stackerdb_contract_id.clone(), + (stacker_db_config, stacker_db_sync), + ); + } + + let stackerdb_contract_ids: Vec<_> = + stackerdb_machines.keys().map(|sc| sc.clone()).collect(); + let peerdb = Self::setup_peer_db(config, &burnchain, &stackerdb_contract_ids); + let local_peer = match PeerDB::get_local_peer(peerdb.conn()) { Ok(local_peer) => local_peer, _ => panic!("Unable to retrieve local peer"), @@ -4036,7 +4115,7 @@ impl StacksNode { burnchain, view, config.connection_options.clone(), - vec![], + stackerdb_machines, epochs, ); @@ -4185,6 +4264,7 @@ impl StacksNode { let _ = Self::setup_mempool_db(&config); let mut p2p_net = Self::setup_peer_network(&config, &atlas_config, burnchain.clone()); + let stackerdbs = StackerDBs::connect(&config.get_stacker_db_file_path(), true) .expect("FATAL: failed to connect to stacker DB"); diff --git a/testnet/stacks-node/src/node.rs b/testnet/stacks-node/src/node.rs index de5564b48c..f88c861ee5 100644 --- a/testnet/stacks-node/src/node.rs +++ b/testnet/stacks-node/src/node.rs @@ -1,7 +1,7 @@ use std::convert::TryFrom; use std::default::Default; use std::net::SocketAddr; -use std::{collections::HashSet, env}; +use std::{collections::HashMap, collections::HashSet, env}; use std::{thread, thread::JoinHandle, time}; use stacks::chainstate::burn::ConsensusHash; @@ -505,7 +505,7 @@ impl Node { burnchain.clone(), view, self.config.connection_options.clone(), - vec![], + HashMap::new(), epochs, ); let _join_handle = spawn_peer( diff --git a/testnet/stacks-node/src/tests/mod.rs b/testnet/stacks-node/src/tests/mod.rs index 94d6401c52..acddd4ff13 100644 --- a/testnet/stacks-node/src/tests/mod.rs +++ b/testnet/stacks-node/src/tests/mod.rs @@ -48,6 +48,7 @@ mod epoch_24; mod integrations; mod mempool; pub mod neon_integrations; +mod stackerdb; // $ cat /tmp/out.clar pub const STORE_CONTRACT: &str = r#"(define-map store { key: (string-ascii 32) } { value: (string-ascii 32) }) diff --git a/testnet/stacks-node/src/tests/neon_integrations.rs b/testnet/stacks-node/src/tests/neon_integrations.rs index 93db8eb6c2..fb3a3efd9f 100644 --- a/testnet/stacks-node/src/tests/neon_integrations.rs +++ b/testnet/stacks-node/src/tests/neon_integrations.rs @@ -205,7 +205,7 @@ pub mod test_observer { use warp; use warp::Filter; - use crate::event_dispatcher::{MinedBlockEvent, MinedMicroblockEvent}; + use crate::event_dispatcher::{MinedBlockEvent, MinedMicroblockEvent, StackerDBChunksEvent}; pub const EVENT_OBSERVER_PORT: u16 = 50303; @@ -214,6 +214,8 @@ pub mod test_observer { pub static ref MINED_BLOCKS: Mutex> = Mutex::new(Vec::new()); pub static ref MINED_MICROBLOCKS: Mutex> = Mutex::new(Vec::new()); pub static ref NEW_MICROBLOCKS: Mutex> = Mutex::new(Vec::new()); + pub static ref NEW_STACKERDB_CHUNKS: Mutex> = + Mutex::new(Vec::new()); pub static ref BURN_BLOCKS: Mutex> = Mutex::new(Vec::new()); pub static ref MEMTXS: Mutex> = Mutex::new(Vec::new()); pub static ref MEMTXS_DROPPED: Mutex> = Mutex::new(Vec::new()); @@ -242,6 +244,18 @@ pub mod test_observer { Ok(warp::http::StatusCode::OK) } + async fn handle_stackerdb_chunks( + chunks: serde_json::Value, + ) -> Result { + debug!( + "Got stackerdb chunks: {}", + serde_json::to_string(&chunks).unwrap() + ); + let mut stackerdb_chunks = NEW_STACKERDB_CHUNKS.lock().unwrap(); + stackerdb_chunks.push(serde_json::from_value(chunks).unwrap()); + Ok(warp::http::StatusCode::OK) + } + async fn handle_mined_block(block: serde_json::Value) -> Result { let mut mined_blocks = MINED_BLOCKS.lock().unwrap(); // assert that the mined transaction events have string-y txids @@ -362,6 +376,10 @@ pub mod test_observer { MINED_MICROBLOCKS.lock().unwrap().clone() } + pub fn get_stackerdb_chunks() -> Vec { + NEW_STACKERDB_CHUNKS.lock().unwrap().clone() + } + /// each path here should correspond to one of the paths listed in `event_dispatcher.rs` async fn serve() { let new_blocks = warp::path!("new_block") @@ -396,6 +414,10 @@ pub mod test_observer { .and(warp::post()) .and(warp::body::json()) .and_then(handle_mined_microblock); + let new_stackerdb_chunks = warp::path!("stackerdb_chunks") + .and(warp::post()) + .and(warp::body::json()) + .and_then(handle_stackerdb_chunks); info!("Spawning warp server"); warp::serve( @@ -406,7 +428,8 @@ pub mod test_observer { .or(new_attachments) .or(new_microblocks) .or(mined_blocks) - .or(mined_microblocks), + .or(mined_microblocks) + .or(new_stackerdb_chunks), ) .run(([127, 0, 0, 1], EVENT_OBSERVER_PORT)) .await @@ -421,12 +444,15 @@ pub mod test_observer { } pub fn clear() { - ATTACHMENTS.lock().unwrap().clear(); - BURN_BLOCKS.lock().unwrap().clear(); NEW_BLOCKS.lock().unwrap().clear(); + MINED_BLOCKS.lock().unwrap().clear(); + MINED_MICROBLOCKS.lock().unwrap().clear(); + NEW_MICROBLOCKS.lock().unwrap().clear(); + NEW_STACKERDB_CHUNKS.lock().unwrap().clear(); + BURN_BLOCKS.lock().unwrap().clear(); MEMTXS.lock().unwrap().clear(); MEMTXS_DROPPED.lock().unwrap().clear(); - MINED_BLOCKS.lock().unwrap().clear(); + ATTACHMENTS.lock().unwrap().clear(); } } diff --git a/testnet/stacks-node/src/tests/stackerdb.rs b/testnet/stacks-node/src/tests/stackerdb.rs new file mode 100644 index 0000000000..13371fd9dd --- /dev/null +++ b/testnet/stacks-node/src/tests/stackerdb.rs @@ -0,0 +1,413 @@ +// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation +// Copyright (C) 2020-2023 Stacks Open Internet Foundation +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use std::env; +use std::thread; + +use super::bitcoin_regtest::BitcoinCoreController; +use crate::{ + burnchains::BurnchainController, + config::EventKeyType, + config::EventObserverConfig, + config::InitialBalance, + neon, + tests::{ + make_contract_publish, + neon_integrations::{ + neon_integration_test_conf, next_block_and_wait, submit_tx, test_observer, + wait_for_runloop, + }, + to_addr, + }, + BitcoinRegtestController, +}; + +use stacks::chainstate::stacks::StacksPrivateKey; + +use clarity::vm::types::QualifiedContractIdentifier; + +use stacks::libstackerdb::{StackerDBChunkAckData, StackerDBChunkData}; + +use stacks_common::types::chainstate::StacksAddress; +use stacks_common::util::hash::Sha512Trunc256Sum; + +use serde_json; + +use reqwest; + +fn post_stackerdb_chunk( + http_origin: &str, + stackerdb_contract_id: &QualifiedContractIdentifier, + data: Vec, + signer: &StacksPrivateKey, + slot_id: u32, + slot_version: u32, +) -> StackerDBChunkAckData { + let mut chunk = StackerDBChunkData::new(slot_id, slot_version, data); + chunk.sign(&signer).unwrap(); + + let chunk_body = serde_json::to_string(&chunk).unwrap(); + + let client = reqwest::blocking::Client::new(); + let path = format!( + "{}/v2/stackerdb/{}/{}/chunks", + http_origin, + &StacksAddress::from(stackerdb_contract_id.issuer.clone()), + stackerdb_contract_id.name + ); + let res = client + .post(&path) + .header("Content-Type", "application/json") + .body(chunk_body.as_bytes().to_vec()) + .send() + .unwrap(); + if res.status().is_success() { + let ack: StackerDBChunkAckData = res.json().unwrap(); + info!("Got stackerdb ack: {:?}", &ack); + return ack; + } else { + eprintln!("StackerDB post error: {}", res.text().unwrap()); + panic!(""); + } +} + +fn get_stackerdb_chunk( + http_origin: &str, + stackerdb_contract_id: &QualifiedContractIdentifier, + slot_id: u32, + slot_version: Option, +) -> Vec { + let path = if let Some(version) = slot_version { + format!( + "{}/v2/stackerdb/{}/{}/{}/{}", + http_origin, + StacksAddress::from(stackerdb_contract_id.issuer.clone()), + stackerdb_contract_id.name, + slot_id, + version + ) + } else { + format!( + "{}/v2/stackerdb/{}/{}/{}", + http_origin, + StacksAddress::from(stackerdb_contract_id.issuer.clone()), + stackerdb_contract_id.name, + slot_id + ) + }; + + let client = reqwest::blocking::Client::new(); + let res = client.get(&path).send().unwrap(); + + if res.status().is_success() { + let chunk_data: Vec = res.bytes().unwrap().to_vec(); + return chunk_data; + } else { + eprintln!("Get chunk error: {}", res.text().unwrap()); + panic!(""); + } +} + +#[test] +#[ignore] +fn test_stackerdb_load_store() { + if env::var("BITCOIND_TEST") != Ok("1".into()) { + return; + } + + let (mut conf, _) = neon_integration_test_conf(); + conf.events_observers.push(EventObserverConfig { + endpoint: format!("localhost:{}", test_observer::EVENT_OBSERVER_PORT), + events_keys: vec![EventKeyType::AnyEvent], + }); + + let privks = vec![ + // ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R + StacksPrivateKey::from_hex( + "9f1f85a512a96a244e4c0d762788500687feb97481639572e3bffbd6860e6ab001", + ) + .unwrap(), + // STVN97YYA10MY5F6KQJHKNYJNM24C4A1AT39WRW + StacksPrivateKey::from_hex( + "94c319327cc5cd04da7147d32d836eb2e4c44f4db39aa5ede7314a761183d0c701", + ) + .unwrap(), + ]; + + let stackerdb_contract = " + ;; stacker DB + (define-read-only (stackerdb-get-signer-slots) + (ok (list + { + signer: 'ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R, + num-slots: u3 + } + { + signer: 'STVN97YYA10MY5F6KQJHKNYJNM24C4A1AT39WRW, + num-slots: u3 + }))) + + (define-read-only (stackerdb-get-config) + (ok { + chunk-size: u4096, + write-freq: u0, + max-writes: u4096, + max-neighbors: u32, + hint-replicas: (list ) + })) + "; + + conf.initial_balances.append(&mut vec![ + InitialBalance { + address: to_addr(&privks[0]).into(), + amount: 10_000_000_000_000, + }, + InitialBalance { + address: to_addr(&privks[1]).into(), + amount: 10_000_000_000_000, + }, + ]); + + conf.node.stacker_dbs.push(QualifiedContractIdentifier::new( + to_addr(&privks[0]).into(), + "hello-world".into(), + )); + let contract_id = conf.node.stacker_dbs[0].clone(); + + test_observer::spawn(); + + let mut btcd_controller = BitcoinCoreController::new(conf.clone()); + btcd_controller + .start_bitcoind() + .map_err(|_e| ()) + .expect("Failed starting bitcoind"); + + let mut btc_regtest_controller = BitcoinRegtestController::new(conf.clone(), None); + + btc_regtest_controller.bootstrap_chain(201); + + eprintln!("Chain bootstrapped..."); + + let mut run_loop = neon::RunLoop::new(conf.clone()); + let blocks_processed = run_loop.get_blocks_processed_arc(); + + thread::spawn(move || run_loop.start(None, 0)); + + // Give the run loop some time to start up! + eprintln!("Wait for runloop..."); + wait_for_runloop(&blocks_processed); + + // First block wakes up the run loop. + eprintln!("Mine first block..."); + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + // Second block will hold our VRF registration. + eprintln!("Mine second block..."); + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + // Third block will be the first mined Stacks block. + eprintln!("Mine third block..."); + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + let http_origin = format!("http://{}", &conf.node.rpc_bind); + + eprintln!("Send contract-publish..."); + let tx = make_contract_publish(&privks[0], 0, 10_000, "hello-world", stackerdb_contract); + submit_tx(&http_origin, &tx); + + // mine it + eprintln!("Mine it..."); + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + // write some chunks and read them back + for i in 0..3 { + let chunk_str = format!("Hello chunks {}", &i); + let ack = post_stackerdb_chunk( + &http_origin, + &contract_id, + chunk_str.as_bytes().to_vec(), + &privks[0], + 0, + (i + 1) as u32, + ); + debug!("ACK: {:?}", &ack); + + let data = get_stackerdb_chunk(&http_origin, &contract_id, 0, Some((i + 1) as u32)); + assert_eq!(data, chunk_str.as_bytes().to_vec()); + + let data = get_stackerdb_chunk(&http_origin, &contract_id, 0, None); + assert_eq!(data, chunk_str.as_bytes().to_vec()); + } +} + +#[test] +#[ignore] +fn test_stackerdb_event_observer() { + if env::var("BITCOIND_TEST") != Ok("1".into()) { + return; + } + + let (mut conf, _) = neon_integration_test_conf(); + conf.events_observers.push(EventObserverConfig { + endpoint: format!("localhost:{}", test_observer::EVENT_OBSERVER_PORT), + events_keys: vec![EventKeyType::StackerDBChunks], + }); + + let privks = vec![ + // ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R + StacksPrivateKey::from_hex( + "9f1f85a512a96a244e4c0d762788500687feb97481639572e3bffbd6860e6ab001", + ) + .unwrap(), + // STVN97YYA10MY5F6KQJHKNYJNM24C4A1AT39WRW + StacksPrivateKey::from_hex( + "94c319327cc5cd04da7147d32d836eb2e4c44f4db39aa5ede7314a761183d0c701", + ) + .unwrap(), + ]; + + let stackerdb_contract = " + ;; stacker DB + (define-read-only (stackerdb-get-signer-slots) + (ok (list + { + signer: 'ST2DS4MSWSGJ3W9FBC6BVT0Y92S345HY8N3T6AV7R, + num-slots: u3 + } + { + signer: 'STVN97YYA10MY5F6KQJHKNYJNM24C4A1AT39WRW, + num-slots: u3 + }))) + + (define-read-only (stackerdb-get-config) + (ok { + chunk-size: u4096, + write-freq: u0, + max-writes: u4096, + max-neighbors: u32, + hint-replicas: (list ) + })) + "; + + conf.initial_balances.append(&mut vec![ + InitialBalance { + address: to_addr(&privks[0]).into(), + amount: 10_000_000_000_000, + }, + InitialBalance { + address: to_addr(&privks[1]).into(), + amount: 10_000_000_000_000, + }, + ]); + + conf.node.stacker_dbs.push(QualifiedContractIdentifier::new( + to_addr(&privks[0]).into(), + "hello-world".into(), + )); + let contract_id = conf.node.stacker_dbs[0].clone(); + + test_observer::spawn(); + + let mut btcd_controller = BitcoinCoreController::new(conf.clone()); + btcd_controller + .start_bitcoind() + .map_err(|_e| ()) + .expect("Failed starting bitcoind"); + + let mut btc_regtest_controller = BitcoinRegtestController::new(conf.clone(), None); + + btc_regtest_controller.bootstrap_chain(201); + + eprintln!("Chain bootstrapped..."); + + let mut run_loop = neon::RunLoop::new(conf.clone()); + let blocks_processed = run_loop.get_blocks_processed_arc(); + + thread::spawn(move || run_loop.start(None, 0)); + + // Give the run loop some time to start up! + eprintln!("Wait for runloop..."); + wait_for_runloop(&blocks_processed); + + // First block wakes up the run loop. + eprintln!("Mine first block..."); + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + // Second block will hold our VRF registration. + eprintln!("Mine second block..."); + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + // Third block will be the first mined Stacks block. + eprintln!("Mine third block..."); + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + let http_origin = format!("http://{}", &conf.node.rpc_bind); + + eprintln!("Send contract-publish..."); + let tx = make_contract_publish(&privks[0], 0, 10_000, "hello-world", stackerdb_contract); + submit_tx(&http_origin, &tx); + + // mine it + eprintln!("Mine it..."); + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + // write some chunks and read them back + for i in 0..6 { + let slot_id = i as u32; + let privk = &privks[i / 3]; + let chunk_str = format!("Hello chunks {}", &i); + let ack = post_stackerdb_chunk( + &http_origin, + &contract_id, + chunk_str.as_bytes().to_vec(), + privk, + slot_id, + 1, + ); + debug!("ACK: {:?}", &ack); + + let data = get_stackerdb_chunk(&http_origin, &contract_id, slot_id, Some(1)); + assert_eq!(data, chunk_str.as_bytes().to_vec()); + + let data = get_stackerdb_chunk(&http_origin, &contract_id, slot_id, None); + assert_eq!(data, chunk_str.as_bytes().to_vec()); + } + + // get events, verifying that they're all for the same contract (i.e. this one) + let stackerdb_events: Vec<_> = test_observer::get_stackerdb_chunks() + .into_iter() + .map(|stackerdb_event| { + assert_eq!(stackerdb_event.contract_id, contract_id); + stackerdb_event.modified_slots + }) + .flatten() + .collect(); + + assert_eq!(stackerdb_events.len(), 6); + for (i, event) in stackerdb_events.iter().enumerate() { + // reported in order + assert_eq!(i as u32, event.slot_id); + assert_eq!(event.slot_version, 1); + + let expected_data = format!("Hello chunks {}", &i); + let expected_hash = Sha512Trunc256Sum::from_data(expected_data.as_bytes()); + + assert_eq!(event.data, expected_data.as_bytes().to_vec()); + assert_eq!(event.data_hash(), expected_hash); + } +}