Merge branch 'develop' into fix/mocknet-events

.github/actions/docsgen/Dockerfile.docsgen

@@ -10,8 +10,8 @@ RUN cargo build
 
 RUN mkdir /out
 
-RUN /src/target/debug/blockstack-core docgen | jq . > /out/clarity-reference.json
-RUN /src/target/debug/blockstack-core docgen_boot | jq . > /out/boot-contracts-reference.json
+RUN /src/target/debug/stacks-inspect docgen | jq . > /out/clarity-reference.json
+RUN /src/target/debug/stacks-inspect docgen_boot | jq . > /out/boot-contracts-reference.json
 
 FROM scratch AS export-stage
 COPY --from=build /out/clarity-reference.json /

.vscode/launch.json

@@ -8,11 +8,11 @@
             "cargo": {
                 "args": [
                     "build",
-                    "--bin=blockstack-core",
+                    "--bin=stacks-inspect",
                     "--package=blockstack-core"
                 ],
                 "filter": {
-                    "name": "blockstack-core",
+                    "name": "stacks-inspect",
                     "kind": "bin"
                 }
             },
@@ -118,11 +118,11 @@
                 "args": [
                     "test",
                     "--no-run",
-                    "--bin=blockstack-core",
+                    "--bin=stacks-inspect",
                     "--package=blockstack-core"
                 ],
                 "filter": {
-                    "name": "blockstack-core",
+                    "name": "stacks-inspect",
                     "kind": "bin"
                 }
             },

CHANGELOG.md

@@ -13,25 +13,27 @@ the node to spend up to 30 minutes migrating the data to a new schema.
 ### Changed
 - The MARF implementation will now defer calculating the root hash of a new trie
   until the moment the trie is committed to disk.  This avoids gratuitous hash
-calculations, and yields a performance improvement of anywhere between 10x and
-200x (#3041).
+  calculations, and yields a performance improvement of anywhere between 10x and
+  200x (#3041).
 - The MARF implementation will now store tries to an external file for instances
   where the tries are expected to exceed the SQLite page size (namely, the
-Clarity database). This improves read performance by a factor of 10x to 14x
-(#3059).
+  Clarity database). This improves read performance by a factor of 10x to 14x
+  (#3059).
 - The MARF implementation may now cache trie nodes in RAM if directed to do so
   by an environment variable (#3042).
 - Sortition processing performance has been improved by about an order of
   magnitude, by avoiding a slew of expensive database reads (#3045).  WARNING:
-applying this change to an existing chainstate directory will take a few
-minutes when the node starts up.
+  applying this change to an existing chainstate directory will take a few
+  minutes when the node starts up.
 - Updated chains coordinator so that before a Stacks block or a burn block is processed, 
   an event is sent through the event dispatcher. This fixes #3015. 
 - Expose a node's public key and public key hash160 (i.e. what appears in
   /v2/neighbors) via the /v2/info API endpoint (#3046)
 - Reduced the default subsequent block attempt timeout from 180 seconds to 30
   seconds, based on benchmarking the new MARF performance data during a period
-of network congestion (#3098)
+  of network congestion (#3098)
+- The `blockstack-core` binary has been renamed to `stacks-inspect`.
+  This binary provides CLI tools for chain and mempool inspection.
 
 ## [2.05.0.1.0]
 

Cargo.toml

@@ -21,7 +21,7 @@ name = "blockstack_lib"
 path = "src/lib.rs"
 
 [[bin]]
-name = "blockstack-core"
+name = "stacks-inspect"
 path = "src/main.rs"
 
 [[bin]]

build-scripts/Dockerfile.linux-arm64

@@ -20,4 +20,4 @@ RUN CC=aarch64-linux-gnu-gcc \
 RUN mkdir /out && cp -R /src/target/aarch64-unknown-linux-gnu/release/. /out
 
 FROM scratch AS export-stage
-COPY --from=build /out/blockstack-core /out/blockstack-cli /out/clarity-cli /out/stacks-node /
+COPY --from=build /out/stacks-inspect /out/blockstack-cli /out/clarity-cli /out/stacks-node /

build-scripts/Dockerfile.linux-armv7

@@ -20,4 +20,4 @@ RUN CC=arm-linux-gnueabihf-gcc \
 RUN mkdir /out && cp -R /src/target/armv7-unknown-linux-gnueabihf/release/. /out
 
 FROM scratch AS export-stage
-COPY --from=build /out/blockstack-core /out/blockstack-cli /out/clarity-cli /out/stacks-node /
+COPY --from=build /out/stacks-inspect /out/blockstack-cli /out/clarity-cli /out/stacks-node /

build-scripts/Dockerfile.linux-musl-x64

@@ -20,4 +20,4 @@ RUN CC=musl-gcc \
 RUN mkdir /out && cp -R /src/target/x86_64-unknown-linux-musl/release/. /out
 
 FROM scratch AS export-stage
-COPY --from=build /out/blockstack-core /out/blockstack-cli /out/clarity-cli /out/stacks-node /
+COPY --from=build /out/stacks-inspect /out/blockstack-cli /out/clarity-cli /out/stacks-node /

build-scripts/Dockerfile.linux-x64

@@ -17,4 +17,4 @@ RUN cargo build --release --workspace --target x86_64-unknown-linux-gnu
 RUN mkdir /out && cp -R /src/target/x86_64-unknown-linux-gnu/release/. /out
 
 FROM scratch AS export-stage
-COPY --from=build /out/blockstack-core /out/blockstack-cli /out/clarity-cli /out/stacks-node /
+COPY --from=build /out/stacks-inspect /out/blockstack-cli /out/clarity-cli /out/stacks-node /

build-scripts/Dockerfile.macos-arm64

@@ -21,4 +21,4 @@ RUN . /opt/osxcross/env-macos-aarch64 && \
 RUN mkdir /out && cp -R /src/target/aarch64-apple-darwin/release/. /out
 
 FROM scratch AS export-stage
-COPY --from=build /out/blockstack-core /out/blockstack-cli /out/clarity-cli /out/stacks-node /
+COPY --from=build /out/stacks-inspect /out/blockstack-cli /out/clarity-cli /out/stacks-node /

build-scripts/Dockerfile.macos-x64

@@ -21,4 +21,4 @@ RUN . /opt/osxcross/env-macos-x86_64 && \
 RUN mkdir /out && cp -R /src/target/x86_64-apple-darwin/release/. /out
 
 FROM scratch AS export-stage
-COPY --from=build /out/blockstack-core /out/blockstack-cli /out/clarity-cli /out/stacks-node /
+COPY --from=build /out/stacks-inspect /out/blockstack-cli /out/clarity-cli /out/stacks-node /

build-scripts/Dockerfile.windows-x64

@@ -19,4 +19,4 @@ RUN CC_x86_64_pc_windows_gnu=x86_64-w64-mingw32-gcc \
 RUN mkdir /out && cp -R /src/target/x86_64-pc-windows-gnu/release/. /out
 
 FROM scratch AS export-stage
-COPY --from=build /out/blockstack-core.exe /out/blockstack-cli.exe /out/clarity-cli.exe /out/stacks-node.exe /
+COPY --from=build /out/stacks-inspect.exe /out/blockstack-cli.exe /out/clarity-cli.exe /out/stacks-node.exe /

circle.yml

@@ -15,15 +15,15 @@ jobs:
             cargo build
       - run:
           command: |
-            ./target/debug/blockstack-core local initialize db &&
-            ./target/debug/blockstack-core local check sample-contracts/tokens.clar db &&
-            ./target/debug/blockstack-core local launch S1G2081040G2081040G2081040G208105NK8PE5.tokens sample-contracts/tokens.clar db &&
-            ./target/debug/blockstack-core local check sample-contracts/names.clar db &&
-            ./target/debug/blockstack-core local launch S1G2081040G2081040G2081040G208105NK8PE5.names sample-contracts/names.clar db &&
-            ./target/debug/blockstack-core local execute db S1G2081040G2081040G2081040G208105NK8PE5.tokens mint! SZ2J6ZY48GV1EZ5V2V5RB9MP66SW86PYKKQ9H6DPR u100000
+            ./target/debug/stacks-inspect local initialize db &&
+            ./target/debug/stacks-inspect local check sample-contracts/tokens.clar db &&
+            ./target/debug/stacks-inspect local launch S1G2081040G2081040G2081040G208105NK8PE5.tokens sample-contracts/tokens.clar db &&
+            ./target/debug/stacks-inspect local check sample-contracts/names.clar db &&
+            ./target/debug/stacks-inspect local launch S1G2081040G2081040G2081040G208105NK8PE5.names sample-contracts/names.clar db &&
+            ./target/debug/stacks-inspect local execute db S1G2081040G2081040G2081040G208105NK8PE5.tokens mint! SZ2J6ZY48GV1EZ5V2V5RB9MP66SW86PYKKQ9H6DPR u100000
       - run:
           command: |
-            echo "(get-balance 'SZ2J6ZY48GV1EZ5V2V5RB9MP66SW86PYKKQ9H6DPR)" | ./target/debug/blockstack-core local eval S1G2081040G2081040G2081040G208105NK8PE5.tokens db
+            echo "(get-balance 'SZ2J6ZY48GV1EZ5V2V5RB9MP66SW86PYKKQ9H6DPR)" | ./target/debug/stacks-inspect local eval S1G2081040G2081040G2081040G208105NK8PE5.tokens db
   unit_tests_with_cov:
     machine: true
     working_directory: ~/blockstack

src/chainstate/burn/db/sortdb.rs

@@ -66,6 +66,7 @@ use crate::net::neighbors::MAX_NEIGHBOR_BLOCK_DELAY;
 use crate::net::{Error as NetError, Error};
 use crate::util_lib::db::tx_begin_immediate;
 use crate::util_lib::db::tx_busy_handler;
+use crate::util_lib::db::DBTx;
 use crate::util_lib::db::Error as db_error;
 use crate::util_lib::db::{
     db_mkdirs, query_count, query_row, query_row_columns, query_row_panic, query_rows, sql_pragma,
@@ -2477,7 +2478,7 @@ impl SortitionDB {
         Ok(version)
     }
 
-    fn apply_schema_2(tx: &SortitionDBTx, epochs: &[StacksEpoch]) -> Result<(), db_error> {
+    fn apply_schema_2(tx: &DBTx, epochs: &[StacksEpoch]) -> Result<(), db_error> {
         for sql_exec in SORTITION_DB_SCHEMA_2 {
             tx.execute_batch(sql_exec)?;
         }
@@ -2492,7 +2493,7 @@ impl SortitionDB {
         Ok(())
     }
 
-    fn apply_schema_3(tx: &SortitionDBTx) -> Result<(), db_error> {
+    fn apply_schema_3(tx: &DBTx) -> Result<(), db_error> {
         for sql_exec in SORTITION_DB_SCHEMA_3 {
             tx.execute_batch(sql_exec)?;
         }
@@ -2510,30 +2511,32 @@ impl SortitionDB {
                 if version == expected_version {
                     Ok(())
                 } else {
-                    Err(db_error::Other(format!(
-                        "The version of the sortition DB {} does not match the expected {} and cannot be updated from SortitionDB::open()",
-                        version, expected_version
-                    )))
+                    let version_u64 = version.parse::<u64>().unwrap();
+                    Err(db_error::OldSchema(version_u64))
                 }
             }
             Ok(None) => panic!("The schema version of the sortition DB is not recorded."),
             Err(e) => panic!("Error obtaining the version of the sortition DB: {:?}", e),
         }
     }
 
-    fn check_schema_version_and_update(&mut self, epochs: &[StacksEpoch]) -> Result<(), db_error> {
+    /// Migrate the sortition DB to its latest version, given the set of system epochs
+    pub fn check_schema_version_and_update(
+        &mut self,
+        epochs: &[StacksEpoch],
+    ) -> Result<(), db_error> {
         let expected_version = SORTITION_DB_VERSION.to_string();
         loop {
             match SortitionDB::get_schema_version(self.conn()) {
                 Ok(Some(version)) => {
                     if version == "1" {
                         let tx = self.tx_begin()?;
-                        SortitionDB::apply_schema_2(&tx, epochs)?;
+                        SortitionDB::apply_schema_2(&tx.deref(), epochs)?;
                         tx.commit()?;
                     } else if version == "2" {
                         // add the tables of schema 3, but do not populate them.
                         let tx = self.tx_begin()?;
-                        SortitionDB::apply_schema_3(&tx)?;
+                        SortitionDB::apply_schema_3(&tx.deref())?;
                         tx.commit()?;
                     } else if version == expected_version {
                         return Ok(());
@@ -2547,6 +2550,24 @@ impl SortitionDB {
         }
     }
 
+    /// Open and migrate the sortition DB if it exists.
+    pub fn migrate_if_exists(path: &str, epochs: &[StacksEpoch]) -> Result<(), db_error> {
+        if let Err(db_error::OldSchema(_)) = SortitionDB::open(path, false) {
+            let index_path = db_mkdirs(path)?;
+            let marf = SortitionDB::open_index(&index_path)?;
+            let mut db = SortitionDB {
+                marf,
+                readwrite: true,
+                // not used by migration logic
+                first_block_height: 0,
+                first_burn_header_hash: BurnchainHeaderHash([0xff; 32]),
+            };
+            db.check_schema_version_and_update(epochs)
+        } else {
+            Ok(())
+        }
+    }
+
     fn add_indexes(&mut self) -> Result<(), db_error> {
         // do we need to instantiate indexes?
         // only do a transaction if we need to, since this gets called each time the sortition DB

src/chainstate/coordinator/mod.rs

@@ -59,6 +59,8 @@ use crate::types::chainstate::{
 };
 use clarity::vm::database::BurnStateDB;
 
+use crate::chainstate::stacks::index::marf::MARFOpenOpts;
+
 pub use self::comm::CoordinatorCommunication;
 
 pub mod comm;
@@ -935,3 +937,35 @@ pub fn check_chainstate_db_versions(
 
     Ok(true)
 }
+
+/// Migrate all databases to their latest schemas.
+/// Verifies that this is possible as well
+pub fn migrate_chainstate_dbs(
+    epochs: &[StacksEpoch],
+    sortdb_path: &str,
+    chainstate_path: &str,
+    chainstate_marf_opts: Option<MARFOpenOpts>,
+) -> Result<(), Error> {
+    if !check_chainstate_db_versions(epochs, sortdb_path, chainstate_path)? {
+        warn!("Unable to migrate chainstate DBs to the latest schemas in the current epoch");
+        return Err(DBError::TooOldForEpoch.into());
+    }
+
+    if fs::metadata(&sortdb_path).is_ok() {
+        info!("Migrating sortition DB to the latest schema version");
+        SortitionDB::migrate_if_exists(&sortdb_path, epochs)?;
+    }
+    if fs::metadata(&chainstate_path).is_ok() {
+        info!("Migrating chainstate DB to the latest schema version");
+        let db_config = StacksChainState::get_db_config_from_path(&chainstate_path)?;
+
+        // this does the migration internally
+        let _ = StacksChainState::open(
+            db_config.mainnet,
+            db_config.chain_id,
+            chainstate_path,
+            chainstate_marf_opts,
+        )?;
+    }
+    Ok(())
+}

src/net/p2p.rs

@@ -1927,7 +1927,7 @@ impl PeerNetwork {
         }
 
         for (event_id, convo) in self.peers.iter() {
-            if convo.is_authenticated() {
+            if convo.is_authenticated() && convo.stats.last_contact_time > 0 {
                 // have handshaked with this remote peer
                 if convo.stats.last_contact_time
                     + (convo.peer_heartbeat as u64)

src/util_lib/db.rs

@@ -106,6 +106,10 @@ pub enum Error {
     IOError(IOError),
     /// MARF index error
     IndexError(MARFError),
+    /// Old schema error
+    OldSchema(u64),
+    /// Database is too old for epoch
+    TooOldForEpoch,
     /// Other error
     Other(String),
 }
@@ -127,6 +131,10 @@ impl fmt::Display for Error {
             Error::IOError(ref e) => fmt::Display::fmt(e, f),
             Error::SqliteError(ref e) => fmt::Display::fmt(e, f),
             Error::IndexError(ref e) => fmt::Display::fmt(e, f),
+            Error::OldSchema(ref s) => write!(f, "Old database schema: {}", s),
+            Error::TooOldForEpoch => {
+                write!(f, "Database is not compatible with current system epoch")
+            }
             Error::Other(ref s) => fmt::Display::fmt(s, f),
         }
     }
@@ -149,6 +157,8 @@ impl error::Error for Error {
             Error::SqliteError(ref e) => Some(e),
             Error::IOError(ref e) => Some(e),
             Error::IndexError(ref e) => Some(e),
+            Error::OldSchema(ref _s) => None,
+            Error::TooOldForEpoch => None,
             Error::Other(ref _s) => None,
         }
     }

testnet/stacks-node/src/neon_node.rs

@@ -1319,6 +1319,8 @@ impl StacksNode {
             // bootstrap nodes *always* allowed
             let mut tx = peerdb.tx_begin().unwrap();
             for initial_neighbor in initial_neighbors.iter() {
+                // update peer in case public key changed
+                PeerDB::update_peer(&mut tx, &initial_neighbor).unwrap();
                 PeerDB::set_allow_peer(
                     &mut tx,
                     initial_neighbor.addr.network_id,