Minimize time between sortitions and block-acceptance

.github/workflows/bitcoin-tests.yml

@@ -62,6 +62,7 @@ jobs:
           - tests::neon_integrations::fuzzed_median_fee_rate_estimation_test_window5
           - tests::neon_integrations::fuzzed_median_fee_rate_estimation_test_window10
           - tests::neon_integrations::use_latest_tip_integration_test
+          - tests::neon_integrations::test_flash_block_skip_tenure
           - tests::epoch_205::test_dynamic_db_method_costs
           - tests::epoch_205::transition_empty_blocks
           - tests::epoch_205::test_cost_limit_switch_version205

CHANGELOG.md

@@ -47,6 +47,22 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE
   sync, the p2p state machine will immediately transition from the inventory sync 
   work state to the block downloader work state, and immediately proceed to fetch 
   the available block or microblock stream. (#2862)
+- Nodes will push recently-obtained blocks and microblock streams to outbound
+  neighbors if their cached inventories indicate that they do not yet have them
+(#2986).
+- Nodes will no longer perform full inventory scans on their peers, except
+  during boot-up, in a bid to minimize block-download stalls (#2986).
+- Nodes will process sortitions in parallel to downloading the Stacks blocks for
+  a reward cycle, instead of doing these tasks sequentially (#2986).
+- The node's runloop will coalesce and expire stale requests to mine blocks on
+  top of parent blocks that are no longer the chain tip (#2969).
+- Several database indexes have been updated to avoid table scans, which
+  significantly improves most RPC endpoint speed and cuts node spin-up time in
+half (#2989, #3005).
+- Fixed a rare denial-of-service bug whereby a node that processes a very deep
+  burnchain reorg can get stuck, and be rendered unable to process further
+sortitions.  This has never happened in production, but it can be replicated in
+tests (#2989).
 
 ### Fixed 
 - Updates the lookup key for contracts in the pessimistic cost estimator. Before, contracts

README.md

@@ -337,12 +337,14 @@ wait_time_for_microblocks = 10000
 [miner]
 # Smallest allowed tx fee, in microSTX
 min_tx_fee = 100
-# Time to spend on the first attempt to make a block.
+# Time to spend on the first attempt to make a block, in milliseconds.
 # This can be small, so your node gets a block-commit into the Bitcoin mempool early.
-first_attempt_time_ms: 1000
-# Time to spend on subsequent attempts to make a block.
+first_attempt_time_ms = 1000
+# Time to spend on subsequent attempts to make a block, in milliseconds.
 # This can be bigger -- new block-commits will be RBF'ed.
-subsequent_attempt_time_ms: 60000
+subsequent_attempt_time_ms = 60000
+# Time to spend mining a microblock, in milliseconds.
+microblock_attempt_time_ms = 30000
 ```
 
 You can verify that your node is operating as a miner by checking its log output

src/burnchains/bitcoin/indexer.rs

@@ -370,6 +370,14 @@ impl BitcoinIndexer {
             true,
             false,
         )?;
+        if let Some(last_block) = last_block.as_ref() {
+            // do we need to do anything?
+            let cur_height = spv_client.get_headers_height()?;
+            if *last_block <= cur_height {
+                debug!("SPV client has all headers up to {}", cur_height);
+                return Ok(cur_height);
+            }
+        }
         spv_client
             .run(self)
             .and_then(|_r| Ok(spv_client.end_block_height.unwrap()))

src/burnchains/burnchain.rs

@@ -1245,6 +1245,9 @@ impl Burnchain {
                     start_block + max_blocks
                 );
                 end_block = start_block + max_blocks;
+
+                // make sure we resume at this height next time
+                indexer.drop_headers(end_block.saturating_sub(1))?;
             }
         }
 

src/burnchains/db.rs

@@ -122,6 +122,7 @@ CREATE TABLE burnchain_db_block_headers (
 
     PRIMARY KEY(block_hash)
 );
+CREATE INDEX index_burnchain_db_block_headers_height_hash ON burnchain_db_block_headers(block_height DESC, block_hash ASC);
 
 CREATE TABLE burnchain_db_block_ops (
     block_hash TEXT NOT NULL,
@@ -130,6 +131,9 @@ CREATE TABLE burnchain_db_block_ops (
     FOREIGN KEY(block_hash) REFERENCES burnchain_db_block_headers(block_hash)
 );
 
+CREATE INDEX index_burnchain_db_block_hash ON burnchain_db_block_ops(block_hash);
+CREATE INDEX index_burnchain_db_txid ON burnchain_db_block_ops(txid);
+
 CREATE TABLE db_config(version TEXT NOT NULL);";
 
 impl<'a> BurnchainDBTransaction<'a> {

src/chainstate/burn/db/sortdb.rs

@@ -515,6 +515,9 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         FOREIGN KEY(sortition_id) REFERENCES snapshots(sortition_id)
     );"#,
     r#"
+    CREATE INDEX index_leader_keys_sortition_id_block_height_vtxindex ON leader_keys(sortition_id,block_height,vtxindex);
+    "#,
+    r#"
     CREATE TABLE block_commits(
         txid TEXT NOT NULL,
         vtxindex INTEGER NOT NULL,
@@ -540,6 +543,10 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         FOREIGN KEY(sortition_id) REFERENCES snapshots(sortition_id)
     );"#,
     r#"
+    CREATE INDEX index_block_commits_sortition_id_vtxindex ON block_commits(sortition_id,vtxindex);
+    CREATE INDEX index_block_commits_sortition_id_block_height_vtxindex ON block_commits(sortition_id,block_height,vtxindex);
+    "#,
+    r#"
     CREATE TABLE user_burn_support(
         txid TEXT NOT NULL,
         vtxindex INTEGER NOT NULL,
@@ -560,6 +567,11 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         FOREIGN KEY(sortition_id) REFERENCES snapshots(sortition_id)
     );"#,
     r#"
+    CREATE INDEX index_user_burn_support_txid ON user_burn_support(txid);
+    CREATE INDEX index_user_burn_support_sortition_id_vtxindex ON user_burn_support(sortition_id,vtxindex);
+    CREATE INDEX index_user_burn_support_sortition_id_hash_160_key_vtxindex_key_block_ptr_vtxindex ON user_burn_support(sortition_id,block_header_hash_160,key_vtxindex,key_block_ptr,vtxindex ASC);
+    "#,
+    r#"
     CREATE TABLE stack_stx (
         txid TEXT NOT NULL,
         vtxindex INTEGER NOT NULL,
@@ -574,6 +586,9 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         PRIMARY KEY(txid)
     );"#,
     r#"
+    CREATE INDEX index_stack_stx_burn_header_hash ON stack_stx(burn_header_hash);
+    "#,
+    r#"
     CREATE TABLE transfer_stx (
         txid TEXT NOT NULL,
         vtxindex INTEGER NOT NULL,
@@ -588,6 +603,9 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         PRIMARY KEY(txid)
     );"#,
     r#"
+    CREATE INDEX index_transfer_stx_burn_header_hash ON transfer_stx(burn_header_hash);
+    "#,
+    r#"
     CREATE TABLE missed_commits (
         txid TEXT NOT NULL,
         input TEXT NOT NULL,
@@ -596,6 +614,9 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         PRIMARY KEY(txid, intended_sortition_id)
     );"#,
     r#"
+    CREATE INDEX index_missed_commits_intended_sortition_id ON missed_commits(intended_sortition_id);
+    "#,
+    r#"
     CREATE TABLE canonical_accepted_stacks_blocks(
         tip_consensus_hash TEXT NOT NULL,
         consensus_hash TEXT NOT NULL,
@@ -1574,8 +1595,8 @@ impl<'a> SortitionHandleConn<'a> {
         let winning_block_hash160 =
             Hash160::from_sha256(snapshot.winning_stacks_block_hash.as_bytes());
 
-        let qry = "SELECT * FROM user_burn_support
-                   WHERE sortition_id = ?1 AND block_header_hash_160 = ?2 AND key_vtxindex = ?3 AND key_block_ptr = ?4
+        let qry = "SELECT * FROM user_burn_support \
+                   WHERE sortition_id = ?1 AND block_header_hash_160 = ?2 AND key_vtxindex = ?3 AND key_block_ptr = ?4 \
                    ORDER BY vtxindex ASC";
         let args: [&dyn ToSql; 4] = [
             &snapshot.sortition_id,
@@ -1587,7 +1608,7 @@ impl<'a> SortitionHandleConn<'a> {
         let mut winning_user_burns: Vec<UserBurnSupportOp> = query_rows(self, qry, &args)?;
 
         // were there multiple miners with the same VRF key and block header hash? (i.e., are these user burns shared?)
-        let qry = "SELECT COUNT(*) FROM block_commits
+        let qry = "SELECT COUNT(*) FROM block_commits \
                    WHERE sortition_id = ?1 AND block_header_hash = ?2 AND key_vtxindex = ?3 AND key_block_ptr = ?4";
         let args: [&dyn ToSql; 4] = [
             &snapshot.sortition_id,

src/chainstate/stacks/db/blocks.rs

@@ -530,15 +530,26 @@ impl StreamCursor {
         }))
     }
 
-    pub fn new_tx_stream(tx_query: MemPoolSyncData, max_txs: u64, height: u64) -> StreamCursor {
+    pub fn new_tx_stream(
+        tx_query: MemPoolSyncData,
+        max_txs: u64,
+        height: u64,
+        page_id_opt: Option<Txid>,
+    ) -> StreamCursor {
+        let last_randomized_txid = page_id_opt.unwrap_or_else(|| {
+            let random_bytes = rand::thread_rng().gen::<[u8; 32]>();
+            Txid(random_bytes)
+        });
+
         StreamCursor::MempoolTxs(TxStreamData {
             tx_query,
-            last_randomized_txid: Txid([0u8; 32]),
+            last_randomized_txid: last_randomized_txid,
             tx_buf: vec![],
             tx_buf_ptr: 0,
             num_txs: 0,
             max_txs: max_txs,
             height: height,
+            corked: false,
         })
     }
 
@@ -2123,6 +2134,7 @@ impl StacksChainState {
                     let block_bench_start = get_epoch_time_ms();
                     let mut parent_microblock_hash = None;
 
+                    // TODO: just do a stat? cache this?
                     match StacksChainState::load_block_header(
                         &self.blocks_path,
                         &consensus_hash,
@@ -2150,6 +2162,7 @@ impl StacksChainState {
 
                     let mblock_bench_begin = get_epoch_time_ms();
                     if let Some(parent_microblock) = parent_microblock_hash {
+                        // TODO: can we cache this?
                         if self.has_processed_microblocks_at_tail(
                             &index_block_hash,
                             &parent_microblock,

src/chainstate/stacks/db/mod.rs

@@ -549,12 +549,15 @@ pub struct TxStreamData {
     /// serialized transaction buffer that's being sent
     pub tx_buf: Vec<u8>,
     pub tx_buf_ptr: usize,
-    /// number of transactions sent so far
+    /// number of transactions visited in the DB so far
     pub num_txs: u64,
-    /// maximum we can send
+    /// maximum we can visit in the query
     pub max_txs: u64,
     /// height of the chain at time of query
     pub height: u64,
+    /// Are we done sending transactions, and are now in the process of sending the trailing page
+    /// ID?
+    pub corked: bool,
 }
 
 pub const CHAINSTATE_VERSION: &'static str = "2";
@@ -597,6 +600,7 @@ const CHAINSTATE_INITIAL_SCHEMA: &'static [&'static str] = &[
     "CREATE INDEX index_block_hash_to_primary_key ON block_headers(index_block_hash,consensus_hash,block_hash);",
     "CREATE INDEX block_headers_hash_index ON block_headers(block_hash,block_height);",
     "CREATE INDEX block_index_hash_index ON block_headers(index_block_hash,consensus_hash,block_hash);",
+    "CREATE INDEX block_headers_burn_header_height ON block_headers(burn_header_height);",
     r#"
     -- scheduled payments
     -- no designated primary key since there can be duplicate entries
@@ -620,6 +624,10 @@ const CHAINSTATE_INITIAL_SCHEMA: &'static [&'static str] = &[
         vtxindex INT NOT NULL               -- user burn support vtxindex
     );"#,
     r#"
+    CREATE INDEX index_payments_block_hash_consensus_hash_vtxindex ON payments(block_hash,consensus_hash,vtxindex ASC);
+    CREATE INDEX index_payments_index_block_hash_vtxindex ON payments(index_block_hash,vtxindex ASC);
+    "#,
+    r#"
     -- users who supported miners
     CREATE TABLE user_supporters(
         address TEXT NOT NULL,
@@ -648,7 +656,11 @@ const CHAINSTATE_INITIAL_SCHEMA: &'static [&'static str] = &[
                                      orphaned INT NOT NULL,
                                      PRIMARY KEY(anchored_block_hash,consensus_hash,microblock_hash)
     );"#,
+    "CREATE INDEX staging_microblocks_processed ON staging_microblocks(processed);",
+    "CREATE INDEX staging_microblocks_orphaned ON staging_microblocks(orphaned);",
     "CREATE INDEX staging_microblocks_index_hash ON staging_microblocks(index_block_hash);",
+    "CREATE INDEX staging_microblocks_index_hash_processed ON staging_microblocks(index_block_hash,processed);",
+    "CREATE INDEX staging_microblocks_index_hash_orphaned ON staging_microblocks(index_block_hash,orphaned);",
     r#"
     -- Staging microblocks data
     CREATE TABLE staging_microblocks_data(block_hash TEXT NOT NULL,
@@ -688,6 +700,7 @@ const CHAINSTATE_INITIAL_SCHEMA: &'static [&'static str] = &[
     "CREATE INDEX parent_blocks ON staging_blocks(parent_anchored_block_hash);",
     "CREATE INDEX parent_consensus_hashes ON staging_blocks(parent_consensus_hash);",
     "CREATE INDEX index_block_hashes ON staging_blocks(index_block_hash);",
+    "CREATE INDEX height_stacks_blocks ON staging_blocks(height);",
     r#"
     -- users who burned in support of a block
     CREATE TABLE staging_user_burn_support(anchored_block_hash TEXT NOT NULL,
@@ -697,6 +710,9 @@ const CHAINSTATE_INITIAL_SCHEMA: &'static [&'static str] = &[
                                            vtxindex INT NOT NULL
     );"#,
     r#"
+    CREATE INDEX index_staging_user_burn_support ON staging_user_burn_support(anchored_block_hash,consensus_hash);
+    "#,
+    r#"
     CREATE TABLE transactions(
         id INTEGER PRIMARY KEY,
         txid TEXT NOT NULL,