2.05 integration testing: exact costs matching in miner and follower: #2913

CHANGELOG.md

@@ -5,7 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to the versioning scheme outlined in the [README.md](README.md).
 
-## [Not yet released]
+## [2.0.11.4.0]
+
+This software update is a point-release to change the transaction
+selection logic in the default miner to prioritize by an estimated fee
+rate instead of raw fee. This release's chainstate directory is
+compatible with chainstate directories from 2.0.11.3.0.
 
 ## Added
 
@@ -18,6 +23,8 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE
 ## Changed
 
 - Prioritize transaction inclusion in blocks by estimated fee rates (#2859).
+- MARF sqlite connections will now use `mmap`'ed connections with up to 256MB
+  space (#2869).
 
 ## [2.0.11.3.0]
 

Cargo.toml

@@ -64,6 +64,15 @@ slog-json = { version = "2.3.0", optional = true }
 chrono = "0.4.19"
 libc = "0.2.82"
 
+[target.'cfg(unix)'.dependencies]
+nix = "0.23"
+
+[target.'cfg(windows)'.dependencies]
+winapi = { version = "0.3", features = ["consoleapi", "handleapi", "synchapi", "winbase"] }
+
+[target.'cfg(windows)'.dev-dependencies]
+winapi = { version = "0.3", features = ["fileapi", "processenv", "winnt"] }
+
 [dependencies.serde_json]
 version = "1.0"
 features = ["arbitrary_precision", "unbounded_depth"]

clarity/Cargo.toml

@@ -41,6 +41,15 @@ slog-json = { version = "2.3.0", optional = true }
 chrono = "0.4.19"
 libc = "0.2.82"
 
+[target.'cfg(unix)'.dependencies]
+nix = "0.23"
+
+[target.'cfg(windows)'.dependencies]
+winapi = { version = "0.3", features = ["consoleapi", "handleapi", "synchapi", "winbase"] }
+
+[target.'cfg(windows)'.dev-dependencies]
+winapi = { version = "0.3", features = ["fileapi", "processenv", "winnt"] }
+
 [dependencies.serde_json]
 version = "1.0"
 features = ["arbitrary_precision", "unbounded_depth"]

docs/event-dispatcher.md

@@ -87,7 +87,21 @@ Example:
       "from_stacks_block_hash": "0xf5d4ce0efe1d42c963d615ce57f0d014f263a985175e4ece766eceff10e0a358",
       "from_index_block_hash": "0x329efcbcc6daf5ac3f264522e0df50eddb5be85df6ee8a9fc2384c54274d7afc",
     }
-   ]
+   ],
+   "anchored_cost": {
+    "runtime": 100,
+    "read_count": 10,
+    "write_count": 5,
+    "read_length": 150,
+    "write_length": 75
+   },
+   "confirmed_microblocks_cost": {
+    "runtime": 100,
+    "read_count": 10,
+    "write_count": 5,
+    "read_length": 150,
+    "write_length": 75
+   }
 }
 ```
 
@@ -216,3 +230,37 @@ Reason can be one of:
 * `ReplaceAcrossFork` - replaced by a transaction with the same nonce but in the canonical fork
 * `TooExpensive` - the transaction is too expensive to include in a block
 * `StaleGarbageCollect` - transaction was dropped because it became stale
+
+### `POST /mined_block`
+
+This payload includes data related to block mined by this Stacks node. This
+will never be invoked if the node is configured only as a follower. This is invoked
+when the miner **assembles** the block; this block may or may not win the sortition.
+
+This endpoint will only broadcast events to observers that explicitly register for
+`MinedBlocks` events, `AnyEvent` observers will not receive the events by default.
+
+Example:
+
+```json
+{
+  "block_hash": "0x4eaabcd105865e471f697eff5dd5bd85d47ecb5a26a3379d74fae0ae87c40904",
+  "staks_height": 3,
+  "target_burn_height": 745000,
+  "block_size": 145000,
+  "anchored_cost": {
+    "runtime": 100,
+    "read_count": 10,
+    "write_count": 5,
+    "read_length": 150,
+    "write_length": 75
+  },
+  "confirmed_microblocks_cost": {
+    "runtime": 100,
+    "read_count": 10,
+    "write_count": 5,
+    "read_length": 150,
+    "write_length": 75
+  }
+}
+```

src/burnchains/bitcoin/spv.rs

@@ -42,8 +42,8 @@ use rusqlite::Transaction;
 use rusqlite::{Connection, OpenFlags, NO_PARAMS};
 
 use util::db::{
-    query_row, query_rows, tx_begin_immediate, tx_busy_handler, u64_to_sql, DBConn, DBTx,
-    Error as db_error, FromColumn, FromRow,
+    query_row, query_rows, sqlite_open, tx_begin_immediate, tx_busy_handler, u64_to_sql, DBConn,
+    DBTx, Error as db_error, FromColumn, FromRow,
 };
 use util::get_epoch_time_secs;
 use util::hash::{hex_bytes, to_hex};
@@ -212,11 +212,8 @@ impl SpvClient {
             }
         };
 
-        let mut conn =
-            Connection::open_with_flags(headers_path, open_flags).map_err(db_error::SqliteError)?;
-
-        conn.busy_handler(Some(tx_busy_handler))
-            .map_err(db_error::SqliteError)?;
+        let mut conn = sqlite_open(headers_path, open_flags, false)
+            .map_err(|e| btc_error::DBError(db_error::SqliteError(e)))?;
 
         if create_flag {
             SpvClient::db_instantiate(&mut conn)?;

src/burnchains/db.rs

@@ -27,8 +27,8 @@ use burnchains::{Burnchain, BurnchainBlock, BurnchainBlockHeader, Error as Burnc
 use chainstate::burn::operations::BlockstackOperationType;
 use chainstate::stacks::index::MarfTrieId;
 use util::db::{
-    query_row, query_rows, sql_pragma, tx_begin_immediate, tx_busy_handler, u64_to_sql,
-    Error as DBError, FromColumn, FromRow,
+    query_row, query_rows, sql_pragma, sqlite_open, tx_begin_immediate, tx_busy_handler,
+    u64_to_sql, Error as DBError, FromColumn, FromRow,
 };
 
 use crate::types::chainstate::BurnchainHeaderHash;
@@ -209,16 +209,11 @@ impl BurnchainDB {
             }
         };
 
-        let conn = Connection::open_with_flags(path, open_flags)
-            .expect(&format!("FAILED to open: {}", path));
-
-        conn.busy_handler(Some(tx_busy_handler))?;
-
+        let conn = sqlite_open(path, open_flags, true)?;
         let mut db = BurnchainDB { conn };
 
         if create_flag {
             let db_tx = db.tx_begin()?;
-            sql_pragma(&db_tx.sql_tx, "PRAGMA journal_mode = WAL;")?;
             db_tx.sql_tx.execute_batch(BURNCHAIN_DB_INITIAL_SCHEMA)?;
 
             db_tx.sql_tx.execute(
@@ -247,9 +242,7 @@ impl BurnchainDB {
         } else {
             OpenFlags::SQLITE_OPEN_READ_ONLY
         };
-        let conn = Connection::open_with_flags(path, open_flags)?;
-        conn.busy_handler(Some(tx_busy_handler))?;
-
+        let conn = sqlite_open(path, open_flags, true)?;
         Ok(BurnchainDB { conn })
     }
 

src/chainstate/burn/db/sortdb.rs

@@ -2011,17 +2011,18 @@ impl SortitionDB {
 
     fn open_index(index_path: &str) -> Result<MARF<SortitionId>, db_error> {
         test_debug!("Open index at {}", index_path);
-        MARF::from_path(index_path).map_err(|_e| db_error::Corruption)
+        let marf = MARF::from_path(index_path).map_err(|_e| db_error::Corruption)?;
+        sql_pragma(marf.sqlite_conn(), "foreign_keys", &true)?;
+        Ok(marf)
     }
 
     /// Open the database on disk.  It must already exist and be instantiated.
     /// It's best not to call this if you are able to call connect().  If you must call this, do so
     /// after you call connect() somewhere else, since connect() performs additional validations.
     pub fn open(path: &str, readwrite: bool) -> Result<SortitionDB, db_error> {
-        let (db_path, index_path) = db_mkdirs(path)?;
+        let index_path = db_mkdirs(path)?;
         debug!(
-            "Open sortdb '{}' as '{}', with index as '{}'",
-            db_path,
+            "Open sortdb as '{}', with index as '{}'",
             if readwrite { "readwrite" } else { "readonly" },
             index_path
         );
@@ -2066,11 +2067,10 @@ impl SortitionDB {
             Ok(_md) => false,
         };
 
-        let (db_path, index_path) = db_mkdirs(path)?;
+        let index_path = db_mkdirs(path)?;
         debug!(
-            "Connect/Open {} sortdb '{}' as '{}', with index as '{}'",
+            "Connect/Open {} sortdb as '{}', with index as '{}'",
             if create_flag { "(create)" } else { "" },
-            db_path,
             if readwrite { "readwrite" } else { "readonly" },
             index_path
         );
@@ -2154,13 +2154,12 @@ impl SortitionDB {
             Ok(_md) => false,
         };
 
-        let (db_path, index_path) = db_mkdirs(path)?;
+        let index_path = db_mkdirs(path)?;
         debug!(
-            "Connect/Open {} sortdb '{}' as '{}', with index as '{}'",
+            "Connect/Open {} sortdb '{}' as '{}'",
             if create_flag { "(create)" } else { "" },
-            db_path,
-            if readwrite { "readwrite" } else { "readonly" },
-            index_path
+            index_path,
+            if readwrite { "readwrite" } else { "readonly" }
         );
 
         let marf = SortitionDB::open_index(&index_path)?;
@@ -2242,7 +2241,8 @@ impl SortitionDB {
     ) -> Result<(), db_error> {
         debug!("Instantiate SortDB");
 
-        sql_pragma(self.conn(), "PRAGMA journal_mode = WAL;")?;
+        sql_pragma(self.conn(), "journal_mode", &"WAL")?;
+        sql_pragma(self.conn(), "foreign_keys", &true)?;
 
         let mut db_tx = SortitionHandleTx::begin(self, &SortitionId::sentinel())?;
 
@@ -2313,7 +2313,8 @@ impl SortitionDB {
     ) -> Result<(), db_error> {
         debug!("Instantiate SortDB");
 
-        sql_pragma(self.conn(), "PRAGMA journal_mode = WAL;")?;
+        sql_pragma(self.conn(), "journal_mode", &"WAL")?;
+        sql_pragma(self.conn(), "foreign_keys", &true)?;
 
         let mut db_tx = SortitionHandleTx::begin(self, &SortitionId::sentinel())?;
 

src/chainstate/coordinator/mod.rs

@@ -126,6 +126,8 @@ pub trait BlockEventDispatcher {
         parent_burn_block_hash: BurnchainHeaderHash,
         parent_burn_block_height: u32,
         parent_burn_block_timestamp: u64,
+        anchored_consumed: &ExecutionCost,
+        mblock_confirmed_consumed: &ExecutionCost,
     );
 
     /// called whenever a burn block is about to be
@@ -833,6 +835,8 @@ impl<
                             block_receipt.parent_burn_block_hash,
                             block_receipt.parent_burn_block_height,
                             block_receipt.parent_burn_block_timestamp,
+                            &block_receipt.anchored_block_cost,
+                            &block_receipt.parent_microblocks_cost,
                         );
                     }