Merge pull request #2900 from blockstack/fix/2869

Use mmap on MARF connections

CHANGELOG.md

@@ -18,6 +18,8 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE
 ## Changed
 
 - Prioritize transaction inclusion in blocks by estimated fee rates (#2859).
+- MARF sqlite connections will now use `mmap`'ed connections with up to 256MB
+  space (#2869).
 
 ## [2.0.11.3.0]
 

Cargo.toml

@@ -64,6 +64,15 @@ slog-json = { version = "2.3.0", optional = true }
 chrono = "0.4.19"
 libc = "0.2.82"
 
+[target.'cfg(unix)'.dependencies]
+nix = "0.23"
+
+[target.'cfg(windows)'.dependencies]
+winapi = { version = "0.3", features = ["consoleapi", "handleapi", "synchapi", "winbase"] }
+
+[target.'cfg(windows)'.dev-dependencies]
+winapi = { version = "0.3", features = ["fileapi", "processenv", "winnt"] }
+
 [dependencies.serde_json]
 version = "1.0"
 features = ["arbitrary_precision", "unbounded_depth"]

clarity/Cargo.toml

@@ -41,6 +41,15 @@ slog-json = { version = "2.3.0", optional = true }
 chrono = "0.4.19"
 libc = "0.2.82"
 
+[target.'cfg(unix)'.dependencies]
+nix = "0.23"
+
+[target.'cfg(windows)'.dependencies]
+winapi = { version = "0.3", features = ["consoleapi", "handleapi", "synchapi", "winbase"] }
+
+[target.'cfg(windows)'.dev-dependencies]
+winapi = { version = "0.3", features = ["fileapi", "processenv", "winnt"] }
+
 [dependencies.serde_json]
 version = "1.0"
 features = ["arbitrary_precision", "unbounded_depth"]

src/chainstate/burn/db/sortdb.rs

@@ -1989,10 +1989,9 @@ impl SortitionDB {
     /// It's best not to call this if you are able to call connect().  If you must call this, do so
     /// after you call connect() somewhere else, since connect() performs additional validations.
     pub fn open(path: &str, readwrite: bool) -> Result<SortitionDB, db_error> {
-        let (db_path, index_path) = db_mkdirs(path)?;
+        let index_path = db_mkdirs(path)?;
         debug!(
-            "Open sortdb '{}' as '{}', with index as '{}'",
-            db_path,
+            "Open sortdb as '{}', with index as '{}'",
             if readwrite { "readwrite" } else { "readonly" },
             index_path
         );
@@ -2035,11 +2034,10 @@ impl SortitionDB {
             Ok(_md) => false,
         };
 
-        let (db_path, index_path) = db_mkdirs(path)?;
+        let index_path = db_mkdirs(path)?;
         debug!(
-            "Connect/Open {} sortdb '{}' as '{}', with index as '{}'",
+            "Connect/Open {} sortdb as '{}', with index as '{}'",
             if create_flag { "(create)" } else { "" },
-            db_path,
             if readwrite { "readwrite" } else { "readonly" },
             index_path
         );

src/chainstate/stacks/index/storage.rs

@@ -45,10 +45,12 @@ use chainstate::stacks::index::node::{
 };
 use chainstate::stacks::index::Error;
 use chainstate::stacks::index::{trie_sql, BlockMap, MarfTrieId};
+use util::db::sql_pragma;
 use util::db::sqlite_open;
 use util::db::tx_begin_immediate;
 use util::db::tx_busy_handler;
 use util::db::Error as db_error;
+use util::db::SQLITE_MMAP_SIZE;
 use util::log;
 
 use crate::types::chainstate::BlockHeaderHash;
@@ -721,6 +723,16 @@ pub struct TrieFileStorage<T: MarfTrieId> {
     pub test_genesis_block: Option<T>,
 }
 
+fn marf_sqlite_open<P: AsRef<Path>>(
+    db_path: P,
+    open_flags: OpenFlags,
+    foreign_keys: bool,
+) -> Result<Connection, db_error> {
+    let db = sqlite_open(db_path, open_flags, foreign_keys)?;
+    sql_pragma(&db, "mmap_size", &SQLITE_MMAP_SIZE)?;
+    Ok(db)
+}
+
 impl<T: MarfTrieId> TrieFileStorage<T> {
     pub fn connection<'a>(&'a mut self) -> TrieStorageConnection<'a, T> {
         TrieStorageConnection {
@@ -796,7 +808,7 @@ impl<T: MarfTrieId> TrieFileStorage<T> {
             }
         };
 
-        let mut db = sqlite_open(db_path, open_flags, false)?;
+        let mut db = marf_sqlite_open(db_path, open_flags, false)?;
         let db_path = db_path.to_string();
 
         if create_flag {
@@ -865,7 +877,7 @@ impl<T: MarfTrieId> TrieFileStorage<T> {
     }
 
     pub fn reopen_readonly(&self) -> Result<TrieFileStorage<T>, Error> {
-        let db = sqlite_open(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY, false)?;
+        let db = marf_sqlite_open(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY, false)?;
 
         trace!("Make read-only view of TrieFileStorage: {}", &self.db_path);
 
@@ -909,7 +921,7 @@ impl<'a, T: MarfTrieId> TrieStorageTransaction<'a, T> {
     /// reopen this transaction as a read-only marf.
     ///  _does not_ preserve the cur_block/open tip
     pub fn reopen_readonly(&self) -> Result<TrieFileStorage<T>, Error> {
-        let db = sqlite_open(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY, false)?;
+        let db = marf_sqlite_open(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY, false)?;
 
         trace!(
             "Make read-only view of TrieStorageTransaction: {}",
@@ -1283,7 +1295,7 @@ impl<'a, T: MarfTrieId> TrieStorageConnection<'a, T> {
     /// Recover from partially-written state -- i.e. blow it away.
     /// Doesn't get called automatically.
     pub fn recover(db_path: &String) -> Result<(), Error> {
-        let conn = sqlite_open(db_path, OpenFlags::SQLITE_OPEN_READ_WRITE, false)?;
+        let conn = marf_sqlite_open(db_path, OpenFlags::SQLITE_OPEN_READ_WRITE, false)?;
         trie_sql::clear_lock_data(&conn)
     }
 

src/deps/README.md

@@ -10,3 +10,5 @@ snapshots of important libraries this codebase depends on.
 * The `bitcoin` package was produced by Andrew Poelstra (https://github.com/rust-bitcoin/rust-bitcoin).  License is CC0.
 * The `httparse` package was produced by Sean McArthur
   (https://github.com/seanmonstar/httparse).  License is MIT.
+* The `ctrlc` package was produced by Antti Keräne
+  (https://github.com/Detegr/rust-ctrlc).  License is MIT.

src/deps/ctrlc/error.rs

@@ -0,0 +1,46 @@
+use deps::ctrlc::platform;
+use std::fmt;
+
+/// Ctrl-C error.
+#[derive(Debug)]
+pub enum Error {
+    /// Ctrl-C signal handler already registered.
+    MultipleHandlers,
+    /// Unexpected system error.
+    System(std::io::Error),
+}
+
+impl Error {
+    fn describe(&self) -> &str {
+        match *self {
+            Error::MultipleHandlers => "Ctrl-C signal handler already registered",
+            Error::System(_) => "Unexpected system error",
+        }
+    }
+}
+
+impl From<platform::Error> for Error {
+    fn from(e: platform::Error) -> Error {
+        let system_error = std::io::Error::new(std::io::ErrorKind::Other, e);
+        Error::System(system_error)
+    }
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "Ctrl-C error: {}", self.describe())
+    }
+}
+
+impl std::error::Error for Error {
+    fn description(&self) -> &str {
+        self.describe()
+    }
+
+    fn cause(&self) -> Option<&dyn std::error::Error> {
+        match *self {
+            Error::System(ref e) => Some(e),
+            _ => None,
+        }
+    }
+}