Merge pull request #3029 from stacks-network/feat/add-all-indexes

Feat/add all indexes

src/burnchains/burnchain.rs

@@ -618,6 +618,7 @@ impl Burnchain {
         db_path
     }
 
+    /// Connect to the burnchain databases.  They may or may not already exist.
     pub fn connect_db<I: BurnchainIndexer>(
         &self,
         indexer: &I,
@@ -651,7 +652,7 @@ impl Burnchain {
         Ok((sortitiondb, burnchaindb))
     }
 
-    /// Open the burn database.  It must already exist.
+    /// Open the burn databases.  They must already exist.
     pub fn open_db(&self, readwrite: bool) -> Result<(SortitionDB, BurnchainDB), burnchain_error> {
         let db_path = self.get_db_path();
         let burnchain_db_path = self.get_burnchaindb_path();
@@ -1145,6 +1146,37 @@ impl Burnchain {
         Ok((block_snapshot, state_transition_opt))
     }
 
+    /// Get the highest burnchain block processed, if we have processed any.
+    /// Return Some(..) if we have processed at least one processed burnchain block; return None
+    /// otherwise.
+    pub fn get_highest_burnchain_block(
+        &self,
+    ) -> Result<Option<BurnchainBlockHeader>, burnchain_error> {
+        let burndb = match self.open_db(true) {
+            Ok((_sortdb, burndb)) => burndb,
+            Err(burnchain_error::DBError(db_error::NoDBError)) => {
+                // databases not yet initialized, so no blocks processed
+                return Ok(None);
+            }
+            Err(e) => {
+                return Err(e);
+            }
+        };
+
+        let burn_chain_tip = match burndb.get_canonical_chain_tip() {
+            Ok(tip) => tip,
+            Err(burnchain_error::MissingParentBlock) => {
+                // database is empty
+                return Ok(None);
+            }
+            Err(e) => {
+                return Err(e);
+            }
+        };
+
+        Ok(Some(burn_chain_tip))
+    }
+
     /// Top-level burnchain sync.
     /// Returns the burnchain block header for the new burnchain tip, which will be _at least_ as
     /// high as target_block_height_opt (if given), or whatever is currently at the tip of the

src/burnchains/db.rs

@@ -122,7 +122,6 @@ CREATE TABLE burnchain_db_block_headers (
 
     PRIMARY KEY(block_hash)
 );
-CREATE INDEX index_burnchain_db_block_headers_height_hash ON burnchain_db_block_headers(block_height DESC, block_hash ASC);
 
 CREATE TABLE burnchain_db_block_ops (
     block_hash TEXT NOT NULL,
@@ -131,11 +130,14 @@ CREATE TABLE burnchain_db_block_ops (
     FOREIGN KEY(block_hash) REFERENCES burnchain_db_block_headers(block_hash)
 );
 
-CREATE INDEX index_burnchain_db_block_hash ON burnchain_db_block_ops(block_hash);
-CREATE INDEX index_burnchain_db_txid ON burnchain_db_block_ops(txid);
-
 CREATE TABLE db_config(version TEXT NOT NULL);";
 
+const BURNCHAIN_DB_INDEXES: &'static [&'static str] = &[
+    "CREATE INDEX IF NOT EXISTS index_burnchain_db_block_headers_height_hash ON burnchain_db_block_headers(block_height DESC, block_hash ASC);",
+    "CREATE INDEX IF NOT EXISTS index_burnchain_db_block_hash ON burnchain_db_block_ops(block_hash);",
+    "CREATE INDEX IF NOT EXISTS index_burnchain_db_txid ON burnchain_db_block_ops(txid);",
+];
+
 impl<'a> BurnchainDBTransaction<'a> {
     fn store_burnchain_db_entry(
         &self,
@@ -181,6 +183,15 @@ impl<'a> BurnchainDBTransaction<'a> {
 }
 
 impl BurnchainDB {
+    fn add_indexes(&mut self) -> Result<(), BurnchainError> {
+        let db_tx = self.tx_begin()?;
+        for index in BURNCHAIN_DB_INDEXES.iter() {
+            db_tx.sql_tx.execute_batch(index)?;
+        }
+        db_tx.commit()?;
+        Ok(())
+    }
+
     pub fn connect(
         path: &str,
         first_block_height: u64,
@@ -237,6 +248,9 @@ impl BurnchainDB {
             db_tx.commit()?;
         }
 
+        if readwrite {
+            db.add_indexes()?;
+        }
         Ok(db)
     }
 
@@ -247,7 +261,12 @@ impl BurnchainDB {
             OpenFlags::SQLITE_OPEN_READ_ONLY
         };
         let conn = sqlite_open(path, open_flags, true)?;
-        Ok(BurnchainDB { conn })
+        let mut db = BurnchainDB { conn };
+
+        if readwrite {
+            db.add_indexes()?;
+        }
+        Ok(db)
     }
 
     fn tx_begin<'a>(&'a mut self) -> Result<BurnchainDBTransaction<'a>, BurnchainError> {
@@ -258,7 +277,7 @@ impl BurnchainDB {
     pub fn get_canonical_chain_tip(&self) -> Result<BurnchainBlockHeader, BurnchainError> {
         let qry = "SELECT * FROM burnchain_db_block_headers ORDER BY block_height DESC, block_hash ASC LIMIT 1";
         let opt = query_row(&self.conn, qry, NO_PARAMS)?;
-        Ok(opt.expect("CORRUPTION: No canonical burnchain tip"))
+        opt.ok_or(BurnchainError::MissingParentBlock)
     }
 
     pub fn get_burnchain_block(

src/chainstate/burn/db/sortdb.rs

@@ -483,13 +483,6 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
 
         PRIMARY KEY(sortition_id)
     );"#,
-    "CREATE INDEX snapshots_block_hashes ON snapshots(block_height,index_root,winning_stacks_block_hash);",
-    "CREATE INDEX snapshots_block_stacks_hashes ON snapshots(num_sortitions,index_root,winning_stacks_block_hash);",
-    "CREATE INDEX snapshots_block_heights ON snapshots(burn_header_hash,block_height);",
-    "CREATE INDEX snapshots_block_winning_hash ON snapshots(winning_stacks_block_hash);",
-    "CREATE INDEX snapshots_canonical_chain_tip ON snapshots(pox_valid,block_height DESC,burn_header_hash ASC);",
-    "CREATE INDEX block_arrivals ON snapshots(arrival_index,burn_header_hash);",
-    "CREATE INDEX arrival_indexes ON snapshots(arrival_index);",
     r#"
     CREATE TABLE snapshot_transition_ops(
       sortition_id TEXT PRIMARY KEY,
@@ -515,9 +508,6 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         FOREIGN KEY(sortition_id) REFERENCES snapshots(sortition_id)
     );"#,
     r#"
-    CREATE INDEX index_leader_keys_sortition_id_block_height_vtxindex ON leader_keys(sortition_id,block_height,vtxindex);
-    "#,
-    r#"
     CREATE TABLE block_commits(
         txid TEXT NOT NULL,
         vtxindex INTEGER NOT NULL,
@@ -543,10 +533,6 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         FOREIGN KEY(sortition_id) REFERENCES snapshots(sortition_id)
     );"#,
     r#"
-    CREATE INDEX index_block_commits_sortition_id_vtxindex ON block_commits(sortition_id,vtxindex);
-    CREATE INDEX index_block_commits_sortition_id_block_height_vtxindex ON block_commits(sortition_id,block_height,vtxindex);
-    "#,
-    r#"
     CREATE TABLE user_burn_support(
         txid TEXT NOT NULL,
         vtxindex INTEGER NOT NULL,
@@ -567,11 +553,6 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         FOREIGN KEY(sortition_id) REFERENCES snapshots(sortition_id)
     );"#,
     r#"
-    CREATE INDEX index_user_burn_support_txid ON user_burn_support(txid);
-    CREATE INDEX index_user_burn_support_sortition_id_vtxindex ON user_burn_support(sortition_id,vtxindex);
-    CREATE INDEX index_user_burn_support_sortition_id_hash_160_key_vtxindex_key_block_ptr_vtxindex ON user_burn_support(sortition_id,block_header_hash_160,key_vtxindex,key_block_ptr,vtxindex ASC);
-    "#,
-    r#"
     CREATE TABLE stack_stx (
         txid TEXT NOT NULL,
         vtxindex INTEGER NOT NULL,
@@ -586,9 +567,6 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         PRIMARY KEY(txid)
     );"#,
     r#"
-    CREATE INDEX index_stack_stx_burn_header_hash ON stack_stx(burn_header_hash);
-    "#,
-    r#"
     CREATE TABLE transfer_stx (
         txid TEXT NOT NULL,
         vtxindex INTEGER NOT NULL,
@@ -603,9 +581,6 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         PRIMARY KEY(txid)
     );"#,
     r#"
-    CREATE INDEX index_transfer_stx_burn_header_hash ON transfer_stx(burn_header_hash);
-    "#,
-    r#"
     CREATE TABLE missed_commits (
         txid TEXT NOT NULL,
         input TEXT NOT NULL,
@@ -614,17 +589,13 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         PRIMARY KEY(txid, intended_sortition_id)
     );"#,
     r#"
-    CREATE INDEX index_missed_commits_intended_sortition_id ON missed_commits(intended_sortition_id);
-    "#,
-    r#"
     CREATE TABLE canonical_accepted_stacks_blocks(
         tip_consensus_hash TEXT NOT NULL,
         consensus_hash TEXT NOT NULL,
         stacks_block_hash TEXT NOT NULL,
         block_height INTEGER NOT NULL,
         PRIMARY KEY(consensus_hash, stacks_block_hash)
     );"#,
-    "CREATE INDEX canonical_stacks_blocks ON canonical_accepted_stacks_blocks(tip_consensus_hash,stacks_block_hash);",
     "CREATE TABLE db_config(version TEXT PRIMARY KEY);",
 ];
 
@@ -638,6 +609,26 @@ const SORTITION_DB_SCHEMA_2: &'static [&'static str] = &[r#"
          PRIMARY KEY(start_block_height,epoch_id)
      );"#];
 
+const SORTITION_DB_INDEXES: &'static [&'static str] = &[
+    "CREATE INDEX IF NOT EXISTS snapshots_block_hashes ON snapshots(block_height,index_root,winning_stacks_block_hash);",
+    "CREATE INDEX IF NOT EXISTS snapshots_block_stacks_hashes ON snapshots(num_sortitions,index_root,winning_stacks_block_hash);",
+    "CREATE INDEX IF NOT EXISTS snapshots_block_heights ON snapshots(burn_header_hash,block_height);",
+    "CREATE INDEX IF NOT EXISTS snapshots_block_winning_hash ON snapshots(winning_stacks_block_hash);",
+    "CREATE INDEX IF NOT EXISTS snapshots_canonical_chain_tip ON snapshots(pox_valid,block_height DESC,burn_header_hash ASC);",
+    "CREATE INDEX IF NOT EXISTS block_arrivals ON snapshots(arrival_index,burn_header_hash);",
+    "CREATE INDEX IF NOT EXISTS arrival_indexes ON snapshots(arrival_index);",
+    "CREATE INDEX IF NOT EXISTS index_leader_keys_sortition_id_block_height_vtxindex ON leader_keys(sortition_id,block_height,vtxindex);",
+    "CREATE INDEX IF NOT EXISTS index_block_commits_sortition_id_vtxindex ON block_commits(sortition_id,vtxindex);",
+    "CREATE INDEX IF NOT EXISTS index_block_commits_sortition_id_block_height_vtxindex ON block_commits(sortition_id,block_height,vtxindex);",
+    "CREATE INDEX IF NOT EXISTS index_user_burn_support_txid ON user_burn_support(txid);",
+    "CREATE INDEX IF NOT EXISTS index_user_burn_support_sortition_id_vtxindex ON user_burn_support(sortition_id,vtxindex);",
+    "CREATE INDEX IF NOT EXISTS index_user_burn_support_sortition_id_hash_160_key_vtxindex_key_block_ptr_vtxindex ON user_burn_support(sortition_id,block_header_hash_160,key_vtxindex,key_block_ptr,vtxindex ASC);",
+    "CREATE INDEX IF NOT EXISTS index_stack_stx_burn_header_hash ON stack_stx(burn_header_hash);",
+    "CREATE INDEX IF NOT EXISTS index_transfer_stx_burn_header_hash ON transfer_stx(burn_header_hash);",
+    "CREATE INDEX IF NOT EXISTS index_missed_commits_intended_sortition_id ON missed_commits(intended_sortition_id);",
+    "CREATE INDEX IF NOT EXISTS canonical_stacks_blocks ON canonical_accepted_stacks_blocks(tip_consensus_hash,stacks_block_hash);"
+];
+
 pub struct SortitionDB {
     pub readwrite: bool,
     pub marf: MARF<SortitionId>,
@@ -2135,6 +2126,9 @@ impl SortitionDB {
         }
 
         db.check_schema_version_and_update(epochs)?;
+        if readwrite {
+            db.add_indexes()?;
+        }
         Ok(db)
     }
 
@@ -2318,6 +2312,8 @@ impl SortitionDB {
         )?;
 
         db_tx.commit()?;
+
+        self.add_indexes()?;
         Ok(())
     }
 
@@ -2516,6 +2512,15 @@ impl SortitionDB {
             Err(e) => panic!("Error obtaining the version of the sortition DB: {:?}", e),
         }
     }
+
+    fn add_indexes(&mut self) -> Result<(), db_error> {
+        let tx = self.tx_begin()?;
+        for row_text in SORTITION_DB_INDEXES {
+            tx.execute_batch(row_text)?;
+        }
+        tx.commit()?;
+        Ok(())
+    }
 }
 
 impl<'a> SortitionDBConn<'a> {