Feat/add all indexes

src/burnchains/burnchain.rs

@@ -618,6 +618,7 @@ impl Burnchain {
         db_path
     }
 
+    /// Connect to the burnchain databases.  They may or may not already exist.
     pub fn connect_db<I: BurnchainIndexer>(
         &self,
         indexer: &I,
@@ -651,7 +652,7 @@ impl Burnchain {
         Ok((sortitiondb, burnchaindb))
     }
 
-    /// Open the burn database.  It must already exist.
+    /// Open the burn databases.  They must already exist.
     pub fn open_db(&self, readwrite: bool) -> Result<(SortitionDB, BurnchainDB), burnchain_error> {
         let db_path = self.get_db_path();
         let burnchain_db_path = self.get_burnchaindb_path();
@@ -1145,6 +1146,37 @@ impl Burnchain {
         Ok((block_snapshot, state_transition_opt))
     }
 
+    /// Get the highest burnchain block processed, if we have processed any.
+    /// Return Some(..) if we have processed at least one processed burnchain block; return None
+    /// otherwise.
+    pub fn get_highest_burnchain_block(
+        &self,
+    ) -> Result<Option<BurnchainBlockHeader>, burnchain_error> {
+        let burndb = match self.open_db(true) {
+            Ok((_sortdb, burndb)) => burndb,
+            Err(burnchain_error::DBError(db_error::NoDBError)) => {
+                // databases not yet initialized, so no blocks processed
+                return Ok(None);
+            }
+            Err(e) => {
+                return Err(e);
+            }
+        };
+
+        let burn_chain_tip = match burndb.get_canonical_chain_tip() {
+            Ok(tip) => tip,
+            Err(burnchain_error::MissingParentBlock) => {
+                // database is empty
+                return Ok(None);
+            }
+            Err(e) => {
+                return Err(e);
+            }
+        };
+
+        Ok(Some(burn_chain_tip))
+    }
+
     /// Top-level burnchain sync.
     /// Returns the burnchain block header for the new burnchain tip, which will be _at least_ as
     /// high as target_block_height_opt (if given), or whatever is currently at the tip of the

src/burnchains/db.rs

@@ -122,7 +122,6 @@ CREATE TABLE burnchain_db_block_headers (
 
     PRIMARY KEY(block_hash)
 );
-CREATE INDEX index_burnchain_db_block_headers_height_hash ON burnchain_db_block_headers(block_height DESC, block_hash ASC);
 
 CREATE TABLE burnchain_db_block_ops (
     block_hash TEXT NOT NULL,
@@ -131,11 +130,14 @@ CREATE TABLE burnchain_db_block_ops (
     FOREIGN KEY(block_hash) REFERENCES burnchain_db_block_headers(block_hash)
 );
 
-CREATE INDEX index_burnchain_db_block_hash ON burnchain_db_block_ops(block_hash);
-CREATE INDEX index_burnchain_db_txid ON burnchain_db_block_ops(txid);
-
 CREATE TABLE db_config(version TEXT NOT NULL);";
 
+const BURNCHAIN_DB_INDEXES: &'static [&'static str] = &[
+    "CREATE INDEX IF NOT EXISTS index_burnchain_db_block_headers_height_hash ON burnchain_db_block_headers(block_height DESC, block_hash ASC);",
+    "CREATE INDEX IF NOT EXISTS index_burnchain_db_block_hash ON burnchain_db_block_ops(block_hash);",
+    "CREATE INDEX IF NOT EXISTS index_burnchain_db_txid ON burnchain_db_block_ops(txid);",
+];
+
 impl<'a> BurnchainDBTransaction<'a> {
     fn store_burnchain_db_entry(
         &self,
@@ -220,6 +222,10 @@ impl BurnchainDB {
             let db_tx = db.tx_begin()?;
             db_tx.sql_tx.execute_batch(BURNCHAIN_DB_INITIAL_SCHEMA)?;
 
+            for index in BURNCHAIN_DB_INDEXES.iter() {
+                db_tx.sql_tx.execute_batch(index)?;
+            }
+
             db_tx.sql_tx.execute(
                 "INSERT INTO db_config (version) VALUES (?1)",
                 &[&BURNCHAIN_DB_VERSION],
@@ -258,7 +264,7 @@ impl BurnchainDB {
     pub fn get_canonical_chain_tip(&self) -> Result<BurnchainBlockHeader, BurnchainError> {
         let qry = "SELECT * FROM burnchain_db_block_headers ORDER BY block_height DESC, block_hash ASC LIMIT 1";
         let opt = query_row(&self.conn, qry, NO_PARAMS)?;
-        Ok(opt.expect("CORRUPTION: No canonical burnchain tip"))
+        opt.ok_or(BurnchainError::MissingParentBlock)
     }
 
     pub fn get_burnchain_block(

src/chainstate/burn/db/sortdb.rs

@@ -483,13 +483,6 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
 
         PRIMARY KEY(sortition_id)
     );"#,
-    "CREATE INDEX snapshots_block_hashes ON snapshots(block_height,index_root,winning_stacks_block_hash);",
-    "CREATE INDEX snapshots_block_stacks_hashes ON snapshots(num_sortitions,index_root,winning_stacks_block_hash);",
-    "CREATE INDEX snapshots_block_heights ON snapshots(burn_header_hash,block_height);",
-    "CREATE INDEX snapshots_block_winning_hash ON snapshots(winning_stacks_block_hash);",
-    "CREATE INDEX snapshots_canonical_chain_tip ON snapshots(pox_valid,block_height DESC,burn_header_hash ASC);",
-    "CREATE INDEX block_arrivals ON snapshots(arrival_index,burn_header_hash);",
-    "CREATE INDEX arrival_indexes ON snapshots(arrival_index);",
     r#"
     CREATE TABLE snapshot_transition_ops(
       sortition_id TEXT PRIMARY KEY,
@@ -515,9 +508,6 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         FOREIGN KEY(sortition_id) REFERENCES snapshots(sortition_id)
     );"#,
     r#"
-    CREATE INDEX index_leader_keys_sortition_id_block_height_vtxindex ON leader_keys(sortition_id,block_height,vtxindex);
-    "#,
-    r#"
     CREATE TABLE block_commits(
         txid TEXT NOT NULL,
         vtxindex INTEGER NOT NULL,
@@ -543,10 +533,6 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         FOREIGN KEY(sortition_id) REFERENCES snapshots(sortition_id)
     );"#,
     r#"
-    CREATE INDEX index_block_commits_sortition_id_vtxindex ON block_commits(sortition_id,vtxindex);
-    CREATE INDEX index_block_commits_sortition_id_block_height_vtxindex ON block_commits(sortition_id,block_height,vtxindex);
-    "#,
-    r#"
     CREATE TABLE user_burn_support(
         txid TEXT NOT NULL,
         vtxindex INTEGER NOT NULL,
@@ -567,11 +553,6 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         FOREIGN KEY(sortition_id) REFERENCES snapshots(sortition_id)
     );"#,
     r#"
-    CREATE INDEX index_user_burn_support_txid ON user_burn_support(txid);
-    CREATE INDEX index_user_burn_support_sortition_id_vtxindex ON user_burn_support(sortition_id,vtxindex);
-    CREATE INDEX index_user_burn_support_sortition_id_hash_160_key_vtxindex_key_block_ptr_vtxindex ON user_burn_support(sortition_id,block_header_hash_160,key_vtxindex,key_block_ptr,vtxindex ASC);
-    "#,
-    r#"
     CREATE TABLE stack_stx (
         txid TEXT NOT NULL,
         vtxindex INTEGER NOT NULL,
@@ -586,9 +567,6 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         PRIMARY KEY(txid)
     );"#,
     r#"
-    CREATE INDEX index_stack_stx_burn_header_hash ON stack_stx(burn_header_hash);
-    "#,
-    r#"
     CREATE TABLE transfer_stx (
         txid TEXT NOT NULL,
         vtxindex INTEGER NOT NULL,
@@ -603,9 +581,6 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         PRIMARY KEY(txid)
     );"#,
     r#"
-    CREATE INDEX index_transfer_stx_burn_header_hash ON transfer_stx(burn_header_hash);
-    "#,
-    r#"
     CREATE TABLE missed_commits (
         txid TEXT NOT NULL,
         input TEXT NOT NULL,
@@ -614,17 +589,13 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         PRIMARY KEY(txid, intended_sortition_id)
     );"#,
     r#"
-    CREATE INDEX index_missed_commits_intended_sortition_id ON missed_commits(intended_sortition_id);
-    "#,
-    r#"
     CREATE TABLE canonical_accepted_stacks_blocks(
         tip_consensus_hash TEXT NOT NULL,
         consensus_hash TEXT NOT NULL,
         stacks_block_hash TEXT NOT NULL,
         block_height INTEGER NOT NULL,
         PRIMARY KEY(consensus_hash, stacks_block_hash)
     );"#,
-    "CREATE INDEX canonical_stacks_blocks ON canonical_accepted_stacks_blocks(tip_consensus_hash,stacks_block_hash);",
     "CREATE TABLE db_config(version TEXT PRIMARY KEY);",
 ];
 
@@ -638,6 +609,26 @@ const SORTITION_DB_SCHEMA_2: &'static [&'static str] = &[r#"
          PRIMARY KEY(start_block_height,epoch_id)
      );"#];
 
+const SORTITION_DB_INDEXES: &'static [&'static str] = &[
+    "CREATE INDEX IF NOT EXISTS snapshots_block_hashes ON snapshots(block_height,index_root,winning_stacks_block_hash);",
+    "CREATE INDEX IF NOT EXISTS snapshots_block_stacks_hashes ON snapshots(num_sortitions,index_root,winning_stacks_block_hash);",
+    "CREATE INDEX IF NOT EXISTS snapshots_block_heights ON snapshots(burn_header_hash,block_height);",
+    "CREATE INDEX IF NOT EXISTS snapshots_block_winning_hash ON snapshots(winning_stacks_block_hash);",
+    "CREATE INDEX IF NOT EXISTS snapshots_canonical_chain_tip ON snapshots(pox_valid,block_height DESC,burn_header_hash ASC);",
+    "CREATE INDEX IF NOT EXISTS block_arrivals ON snapshots(arrival_index,burn_header_hash);",
+    "CREATE INDEX IF NOT EXISTS arrival_indexes ON snapshots(arrival_index);",
+    "CREATE INDEX IF NOT EXISTS index_leader_keys_sortition_id_block_height_vtxindex ON leader_keys(sortition_id,block_height,vtxindex);",
+    "CREATE INDEX IF NOT EXISTS index_block_commits_sortition_id_vtxindex ON block_commits(sortition_id,vtxindex);",
+    "CREATE INDEX IF NOT EXISTS index_block_commits_sortition_id_block_height_vtxindex ON block_commits(sortition_id,block_height,vtxindex);",
+    "CREATE INDEX IF NOT EXISTS index_user_burn_support_txid ON user_burn_support(txid);",
+    "CREATE INDEX IF NOT EXISTS index_user_burn_support_sortition_id_vtxindex ON user_burn_support(sortition_id,vtxindex);",
+    "CREATE INDEX IF NOT EXISTS index_user_burn_support_sortition_id_hash_160_key_vtxindex_key_block_ptr_vtxindex ON user_burn_support(sortition_id,block_header_hash_160,key_vtxindex,key_block_ptr,vtxindex ASC);",
+    "CREATE INDEX IF NOT EXISTS index_stack_stx_burn_header_hash ON stack_stx(burn_header_hash);",
+    "CREATE INDEX IF NOT EXISTS index_transfer_stx_burn_header_hash ON transfer_stx(burn_header_hash);",
+    "CREATE INDEX IF NOT EXISTS index_missed_commits_intended_sortition_id ON missed_commits(intended_sortition_id);",
+    "CREATE INDEX IF NOT EXISTS canonical_stacks_blocks ON canonical_accepted_stacks_blocks(tip_consensus_hash,stacks_block_hash);"
+];
+
 pub struct SortitionDB {
     pub readwrite: bool,
     pub marf: MARF<SortitionId>,
@@ -2295,6 +2286,9 @@ impl SortitionDB {
         for row_text in SORTITION_DB_SCHEMA_2 {
             db_tx.execute_batch(row_text)?;
         }
+        for row_text in SORTITION_DB_INDEXES {
+            db_tx.execute_batch(row_text)?;
+        }
 
         SortitionDB::validate_and_insert_epochs(&db_tx, epochs_ref)?;
 

src/chainstate/stacks/db/mod.rs

@@ -597,10 +597,6 @@ const CHAINSTATE_INITIAL_SCHEMA: &'static [&'static str] = &[
 
         PRIMARY KEY(consensus_hash,block_hash)
     );"#,
-    "CREATE INDEX index_block_hash_to_primary_key ON block_headers(index_block_hash,consensus_hash,block_hash);",
-    "CREATE INDEX block_headers_hash_index ON block_headers(block_hash,block_height);",
-    "CREATE INDEX block_index_hash_index ON block_headers(index_block_hash,consensus_hash,block_hash);",
-    "CREATE INDEX block_headers_burn_header_height ON block_headers(burn_header_height);",
     r#"
     -- scheduled payments
     -- no designated primary key since there can be duplicate entries
@@ -624,10 +620,6 @@ const CHAINSTATE_INITIAL_SCHEMA: &'static [&'static str] = &[
         vtxindex INT NOT NULL               -- user burn support vtxindex
     );"#,
     r#"
-    CREATE INDEX index_payments_block_hash_consensus_hash_vtxindex ON payments(block_hash,consensus_hash,vtxindex ASC);
-    CREATE INDEX index_payments_index_block_hash_vtxindex ON payments(index_block_hash,vtxindex ASC);
-    "#,
-    r#"
     -- users who supported miners
     CREATE TABLE user_supporters(
         address TEXT NOT NULL,
@@ -656,11 +648,6 @@ const CHAINSTATE_INITIAL_SCHEMA: &'static [&'static str] = &[
                                      orphaned INT NOT NULL,
                                      PRIMARY KEY(anchored_block_hash,consensus_hash,microblock_hash)
     );"#,
-    "CREATE INDEX staging_microblocks_processed ON staging_microblocks(processed);",
-    "CREATE INDEX staging_microblocks_orphaned ON staging_microblocks(orphaned);",
-    "CREATE INDEX staging_microblocks_index_hash ON staging_microblocks(index_block_hash);",
-    "CREATE INDEX staging_microblocks_index_hash_processed ON staging_microblocks(index_block_hash,processed);",
-    "CREATE INDEX staging_microblocks_index_hash_orphaned ON staging_microblocks(index_block_hash,orphaned);",
     r#"
     -- Staging microblocks data
     CREATE TABLE staging_microblocks_data(block_hash TEXT NOT NULL,
@@ -695,12 +682,6 @@ const CHAINSTATE_INITIAL_SCHEMA: &'static [&'static str] = &[
                                 processed_time INT NOT NULL,              -- when this block was processed
                                 PRIMARY KEY(anchored_block_hash,consensus_hash)
     );"#,
-    "CREATE INDEX processed_stacks_blocks ON staging_blocks(processed,anchored_block_hash,consensus_hash);",
-    "CREATE INDEX orphaned_stacks_blocks ON staging_blocks(orphaned,anchored_block_hash,consensus_hash);",
-    "CREATE INDEX parent_blocks ON staging_blocks(parent_anchored_block_hash);",
-    "CREATE INDEX parent_consensus_hashes ON staging_blocks(parent_consensus_hash);",
-    "CREATE INDEX index_block_hashes ON staging_blocks(index_block_hash);",
-    "CREATE INDEX height_stacks_blocks ON staging_blocks(height);",
     r#"
     -- users who burned in support of a block
     CREATE TABLE staging_user_burn_support(anchored_block_hash TEXT NOT NULL,
@@ -710,9 +691,6 @@ const CHAINSTATE_INITIAL_SCHEMA: &'static [&'static str] = &[
                                            vtxindex INT NOT NULL
     );"#,
     r#"
-    CREATE INDEX index_staging_user_burn_support ON staging_user_burn_support(anchored_block_hash,consensus_hash);
-    "#,
-    r#"
     CREATE TABLE transactions(
         id INTEGER PRIMARY KEY,
         txid TEXT NOT NULL,
@@ -721,8 +699,6 @@ const CHAINSTATE_INITIAL_SCHEMA: &'static [&'static str] = &[
         result TEXT NOT NULL,
         UNIQUE (txid,index_block_hash)
     );"#,
-    "CREATE INDEX txid_tx_index ON transactions(txid);",
-    "CREATE INDEX index_block_hash_tx_index ON transactions(index_block_hash);",
 ];
 
 const CHAINSTATE_SCHEMA_2: &'static [&'static str] = &[
@@ -737,6 +713,29 @@ const CHAINSTATE_SCHEMA_2: &'static [&'static str] = &[
     "#,
 ];
 
+const CHAINSTATE_INDEXES: &'static [&'static str] = &[
+    "CREATE INDEX IF NOT EXISTS index_block_hash_to_primary_key ON block_headers(index_block_hash,consensus_hash,block_hash);",
+    "CREATE INDEX IF NOT EXISTS block_headers_hash_index ON block_headers(block_hash,block_height);",
+    "CREATE INDEX IF NOT EXISTS block_index_hash_index ON block_headers(index_block_hash,consensus_hash,block_hash);",
+    "CREATE INDEX IF NOT EXISTS block_headers_burn_header_height ON block_headers(burn_header_height);",
+    "CREATE INDEX IF NOT EXISTS index_payments_block_hash_consensus_hash_vtxindex ON payments(block_hash,consensus_hash,vtxindex ASC);",
+    "CREATE INDEX IF NOT EXISTS index_payments_index_block_hash_vtxindex ON payments(index_block_hash,vtxindex ASC);",
+    "CREATE INDEX IF NOT EXISTS staging_microblocks_processed ON staging_microblocks(processed);",
+    "CREATE INDEX IF NOT EXISTS staging_microblocks_orphaned ON staging_microblocks(orphaned);",
+    "CREATE INDEX IF NOT EXISTS staging_microblocks_index_hash ON staging_microblocks(index_block_hash);",
+    "CREATE INDEX IF NOT EXISTS staging_microblocks_index_hash_processed ON staging_microblocks(index_block_hash,processed);",
+    "CREATE INDEX IF NOT EXISTS staging_microblocks_index_hash_orphaned ON staging_microblocks(index_block_hash,orphaned);",
+    "CREATE INDEX IF NOT EXISTS processed_stacks_blocks ON staging_blocks(processed,anchored_block_hash,consensus_hash);",
+    "CREATE INDEX IF NOT EXISTS orphaned_stacks_blocks ON staging_blocks(orphaned,anchored_block_hash,consensus_hash);",
+    "CREATE INDEX IF NOT EXISTS parent_blocks ON staging_blocks(parent_anchored_block_hash);",
+    "CREATE INDEX IF NOT EXISTS parent_consensus_hashes ON staging_blocks(parent_consensus_hash);",
+    "CREATE INDEX IF NOT EXISTS index_block_hashes ON staging_blocks(index_block_hash);",
+    "CREATE INDEX IF NOT EXISTS height_stacks_blocks ON staging_blocks(height);",
+    "CREATE INDEX IF NOT EXISTS index_staging_user_burn_support ON staging_user_burn_support(anchored_block_hash,consensus_hash);",
+    "CREATE INDEX IF NOT EXISTS txid_tx_index ON transactions(txid);",
+    "CREATE INDEX IF NOT EXISTS index_block_hash_tx_index ON transactions(index_block_hash);",
+];
+
 #[cfg(test)]
 pub const MINER_REWARD_MATURITY: u64 = 2; // small for testing purposes
 
@@ -858,6 +857,10 @@ impl StacksChainState {
             if migrate {
                 StacksChainState::apply_schema_migrations(&tx, mainnet, chain_id)?;
             }
+
+            for cmd in CHAINSTATE_INDEXES {
+                tx.execute_batch(cmd)?;
+            }
         }
 
         dbtx.instantiate_index()?;