Use fee rate estimation in block assembly

.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests

@@ -1,36 +1,7 @@
-FROM rust:bullseye
-
-WORKDIR /src/
-
-COPY . .
-
-WORKDIR /src/testnet/stacks-node
-RUN cargo test --no-run
-
-RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-0.20.0/bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
-RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
-
-RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/
+FROM stacks-node:integrations
 
+ARG test_name
 ENV BITCOIND_TEST 1
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::microblock_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_check_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::cost_voting_integration
-RUN cargo test -- --test-threads 1 --ignored tests::integrations::integration_test_get_info
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::bitcoind_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::liquid_ustx_integration
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::stx_transfer_btc_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::bitcoind_forking_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::should_fix_2771
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::pox_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::bitcoin_regtest::bitcoind_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::should_succeed_handling_malformed_and_valid_txs
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_overflow_unconfirmed_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_overflow_unconfirmed_stream_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_overflow_unconfirmed_invalid_stream_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::runtime_overflow_unconfirmed_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::antientropy_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::filter_low_fee_tx_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::filter_long_runtime_tx_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::mining_transactions_is_fair
+
+RUN cargo test -- --test-threads 1 --ignored "$test_name"
 

.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests

@@ -0,0 +1,15 @@
+FROM rust:bullseye
+
+WORKDIR /src/
+
+COPY . .
+
+WORKDIR /src/testnet/stacks-node
+RUN cargo test --no-run
+
+RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-0.20.0/bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
+RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
+
+RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/
+
+ENV BITCOIND_TEST 1

.github/workflows/bitcoin-tests.yml

@@ -6,15 +6,63 @@ on:
   pull_request:
 
 jobs:
+  build-integration-image:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build bitcoin integration testing image
+        env:
+          DOCKER_BUILDKIT: 1
+        run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests -t stacks-node:integrations .
+      - name: Export docker image as tarball
+        run: docker save -o integration-image.tar stacks-node:integrations
+      - name: Upload built docker image
+        uses: actions/upload-artifact@v2
+        with:
+          name: integration-image.tar
+          path: integration-image.tar
   # Run sampled genesis tests
   sampled-genesis:
     runs-on: ubuntu-latest
+    needs:
+      - build-integration-image
+    strategy:
+      matrix:
+        test-name:
+          - tests::neon_integrations::microblock_integration_test
+          - tests::neon_integrations::size_check_integration_test
+          - tests::neon_integrations::cost_voting_integration
+          - tests::integrations::integration_test_get_info
+          - tests::neon_integrations::bitcoind_integration_test
+          - tests::neon_integrations::liquid_ustx_integration
+          - tests::neon_integrations::stx_transfer_btc_integration_test
+          - tests::neon_integrations::bitcoind_forking_test
+          - tests::neon_integrations::should_fix_2771
+          - tests::neon_integrations::pox_integration_test
+          - tests::bitcoin_regtest::bitcoind_integration_test
+          - tests::should_succeed_handling_malformed_and_valid_txs
+          - tests::neon_integrations::size_overflow_unconfirmed_microblocks_integration_test
+          - tests::neon_integrations::size_overflow_unconfirmed_stream_microblocks_integration_test
+          - tests::neon_integrations::size_overflow_unconfirmed_invalid_stream_microblocks_integration_test
+          - tests::neon_integrations::runtime_overflow_unconfirmed_microblocks_integration_test
+          - tests::neon_integrations::antientropy_integration_test
+          - tests::neon_integrations::filter_low_fee_tx_integration_test
+          - tests::neon_integrations::filter_long_runtime_tx_integration_test
+          - tests::neon_integrations::mining_transactions_is_fair
     steps:
       - uses: actions/checkout@v2
+      - name: Download math result for job 1
+        uses: actions/download-artifact@v2
+        with:
+          name: integration-image.tar
+      - name: Load docker image
+        run: docker load -i integration-image.tar && rm integration-image.tar
       - name: All integration tests with sampled genesis
+        timeout-minutes: 30
         env:
           DOCKER_BUILDKIT: 1
-        run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests .
+          TEST_NAME: ${{ matrix.test-name }}
+        run: docker build --build-arg test_name=${{ matrix.test-name }} -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests .
   atlas-test:
     # disable this job/test for now, as the atlas endpoints are currently disabled.
     if: ${{ true }}

src/chainstate/coordinator/mod.rs

@@ -148,8 +148,8 @@ pub struct ChainsCoordinator<
     T: BlockEventDispatcher,
     N: CoordinatorNotices,
     R: RewardSetProvider,
-    CE: CostEstimator,
-    FE: FeeEstimator,
+    CE: CostEstimator + ?Sized,
+    FE: FeeEstimator + ?Sized,
 > {
     canonical_sortition_tip: Option<SortitionId>,
     canonical_chain_tip: Option<StacksBlockId>,
@@ -160,8 +160,8 @@ pub struct ChainsCoordinator<
     burnchain: Burnchain,
     attachments_tx: SyncSender<HashSet<AttachmentInstance>>,
     dispatcher: Option<&'a T>,
-    cost_estimator: Option<CE>,
-    fee_estimator: Option<FE>,
+    cost_estimator: Option<&'a mut CE>,
+    fee_estimator: Option<&'a mut FE>,
     reward_set_provider: R,
     notifier: N,
     atlas_config: AtlasConfig,
@@ -256,7 +256,7 @@ impl RewardSetProvider for OnChainRewardSetProvider {
     }
 }
 
-impl<'a, T: BlockEventDispatcher, CE: CostEstimator, FE: FeeEstimator>
+impl<'a, T: BlockEventDispatcher, CE: CostEstimator + ?Sized, FE: FeeEstimator + ?Sized>
     ChainsCoordinator<'a, T, ArcCounterCoordinatorNotices, OnChainRewardSetProvider, CE, FE>
 {
     pub fn run(
@@ -266,8 +266,8 @@ impl<'a, T: BlockEventDispatcher, CE: CostEstimator, FE: FeeEstimator>
         dispatcher: &'a mut T,
         comms: CoordinatorReceivers,
         atlas_config: AtlasConfig,
-        cost_estimator: Option<CE>,
-        fee_estimator: Option<FE>,
+        cost_estimator: Option<&mut CE>,
+        fee_estimator: Option<&mut FE>,
     ) where
         T: BlockEventDispatcher,
     {
@@ -517,8 +517,8 @@ impl<
         T: BlockEventDispatcher,
         N: CoordinatorNotices,
         U: RewardSetProvider,
-        CE: CostEstimator,
-        FE: FeeEstimator,
+        CE: CostEstimator + ?Sized,
+        FE: FeeEstimator + ?Sized,
     > ChainsCoordinator<'a, T, N, U, CE, FE>
 {
     pub fn handle_new_stacks_block(&mut self) -> Result<(), Error> {

src/chainstate/stacks/db/blocks.rs

@@ -5545,6 +5545,8 @@ pub mod test {
     use util::hash::*;
     use util::retry::*;
 
+    use crate::cost_estimates::metrics::UnitMetric;
+    use crate::cost_estimates::UnitEstimator;
     use crate::types::chainstate::{BlockHeaderHash, StacksWorkScore};
 
     use super::*;
@@ -9166,7 +9168,8 @@ pub mod test {
                         }
                     };
 
-                    let mut mempool = MemPoolDB::open(false, 0x80000000, &chainstate_path).unwrap();
+                    let mut mempool =
+                        MemPoolDB::open_test(false, 0x80000000, &chainstate_path).unwrap();
                     let coinbase_tx = make_coinbase(miner, tenure_id);
 
                     let anchored_block = StacksBlockBuilder::build_anchored_block(

src/chainstate/stacks/db/mod.rs

@@ -237,11 +237,11 @@ impl FromRow<DBConfig> for DBConfig {
 
 impl FromRow<StacksHeaderInfo> for StacksHeaderInfo {
     fn from_row<'a>(row: &'a Row) -> Result<StacksHeaderInfo, db_error> {
-        let block_height = u64::from_column(row, "block_height")?;
+        let block_height: u64 = u64::from_column(row, "block_height")?;
         let index_root = TrieHash::from_column(row, "index_root")?;
         let consensus_hash = ConsensusHash::from_column(row, "consensus_hash")?;
         let burn_header_hash = BurnchainHeaderHash::from_column(row, "burn_header_hash")?;
-        let burn_header_height = u64::from_column(row, "burn_header_height")? as u32;
+        let burn_header_height: u64 = u64::from_column(row, "burn_header_height")?;
         let burn_header_timestamp = u64::from_column(row, "burn_header_timestamp")?;
         let stacks_header = StacksBlockHeader::from_row(row)?;
         let anchored_block_size_str: String = row.get_unwrap("block_size");
@@ -256,13 +256,13 @@ impl FromRow<StacksHeaderInfo> for StacksHeaderInfo {
         Ok(StacksHeaderInfo {
             anchored_header: stacks_header,
             microblock_tail: None,
-            block_height: block_height,
-            index_root: index_root,
-            consensus_hash: consensus_hash,
-            burn_header_hash: burn_header_hash,
-            burn_header_height: burn_header_height,
-            burn_header_timestamp: burn_header_timestamp,
-            anchored_block_size: anchored_block_size,
+            block_height,
+            index_root,
+            consensus_hash,
+            burn_header_hash,
+            burn_header_height: burn_header_height as u32,
+            burn_header_timestamp,
+            anchored_block_size,
         })
     }
 }