Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Nakamoto mempool sync #4966

Merged
merged 14 commits into from
Jul 17, 2024
Merged

Nakamoto mempool sync #4966

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
f2a2bbf
fix: #4938
jcnelson Jul 1, 2024
23b450f
Merge branch 'develop' of https://github.com/stacks-network/stacks-co…
jcnelson Jul 12, 2024
e9b23a0
chore: move mempool sync state machine into its own file
jcnelson Jul 12, 2024
8b2a5cf
chore: API sync
jcnelson Jul 12, 2024
b93b103
feat: make mempool state compatible with nakamoto by treating `height…
jcnelson Jul 12, 2024
c199312
chore: API sync
jcnelson Jul 12, 2024
22ecca9
feat: make /v2/mempool/query compatible with Nakamoto by querying Nak…
jcnelson Jul 12, 2024
a999518
chore: move mempool tests into their own file and API sync
jcnelson Jul 12, 2024
85309c3
chore: don't talk to unroutable nodes if not configured to do so
jcnelson Jul 16, 2024
5ee77d1
fix: a request is inflight if the response is inflight as well
jcnelson Jul 16, 2024
d0f056b
feat: take additional balances
jcnelson Jul 16, 2024
1d2dff0
chore: nakamoto-specific mempool testing
jcnelson Jul 16, 2024
8182591
fix: document `false` usage in `MemPoolDB::try_add_tx()` and remove TODO
jcnelson Jul 17, 2024
d47c4ab
Merge branch 'develop' into feat/nakamoto-mempool-sync
jcnelson Jul 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions stackslib/src/chainstate/stacks/miner.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1203,7 +1203,6 @@ impl<'a> StacksMicroblockBuilder<'a> {
intermediate_result = mem_pool.iterate_candidates(
&mut clarity_tx,
&mut tx_events,
self.anchor_block_height,
mempool_settings.clone(),
|clarity_tx, to_consider, estimator| {
let mempool_tx = &to_consider.tx;
Expand Down Expand Up @@ -2210,7 +2209,6 @@ impl StacksBlockBuilder {
intermediate_result = mempool.iterate_candidates(
epoch_tx,
&mut tx_events,
tip_height,
mempool_settings.clone(),
|epoch_tx, to_consider, estimator| {
// first, have we been preempted?
Expand Down
4 changes: 2 additions & 2 deletions stackslib/src/chainstate/stacks/tests/block_construction.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3060,7 +3060,7 @@ fn test_build_microblock_stream_forks_with_descendants() {

// erase any pending transactions -- this is a "worse" poison-microblock,
// and we want to avoid mining the "better" one
mempool.clear_before_height(10).unwrap();
mempool.clear_before_coinbase_height(10).unwrap();

let mut tx_bytes = vec![];
poison_microblock_tx
Expand Down Expand Up @@ -4784,6 +4784,7 @@ fn paramaterized_mempool_walk_test(
&mut chainstate,
&b_1.0,
&b_1.1,
true,
txid,
tx_bytes,
tx_fee,
Expand Down Expand Up @@ -4832,7 +4833,6 @@ fn paramaterized_mempool_walk_test(
.iterate_candidates::<_, ChainstateError, _>(
clarity_conn,
&mut tx_events,
2,
mempool_settings.clone(),
|_, available_tx, _| {
count_txs += 1;
Expand Down
299 changes: 148 additions & 151 deletions stackslib/src/core/mempool.rs
View file
Open in desktop

Large diffs are not rendered by default.

58 changes: 36 additions & 22 deletions stackslib/src/core/tests/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,7 @@ fn mempool_walk_over_fork() {
&mut chainstate,
&block.0,
&block.1,
true,
txid,
tx_bytes,
tx_fee,
Expand Down Expand Up @@ -275,7 +276,6 @@ fn mempool_walk_over_fork() {
.iterate_candidates::<_, ChainstateError, _>(
clarity_conn,
&mut tx_events,
2,
mempool_settings.clone(),
|_, available_tx, _| {
count_txs += 1;
Expand Down Expand Up @@ -314,7 +314,6 @@ fn mempool_walk_over_fork() {
.iterate_candidates::<_, ChainstateError, _>(
clarity_conn,
&mut tx_events,
2,
mempool_settings.clone(),
|_, available_tx, _| {
count_txs += 1;
Expand Down Expand Up @@ -352,7 +351,6 @@ fn mempool_walk_over_fork() {
.iterate_candidates::<_, ChainstateError, _>(
clarity_conn,
&mut tx_events,
3,
mempool_settings.clone(),
|_, available_tx, _| {
count_txs += 1;
Expand Down Expand Up @@ -395,7 +393,6 @@ fn mempool_walk_over_fork() {
.iterate_candidates::<_, ChainstateError, _>(
clarity_conn,
&mut tx_events,
2,
mempool_settings.clone(),
|_, available_tx, _| {
count_txs += 1;
Expand Down Expand Up @@ -436,7 +433,6 @@ fn mempool_walk_over_fork() {
.iterate_candidates::<_, ChainstateError, _>(
clarity_conn,
&mut tx_events,
3,
mempool_settings.clone(),
|_, available_tx, _| {
count_txs += 1;
Expand Down Expand Up @@ -498,6 +494,7 @@ fn mempool_walk_over_fork() {
&mut chainstate,
&block.0,
&block.1,
true,
txid,
tx_bytes,
tx_fee,
Expand Down Expand Up @@ -551,6 +548,7 @@ fn mempool_walk_over_fork() {
&mut chainstate,
&block.0,
&block.1,
true,
txid,
tx_bytes,
tx_fee,
Expand Down Expand Up @@ -630,6 +628,7 @@ fn test_iterate_candidates_consider_no_estimate_tx_prob() {
&mut chainstate,
&b_1.0,
&b_1.1,
true,
txid,
tx_bytes,
tx_fee,
Expand Down Expand Up @@ -672,7 +671,6 @@ fn test_iterate_candidates_consider_no_estimate_tx_prob() {
.iterate_candidates::<_, ChainstateError, _>(
clarity_conn,
&mut tx_events,
2,
mempool_settings.clone(),
|_, available_tx, _| {
count_txs += 1;
Expand Down Expand Up @@ -710,7 +708,6 @@ fn test_iterate_candidates_consider_no_estimate_tx_prob() {
.iterate_candidates::<_, ChainstateError, _>(
clarity_conn,
&mut tx_events,
2,
mempool_settings.clone(),
|_, available_tx, _| {
count_txs += 1;
Expand Down Expand Up @@ -748,7 +745,6 @@ fn test_iterate_candidates_consider_no_estimate_tx_prob() {
.iterate_candidates::<_, ChainstateError, _>(
clarity_conn,
&mut tx_events,
2,
mempool_settings.clone(),
|_, available_tx, _| {
count_txs += 1;
Expand Down Expand Up @@ -826,6 +822,7 @@ fn test_iterate_candidates_skipped_transaction() {
&mut chainstate,
&b_1.0,
&b_1.1,
true,
txid,
tx_bytes,
tx_fee,
Expand All @@ -850,7 +847,6 @@ fn test_iterate_candidates_skipped_transaction() {
.iterate_candidates::<_, ChainstateError, _>(
clarity_conn,
&mut tx_events,
2,
mempool_settings.clone(),
|_, available_tx, _| {
count_txs += 1;
Expand Down Expand Up @@ -939,6 +935,7 @@ fn test_iterate_candidates_processing_error_transaction() {
&mut chainstate,
&b_1.0,
&b_1.1,
true,
txid,
tx_bytes,
tx_fee,
Expand All @@ -963,7 +960,6 @@ fn test_iterate_candidates_processing_error_transaction() {
.iterate_candidates::<_, ChainstateError, _>(
clarity_conn,
&mut tx_events,
2,
mempool_settings.clone(),
|_, available_tx, _| {
count_txs += 1;
Expand Down Expand Up @@ -1054,6 +1050,7 @@ fn test_iterate_candidates_problematic_transaction() {
&mut chainstate,
&b_1.0,
&b_1.1,
true,
txid,
tx_bytes,
tx_fee,
Expand All @@ -1078,7 +1075,6 @@ fn test_iterate_candidates_problematic_transaction() {
.iterate_candidates::<_, ChainstateError, _>(
clarity_conn,
&mut tx_events,
2,
mempool_settings.clone(),
|_, available_tx, _| {
count_txs += 1;
Expand Down Expand Up @@ -1183,6 +1179,7 @@ fn test_iterate_candidates_concurrent_write_lock() {
&mut chainstate,
&b_1.0,
&b_1.1,
true,
txid,
tx_bytes,
tx_fee,
Expand Down Expand Up @@ -1239,7 +1236,6 @@ fn test_iterate_candidates_concurrent_write_lock() {
.iterate_candidates::<_, ChainstateError, _>(
clarity_conn,
&mut tx_events,
2,
mempool_settings.clone(),
|_, available_tx, _| {
count_txs += 1;
Expand Down Expand Up @@ -1342,6 +1338,7 @@ fn mempool_do_not_replace_tx() {
&mut chainstate,
&b_1.0,
&b_1.1,
true,
txid,
tx_bytes,
tx_fee,
Expand Down Expand Up @@ -1370,6 +1367,7 @@ fn mempool_do_not_replace_tx() {
&mut chainstate,
&b_2.0,
&b_2.1,
true,
txid,
tx_bytes,
tx_fee,
Expand Down Expand Up @@ -1446,6 +1444,7 @@ fn mempool_db_load_store_replace_tx(#[case] behavior: MempoolCollectionBehavior)
&mut chainstate,
&ConsensusHash([0x1; 20]),
&BlockHeaderHash([0x2; 32]),
false, // don't resolve the above chain tip since it doesn't exist
txid,
tx_bytes,
tx_fee,
Expand All @@ -1471,12 +1470,15 @@ fn mempool_db_load_store_replace_tx(#[case] behavior: MempoolCollectionBehavior)
assert_eq!(tx_info.metadata.origin_nonce, origin_nonce);
assert_eq!(tx_info.metadata.sponsor_address, sponsor_address);
assert_eq!(tx_info.metadata.sponsor_nonce, sponsor_nonce);
assert_eq!(tx_info.metadata.consensus_hash, ConsensusHash([0x1; 20]));
assert_eq!(
tx_info.metadata.block_header_hash,
tx_info.metadata.tenure_consensus_hash,
ConsensusHash([0x1; 20])
);
assert_eq!(
tx_info.metadata.tenure_block_header_hash,
BlockHeaderHash([0x2; 32])
);
assert_eq!(tx_info.metadata.block_height, height);
assert_eq!(tx_info.metadata.coinbase_height, height);

// test replace-by-fee with a higher fee
let old_txid = txid;
Expand All @@ -1503,6 +1505,7 @@ fn mempool_db_load_store_replace_tx(#[case] behavior: MempoolCollectionBehavior)
&mut chainstate,
&ConsensusHash([0x1; 20]),
&BlockHeaderHash([0x2; 32]),
false, // don't resolve the above chain tip since it doesn't exist
txid,
tx_bytes,
tx_fee,
Expand Down Expand Up @@ -1539,12 +1542,15 @@ fn mempool_db_load_store_replace_tx(#[case] behavior: MempoolCollectionBehavior)
assert_eq!(tx_info.metadata.origin_nonce, origin_nonce);
assert_eq!(tx_info.metadata.sponsor_address, sponsor_address);
assert_eq!(tx_info.metadata.sponsor_nonce, sponsor_nonce);
assert_eq!(tx_info.metadata.consensus_hash, ConsensusHash([0x1; 20]));
assert_eq!(
tx_info.metadata.block_header_hash,
tx_info.metadata.tenure_consensus_hash,
ConsensusHash([0x1; 20])
);
assert_eq!(
tx_info.metadata.tenure_block_header_hash,
BlockHeaderHash([0x2; 32])
);
assert_eq!(tx_info.metadata.block_height, height);
assert_eq!(tx_info.metadata.coinbase_height, height);

// test replace-by-fee with a lower fee
let old_txid = txid;
Expand All @@ -1563,6 +1569,7 @@ fn mempool_db_load_store_replace_tx(#[case] behavior: MempoolCollectionBehavior)
&mut chainstate,
&ConsensusHash([0x1; 20]),
&BlockHeaderHash([0x2; 32]),
false, // don't resolve the above chain tip since it doesn't exist
txid,
tx_bytes,
tx_fee,
Expand Down Expand Up @@ -1622,7 +1629,7 @@ fn mempool_db_load_store_replace_tx(#[case] behavior: MempoolCollectionBehavior)
let mut mempool_tx = mempool.tx_begin().unwrap();
match behavior {
MempoolCollectionBehavior::ByStacksHeight => {
MemPoolDB::garbage_collect_by_height(&mut mempool_tx, 101, None)
MemPoolDB::garbage_collect_by_coinbase_height(&mut mempool_tx, 101, None)
}
MempoolCollectionBehavior::ByReceiveTime => {
let test_max_age = Duration::from_secs(1);
Expand Down Expand Up @@ -1712,6 +1719,7 @@ fn mempool_db_test_rbf() {
&mut chainstate,
&ConsensusHash([0x1; 20]),
&BlockHeaderHash([0x2; 32]),
false, // don't resolve the above chain tip since it doesn't exist
txid,
tx_bytes,
tx_fee,
Expand Down Expand Up @@ -1761,6 +1769,7 @@ fn mempool_db_test_rbf() {
&mut chainstate,
&ConsensusHash([0x1; 20]),
&BlockHeaderHash([0x2; 32]),
false, // don't resolve the above chain tip since it doesn't exist
txid,
tx_bytes,
tx_fee,
Expand Down Expand Up @@ -1843,6 +1852,7 @@ fn test_add_txs_bloom_filter() {
&mut chainstate,
&ConsensusHash([0x1 + (block_height as u8); 20]),
&BlockHeaderHash([0x2 + (block_height as u8); 32]),
false, // don't resolve the above chain tip since it doesn't exist
txid,
tx_bytes,
tx_fee,
Expand Down Expand Up @@ -1953,6 +1963,7 @@ fn test_txtags() {
&mut chainstate,
&ConsensusHash([0x1 + (block_height as u8); 20]),
&BlockHeaderHash([0x2 + (block_height as u8); 32]),
false, // don't resolve the above chain tip since it doesn't exist
txid,
tx_bytes,
tx_fee,
Expand Down Expand Up @@ -2046,6 +2057,7 @@ fn test_make_mempool_sync_data() {
&mut chainstate,
&ConsensusHash([0x1 + (block_height as u8); 20]),
&BlockHeaderHash([0x2 + (block_height as u8); 32]),
false, // don't resolve the above chain tip since it doesn't exist
txid.clone(),
tx_bytes,
tx_fee,
Expand Down Expand Up @@ -2084,7 +2096,7 @@ fn test_make_mempool_sync_data() {
let recent_txids = mempool.get_bloom_txids().unwrap();
assert!(recent_txids.len() <= MAX_BLOOM_COUNTER_TXS as usize);

let max_height = MemPoolDB::get_max_height(mempool.conn())
let max_height = MemPoolDB::get_max_coinbase_height(mempool.conn())
.unwrap()
.unwrap_or(0);
eprintln!(
Expand Down Expand Up @@ -2223,6 +2235,7 @@ fn test_find_next_missing_transactions() {
&mut chainstate,
&ConsensusHash([0x1 + (block_height as u8); 20]),
&BlockHeaderHash([0x2 + (block_height as u8); 32]),
false, // don't resolve the above chain tip since it doesn't exist
txid.clone(),
tx_bytes,
tx_fee,
Expand Down Expand Up @@ -2492,6 +2505,7 @@ fn test_drop_and_blacklist_txs_by_time() {
&mut chainstate,
&ConsensusHash([0x1 + (block_height as u8); 20]),
&BlockHeaderHash([0x2 + (block_height as u8); 32]),
false, // don't resolve the above chain tip since it doesn't exist
txid.clone(),
tx_bytes,
tx_fee,
Expand Down Expand Up @@ -2611,6 +2625,7 @@ fn test_drop_and_blacklist_txs_by_size() {
&mut chainstate,
&ConsensusHash([0x1 + (block_height as u8); 20]),
&BlockHeaderHash([0x2 + (block_height as u8); 32]),
false, // don't resolve the above chain tip since it doesn't exist
txid.clone(),
tx_bytes,
tx_fee,
Expand Down Expand Up @@ -2728,6 +2743,7 @@ fn test_filter_txs_by_type() {
&mut chainstate,
&b_2.0,
&b_2.1,
true,
txid.clone(),
tx_bytes,
tx_fee,
Expand Down Expand Up @@ -2763,7 +2779,6 @@ fn test_filter_txs_by_type() {
.iterate_candidates::<_, ChainstateError, _>(
clarity_conn,
&mut tx_events,
2,
mempool_settings.clone(),
|_, available_tx, _| {
count_txs += 1;
Expand Down Expand Up @@ -2799,7 +2814,6 @@ fn test_filter_txs_by_type() {
.iterate_candidates::<_, ChainstateError, _>(
clarity_conn,
&mut tx_events,
2,
mempool_settings.clone(),
|_, available_tx, _| {
count_txs += 1;
Expand Down
Loading