diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp index 0ee3790867d..8c34636f441 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp @@ -46,6 +46,9 @@ template class ContentAddressedAppendOn using GetLeafCallback = std::function&)>; using CommitCallback = std::function; using RollbackCallback = std::function; + using RemoveHistoricBlockCallback = std::function; + using UnwindBlockCallback = std::function; + using FinaliseBlockCallback = std::function; // Only construct from provided store and thread pool, no copies or moves ContentAddressedAppendOnlyTree(std::unique_ptr store, @@ -167,7 +170,7 @@ template class ContentAddressedAppendOn * @brief Returns the index of the provided leaf in the tree only if it exists after the index value provided */ void find_leaf_index_from(const fr& leaf, - index_t start_index, + const index_t& start_index, bool includeUncommitted, const FindLeafCallback& on_completion) const; @@ -175,7 +178,7 @@ template class ContentAddressedAppendOn * @brief Returns the index of the provided leaf in the tree only if it exists after the index value provided */ void find_leaf_index_from(const fr& leaf, - index_t start_index, + const index_t& start_index, const index_t& blockNumber, bool includeUncommitted, const FindLeafCallback& on_completion) const; @@ -195,6 +198,12 @@ template class ContentAddressedAppendOn */ uint32_t depth() const { return depth_; } + void remove_historic_block(const index_t& blockNumber, const RemoveHistoricBlockCallback& on_completion); + + void unwind_block(const index_t& blockNumber, const UnwindBlockCallback& on_completion); + + void finalise_block(const index_t& blockNumber, const FinaliseBlockCallback& on_completion); + protected: using ReadTransaction = typename Store::ReadTransaction; using ReadTransactionPtr = typename Store::ReadTransactionPtr; @@ -355,6 +364,9 @@ void ContentAddressedAppendOnlyTree::get_sibling_path(cons auto job = [=, this]() { execute_and_report( [=, this](TypedResponse& response) { + if (blockNumber == 0) { + throw std::runtime_error("Invalid block number"); + } ReadTransactionPtr tx = store_->create_read_transaction(); BlockPayload blockData; if (!store_->get_block_data(blockNumber, blockData, *tx)) { @@ -409,6 +421,7 @@ void ContentAddressedAppendOnlyTree::get_subtree_sibling_p RequestContext requestContext; requestContext.includeUncommitted = includeUncommitted; requestContext.root = store_->get_current_root(*tx, includeUncommitted); + // std::cout << "Current root: " << requestContext.root << std::endl; OptionalSiblingPath optional_path = get_subtree_sibling_path_internal(leaf_index, subtree_depth, requestContext, *tx); response.inner.path = optional_sibling_path_to_full_sibling_path(optional_path); @@ -467,7 +480,7 @@ std::optional ContentAddressedAppendOnlyTree::find_lea // std::cout << "No child" << std::endl; return std::nullopt; } - // std::cout << "Found child" << std::endl; + // std::cout << "Found child " << child.value() << std::endl; hash = child.value(); @@ -572,6 +585,9 @@ void ContentAddressedAppendOnlyTree::get_leaf(const index_ auto job = [=, this]() { execute_and_report( [=, this](TypedResponse& response) { + if (blockNumber == 0) { + throw std::runtime_error("Invalid block number"); + } ReadTransactionPtr tx = store_->create_read_transaction(); BlockPayload blockData; if (!store_->get_block_data(blockNumber, blockData, *tx)) { @@ -616,12 +632,12 @@ void ContentAddressedAppendOnlyTree::find_leaf_index(const template void ContentAddressedAppendOnlyTree::find_leaf_index_from( - const fr& leaf, index_t start_index, bool includeUncommitted, const FindLeafCallback& on_completion) const + const fr& leaf, const index_t& start_index, bool includeUncommitted, const FindLeafCallback& on_completion) const { auto job = [=, this]() -> void { execute_and_report( [=, this](TypedResponse& response) { - typename Store::ReadTransactionPtr tx = store_->create_read_transaction(); + ReadTransactionPtr tx = store_->create_read_transaction(); RequestContext requestContext; requestContext.includeUncommitted = includeUncommitted; requestContext.root = store_->get_current_root(*tx, includeUncommitted); @@ -640,7 +656,7 @@ void ContentAddressedAppendOnlyTree::find_leaf_index_from( template void ContentAddressedAppendOnlyTree::find_leaf_index_from( const fr& leaf, - index_t start_index, + const index_t& start_index, const index_t& blockNumber, bool includeUncommitted, const FindLeafCallback& on_completion) const @@ -648,7 +664,10 @@ void ContentAddressedAppendOnlyTree::find_leaf_index_from( auto job = [=, this]() -> void { execute_and_report( [=, this](TypedResponse& response) { - typename Store::ReadTransactionPtr tx = store_->create_read_transaction(); + if (blockNumber == 0) { + throw std::runtime_error("Invalid block number"); + } + ReadTransactionPtr tx = store_->create_read_transaction(); BlockPayload blockData; if (!store_->get_block_data(blockNumber, blockData, *tx)) { throw std::runtime_error("Data for block unavailable"); @@ -712,6 +731,57 @@ void ContentAddressedAppendOnlyTree::rollback(const Rollba workers_->enqueue(job); } +template +void ContentAddressedAppendOnlyTree::remove_historic_block( + const index_t& blockNumber, const RemoveHistoricBlockCallback& on_completion) +{ + auto job = [=, this]() { + execute_and_report( + [=, this]() { + if (blockNumber == 0) { + throw std::runtime_error("Invalid block number"); + } + store_->remove_historical_block(blockNumber); + }, + on_completion); + }; + workers_->enqueue(job); +} + +template +void ContentAddressedAppendOnlyTree::unwind_block( + const index_t& blockNumber, const RemoveHistoricBlockCallback& on_completion) +{ + auto job = [=, this]() { + execute_and_report( + [=, this]() { + if (blockNumber == 0) { + throw std::runtime_error("Invalid block number"); + } + store_->unwind_block(blockNumber); + }, + on_completion); + }; + workers_->enqueue(job); +} + +template +void ContentAddressedAppendOnlyTree::finalise_block(const index_t& blockNumber, + const FinaliseBlockCallback& on_completion) +{ + auto job = [=, this]() { + execute_and_report( + [=, this]() { + if (blockNumber == 0) { + throw std::runtime_error("Invalid block number"); + } + store_->advance_finalised_block(blockNumber); + }, + on_completion); + }; + workers_->enqueue(job); +} + template index_t ContentAddressedAppendOnlyTree::get_batch_insertion_size(index_t treeSize, index_t remainingAppendSize) @@ -738,7 +808,7 @@ void ContentAddressedAppendOnlyTree::add_values_internal(s index_t& new_size, bool update_index) { - typename Store::ReadTransactionPtr tx = store_->create_read_transaction(); + ReadTransactionPtr tx = store_->create_read_transaction(); TreeMeta meta; store_->get_meta(meta, *tx, true); index_t sizeToAppend = values->size(); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp index 2b276d1b503..b15c3c4df61 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp @@ -1,6 +1,7 @@ #include "barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" #include "../fixtures.hpp" #include "../memory_tree.hpp" +#include "../test_fixtures.hpp" #include "barretenberg/common/streams.hpp" #include "barretenberg/common/test.hpp" #include "barretenberg/common/thread_pool.hpp" @@ -13,12 +14,16 @@ #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/crypto/merkle_tree/signal.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" -#include "gtest/gtest.h" +#include "barretenberg/ecc/curves/bn254/fr.hpp" +#include "barretenberg/relations/relation_parameters.hpp" +#include +#include #include #include #include #include #include +#include #include using namespace bb; @@ -61,7 +66,19 @@ void check_size(TreeType& tree, index_t expected_size, bool includeUncommitted = signal.wait_for_level(); } -void check_unfinalised_block_height(TreeType& tree, index_t expected_block_height) +void check_finalised_block_height(TreeType& tree, index_t expected_finalised_block) +{ + Signal signal; + auto completion = [&](const TypedResponse& response) -> void { + EXPECT_EQ(response.success, true); + EXPECT_EQ(response.inner.meta.finalisedBlockHeight, expected_finalised_block); + signal.signal_level(); + }; + tree.get_meta_data(false, completion); + signal.wait_for_level(); +} + +void check_block_height(TreeType& tree, index_t expected_block_height) { Signal signal; auto completion = [&](const TypedResponse& response) -> void { @@ -106,12 +123,15 @@ void check_sibling_path(TreeType& tree, void check_historic_sibling_path(TreeType& tree, index_t index, fr_sibling_path expected_sibling_path, - index_t blockNumber) + index_t blockNumber, + bool expected_success = true) { Signal signal; auto completion = [&](const TypedResponse& response) -> void { - EXPECT_EQ(response.success, true); - EXPECT_EQ(response.inner.path, expected_sibling_path); + EXPECT_EQ(response.success, expected_success); + if (response.success) { + EXPECT_EQ(response.inner.path, expected_sibling_path); + } signal.signal_level(); }; tree.get_sibling_path(index, blockNumber, completion, false); @@ -140,6 +160,28 @@ void rollback_tree(TreeType& tree) signal.wait_for_level(); } +void remove_historic_block(TreeType& tree, const index_t& blockNumber, bool expected_success = true) +{ + Signal signal; + auto completion = [&](const Response& response) -> void { + EXPECT_EQ(response.success, expected_success); + signal.signal_level(); + }; + tree.remove_historic_block(blockNumber, completion); + signal.wait_for_level(); +} + +void unwind_block(TreeType& tree, const index_t& blockNumber, bool expected_success = true) +{ + Signal signal; + auto completion = [&](const Response& response) -> void { + EXPECT_EQ(response.success, expected_success); + signal.signal_level(); + }; + tree.unwind_block(blockNumber, completion); + signal.wait_for_level(); +} + void add_value(TreeType& tree, const fr& value) { Signal signal; @@ -164,6 +206,20 @@ void add_values(TreeType& tree, const std::vector& values) signal.wait_for_level(); } +void finalise_block(TreeType& tree, const index_t& blockNumber, bool expected_success = true) +{ + Signal signal; + auto completion = [&](const Response& response) -> void { + EXPECT_EQ(response.success, expected_success); + if (!response.success && expected_success) { + std::cout << response.message << std::endl; + } + signal.signal_level(); + }; + tree.finalise_block(blockNumber, completion); + signal.wait_for_level(); +} + void check_find_leaf_index( TreeType& tree, const fr& leaf, index_t expected_index, bool expected_success, bool includeUncommitted = true) { @@ -275,13 +331,15 @@ void check_historic_leaf(TreeType& tree, void check_sibling_path(fr expected_root, fr node, index_t index, fr_sibling_path sibling_path) { - fr left, right, hash = node; - for (size_t i = 0; i < sibling_path.size(); ++i) { + fr left; + fr right; + fr hash = node; + for (const auto& i : sibling_path) { if (index % 2 == 0) { left = hash; - right = sibling_path[i]; + right = i; } else { - left = sibling_path[i]; + left = i; right = hash; } @@ -308,6 +366,45 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_create) check_root(tree, memdb.root()); } +TEST_F(PersistedContentAddressedAppendOnlyTreeTest, committing_with_no_changes_should_succeed) +{ + constexpr size_t depth = 10; + std::string name = random_string(); + LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); + EXPECT_NO_THROW(Store store(name, depth, db)); + std::unique_ptr store = std::make_unique(name, depth, db); + + ThreadPoolPtr pool = make_thread_pool(1); + TreeType tree(std::move(store), pool); + MemoryTree memdb(depth); + + add_value(tree, VALUES[0]); + memdb.update_element(0, VALUES[0]); + + commit_tree(tree, true); + check_root(tree, memdb.root()); + check_size(tree, 1, false); + commit_tree(tree, true); + check_root(tree, memdb.root()); + check_size(tree, 1, false); + // rollbacks should do nothing + rollback_tree(tree); + check_root(tree, memdb.root()); + check_size(tree, 1, false); + add_value(tree, VALUES[1]); + + // committed should be the same + check_root(tree, memdb.root(), false); + check_size(tree, 1, false); + + // rollback + rollback_tree(tree); + // commit should do nothing + commit_tree(tree, true); + check_root(tree, memdb.root()); + check_size(tree, 1, false); +} + TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_only_recreate_with_same_name_and_depth) { constexpr size_t depth = 10; @@ -372,13 +469,11 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, errors_are_caught_and_handle { // We use a deep tree with a small amount of storage (100 * 1024) bytes constexpr size_t depth = 16; - constexpr uint32_t numDbs = 4; std::string name = random_string(); std::string directory = random_temp_directory(); std::filesystem::create_directories(directory); { - auto environment = std::make_unique(directory, 50, numDbs, 2); LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, 50, _maxReaders); std::unique_ptr store = std::make_unique(name, depth, db); @@ -438,7 +533,6 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, errors_are_caught_and_handle } { - auto environment = std::make_unique(directory, 500, numDbs, 2); LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, 500, _maxReaders); std::unique_ptr store = std::make_unique(name, depth, db); @@ -507,6 +601,8 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_commit_and_restore) // commit the changes commit_tree(tree); + + check_block_and_root_data(db, 1, memdb.root(), true); // now committed and uncommitted should be the same // check uncommitted state @@ -533,6 +629,8 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_commit_and_restore) check_root(tree, memdb.root()); check_sibling_path(tree, 0, memdb.get_sibling_path(0)); + check_block_and_root_data(db, 1, memdb.root(), true); + // check committed state check_size(tree, 1, false); check_root(tree, memdb.root(), false); @@ -710,7 +808,7 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_commit_multiple_blocks) auto check = [&](index_t expected_size, index_t expected_unfinalised_block_height) { check_size(tree, expected_size); - check_unfinalised_block_height(tree, expected_unfinalised_block_height); + check_block_height(tree, expected_unfinalised_block_height); check_root(tree, memdb.root()); check_sibling_path(tree, 0, memdb.get_sibling_path(0)); check_sibling_path(tree, expected_size - 1, memdb.get_sibling_path(expected_size - 1)); @@ -731,6 +829,7 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_commit_multiple_blocks) check(expected_size, i); commit_tree(tree); check(expected_size, i + 1); + check_block_and_root_data(db, 1 + i, memdb.root(), true); } } @@ -746,7 +845,7 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_add_varying_size_blocks) auto check = [&](index_t expected_size, index_t expected_unfinalised_block_height) { check_size(tree, expected_size); - check_unfinalised_block_height(tree, expected_unfinalised_block_height); + check_block_height(tree, expected_unfinalised_block_height); check_root(tree, memdb.root()); check_sibling_path(tree, 0, memdb.get_sibling_path(0)); check_sibling_path(tree, expected_size - 1, memdb.get_sibling_path(expected_size - 1)); @@ -768,6 +867,7 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_add_varying_size_blocks) check(expected_size, i); commit_tree(tree); check(expected_size, i + 1); + check_block_and_root_data(db, 1 + i, memdb.root(), true); } } @@ -789,7 +889,7 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_retrieve_historic_siblin auto check = [&](index_t expected_size, index_t expected_unfinalised_block_height) { check_size(tree, expected_size); - check_unfinalised_block_height(tree, expected_unfinalised_block_height); + check_block_height(tree, expected_unfinalised_block_height); check_root(tree, memdb.root()); check_sibling_path(tree, 0, memdb.get_sibling_path(0)); check_sibling_path(tree, expected_size - 1, memdb.get_sibling_path(expected_size - 1)); @@ -1043,6 +1143,8 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_create_images_at_histori commit_tree(tree1); + check_block_and_root_data(db, 1, memdb.root(), true); + fr_sibling_path block1SiblingPathIndex3 = memdb.get_sibling_path(3); values = { 15, 18, 26, 2 }; @@ -1053,6 +1155,8 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_create_images_at_histori commit_tree(tree1); + check_block_and_root_data(db, 2, memdb.root(), true); + fr block2Root = memdb.root(); fr_sibling_path block2SiblingPathIndex7 = memdb.get_sibling_path(7); @@ -1066,6 +1170,8 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_create_images_at_histori commit_tree(tree1); + check_block_and_root_data(db, 3, memdb.root(), true); + fr_sibling_path block3SiblingPathIndex11 = memdb.get_sibling_path(11); fr_sibling_path block3SiblingPathIndex7 = memdb.get_sibling_path(7); fr_sibling_path block3SiblingPathIndex3 = memdb.get_sibling_path(3); @@ -1104,8 +1210,579 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_create_images_at_histori check_find_historic_leaf_index_from(treeAtBlock2, 1, 18, 3, 0, false, false); check_find_historic_leaf_index_from(treeAtBlock2, 1, 20, 0, 2, true, false); - check_unfinalised_block_height(treeAtBlock2, 2); + check_block_height(treeAtBlock2, 2); // It should be impossible to commit using the image commit_tree(treeAtBlock2, false); } + +TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_remove_historic_block_data) +{ + constexpr size_t depth = 10; + std::string name = random_string(); + LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); + std::unique_ptr store = std::make_unique(name, depth, db); + ThreadPoolPtr pool = make_thread_pool(1); + TreeType tree(std::move(store), pool); + MemoryTree memdb(depth); + + constexpr uint32_t numBlocks = 50; + constexpr uint32_t batchSize = 16; + constexpr uint32_t windowSize = 8; + + std::vector historicPathsZeroIndex; + std::vector historicPathsMaxIndex; + std::vector roots; + + auto check = [&](index_t expectedSize, index_t expectedBlockHeight) { + check_size(tree, expectedSize); + check_block_height(tree, expectedBlockHeight); + check_root(tree, memdb.root()); + check_sibling_path(tree, 0, memdb.get_sibling_path(0)); + check_sibling_path(tree, expectedSize - 1, memdb.get_sibling_path(expectedSize - 1)); + + for (uint32_t i = 0; i < historicPathsZeroIndex.size(); i++) { + // retrieving historic data should fail if the block is outside of the window + const index_t blockNumber = i + 1; + const bool expectedSuccess = + expectedBlockHeight <= windowSize || blockNumber > (expectedBlockHeight - windowSize); + check_historic_sibling_path(tree, 0, historicPathsZeroIndex[i], blockNumber, expectedSuccess); + index_t maxSizeAtBlock = ((i + 1) * batchSize) - 1; + check_historic_sibling_path(tree, maxSizeAtBlock, historicPathsMaxIndex[i], blockNumber, expectedSuccess); + + const index_t leafIndex = 6; + check_historic_leaf(tree, blockNumber, VALUES[leafIndex], leafIndex, expectedSuccess); + check_find_historic_leaf_index(tree, blockNumber, VALUES[leafIndex], leafIndex, expectedSuccess); + check_find_historic_leaf_index_from(tree, blockNumber, VALUES[leafIndex], 0, leafIndex, expectedSuccess); + } + }; + + for (uint32_t i = 0; i < numBlocks; i++) { + std::vector to_add; + + for (size_t j = 0; j < batchSize; ++j) { + size_t ind = i * batchSize + j; + memdb.update_element(ind, VALUES[ind]); + to_add.push_back(VALUES[ind]); + } + index_t expected_size = (i + 1) * batchSize; + add_values(tree, to_add); + check(expected_size, i); + commit_tree(tree); + + // immediately finalise the block + finalise_block(tree, i + 1); + + historicPathsZeroIndex.push_back(memdb.get_sibling_path(0)); + historicPathsMaxIndex.push_back(memdb.get_sibling_path(expected_size - 1)); + roots.push_back(memdb.root()); + + // Now remove the oldest block if outside of the window + if (i >= windowSize) { + const index_t oldestBlock = (i + 1) - windowSize; + // trying to remove a block that is not the most historic should fail + remove_historic_block(tree, oldestBlock + 1, false); + + fr rootToRemove = roots[oldestBlock - 1]; + check_block_and_root_data(db, oldestBlock, rootToRemove, true); + + // removing the most historic should succeed + remove_historic_block(tree, oldestBlock, true); + + // the block data should have been removed + check_block_and_root_data(db, oldestBlock, rootToRemove, false); + } + check(expected_size, i + 1); + } + + // Attempting to remove block 0 should fail as there isn't one + remove_historic_block(tree, 0, false); +} + +void test_unwind(std::string directory, + std::string name, + uint64_t mapSize, + uint64_t maxReaders, + uint32_t depth, + uint32_t blockSize, + uint32_t numBlocks, + uint32_t numBlocksToUnwind, + std::vector values) +{ + LMDBTreeStore::SharedPtr db = std::make_shared(directory, name, mapSize, maxReaders); + std::unique_ptr store = std::make_unique(name, depth, db); + ThreadPoolPtr pool = make_thread_pool(1); + TreeType tree(std::move(store), pool); + MemoryTree memdb(depth); + + uint32_t batchSize = blockSize; + + std::vector historicPathsZeroIndex; + std::vector historicPathsMaxIndex; + std::vector roots; + + fr initialRoot = memdb.root(); + fr_sibling_path initialPath = memdb.get_sibling_path(0); + + for (uint32_t i = 0; i < numBlocks; i++) { + std::vector to_add; + + for (size_t j = 0; j < batchSize; ++j) { + size_t ind = i * batchSize + j; + memdb.update_element(ind, values[ind]); + to_add.push_back(values[ind]); + } + index_t expected_size = (i + 1) * batchSize; + add_values(tree, to_add); + + // attempting an unwind of the block being built should fail + unwind_block(tree, i + 1, false); + + if (i > 0) { + // attemnpting an unwind of the most recent committed block should fail as we have uncommitted changes + unwind_block(tree, i, false); + } + + commit_tree(tree); + + historicPathsZeroIndex.push_back(memdb.get_sibling_path(0)); + historicPathsMaxIndex.push_back(memdb.get_sibling_path(expected_size - 1)); + roots.push_back(memdb.root()); + check_root(tree, memdb.root()); + check_sibling_path(tree, 0, memdb.get_sibling_path(0)); + check_sibling_path(tree, expected_size - 1, memdb.get_sibling_path(expected_size - 1)); + check_size(tree, expected_size); + check_block_and_size_data(db, i + 1, expected_size, true); + check_block_and_root_data(db, i + 1, memdb.root(), true); + } + + const uint32_t blocksToRemove = numBlocksToUnwind; + for (uint32_t i = 0; i < blocksToRemove; i++) { + const index_t blockNumber = numBlocks - i; + + check_block_and_root_data(db, blockNumber, roots[blockNumber - 1], true); + // attempting to unwind a block that is not the tip should fail + unwind_block(tree, blockNumber + 1, false); + unwind_block(tree, blockNumber); + check_block_and_root_data(db, blockNumber, roots[blockNumber - 1], false); + + const index_t previousValidBlock = blockNumber - 1; + index_t deletedBlockStartIndex = previousValidBlock * batchSize; + + check_block_height(tree, previousValidBlock); + check_size(tree, deletedBlockStartIndex); + check_root(tree, previousValidBlock == 0 ? initialRoot : roots[previousValidBlock - 1]); + + // The zero index sibling path should be as it was at the previous block + check_sibling_path(tree, + 0, + previousValidBlock == 0 ? initialPath : historicPathsZeroIndex[previousValidBlock - 1], + false, + true); + + // Trying to find leaves appended in the block that was removed should fail + check_leaf(tree, values[1 + deletedBlockStartIndex], 1 + deletedBlockStartIndex, false); + check_find_leaf_index(tree, values[1 + deletedBlockStartIndex], 1 + deletedBlockStartIndex, false); + + for (index_t j = 0; j < numBlocks; j++) { + index_t historicBlockNumber = j + 1; + bool expectedSuccess = historicBlockNumber <= previousValidBlock; + check_historic_sibling_path(tree, 0, historicPathsZeroIndex[j], historicBlockNumber, expectedSuccess); + index_t maxSizeAtBlock = ((j + 1) * batchSize) - 1; + check_historic_sibling_path( + tree, maxSizeAtBlock, historicPathsMaxIndex[j], historicBlockNumber, expectedSuccess); + + const index_t leafIndex = 1; + check_historic_leaf(tree, historicBlockNumber, values[leafIndex], leafIndex, expectedSuccess); + check_find_historic_leaf_index(tree, historicBlockNumber, values[leafIndex], leafIndex, expectedSuccess); + check_find_historic_leaf_index_from( + tree, historicBlockNumber, values[leafIndex], 0, leafIndex, expectedSuccess); + } + } +} + +TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_unwind_blocks) +{ + std::vector first = create_values(1024); + test_unwind(_directory, "DB", _mapSize, _maxReaders, 10, 16, 16, 8, first); +} + +TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_unwind_all_blocks) +{ + std::vector first = create_values(1024); + test_unwind(_directory, "DB", _mapSize, _maxReaders, 10, 16, 16, 16, first); + std::vector second = create_values(1024); + test_unwind(_directory, "DB", _mapSize, _maxReaders, 10, 16, 16, 16, second); +} + +TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_unwind_blocks_with_duplicate_leaves) +{ + constexpr size_t depth = 4; + std::string name = random_string(); + LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); + std::unique_ptr store = std::make_unique(name, depth, db); + ThreadPoolPtr pool = make_thread_pool(1); + TreeType tree(std::move(store), pool); + MemoryTree memdb(depth); + + constexpr size_t blockSize = 2; + constexpr size_t numBlocks = 2; + constexpr size_t numBlocksToUnwind = 1; + + std::vector values = create_values(blockSize); + + // Add the same batch of values many times + for (size_t i = 0; i < numBlocks; i++) { + for (size_t j = 0; j < values.size(); j++) { + size_t ind = i * blockSize + j; + memdb.update_element(ind, values[j]); + } + add_values(tree, values); + commit_tree(tree); + check_block_and_root_data(db, i + 1, memdb.root(), true); + + for (size_t j = 0; j < values.size(); j++) { + size_t ind = i * blockSize + j; + // query the indices db directly + check_indices_data(db, values[j], ind, true, true); + } + } + + for (size_t i = 0; i < numBlocks; i++) { + index_t startIndex = i * blockSize; + index_t expectedIndex = startIndex + 1; + + // search for the leaf from start of each batch + check_find_leaf_index_from(tree, values[1], startIndex, expectedIndex, true); + // search for the leaf from start of the next batch + check_find_leaf_index_from(tree, values[1], startIndex + 2, expectedIndex + blockSize, i < (numBlocks - 1)); + } + + const uint32_t blocksToRemove = numBlocksToUnwind; + for (uint32_t i = 0; i < blocksToRemove; i++) { + const index_t blockNumber = numBlocks - i; + unwind_block(tree, blockNumber); + + const index_t previousValidBlock = blockNumber - 1; + index_t deletedBlockStartIndex = previousValidBlock * blockSize; + + check_block_height(tree, previousValidBlock); + check_size(tree, deletedBlockStartIndex); + + for (size_t j = 0; j < numBlocks; j++) { + index_t startIndex = j * blockSize; + index_t expectedIndex = startIndex + 1; + + // search for the leaf from start of each batch + check_find_leaf_index_from(tree, values[1], startIndex, expectedIndex, j < previousValidBlock); + // search for the leaf from start of the next batch + check_find_leaf_index_from( + tree, values[1], startIndex + 2, expectedIndex + blockSize, j < (previousValidBlock - 1)); + + for (size_t k = 0; k < values.size(); k++) { + size_t ind = j * blockSize + k; + // query the indices db directly. If block number == 1 that means the entry should not be present + check_indices_data(db, values[k], ind, blockNumber > 1, ind < deletedBlockStartIndex); + } + } + } +} + +TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_sync_and_unwind_large_blocks) +{ + + constexpr uint32_t numBlocks = 4; + constexpr uint32_t numBlocksToUnwind = 2; + std::vector blockSizes = { 2, 4, 8, 16, 32 }; + for (const uint32_t& size : blockSizes) { + uint32_t actualSize = size * 1024; + std::vector values = create_values(actualSize * numBlocks); + std::stringstream ss; + ss << "DB " << actualSize; + test_unwind(_directory, ss.str(), _mapSize, _maxReaders, 20, actualSize, numBlocks, numBlocksToUnwind, values); + } +} + +TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_advance_finalised_blocks) +{ + std::string name = random_string(); + constexpr uint32_t depth = 10; + LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); + std::unique_ptr store = std::make_unique(name, depth, db); + ThreadPoolPtr pool = make_thread_pool(1); + TreeType tree(std::move(store), pool); + MemoryTree memdb(depth); + + uint32_t blockSize = 16; + uint32_t numBlocks = 16; + uint32_t finalisedBlockDelay = 4; + std::vector values = create_values(blockSize * numBlocks); + + for (uint32_t i = 0; i < numBlocks; i++) { + std::vector to_add; + + for (size_t j = 0; j < blockSize; ++j) { + size_t ind = i * blockSize + j; + memdb.update_element(ind, values[ind]); + to_add.push_back(values[ind]); + } + add_values(tree, to_add); + commit_tree(tree); + + index_t expectedFinalisedBlock = i < finalisedBlockDelay ? 0 : i - finalisedBlockDelay; + check_finalised_block_height(tree, expectedFinalisedBlock); + index_t expectedPresentStart = i < finalisedBlockDelay ? 0 : (expectedFinalisedBlock * blockSize); + index_t expectedPresentEnd = ((i + 1) * blockSize) - 1; + std::vector toTest(values.begin() + static_cast(expectedPresentStart), + values.begin() + static_cast(expectedPresentEnd + 1)); + check_leaf_keys_are_present(db, expectedPresentStart, expectedPresentEnd, toTest); + + if (i >= finalisedBlockDelay) { + + index_t blockToFinalise = expectedFinalisedBlock + 1; + + // attemnpting to finalise a block that doesn't exist should fail + finalise_block(tree, blockToFinalise + numBlocks, false); + + finalise_block(tree, blockToFinalise, true); + + index_t expectedNotPresentEnd = (blockToFinalise * blockSize) - 1; + check_leaf_keys_are_not_present(db, 0, expectedNotPresentEnd); + } + } +} + +TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_finalise_multiple_blocks) +{ + std::string name = random_string(); + constexpr uint32_t depth = 10; + LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); + std::unique_ptr store = std::make_unique(name, depth, db); + ThreadPoolPtr pool = make_thread_pool(1); + TreeType tree(std::move(store), pool); + MemoryTree memdb(depth); + + uint32_t blockSize = 16; + uint32_t numBlocks = 16; + std::vector values = create_values(blockSize * numBlocks); + + for (uint32_t i = 0; i < numBlocks; i++) { + std::vector to_add; + + for (size_t j = 0; j < blockSize; ++j) { + size_t ind = i * blockSize + j; + memdb.update_element(ind, values[ind]); + to_add.push_back(values[ind]); + } + add_values(tree, to_add); + commit_tree(tree); + } + + check_block_height(tree, numBlocks); + + index_t blockToFinalise = 8; + + check_leaf_keys_are_present(db, 0, (numBlocks * blockSize) - 1, values); + + finalise_block(tree, blockToFinalise); + + index_t expectedNotPresentEnd = (blockToFinalise * blockSize) - 1; + check_leaf_keys_are_not_present(db, 0, expectedNotPresentEnd); +} + +TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_not_finalise_block_beyond_pending_chain) +{ + std::string name = random_string(); + constexpr uint32_t depth = 10; + LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); + std::unique_ptr store = std::make_unique(name, depth, db); + ThreadPoolPtr pool = make_thread_pool(1); + TreeType tree(std::move(store), pool); + MemoryTree memdb(depth); + + uint32_t blockSize = 16; + uint32_t numBlocks = 16; + std::vector values = create_values(blockSize * numBlocks); + + // finalising block 1 should fail + finalise_block(tree, 1, false); + + for (uint32_t i = 0; i < numBlocks; i++) { + std::vector to_add; + + for (size_t j = 0; j < blockSize; ++j) { + size_t ind = i * blockSize + j; + memdb.update_element(ind, values[ind]); + to_add.push_back(values[ind]); + } + add_values(tree, to_add); + commit_tree(tree); + } + + check_block_height(tree, numBlocks); + + // should fail + finalise_block(tree, numBlocks + 1, false); + + // finalise the entire chain + index_t blockToFinalise = numBlocks; + + check_leaf_keys_are_present(db, 0, (numBlocks * blockSize) - 1, values); + + finalise_block(tree, blockToFinalise); + + index_t expectedNotPresentEnd = (blockToFinalise * blockSize) - 1; + check_leaf_keys_are_not_present(db, 0, expectedNotPresentEnd); +} + +TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_not_fork_from_unwound_blocks) +{ + std::string name = random_string(); + uint32_t depth = 20; + LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); + std::unique_ptr store = std::make_unique(name, depth, db); + ThreadPoolPtr pool = make_thread_pool(1); + TreeType tree(std::move(store), pool); + + for (uint32_t i = 0; i < 5; i++) { + std::vector values = create_values(1024); + add_values(tree, values); + commit_tree(tree); + } + + unwind_block(tree, 5); + unwind_block(tree, 4); + + EXPECT_THROW(Store(name, depth, 5, db), std::runtime_error); + EXPECT_THROW(Store(name, depth, 4, db), std::runtime_error); +} + +TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_not_fork_from_expired_historical_blocks) +{ + std::string name = random_string(); + uint32_t depth = 20; + LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); + std::unique_ptr store = std::make_unique(name, depth, db); + ThreadPoolPtr pool = make_thread_pool(1); + TreeType tree(std::move(store), pool); + + for (uint32_t i = 0; i < 5; i++) { + std::vector values = create_values(1024); + add_values(tree, values); + commit_tree(tree); + } + finalise_block(tree, 3); + + remove_historic_block(tree, 1); + remove_historic_block(tree, 2); + + EXPECT_THROW(Store(name, depth, 1, db), std::runtime_error); + EXPECT_THROW(Store(name, depth, 2, db), std::runtime_error); +} +TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_fork_from_block_zero_when_not_latest) +{ + std::string name = random_string(); + uint32_t depth = 20; + LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); + std::unique_ptr store = std::make_unique(name, depth, db); + ThreadPoolPtr pool = make_thread_pool(1); + TreeType tree(std::move(store), pool); + MemoryTree memdb(depth); + + uint32_t numBlocks = 5; + + const fr initialRoot = memdb.root(); + const fr_sibling_path path = memdb.get_sibling_path(0); + + for (uint32_t i = 0; i < numBlocks; i++) { + std::vector values = create_values(1024); + add_values(tree, values); + commit_tree(tree); + } + + check_block_height(tree, numBlocks); + + EXPECT_NO_THROW(Store(name, depth, 0, db)); + + std::unique_ptr store2 = std::make_unique(name, depth, 0, db); + TreeType tree2(std::move(store2), pool); + + check_root(tree2, initialRoot, false); + check_sibling_path(tree2, 0, path, false, true); +} + +TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_not_unwind_finalised_block) +{ + std::string name = random_string(); + constexpr uint32_t depth = 10; + LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); + std::unique_ptr store = std::make_unique(name, depth, db); + ThreadPoolPtr pool = make_thread_pool(1); + TreeType tree(std::move(store), pool); + MemoryTree memdb(depth); + + uint32_t blockSize = 16; + uint32_t numBlocks = 16; + std::vector values = create_values(blockSize * numBlocks); + + for (uint32_t i = 0; i < numBlocks; i++) { + std::vector to_add; + + for (size_t j = 0; j < blockSize; ++j) { + size_t ind = i * blockSize + j; + memdb.update_element(ind, values[ind]); + to_add.push_back(values[ind]); + } + add_values(tree, to_add); + commit_tree(tree); + } + + check_block_height(tree, numBlocks); + + index_t blockToFinalise = 8; + + finalise_block(tree, blockToFinalise); + + for (uint32_t i = numBlocks; i > blockToFinalise; i--) { + unwind_block(tree, i); + } + unwind_block(tree, blockToFinalise, false); +} + +TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_not_historically_remove_finalised_block) +{ + std::string name = random_string(); + constexpr uint32_t depth = 10; + LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); + std::unique_ptr store = std::make_unique(name, depth, db); + ThreadPoolPtr pool = make_thread_pool(1); + TreeType tree(std::move(store), pool); + MemoryTree memdb(depth); + + uint32_t blockSize = 16; + uint32_t numBlocks = 16; + std::vector values = create_values(blockSize * numBlocks); + + for (uint32_t i = 0; i < numBlocks; i++) { + std::vector to_add; + + for (size_t j = 0; j < blockSize; ++j) { + size_t ind = i * blockSize + j; + memdb.update_element(ind, values[ind]); + to_add.push_back(values[ind]); + } + add_values(tree, to_add); + commit_tree(tree); + } + + check_block_height(tree, numBlocks); + + index_t blockToFinalise = 8; + + finalise_block(tree, blockToFinalise); + + for (uint32_t i = 0; i < blockToFinalise - 1; i++) { + remove_historic_block(tree, i + 1); + } + remove_historic_block(tree, blockToFinalise, false); +} diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp index a2a3c7c704f..d20378b1ac4 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp @@ -1,8 +1,11 @@ #pragma once #include "barretenberg/common/thread_pool.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/numeric/random/engine.hpp" +#include +#include #include #include #include @@ -13,13 +16,15 @@ const uint32_t NUM_VALUES = 1024; inline auto& engine = numeric::get_debug_randomness(); inline auto& random_engine = numeric::get_randomness(); -static std::vector VALUES = []() { - std::vector values(NUM_VALUES); - for (uint32_t i = 0; i < NUM_VALUES; ++i) { +static auto create_values = [](uint32_t num_values = NUM_VALUES) { + std::vector values(num_values); + for (uint32_t i = 0; i < num_values; ++i) { values[i] = fr(random_engine.get_random_uint256()); } return values; -}(); +}; + +static std::vector VALUES = create_values(); inline std::string random_string() { @@ -55,4 +60,15 @@ inline ThreadPoolPtr make_thread_pool(uint64_t numThreads) { return std::make_shared(numThreads); } + +void inline print_store_data(LMDBTreeStore::SharedPtr db, std::ostream& os) +{ + LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction(); + StatsMap stats; + db->get_stats(stats, *tx); + + for (const auto& m : stats) { + os << m.first << m.second << std::endl; + } +} } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp index fb3c527f160..e2e1434bcee 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp @@ -10,6 +10,7 @@ #include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" +#include "barretenberg/numeric/bitop/get_msb.hpp" #include "barretenberg/numeric/uint256/uint256.hpp" #include "indexed_leaf.hpp" #include @@ -361,6 +362,9 @@ void ContentAddressedIndexedTree::get_leaf(const index_t& auto job = [=, this]() { execute_and_report>( [=, this](TypedResponse>& response) { + if (blockNumber == 0) { + throw std::runtime_error("Invalid block number"); + } ReadTransactionPtr tx = store_->create_read_transaction(); BlockPayload blockData; if (!store_->get_block_data(blockNumber, blockData, *tx)) { @@ -445,6 +449,9 @@ void ContentAddressedIndexedTree::find_leaf_index_from( auto job = [=, this]() -> void { execute_and_report( [=, this](TypedResponse& response) { + if (blockNumber == 0) { + throw std::runtime_error("Invalid block number"); + } typename Store::ReadTransactionPtr tx = store_->create_read_transaction(); BlockPayload blockData; if (!store_->get_block_data(blockNumber, blockData, *tx)) { @@ -497,6 +504,9 @@ void ContentAddressedIndexedTree::find_low_leaf(const fr& auto job = [=, this]() { execute_and_report( [=, this](TypedResponse& response) { + if (blockNumber == 0) { + throw std::runtime_error("Invalid block number"); + } typename Store::ReadTransactionPtr tx = store_->create_read_transaction(); BlockPayload blockData; if (!store_->get_block_data(blockNumber, blockData, *tx)) { @@ -849,14 +859,25 @@ void ContentAddressedIndexedTree::perform_insertions_witho std::shared_ptr status = std::make_shared(); - uint32_t p = static_cast(std::log2(highest_index + 1)) + 1; - index_t span = static_cast(std::pow(2, p)); - uint64_t numBatches = workers_->num_threads(); + auto log2Ceil = [=](uint64_t value) { + uint64_t log = numeric::get_msb(value); + uint64_t temp = static_cast(1) << log; + return temp == value ? log : log + 1; + }; + + uint64_t indexPower2Ceil = log2Ceil(highest_index + 1); + index_t span = static_cast(std::pow(2UL, indexPower2Ceil)); + uint64_t numBatchesPower2Floor = numeric::get_msb(workers_->num_threads()); + index_t numBatches = static_cast(std::pow(2UL, numBatchesPower2Floor)); index_t batchSize = span / numBatches; - batchSize = std::max(batchSize, 1UL); + batchSize = std::max(batchSize, 2UL); index_t startIndex = 0; - p = static_cast(std::log2(batchSize)); - uint32_t rootLevel = depth_ - p; + indexPower2Ceil = log2Ceil(batchSize); + uint32_t rootLevel = depth_ - static_cast(indexPower2Ceil); + + // std::cout << "HIGHEST INDEX " << highest_index << " SPAN " << span << " NUM BATCHES " << numBatches + // << " BATCH SIZE " << batchSize << " NUM THREADS " << workers_->num_threads() << " ROOT LEVEL " + // << rootLevel << std::endl; struct BatchInsertResults { std::atomic_uint32_t count; @@ -1034,6 +1055,9 @@ void ContentAddressedIndexedTree::generate_insertions( // std::cout << "NEW LEAf TO BE INSERTED at index: " << index_of_new_leaf << " : " << new_leaf // << std::endl; + // std::cout << "Low leaf found at index " << low_leaf_index << " index of new leaf " + // << index_of_new_leaf << std::endl; + store_->put_cached_leaf_by_index(low_leaf_index, low_leaf); // leaves_pre[low_leaf_index] = low_leaf; insertion.low_leaf = low_leaf; @@ -1044,9 +1068,11 @@ void ContentAddressedIndexedTree::generate_insertions( // Update the current leaf's value, don't change it's link IndexedLeafValueType replacement_leaf = IndexedLeafValueType(value_pair.first, low_leaf.nextIndex, low_leaf.nextValue); - IndexedLeafValueType empty_leaf = IndexedLeafValueType::empty(); - // don't update the index for this empty leaf - store_->set_leaf_key_at_index(index_of_new_leaf, empty_leaf); + // IndexedLeafValueType empty_leaf = IndexedLeafValueType::empty(); + // don't update the index for this empty leaf + // std::cout << "Low leaf updated at index " << low_leaf_index << " index of new leaf " + // << index_of_new_leaf << std::endl; + // store_->set_leaf_key_at_index(index_of_new_leaf, empty_leaf); store_->put_cached_leaf_by_index(low_leaf_index, replacement_leaf); insertion.low_leaf = replacement_leaf; // The set of appended leaves already has an empty leaf in the slot at index diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp index 1d913590a49..c7618d62031 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp @@ -3,6 +3,7 @@ #include "../hash.hpp" #include "../node_store/array_store.hpp" #include "../nullifier_tree/nullifier_memory_tree.hpp" +#include "../test_fixtures.hpp" #include "./fixtures.hpp" #include "barretenberg/common/streams.hpp" #include "barretenberg/common/test.hpp" @@ -14,6 +15,7 @@ #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/numeric/random/engine.hpp" +#include #include #include #include @@ -53,6 +55,22 @@ std::string PersistedContentAddressedIndexedTreeTest::_directory; uint64_t PersistedContentAddressedIndexedTreeTest::_maxReaders; uint64_t PersistedContentAddressedIndexedTreeTest::_mapSize; +std::unique_ptr create_tree(const std::string& rootDirectory, + uint64_t mapSize, + uint64_t maxReaders, + uint32_t depth, + uint32_t batchSize, + ThreadPoolPtr workers) +{ + std::string name = random_string(); + std::filesystem::path directory = rootDirectory; + directory.append(name); + std::filesystem::create_directories(directory); + LMDBTreeStore::SharedPtr db = std::make_shared(directory, name, mapSize, maxReaders); + std::unique_ptr store = std::make_unique(name, depth, db); + return std::make_unique(std::move(store), workers, batchSize); +} + template void check_size(TypeOfTree& tree, index_t expected_size, bool includeUncommitted = true) { Signal signal; @@ -283,10 +301,13 @@ void check_historic_sibling_path(TypeOfTree& tree, index_t index, index_t blockNumber, const fr_sibling_path& expected_sibling_path, - bool includeUncommitted = true) + bool includeUncommitted = true, + bool expected_success = true) { - fr_sibling_path path = get_historic_sibling_path(tree, blockNumber, index, includeUncommitted); - EXPECT_EQ(path, expected_sibling_path); + fr_sibling_path path = get_historic_sibling_path(tree, blockNumber, index, includeUncommitted, expected_success); + if (expected_success) { + EXPECT_EQ(path, expected_sibling_path); + } } template @@ -337,10 +358,24 @@ void add_value(TypeOfTree& tree, const LeafValueType& value, bool expectedSucces } template -void add_values(TypeOfTree& tree, const std::vector& values) +void add_values(TypeOfTree& tree, const std::vector& values, bool expectedSuccess = true) +{ + Signal signal; + auto completion = [&](const TypedResponse>& response) -> void { + EXPECT_EQ(response.success, expectedSuccess); + signal.signal_level(); + }; + + tree.add_or_update_values(values, completion); + signal.wait_for_level(); +} + +template +void block_sync_values(TypeOfTree& tree, const std::vector& values, bool expectedSuccess = true) { Signal signal; - auto completion = [&](const TypedResponse>&) -> void { + auto completion = [&](const TypedResponse& response) -> void { + EXPECT_EQ(response.success, expectedSuccess); signal.signal_level(); }; @@ -348,6 +383,54 @@ void add_values(TypeOfTree& tree, const std::vector& values) signal.wait_for_level(); } +template +void remove_historic_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_success = true) +{ + Signal signal; + auto completion = [&](const Response& response) -> void { + EXPECT_EQ(response.success, expected_success); + signal.signal_level(); + }; + tree.remove_historic_block(blockNumber, completion); + signal.wait_for_level(); +} + +template +void finalise_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_success = true) +{ + Signal signal; + auto completion = [&](const Response& response) -> void { + EXPECT_EQ(response.success, expected_success); + signal.signal_level(); + }; + tree.finalise_block(blockNumber, completion); + signal.wait_for_level(); +} + +template +void unwind_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_success = true) +{ + Signal signal; + auto completion = [&](const Response& response) -> void { + EXPECT_EQ(response.success, expected_success); + signal.signal_level(); + }; + tree.unwind_block(blockNumber, completion); + signal.wait_for_level(); +} + +template void check_block_height(TypeOfTree& tree, index_t expected_block_height) +{ + Signal signal; + auto completion = [&](const TypedResponse& response) -> void { + EXPECT_EQ(response.success, true); + EXPECT_EQ(response.inner.meta.unfinalisedBlockHeight, expected_block_height); + signal.signal_level(); + }; + tree.get_meta_data(true, completion); + signal.wait_for_level(); +} + TEST_F(PersistedContentAddressedIndexedTreeTest, can_create) { constexpr size_t depth = 10; @@ -439,7 +522,6 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, test_get_sibling_path) NullifierMemoryTree memdb(depth, current_size); ThreadPoolPtr workers = make_thread_pool(1); - ; std::string name = random_string(); LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); @@ -608,33 +690,22 @@ void test_batch_insert(uint32_t batchSize, std::string directory, uint64_t mapSi ThreadPoolPtr multi_workers = make_thread_pool(8); NullifierMemoryTree memdb(depth, batch_size); - std::string name1 = random_string(); - LMDBTreeStore::SharedPtr db1 = std::make_shared(directory, name1, mapSize, maxReaders); - std::unique_ptr store1 = std::make_unique(name1, depth, db1); - auto tree1 = TreeType(std::move(store1), workers, batch_size); - - std::string name2 = random_string(); - LMDBTreeStore::SharedPtr db2 = std::make_shared(directory, name2, mapSize, maxReaders); - std::unique_ptr store2 = std::make_unique(name2, depth, db2); - auto tree2 = TreeType(std::move(store2), multi_workers, batch_size); - - std::string name3 = random_string(); - LMDBTreeStore::SharedPtr db3 = std::make_shared(directory, name3, mapSize, maxReaders); - std::unique_ptr store3 = std::make_unique(name3, depth, db3); - auto tree3 = TreeType(std::move(store3), multi_workers, batch_size); + auto tree1 = create_tree(directory, mapSize, maxReaders, depth, batch_size, workers); + auto tree2 = create_tree(directory, mapSize, maxReaders, depth, batch_size, multi_workers); + auto tree3 = create_tree(directory, mapSize, maxReaders, depth, batch_size, multi_workers); for (uint32_t i = 0; i < num_batches; i++) { - check_root(tree1, memdb.root()); - check_root(tree2, memdb.root()); - check_root(tree3, memdb.root()); - check_sibling_path(tree1, 0, memdb.get_sibling_path(0)); - check_sibling_path(tree2, 0, memdb.get_sibling_path(0)); - check_sibling_path(tree3, 0, memdb.get_sibling_path(0)); + check_root(*tree1, memdb.root()); + check_root(*tree2, memdb.root()); + check_root(*tree3, memdb.root()); + check_sibling_path(*tree1, 0, memdb.get_sibling_path(0)); + check_sibling_path(*tree2, 0, memdb.get_sibling_path(0)); + check_sibling_path(*tree3, 0, memdb.get_sibling_path(0)); - check_sibling_path(tree1, 512, memdb.get_sibling_path(512)); - check_sibling_path(tree2, 512, memdb.get_sibling_path(512)); - check_sibling_path(tree3, 512, memdb.get_sibling_path(512)); + check_sibling_path(*tree1, 512, memdb.get_sibling_path(512)); + check_sibling_path(*tree2, 512, memdb.get_sibling_path(512)); + check_sibling_path(*tree3, 512, memdb.get_sibling_path(512)); std::vector batch; std::vector memory_tree_sibling_paths; @@ -652,7 +723,7 @@ void test_batch_insert(uint32_t batchSize, std::string directory, uint64_t mapSi tree1_low_leaf_witness_data = response.inner.low_leaf_witness_data; signal.signal_level(); }; - tree1.add_or_update_values(batch, completion); + tree1->add_or_update_values(batch, completion); signal.wait_for_level(); } @@ -663,27 +734,27 @@ void test_batch_insert(uint32_t batchSize, std::string directory, uint64_t mapSi tree2_low_leaf_witness_data = response.inner.low_leaf_witness_data; signal.signal_level(); }; - tree2.add_or_update_values(batch, completion); + tree2->add_or_update_values(batch, completion); signal.wait_for_level(); } { Signal signal; auto completion = [&](const TypedResponse&) { signal.signal_level(); }; - tree3.add_or_update_values(batch, completion); + tree3->add_or_update_values(batch, completion); signal.wait_for_level(); } - check_root(tree1, memdb.root()); - check_root(tree2, memdb.root()); - check_root(tree3, memdb.root()); + check_root(*tree1, memdb.root()); + check_root(*tree2, memdb.root()); + check_root(*tree3, memdb.root()); - check_sibling_path(tree1, 0, memdb.get_sibling_path(0)); - check_sibling_path(tree2, 0, memdb.get_sibling_path(0)); - check_sibling_path(tree3, 0, memdb.get_sibling_path(0)); + check_sibling_path(*tree1, 0, memdb.get_sibling_path(0)); + check_sibling_path(*tree2, 0, memdb.get_sibling_path(0)); + check_sibling_path(*tree3, 0, memdb.get_sibling_path(0)); - check_sibling_path(tree1, 512, memdb.get_sibling_path(512)); - check_sibling_path(tree2, 512, memdb.get_sibling_path(512)); - check_sibling_path(tree3, 512, memdb.get_sibling_path(512)); + check_sibling_path(*tree1, 512, memdb.get_sibling_path(512)); + check_sibling_path(*tree2, 512, memdb.get_sibling_path(512)); + check_sibling_path(*tree3, 512, memdb.get_sibling_path(512)); for (uint32_t j = 0; j < batch_size; j++) { EXPECT_EQ(tree1_low_leaf_witness_data->at(j).leaf, tree2_low_leaf_witness_data->at(j).leaf); @@ -708,31 +779,20 @@ void test_batch_insert_with_commit_restore(uint32_t batchSize, for (uint32_t i = 0; i < num_batches; i++) { - std::string name1 = random_string(); - LMDBTreeStore::SharedPtr db1 = std::make_shared(directory, name1, mapSize, maxReaders); - std::unique_ptr store1 = std::make_unique(name1, depth, db1); - auto tree1 = TreeType(std::move(store1), workers, batch_size); + auto tree1 = create_tree(directory, mapSize, maxReaders, depth, batch_size, workers); + auto tree2 = create_tree(directory, mapSize, maxReaders, depth, batch_size, multi_workers); + auto tree3 = create_tree(directory, mapSize, maxReaders, depth, batch_size, multi_workers); - std::string name2 = random_string(); - LMDBTreeStore::SharedPtr db2 = std::make_shared(directory, name2, mapSize, maxReaders); - std::unique_ptr store2 = std::make_unique(name2, depth, db2); - auto tree2 = TreeType(std::move(store2), multi_workers, batch_size); - - std::string name3 = random_string(); - LMDBTreeStore::SharedPtr db3 = std::make_shared(directory, name3, mapSize, maxReaders); - std::unique_ptr store3 = std::make_unique(name3, depth, db3); - auto tree3 = TreeType(std::move(store3), multi_workers, batch_size); - - check_root(tree1, memdb.root()); - check_root(tree2, memdb.root()); - check_root(tree3, memdb.root()); - check_sibling_path(tree1, 0, memdb.get_sibling_path(0)); - check_sibling_path(tree2, 0, memdb.get_sibling_path(0)); - check_sibling_path(tree3, 0, memdb.get_sibling_path(0)); + check_root(*tree1, memdb.root()); + check_root(*tree2, memdb.root()); + check_root(*tree3, memdb.root()); + check_sibling_path(*tree1, 0, memdb.get_sibling_path(0)); + check_sibling_path(*tree2, 0, memdb.get_sibling_path(0)); + check_sibling_path(*tree3, 0, memdb.get_sibling_path(0)); - check_sibling_path(tree1, 512, memdb.get_sibling_path(512)); - check_sibling_path(tree2, 512, memdb.get_sibling_path(512)); - check_sibling_path(tree3, 512, memdb.get_sibling_path(512)); + check_sibling_path(*tree1, 512, memdb.get_sibling_path(512)); + check_sibling_path(*tree2, 512, memdb.get_sibling_path(512)); + check_sibling_path(*tree3, 512, memdb.get_sibling_path(512)); std::vector batch; std::vector memory_tree_sibling_paths; @@ -750,7 +810,7 @@ void test_batch_insert_with_commit_restore(uint32_t batchSize, tree1_low_leaf_witness_data = response.inner.low_leaf_witness_data; signal.signal_level(); }; - tree1.add_or_update_values(batch, completion); + tree1->add_or_update_values(batch, completion); signal.wait_for_level(); } @@ -761,27 +821,27 @@ void test_batch_insert_with_commit_restore(uint32_t batchSize, tree2_low_leaf_witness_data = response.inner.low_leaf_witness_data; signal.signal_level(); }; - tree2.add_or_update_values(batch, completion); + tree2->add_or_update_values(batch, completion); signal.wait_for_level(); } { Signal signal; auto completion = [&](const TypedResponse&) { signal.signal_level(); }; - tree3.add_or_update_values(batch, completion); + tree3->add_or_update_values(batch, completion); signal.wait_for_level(); } - check_root(tree1, memdb.root()); - check_root(tree2, memdb.root()); - check_root(tree3, memdb.root()); + check_root(*tree1, memdb.root()); + check_root(*tree2, memdb.root()); + check_root(*tree3, memdb.root()); - check_sibling_path(tree1, 0, memdb.get_sibling_path(0)); - check_sibling_path(tree2, 0, memdb.get_sibling_path(0)); - check_sibling_path(tree3, 0, memdb.get_sibling_path(0)); + check_sibling_path(*tree1, 0, memdb.get_sibling_path(0)); + check_sibling_path(*tree2, 0, memdb.get_sibling_path(0)); + check_sibling_path(*tree3, 0, memdb.get_sibling_path(0)); - check_sibling_path(tree1, 512, memdb.get_sibling_path(512)); - check_sibling_path(tree2, 512, memdb.get_sibling_path(512)); - check_sibling_path(tree3, 512, memdb.get_sibling_path(512)); + check_sibling_path(*tree1, 512, memdb.get_sibling_path(512)); + check_sibling_path(*tree2, 512, memdb.get_sibling_path(512)); + check_sibling_path(*tree3, 512, memdb.get_sibling_path(512)); for (uint32_t j = 0; j < batch_size; j++) { EXPECT_EQ(tree1_low_leaf_witness_data->at(j).leaf, tree2_low_leaf_witness_data->at(j).leaf); @@ -789,9 +849,9 @@ void test_batch_insert_with_commit_restore(uint32_t batchSize, EXPECT_EQ(tree1_low_leaf_witness_data->at(j).path, tree2_low_leaf_witness_data->at(j).path); } - commit_tree(tree1); - commit_tree(tree2); - commit_tree(tree3); + commit_tree(*tree1); + commit_tree(*tree2); + commit_tree(*tree3); } } @@ -813,6 +873,55 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, test_batch_insert_with_commit_r } } +TEST_F(PersistedContentAddressedIndexedTreeTest, test_compare_batch_inserts_different_sized_thread_pools) +{ + const uint32_t batch_size = 128; + uint32_t depth = 20; + ThreadPoolPtr workers = make_thread_pool(1); + NullifierMemoryTree memdb(depth, batch_size); + + auto tree1 = create_tree(_directory, _mapSize, _maxReaders, depth, batch_size, workers); + auto tree2 = create_tree(_directory, _mapSize, _maxReaders, depth, batch_size, workers); + + std::vector> trees; + for (uint32_t i = 1; i <= 12; i++) { + ThreadPoolPtr multiWorkers = make_thread_pool(i); + auto tree = create_tree(_directory, _mapSize, _maxReaders, depth, batch_size, multiWorkers); + trees.emplace_back(std::move(tree)); + } + + std::vector tree1Roots; + std::vector tree2Roots; + + for (uint32_t round = 0; round < 10; round++) { + std::vector frValues1 = create_values(3); + std::vector frValues2 = create_values(3); + std::vector leaves(128, NullifierLeafValue(fr::zero())); + for (uint32_t i = 0; i < 3; i++) { + leaves[i] = frValues1[i]; + leaves[i + 64] = frValues2[i]; + } + + std::vector first(leaves.begin(), leaves.begin() + 64); + std::vector second(leaves.begin() + 64, leaves.end()); + + add_values(*tree1, first); + add_values(*tree1, second); + + block_sync_values(*tree2, leaves); + + tree1Roots.push_back(get_root(*tree1)); + tree2Roots.push_back(get_root(*tree2, true)); + EXPECT_EQ(tree1Roots[round], tree2Roots[round]); + + for (const auto& tree : trees) { + block_sync_values(*tree, leaves); + const fr treeRoot = get_root(*tree, true); + EXPECT_EQ(treeRoot, tree1Roots[round]); + } + } +} + TEST_F(PersistedContentAddressedIndexedTreeTest, reports_an_error_if_batch_contains_duplicate) { index_t current_size = 2; @@ -1279,7 +1388,6 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, returns_low_leaves) constexpr uint32_t depth = 8; ThreadPoolPtr workers = make_thread_pool(1); - ; std::string name = random_string(); LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); std::unique_ptr store = std::make_unique(name, depth, db); @@ -1518,7 +1626,68 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, test_historical_leaves) EXPECT_EQ(lowLeaf.index, 2); } -TEST_F(PersistedContentAddressedIndexedTreeTest, test_can_create_images_at_historic_blocks) +TEST_F(PersistedContentAddressedIndexedTreeTest, test_inserting_a_duplicate_committed_nullifier_should_fail) +{ + const uint32_t batch_size = 16; + uint32_t depth = 10; + ThreadPoolPtr multi_workers = make_thread_pool(1); + NullifierMemoryTree memdb(depth, batch_size); + + std::string name1 = random_string(); + LMDBTreeStore::SharedPtr db1 = std::make_shared(_directory, name1, _mapSize, _maxReaders); + std::unique_ptr store1 = std::make_unique(name1, depth, db1); + auto tree = TreeType(std::move(store1), multi_workers, batch_size); + + std::vector values = create_values(batch_size); + std::vector nullifierValues(batch_size); + std::transform( + values.begin(), values.end(), nullifierValues.begin(), [](const fr& v) { return NullifierLeafValue(v); }); + + add_values(tree, nullifierValues); + commit_tree(tree); + + // create a new set of values + std::vector values2 = create_values(batch_size); + + // copy one of the previous values into the middle of the batch + values2[batch_size / 2] = values[0]; + std::vector nullifierValues2(batch_size); + std::transform( + values2.begin(), values2.end(), nullifierValues2.begin(), [](const fr& v) { return NullifierLeafValue(v); }); + add_values(tree, nullifierValues2, false); +} + +TEST_F(PersistedContentAddressedIndexedTreeTest, test_inserting_a_duplicate_uncommitted_nullifier_should_fail) +{ + const uint32_t batch_size = 16; + uint32_t depth = 10; + ThreadPoolPtr multi_workers = make_thread_pool(1); + NullifierMemoryTree memdb(depth, batch_size); + + std::string name1 = random_string(); + LMDBTreeStore::SharedPtr db1 = std::make_shared(_directory, name1, _mapSize, _maxReaders); + std::unique_ptr store1 = std::make_unique(name1, depth, db1); + auto tree = TreeType(std::move(store1), multi_workers, batch_size); + + std::vector values = create_values(batch_size); + std::vector nullifierValues(batch_size); + std::transform( + values.begin(), values.end(), nullifierValues.begin(), [](const fr& v) { return NullifierLeafValue(v); }); + + add_values(tree, nullifierValues); + + // create a new set of values + std::vector values2 = create_values(batch_size); + + // copy one of the previous values into the middle of the batch + values2[batch_size / 2] = values[0]; + std::vector nullifierValues2(batch_size); + std::transform( + values2.begin(), values2.end(), nullifierValues2.begin(), [](const fr& v) { return NullifierLeafValue(v); }); + add_values(tree, nullifierValues2, false); +} + +TEST_F(PersistedContentAddressedIndexedTreeTest, test_can_create_forks_at_historic_blocks) { auto& random_engine = numeric::get_randomness(); const uint32_t batch_size = 16; @@ -1618,3 +1787,725 @@ TEST_F(PersistedContentAddressedIndexedTreeTest, test_can_create_images_at_histo // It should be impossible to commit using the image commit_tree(treeAtBlock2, false); } + +TEST_F(PersistedContentAddressedIndexedTreeTest, test_remove_historical_blocks) +{ + index_t current_size = 2; + ThreadPoolPtr workers = make_thread_pool(8); + // Create a depth-3 indexed merkle tree + constexpr size_t depth = 3; + std::string name = random_string(); + LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); + std::unique_ptr> store = + std::make_unique>(name, depth, db); + auto tree = ContentAddressedIndexedTree, Poseidon2HashPolicy>( + std::move(store), workers, current_size); + + /** + * Intial state: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * slot 0 1 0 0 0 0 0 0 + * val 0 0 0 0 0 0 0 0 + * nextIdx 1 0 0 0 0 0 0 0 + * nextVal 1 0 0 0 0 0 0 0 + */ + IndexedPublicDataLeafType zero_leaf = create_indexed_public_data_leaf(0, 0, 1, 1); + IndexedPublicDataLeafType one_leaf = create_indexed_public_data_leaf(1, 0, 0, 0); + check_size(tree, current_size); + EXPECT_EQ(get_leaf(tree, 0), zero_leaf); + EXPECT_EQ(get_leaf(tree, 1), one_leaf); + + /** + * Add new slot:value 30:5: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * slot 0 1 30 0 0 0 0 0 + * val 0 0 5 0 0 0 0 0 + * nextIdx 1 2 0 0 0 0 0 0 + * nextVal 1 30 0 0 0 0 0 0 + */ + add_value(tree, PublicDataLeafValue(30, 5)); + commit_tree(tree); + check_size(tree, ++current_size); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_public_data_leaf(0, 0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_public_data_leaf(1, 0, 2, 30)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 5, 0, 0)); + + auto leaf1AtBlock1 = PublicDataLeafValue(1, 0); + check_block_and_size_data(db, 1, current_size, true); + + /** + * Add new slot:value 10:20: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * slot 0 1 30 10 0 0 0 0 + * val 0 0 5 20 0 0 0 0 + * nextIdx 1 3 0 2 0 0 0 0 + * nextVal 1 10 0 30 0 0 0 0 + */ + add_value(tree, PublicDataLeafValue(10, 20)); + check_size(tree, ++current_size); + commit_tree(tree); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_public_data_leaf(0, 0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_public_data_leaf(1, 0, 3, 10)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 5, 0, 0)); + EXPECT_EQ(get_leaf(tree, 3), create_indexed_public_data_leaf(10, 20, 2, 30)); + + check_block_and_size_data(db, 2, current_size, true); + + auto leaf2AtBlock2 = PublicDataLeafValue(30, 5); + check_historic_leaf(tree, leaf1AtBlock1, 1, 1, true); + + // shoudl find this leaf at both blocks 1 and 2 as it looks for the slot which doesn't change + check_historic_find_leaf_index(tree, leaf1AtBlock1, 1, 1, true); + check_historic_find_leaf_index(tree, leaf1AtBlock1, 2, 1, true); + + /** + * Update value at slot 30 to 6: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * slot 0 1 30 10 0 0 0 0 + * val 0 0 6 20 0 0 0 0 + * nextIdx 1 3 0 2 0 0 0 0 + * nextVal 1 10 0 30 0 0 0 0 + */ + add_value(tree, PublicDataLeafValue(30, 6)); + // The size still increases as we pad with an empty leaf + check_size(tree, ++current_size); + commit_tree(tree); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_public_data_leaf(0, 0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_public_data_leaf(1, 0, 3, 10)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 6, 0, 0)); + EXPECT_EQ(get_leaf(tree, 3), create_indexed_public_data_leaf(10, 20, 2, 30)); + EXPECT_EQ(get_leaf(tree, 4), create_indexed_public_data_leaf(0, 0, 0, 0)); + + check_block_and_size_data(db, 3, current_size, true); + + auto leaf2AtBlock3 = PublicDataLeafValue(30, 6); + check_historic_leaf(tree, leaf2AtBlock2, 2, 2, true); + + // should find this leaf at both blocks 1 and 2 as it looks for the slot which doesn't change + check_historic_find_leaf_index(tree, leaf1AtBlock1, 1, 1, true); + check_historic_find_leaf_index(tree, leaf1AtBlock1, 2, 1, true); + + /** + * Add new value slot:value 50:8: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * slot 0 1 30 10 0 50 0 0 + * val 0 0 6 20 0 8 0 0 + * nextIdx 1 3 5 2 0 0 0 0 + * nextVal 1 10 50 30 0 0 0 0 + */ + add_value(tree, PublicDataLeafValue(50, 8)); + check_size(tree, ++current_size); + commit_tree(tree); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_public_data_leaf(0, 0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_public_data_leaf(1, 0, 3, 10)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 6, 5, 50)); + EXPECT_EQ(get_leaf(tree, 3), create_indexed_public_data_leaf(10, 20, 2, 30)); + EXPECT_EQ(get_leaf(tree, 4), create_indexed_public_data_leaf(0, 0, 0, 0)); + EXPECT_EQ(get_leaf(tree, 5), create_indexed_public_data_leaf(50, 8, 0, 0)); + + check_block_and_size_data(db, 4, current_size, true); + + check_historic_leaf(tree, leaf2AtBlock3, 2, 3, true); + + // should not be found at block 1 + check_historic_find_leaf_index_from(tree, PublicDataLeafValue(10, 20), 1, 0, 0, false); + // should be found at block + check_historic_find_leaf_index_from(tree, PublicDataLeafValue(10, 20), 2, 0, 3, true); + + GetLowIndexedLeafResponse lowLeaf = get_historic_low_leaf(tree, 1, PublicDataLeafValue(20, 0)); + EXPECT_EQ(lowLeaf.index, 1); + + lowLeaf = get_historic_low_leaf(tree, 2, PublicDataLeafValue(20, 0)); + EXPECT_EQ(lowLeaf.index, 3); + + lowLeaf = get_historic_low_leaf(tree, 2, PublicDataLeafValue(60, 0)); + EXPECT_EQ(lowLeaf.index, 2); + + finalise_block(tree, 3); + + // remove historical block 1 + remove_historic_block(tree, 1); + + // Historic queries against block 1 should no longer work + check_historic_leaf(tree, leaf1AtBlock1, 1, 1, false); + check_historic_find_leaf_index(tree, leaf1AtBlock1, 1, 1, false); + + // Queries against block 2 should work + check_historic_leaf(tree, leaf2AtBlock2, 2, 2, true); + check_historic_find_leaf_index(tree, leaf1AtBlock1, 2, 1, true); + + // now remove block 2 and queries against it should no longer work + remove_historic_block(tree, 2); + check_historic_leaf(tree, leaf2AtBlock2, 2, 2, false); + + // size doesn't matter, should fail to find the data + check_block_and_size_data(db, 1, current_size, false); +} + +TEST_F(PersistedContentAddressedIndexedTreeTest, test_unwind_blocks) +{ + index_t current_size = 2; + ThreadPoolPtr workers = make_thread_pool(8); + // Create a depth-3 indexed merkle tree + constexpr size_t depth = 3; + std::string name = random_string(); + LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); + std::unique_ptr> store = + std::make_unique>(name, depth, db); + auto tree = ContentAddressedIndexedTree, Poseidon2HashPolicy>( + std::move(store), workers, current_size); + + /** + * Intial state: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * slot 0 1 0 0 0 0 0 0 + * val 0 0 0 0 0 0 0 0 + * nextIdx 1 0 0 0 0 0 0 0 + * nextVal 1 0 0 0 0 0 0 0 + */ + IndexedPublicDataLeafType zero_leaf = create_indexed_public_data_leaf(0, 0, 1, 1); + IndexedPublicDataLeafType one_leaf = create_indexed_public_data_leaf(1, 0, 0, 0); + check_size(tree, current_size); + EXPECT_EQ(get_leaf(tree, 0), zero_leaf); + EXPECT_EQ(get_leaf(tree, 1), one_leaf); + + check_leaf_by_hash(db, zero_leaf, true); + check_leaf_by_hash(db, one_leaf, true); + + check_indices_data(db, 0, 0, true, true); + check_indices_data(db, 1, 1, true, true); + + /** + * Add new slot:value 30:5: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * slot 0 1 30 0 0 0 0 0 + * val 0 0 5 0 0 0 0 0 + * nextIdx 1 2 0 0 0 0 0 0 + * nextVal 1 30 0 0 0 0 0 0 + */ + add_value(tree, PublicDataLeafValue(30, 5)); + commit_tree(tree); + check_size(tree, ++current_size); + + EXPECT_EQ(get_leaf(tree, 0), create_indexed_public_data_leaf(0, 0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_public_data_leaf(1, 0, 2, 30)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 5, 0, 0)); + + check_block_and_size_data(db, 1, current_size, true); + + // All historical pre-images should be present + check_leaf_by_hash(db, zero_leaf, true); + check_leaf_by_hash(db, one_leaf, true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(1, 0, 2, 30), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 5, 0, 0), true); + + check_indices_data(db, 30, 2, true, true); + + auto leaf1AtBlock1 = PublicDataLeafValue(1, 0); + + /** + * Add new slot:value 10:20: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * slot 0 1 30 10 0 0 0 0 + * val 0 0 5 20 0 0 0 0 + * nextIdx 1 3 0 2 0 0 0 0 + * nextVal 1 10 0 30 0 0 0 0 + */ + add_value(tree, PublicDataLeafValue(10, 20)); + check_size(tree, ++current_size); + commit_tree(tree); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_public_data_leaf(0, 0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_public_data_leaf(1, 0, 3, 10)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 5, 0, 0)); + EXPECT_EQ(get_leaf(tree, 3), create_indexed_public_data_leaf(10, 20, 2, 30)); + + // All historical pre-images should be present + check_leaf_by_hash(db, zero_leaf, true); + check_leaf_by_hash(db, one_leaf, true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(1, 0, 3, 10), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 5, 0, 0), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(10, 20, 2, 30), true); + + check_indices_data(db, 10, 3, true, true); + + check_block_and_size_data(db, 2, current_size, true); + + auto leaf2AtBlock2 = PublicDataLeafValue(30, 5); + check_historic_leaf(tree, leaf1AtBlock1, 1, 1, true); + + // shoudl find this leaf at both blocks 1 and 2 as it looks for the slot which doesn't change + check_historic_find_leaf_index(tree, leaf1AtBlock1, 1, 1, true); + check_historic_find_leaf_index(tree, leaf1AtBlock1, 2, 1, true); + + /** + * Update value at slot 30 to 6: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * slot 0 1 30 10 0 0 0 0 + * val 0 0 6 20 0 0 0 0 + * nextIdx 1 3 0 2 0 0 0 0 + * nextVal 1 10 0 30 0 0 0 0 + */ + add_value(tree, PublicDataLeafValue(30, 6)); + // The size still increases as we pad with an empty leaf + check_size(tree, ++current_size); + commit_tree(tree); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_public_data_leaf(0, 0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_public_data_leaf(1, 0, 3, 10)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 6, 0, 0)); + EXPECT_EQ(get_leaf(tree, 3), create_indexed_public_data_leaf(10, 20, 2, 30)); + EXPECT_EQ(get_leaf(tree, 4), create_indexed_public_data_leaf(0, 0, 0, 0)); + + // All historical pre-images should be present + check_leaf_by_hash(db, zero_leaf, true); + check_leaf_by_hash(db, one_leaf, true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(1, 0, 3, 10), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 5, 0, 0), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(10, 20, 2, 30), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 5, 0, 0), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 6, 0, 0), true); + + // Zero leaves should not have their indices added + check_indices_data(db, 0, 4, true, false); + + check_block_and_size_data(db, 3, current_size, true); + + auto leaf2AtBlock3 = PublicDataLeafValue(30, 6); + check_historic_leaf(tree, leaf2AtBlock2, 2, 2, true); + + // should find this leaf at both blocks 1 and 2 as it looks for the slot which doesn't change + check_historic_find_leaf_index(tree, leaf1AtBlock1, 1, 1, true); + check_historic_find_leaf_index(tree, leaf1AtBlock1, 2, 1, true); + + /** + * Add new value slot:value 50:8: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * slot 0 1 30 10 0 50 0 0 + * val 0 0 6 20 0 8 0 0 + * nextIdx 1 3 5 2 0 0 0 0 + * nextVal 1 10 50 30 0 0 0 0 + */ + add_value(tree, PublicDataLeafValue(50, 8)); + check_size(tree, ++current_size); + commit_tree(tree); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_public_data_leaf(0, 0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_public_data_leaf(1, 0, 3, 10)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 6, 5, 50)); + EXPECT_EQ(get_leaf(tree, 3), create_indexed_public_data_leaf(10, 20, 2, 30)); + EXPECT_EQ(get_leaf(tree, 4), create_indexed_public_data_leaf(0, 0, 0, 0)); + EXPECT_EQ(get_leaf(tree, 5), create_indexed_public_data_leaf(50, 8, 0, 0)); + + check_indices_data(db, 50, 5, true, true); + // All historical pre-images should be present + check_leaf_by_hash(db, zero_leaf, true); + check_leaf_by_hash(db, one_leaf, true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(1, 0, 3, 10), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 5, 0, 0), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(10, 20, 2, 30), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 6, 0, 0), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(50, 8, 0, 0), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 6, 5, 50), true); + + check_block_and_size_data(db, 4, current_size, true); + + check_historic_leaf(tree, leaf2AtBlock3, 2, 3, true); + + // should not be found at block 1 + check_historic_find_leaf_index_from(tree, PublicDataLeafValue(10, 20), 1, 0, 0, false); + // should be found at block + check_historic_find_leaf_index_from(tree, PublicDataLeafValue(10, 20), 2, 0, 3, true); + + GetLowIndexedLeafResponse lowLeaf = get_historic_low_leaf(tree, 1, PublicDataLeafValue(20, 0)); + EXPECT_EQ(lowLeaf.index, 1); + + lowLeaf = get_historic_low_leaf(tree, 2, PublicDataLeafValue(20, 0)); + EXPECT_EQ(lowLeaf.index, 3); + + lowLeaf = get_historic_low_leaf(tree, 2, PublicDataLeafValue(60, 0)); + EXPECT_EQ(lowLeaf.index, 2); + + unwind_block(tree, 4); + + // Index 5 should be removed + check_indices_data(db, 50, 5, false, false); + // The pre-images created before block 4 should be present + check_leaf_by_hash(db, zero_leaf, true); + check_leaf_by_hash(db, one_leaf, true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(1, 0, 3, 10), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 5, 0, 0), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(10, 20, 2, 30), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 5, 0, 0), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 6, 0, 0), true); + + // The pre-images created in block 4 should be gone + check_leaf_by_hash(db, create_indexed_public_data_leaf(50, 8, 0, 0), false); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 6, 5, 50), false); + + check_size(tree, --current_size); + + // should fail to find block 4 + check_block_and_size_data(db, 4, current_size, false); + + // block 3 should work + check_block_and_size_data(db, 3, current_size, true); + + // should fail to find the leaf at index 5 + check_find_leaf_index(tree, PublicDataLeafValue(50, 8), 5, false); + check_find_leaf_index_from(tree, PublicDataLeafValue(50, 8), 0, 5, false); + + // the leaf at index 2 should no longer be as it was after block 5 + EXPECT_NE(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 6, 5, 50)); + + // it should be as it was after block 4 + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 6, 0, 0)); + + unwind_block(tree, 3); + + // The pre-images created before block 3 should be present + check_leaf_by_hash(db, zero_leaf, true); + check_leaf_by_hash(db, one_leaf, true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(1, 0, 3, 10), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 5, 0, 0), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(10, 20, 2, 30), true); + + check_size(tree, --current_size); + + // the leaf at index 2 should no longer be as it was after block 4 + EXPECT_NE(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 6, 0, 0)); + + // it should be as it was after block 3 + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 5, 0, 0)); +} + +TEST_F(PersistedContentAddressedIndexedTreeTest, test_unwind_duplicate_block) +{ + index_t current_size = 2; + ThreadPoolPtr workers = make_thread_pool(8); + // Create a depth-3 indexed merkle tree + constexpr size_t depth = 3; + std::string name = random_string(); + LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); + std::unique_ptr> store = + std::make_unique>(name, depth, db); + auto tree = ContentAddressedIndexedTree, Poseidon2HashPolicy>( + std::move(store), workers, current_size); + + /** + * Intial state: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * slot 0 1 0 0 0 0 0 0 + * val 0 0 0 0 0 0 0 0 + * nextIdx 1 0 0 0 0 0 0 0 + * nextVal 1 0 0 0 0 0 0 0 + */ + IndexedPublicDataLeafType zero_leaf = create_indexed_public_data_leaf(0, 0, 1, 1); + IndexedPublicDataLeafType one_leaf = create_indexed_public_data_leaf(1, 0, 0, 0); + check_size(tree, current_size); + EXPECT_EQ(get_leaf(tree, 0), zero_leaf); + EXPECT_EQ(get_leaf(tree, 1), one_leaf); + + /** + * Add new slot:value 30:5: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * slot 0 1 30 0 0 0 0 0 + * val 0 0 5 0 0 0 0 0 + * nextIdx 1 2 0 0 0 0 0 0 + * nextVal 1 30 0 0 0 0 0 0 + */ + add_value(tree, PublicDataLeafValue(30, 5)); + commit_tree(tree); + check_size(tree, ++current_size); + fr rootAfterBlock1 = get_root(tree, false); + fr_sibling_path pathAfterBlock1 = get_sibling_path(tree, 0, false); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_public_data_leaf(0, 0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_public_data_leaf(1, 0, 2, 30)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 5, 0, 0)); + + check_block_and_size_data(db, 1, current_size, true); + + // All historical pre-images should be present + check_leaf_by_hash(db, zero_leaf, true); + check_leaf_by_hash(db, one_leaf, true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(1, 0, 2, 30), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 5, 0, 0), true); + + check_indices_data(db, 30, 2, true, true); + + /** + * Update slot:value 30:8: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * slot 0 1 30 0 0 0 0 0 + * val 0 0 8 0 0 0 0 0 + * nextIdx 1 2 0 0 0 0 0 0 + * nextVal 1 30 0 0 0 0 0 0 + */ + add_value(tree, PublicDataLeafValue(30, 8)); + commit_tree(tree); + check_size(tree, ++current_size); + fr rootAfterBlock2 = get_root(tree, false); + fr_sibling_path pathAfterBlock2 = get_sibling_path(tree, 0, false); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_public_data_leaf(0, 0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_public_data_leaf(1, 0, 2, 30)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 8, 0, 0)); + + // All historical pre-images should be present + check_leaf_by_hash(db, zero_leaf, true); + check_leaf_by_hash(db, one_leaf, true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(1, 0, 2, 30), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 5, 0, 0), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 8, 0, 0), true); + + check_indices_data(db, 30, 2, true, true); + + /** + * Revert slot:value 30:5: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * slot 0 1 30 0 0 0 0 0 + * val 0 0 5 0 0 0 0 0 + * nextIdx 1 2 0 0 0 0 0 0 + * nextVal 1 30 0 0 0 0 0 0 + */ + add_value(tree, PublicDataLeafValue(30, 5)); + commit_tree(tree); + check_size(tree, ++current_size); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_public_data_leaf(0, 0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_public_data_leaf(1, 0, 2, 30)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 5, 0, 0)); + + // All historical pre-images should be present + check_leaf_by_hash(db, zero_leaf, true); + check_leaf_by_hash(db, one_leaf, true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(1, 0, 2, 30), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 5, 0, 0), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 8, 0, 0), true); + + check_indices_data(db, 30, 2, true, true); + + // Unwind block 3 and the state should be reverted back to block 2 + unwind_block(tree, 3); + + check_root(tree, rootAfterBlock2); + check_sibling_path(tree, 0, pathAfterBlock2, false); + check_size(tree, --current_size); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_public_data_leaf(0, 0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_public_data_leaf(1, 0, 2, 30)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 8, 0, 0)); + + // All historical pre-images should be present + check_leaf_by_hash(db, zero_leaf, true); + check_leaf_by_hash(db, one_leaf, true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(1, 0, 2, 30), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 5, 0, 0), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 8, 0, 0), true); + + check_indices_data(db, 30, 2, true, true); + + // Unwind block 2 and the state should be reverted back to block 1 + unwind_block(tree, 2); + + check_root(tree, rootAfterBlock1); + check_sibling_path(tree, 0, pathAfterBlock1, false); + check_size(tree, --current_size); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_public_data_leaf(0, 0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_public_data_leaf(1, 0, 2, 30)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 5, 0, 0)); + + // All historical pre-images should be present + check_leaf_by_hash(db, zero_leaf, true); + check_leaf_by_hash(db, one_leaf, true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(1, 0, 2, 30), true); + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 5, 0, 0), true); + + // Check the pre-image was removed + check_leaf_by_hash(db, create_indexed_public_data_leaf(30, 8, 0, 0), false); + + check_indices_data(db, 30, 2, true, true); + + // Now apply block 2 again and it should be moved forward back tot where it was + add_value(tree, PublicDataLeafValue(30, 8)); + commit_tree(tree); + check_size(tree, ++current_size); + check_root(tree, rootAfterBlock2); + EXPECT_EQ(get_leaf(tree, 0), create_indexed_public_data_leaf(0, 0, 1, 1)); + EXPECT_EQ(get_leaf(tree, 1), create_indexed_public_data_leaf(1, 0, 2, 30)); + EXPECT_EQ(get_leaf(tree, 2), create_indexed_public_data_leaf(30, 8, 0, 0)); +} + +void test_nullifier_tree_unwind(std::string directory, + std::string name, + uint64_t mapSize, + uint64_t maxReaders, + uint32_t depth, + uint32_t blockSize, + uint32_t numBlocks, + uint32_t numBlocksToUnwind, + std::vector values) +{ + LMDBTreeStore::SharedPtr db = std::make_shared(directory, name, mapSize, maxReaders); + std::unique_ptr store = std::make_unique(name, depth, db); + ThreadPoolPtr pool = make_thread_pool(8); + TreeType tree(std::move(store), pool, blockSize); + NullifierMemoryTree memdb(depth, blockSize); + + auto it = std::find_if(values.begin(), values.end(), [&](const fr& v) { return v != fr::zero(); }); + bool emptyBlocks = it == values.end(); + + uint32_t batchSize = blockSize; + + std::vector historicPathsZeroIndex; + std::vector historicPathsMaxIndex; + std::vector roots; + + fr initialRoot = memdb.root(); + fr_sibling_path initialPath = memdb.get_sibling_path(0); + + std::vector leafValues; + leafValues.reserve(values.size()); + for (const fr& v : values) { + leafValues.emplace_back(v); + } + + for (uint32_t i = 0; i < numBlocks; i++) { + std::vector to_add; + + for (size_t j = 0; j < batchSize; ++j) { + size_t ind = i * batchSize + j; + memdb.update_element(values[ind]); + to_add.push_back(leafValues[ind]); + } + // Indexed trees have an initial 'batch' inserted at startup + index_t expected_size = (i + 2) * batchSize; + add_values(tree, to_add); + commit_tree(tree); + + historicPathsZeroIndex.push_back(memdb.get_sibling_path(0)); + historicPathsMaxIndex.push_back(memdb.get_sibling_path(expected_size - 1)); + roots.push_back(memdb.root()); + check_root(tree, memdb.root()); + check_sibling_path(tree, 0, memdb.get_sibling_path(0)); + check_sibling_path(tree, expected_size - 1, memdb.get_sibling_path(expected_size - 1)); + check_size(tree, expected_size); + check_block_and_size_data(db, i + 1, expected_size, true); + check_block_and_root_data(db, i + 1, memdb.root(), true); + } + + const uint32_t blocksToRemove = numBlocksToUnwind; + for (uint32_t i = 0; i < blocksToRemove; i++) { + const index_t blockNumber = numBlocks - i; + + check_block_and_root_data(db, blockNumber, roots[blockNumber - 1], true); + unwind_block(tree, blockNumber); + if (emptyBlocks) { + // with empty blocks, we should not find the block data but we do find the root + check_block_and_root_data(db, blockNumber, roots[blockNumber - 1], false, true); + } else { + // if blocks are not empty, this query should fail + check_block_and_root_data(db, blockNumber, roots[blockNumber - 1], false); + } + + const index_t previousValidBlock = blockNumber - 1; + // Indexed trees have an initial 'batch' inserted at startup + index_t deletedBlockStartIndex = (1 + previousValidBlock) * batchSize; + index_t deletedBlockStartIndexIntoLocalValues = previousValidBlock * batchSize; + + check_block_height(tree, previousValidBlock); + check_size(tree, deletedBlockStartIndex); + check_root(tree, previousValidBlock == 0 ? initialRoot : roots[previousValidBlock - 1]); + + // The zero index sibling path should be as it was at the previous block + check_sibling_path(tree, + 0, + previousValidBlock == 0 ? initialPath : historicPathsZeroIndex[previousValidBlock - 1], + false, + true); + + if (!emptyBlocks) { + // Trying to find leaves appended in the block that was removed should fail + get_leaf(tree, 1 + deletedBlockStartIndex, false, false); + + check_find_leaf_index( + tree, leafValues[1 + deletedBlockStartIndexIntoLocalValues], 1 + deletedBlockStartIndex, false); + } + + for (index_t j = 0; j < numBlocks; j++) { + index_t historicBlockNumber = j + 1; + bool expectedSuccess = historicBlockNumber <= previousValidBlock; + check_historic_sibling_path( + tree, 0, historicBlockNumber, historicPathsZeroIndex[j], false, expectedSuccess); + index_t maxSizeAtBlock = ((j + 2) * batchSize) - 1; + check_historic_sibling_path( + tree, maxSizeAtBlock, historicBlockNumber, historicPathsMaxIndex[j], false, expectedSuccess); + + if (emptyBlocks) { + continue; + } + const index_t leafIndex = 1; + const index_t expectedIndexInTree = leafIndex + batchSize; + check_historic_leaf( + tree, leafValues[leafIndex], expectedIndexInTree, historicBlockNumber, expectedSuccess, false); + check_historic_find_leaf_index( + tree, leafValues[leafIndex], historicBlockNumber, expectedIndexInTree, expectedSuccess, false); + check_historic_find_leaf_index_from( + tree, leafValues[leafIndex], historicBlockNumber, 0, expectedIndexInTree, expectedSuccess, false); + } + } +} + +TEST_F(PersistedContentAddressedIndexedTreeTest, can_sync_and_unwind_blocks) +{ + + constexpr uint32_t numBlocks = 8; + constexpr uint32_t numBlocksToUnwind = 4; + std::vector blockSizes = { 2, 4, 8, 16, 32 }; + for (const uint32_t& size : blockSizes) { + uint32_t actualSize = size; + std::vector values = create_values(actualSize * numBlocks); + std::stringstream ss; + ss << "DB " << actualSize; + test_nullifier_tree_unwind( + _directory, ss.str(), _mapSize, _maxReaders, 20, actualSize, numBlocks, numBlocksToUnwind, values); + } +} + +TEST_F(PersistedContentAddressedIndexedTreeTest, can_sync_and_unwind_empty_blocks) +{ + + constexpr uint32_t numBlocks = 8; + constexpr uint32_t numBlocksToUnwind = 4; + std::vector blockSizes = { 2, 4, 8, 16, 32 }; + for (const uint32_t& size : blockSizes) { + uint32_t actualSize = size; + std::vector values = std::vector(actualSize * numBlocks, fr::zero()); + std::stringstream ss; + ss << "DB " << actualSize; + test_nullifier_tree_unwind( + _directory, ss.str(), _mapSize, _maxReaders, 20, actualSize, numBlocks, numBlocksToUnwind, values); + } +} diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp index 33b1b35ffd1..bd9c8f56df6 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp @@ -49,6 +49,11 @@ template bool call_lmdb_func(int (*f)(TArgs...), TArgs... ar return error == 0; } +template int call_lmdb_func_with_return(int (*f)(TArgs...), TArgs... args) +{ + return f(args...); +} + template void call_lmdb_func(const std::string& errorString, int (*f)(TArgs...), TArgs... args) { int error = f(args...); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp index 52f2ac8867c..ab4a2b188fb 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.cpp @@ -1,5 +1,6 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" +#include "lmdb.h" #include #include @@ -56,4 +57,5 @@ MDB_env* LMDBEnvironment::underlying() const { return _mdbEnv; } + } // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_read_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_read_transaction.hpp index 3a64d179d95..68e5b56aa2a 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_read_transaction.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_read_transaction.hpp @@ -4,9 +4,11 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" #include #include +#include #include #include @@ -39,6 +41,16 @@ class LMDBTreeReadTransaction : public LMDBTransaction { template bool get_value(T& key, std::vector& data, const LMDBDatabase& db) const; + template + void get_all_values_greater_or_equal_key(const T& key, + std::vector>& data, + const LMDBDatabase& db) const; + + template + void get_all_values_lesser_or_equal_key(const T& key, + std::vector>& data, + const LMDBDatabase& db) const; + bool get_value(std::vector& key, std::vector& data, const LMDBDatabase& db) const; void abort() override; @@ -54,70 +66,7 @@ bool LMDBTreeReadTransaction::get_value(T& key, std::vector& data, cons template bool LMDBTreeReadTransaction::get_value_or_previous(T& key, std::vector& data, const LMDBDatabase& db) const { - std::vector keyBuffer = serialise_key(key); - uint32_t keySize = static_cast(keyBuffer.size()); - MDB_cursor* cursor = nullptr; - call_lmdb_func("mdb_cursor_open", mdb_cursor_open, underlying(), db.underlying(), &cursor); - - MDB_val dbKey; - dbKey.mv_size = keySize; - dbKey.mv_data = (void*)keyBuffer.data(); - - MDB_val dbVal; - - bool success = false; - - // Look for the key >= to that provided - int code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_SET_RANGE); - if (code == 0) { - // we found the key, now determine if it is the exact key - std::vector temp = mdb_val_to_vector(dbKey); - if (keyBuffer == temp) { - // we have the exact key - copy_to_vector(dbVal, data); - success = true; - } else { - // We have a key of the same size but larger value OR a larger size - // either way we now need to find the previous key - code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_PREV); - if (code == 0) { - // We have found a previous key. It could be of the same size but smaller value, or smaller size which - // is equal to not found - if (dbKey.mv_size != keySize) { - // There is no previous key, do nothing - } else { - copy_to_vector(dbVal, data); - deserialise_key(dbKey.mv_data, key); - success = true; - } - } else if (code == MDB_NOTFOUND) { - // There is no previous key, do nothing - } else { - throw_error("get_value_or_previous::mdb_cursor_get", code); - } - } - } else if (code == MDB_NOTFOUND) { - // The key was not found, use the last key in the db - code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_PREV); - if (code == 0) { - // We found the last key, but we need to ensure it is the same size - if (dbKey.mv_size != keySize) { - // The key is not the same size, same as not found, do nothing - } else { - copy_to_vector(dbVal, data); - deserialise_key(dbKey.mv_data, key); - success = true; - } - } else if (code == MDB_NOTFOUND) { - // DB is empty? - } else { - throw_error("get_value_or_previous::mdb_cursor_get", code); - } - } else { - throw_error("get_value_or_previous::mdb_cursor_get", code); - } - call_lmdb_func(mdb_cursor_close, cursor); - return success; + return lmdb_queries::get_value_or_previous(key, data, db, *this); } template @@ -127,83 +76,22 @@ bool LMDBTreeReadTransaction::get_value_or_previous( const LMDBDatabase& db, const std::function&)>& is_valid) const { - std::vector keyBuffer = serialise_key(key); - uint32_t keySize = static_cast(keyBuffer.size()); - MDB_cursor* cursor = nullptr; - call_lmdb_func("mdb_cursor_open", mdb_cursor_open, underlying(), db.underlying(), &cursor); - - MDB_val dbKey; - dbKey.mv_size = keySize; - dbKey.mv_data = (void*)keyBuffer.data(); - - MDB_val dbVal; - - bool success = false; - - // Look for the key >= to that provided - int code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_SET_RANGE); - if (code == 0) { - bool lower = false; - while (!success) { - // We found the key, now determine if it is the exact key - std::vector temp = mdb_val_to_vector(dbKey); - if (keyBuffer == temp || lower) { - // We have the exact key, we need to determine if it is valid - copy_to_vector(dbVal, data); - if (is_valid(data)) { - deserialise_key(dbKey.mv_data, key); - success = true; - // It's valid - break; - } - } else if (dbKey.mv_size < keySize) { - // We have a key of a smaller size, this means what we are looking for doesn't exist - break; - } - // At this point one of the following is true - // 1. We have a key of the same size but larger value - // 2. A larger size - // 3. The exact key but it is not valid - // either way we now need to find the previous key - code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_PREV); - if (code == 0) { - // Success, go round the loop again, this time we will definitely have a lower key - lower = true; - } else if (code == MDB_NOTFOUND) { - // There is no previous key, do nothing - break; - } else { - throw_error("get_value_or_previous::mdb_cursor_get", code); - } - } - } else if (code == MDB_NOTFOUND) { - while (!success) { - // The key was not found, walk down from the end of the db - code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_PREV); - if (code == 0) { - // We found the last key, but we need to ensure it is the same size - if (dbKey.mv_size != keySize) { - // The key is not the same size, same as not found, exit - break; - } - copy_to_vector(dbVal, data); - if (is_valid(data)) { - deserialise_key(dbKey.mv_data, key); - success = true; - // It's valid - break; - } - // we will need to go round again - } else if (code == MDB_NOTFOUND) { - // DB is empty? - } else { - throw_error("get_value_or_previous::mdb_cursor_get", code); - } - } - } else { - throw_error("get_value_or_previous::mdb_cursor_get", code); - } - call_lmdb_func(mdb_cursor_close, cursor); - return success; + return lmdb_queries::get_value_or_previous(key, data, db, is_valid, *this); +} + +template +void LMDBTreeReadTransaction::get_all_values_greater_or_equal_key(const T& key, + std::vector>& data, + const LMDBDatabase& db) const +{ + lmdb_queries::get_all_values_greater_or_equal_key(key, data, db, *this); +} + +template +void LMDBTreeReadTransaction::get_all_values_lesser_or_equal_key(const T& key, + std::vector>& data, + const LMDBDatabase& db) const +{ + lmdb_queries::get_all_values_lesser_or_equal_key(key, data, db, *this); } } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp index bd81a1fb563..c9fdef9a9f1 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp @@ -1,4 +1,5 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "barretenberg/common/serialize.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" @@ -13,6 +14,8 @@ #include #include #include +#include +#include #include namespace bb::crypto::merkle_tree { @@ -38,10 +41,23 @@ int block_key_cmp(const MDB_val* a, const MDB_val* b) return value_cmp(a, b); } +int index_key_cmp(const MDB_val* a, const MDB_val* b) +{ + return value_cmp(a, b); +} + +std::ostream& operator<<(std::ostream& os, const StatsMap& stats) +{ + for (const auto& it : stats) { + os << it.second << std::endl; + } + return os; +} + LMDBTreeStore::LMDBTreeStore(std::string directory, std::string name, uint64_t mapSizeKb, uint64_t maxNumReaders) : _name(std::move(name)) , _directory(std::move(directory)) - , _environment(std::make_shared(_directory, mapSizeKb, 4, maxNumReaders)) + , _environment(std::make_shared(_directory, mapSizeKb, 5, maxNumReaders)) { { @@ -71,6 +87,13 @@ LMDBTreeStore::LMDBTreeStore(std::string directory, std::string name, uint64_t m _environment, tx, _name + std::string("leaf pre-images"), false, false, fr_key_cmp); tx.commit(); } + + { + LMDBDatabaseCreationTransaction tx(_environment); + _leafIndexToKeyDatabase = std::make_unique( + _environment, tx, _name + std::string("leaf keys"), false, false, index_key_cmp); + tx.commit(); + } } LMDBTreeStore::WriteTransaction::Ptr LMDBTreeStore::create_write_transaction() const @@ -83,6 +106,24 @@ LMDBTreeStore::ReadTransaction::Ptr LMDBTreeStore::create_read_transaction() return std::make_unique(_environment); } +void LMDBTreeStore::get_stats(StatsMap& stats, ReadTransaction& tx) +{ + + MDB_stat stat; + MDB_envinfo info; + call_lmdb_func(mdb_env_info, _environment->underlying(), &info); + call_lmdb_func(mdb_stat, tx.underlying(), _blockDatabase->underlying(), &stat); + stats["blocks"] = DBStats("block", info, stat); + call_lmdb_func(mdb_stat, tx.underlying(), _leafHashToPreImageDatabase->underlying(), &stat); + stats["leaf preimages"] = DBStats("leaf preimages", info, stat); + call_lmdb_func(mdb_stat, tx.underlying(), _leafValueToIndexDatabase->underlying(), &stat); + stats["leaf indices"] = DBStats("leaf indices", info, stat); + call_lmdb_func(mdb_stat, tx.underlying(), _nodeDatabase->underlying(), &stat); + stats["nodes"] = DBStats("nodes", info, stat); + call_lmdb_func(mdb_stat, tx.underlying(), _leafIndexToKeyDatabase->underlying(), &stat); + stats["leaf keys"] = DBStats("leaf keys", info, stat); +} + void LMDBTreeStore::write_block_data(uint64_t blockNumber, const BlockPayload& blockData, LMDBTreeStore::WriteTransaction& tx) @@ -94,6 +135,12 @@ void LMDBTreeStore::write_block_data(uint64_t blockNumber, tx.put_value(key, encoded, *_blockDatabase); } +void LMDBTreeStore::delete_block_data(uint64_t blockNumber, LMDBTreeStore::WriteTransaction& tx) +{ + BlockMetaKeyType key(blockNumber); + tx.delete_value(key, *_blockDatabase); +} + bool LMDBTreeStore::read_block_data(uint64_t blockNumber, BlockPayload& blockData, LMDBTreeStore::ReadTransaction& tx) { BlockMetaKeyType key(blockNumber); @@ -125,44 +172,67 @@ bool LMDBTreeStore::read_meta_data(TreeMeta& metaData, LMDBTreeStore::ReadTransa return success; } -bool LMDBTreeStore::read_leaf_indices(const fr& leafValue, Indices& indices, LMDBTreeStore::ReadTransaction& tx) -{ - FrKeyType key(leafValue); - std::vector data; - bool success = tx.get_value(key, data, *_leafValueToIndexDatabase); - if (success) { - msgpack::unpack((const char*)data.data(), data.size()).get().convert(indices); - } - return success; -} - void LMDBTreeStore::write_leaf_indices(const fr& leafValue, const Indices& indices, LMDBTreeStore::WriteTransaction& tx) { msgpack::sbuffer buffer; msgpack::pack(buffer, indices); std::vector encoded(buffer.data(), buffer.data() + buffer.size()); FrKeyType key(leafValue); + // std::cout << "Writing leaf indices by key " << key << std::endl; tx.put_value(key, encoded, *_leafValueToIndexDatabase); } -bool LMDBTreeStore::read_node(const fr& nodeHash, NodePayload& nodeData, ReadTransaction& tx) +void LMDBTreeStore::delete_leaf_indices(const fr& leafValue, LMDBTreeStore::WriteTransaction& tx) { - FrKeyType key(nodeHash); - std::vector data; - bool success = tx.get_value(key, data, *_nodeDatabase); - if (success) { - msgpack::unpack((const char*)data.data(), data.size()).get().convert(nodeData); + FrKeyType key(leafValue); + // std::cout << "Deleting leaf indices by key " << key << std::endl; + tx.delete_value(key, *_leafValueToIndexDatabase); +} + +void LMDBTreeStore::increment_node_reference_count(const fr& nodeHash, WriteTransaction& tx) +{ + NodePayload nodePayload; + bool success = get_node_data(nodeHash, nodePayload, tx); + if (!success) { + throw std::runtime_error("Failed to find node when attempting to increases reference count"); } - return success; + ++nodePayload.ref; + // std::cout << "Incrementing siblng at " << nodeHash << ", to " << nodePayload.ref << std::endl; + write_node(nodeHash, nodePayload, tx); } -void LMDBTreeStore::write_node(const fr& nodeHash, const NodePayload& nodeData, WriteTransaction& tx) +void LMDBTreeStore::set_or_increment_node_reference_count(const fr& nodeHash, + NodePayload& nodeData, + WriteTransaction& tx) { - msgpack::sbuffer buffer; - msgpack::pack(buffer, nodeData); - std::vector encoded(buffer.data(), buffer.data() + buffer.size()); - FrKeyType key(nodeHash); - tx.put_value(key, encoded, *_nodeDatabase); + // Set to zero here and enrich from DB if present + nodeData.ref = 0; + get_node_data(nodeHash, nodeData, tx); + // Increment now to the correct value + ++nodeData.ref; + // std::cout << "Setting node at " << nodeHash << ", to " << nodeData.ref << std::endl; + write_node(nodeHash, nodeData, tx); +} + +void LMDBTreeStore::decrement_node_reference_count(const fr& nodeHash, NodePayload& nodeData, WriteTransaction& tx) +{ + bool success = get_node_data(nodeHash, nodeData, tx); + if (!success) { + throw std::runtime_error("Failed to find node when attempting to increases reference count"); + } + if (--nodeData.ref == 0) { + // std::cout << "Deleting node at " << nodeHash << std::endl; + tx.delete_value(nodeHash, *_nodeDatabase); + return; + } + // std::cout << "Updating node at " << nodeHash << " ref is now " << nodeData.ref << std::endl; + write_node(nodeHash, nodeData, tx); +} + +void LMDBTreeStore::delete_leaf_by_hash(const fr& leafHash, WriteTransaction& tx) +{ + FrKeyType key(leafHash); + tx.delete_value(key, *_leafHashToPreImageDatabase); } fr LMDBTreeStore::find_low_leaf(const fr& leafValue, @@ -187,4 +257,43 @@ fr LMDBTreeStore::find_low_leaf(const fr& leafValue, return key; } +void LMDBTreeStore::write_leaf_key_by_index(const fr& leafKey, const index_t& index, WriteTransaction& tx) +{ + std::vector data = to_buffer(leafKey); + LeafIndexKeyType key(index); + tx.put_value(key, data, *_leafIndexToKeyDatabase); +} + +void LMDBTreeStore::delete_all_leaf_keys_after_or_equal_index(const index_t& index, WriteTransaction& tx) +{ + LeafIndexKeyType key(index); + tx.delete_all_values_greater_or_equal_key(key, *_leafIndexToKeyDatabase); +} + +void LMDBTreeStore::delete_all_leaf_keys_before_or_equal_index(const index_t& index, WriteTransaction& tx) +{ + LeafIndexKeyType key(index); + tx.delete_all_values_lesser_or_equal_key(key, *_leafIndexToKeyDatabase); +} + +bool LMDBTreeStore::read_node(const fr& nodeHash, NodePayload& nodeData, ReadTransaction& tx) +{ + FrKeyType key(nodeHash); + std::vector data; + bool success = tx.get_value(key, data, *_nodeDatabase); + if (success) { + msgpack::unpack((const char*)data.data(), data.size()).get().convert(nodeData); + } + return success; +} + +void LMDBTreeStore::write_node(const fr& nodeHash, const NodePayload& nodeData, WriteTransaction& tx) +{ + msgpack::sbuffer buffer; + msgpack::pack(buffer, nodeData); + std::vector encoded(buffer.data(), buffer.data() + buffer.size()); + FrKeyType key(nodeHash); + tx.put_value(key, encoded, *_nodeDatabase); +} + } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp index 08d32f8ad6b..e1cbeddb4ff 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp @@ -1,5 +1,7 @@ #pragma once +#include "barretenberg/common/serialize.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_read_transaction.hpp" @@ -8,9 +10,14 @@ #include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/serialize/msgpack.hpp" +#include "lmdb.h" #include #include #include +#include +#include +#include +#include namespace bb::crypto::merkle_tree { @@ -56,6 +63,50 @@ struct NodePayload { } }; +struct DBStats { + std::string name; + uint64_t mapSize; + uint64_t numDataItems; + uint64_t totalUsedSize; + + DBStats() = default; + DBStats(const DBStats& other) = default; + DBStats(DBStats&& other) noexcept + : name(std::move(other.name)) + , mapSize(other.mapSize) + , numDataItems(other.numDataItems) + , totalUsedSize(other.totalUsedSize) + {} + ~DBStats() = default; + DBStats(std::string name, MDB_envinfo& env, MDB_stat& stat) + : name(std::move(name)) + , mapSize(env.me_mapsize) + , numDataItems(stat.ms_entries) + , totalUsedSize(stat.ms_psize * (stat.ms_branch_pages + stat.ms_leaf_pages + stat.ms_overflow_pages)) + {} + + MSGPACK_FIELDS(name, mapSize, numDataItems, totalUsedSize) + + bool operator==(const DBStats& other) const + { + return name == other.name && mapSize == other.mapSize && numDataItems == other.numDataItems && + totalUsedSize == other.totalUsedSize; + } + + DBStats& operator=(const DBStats& other) = default; + + friend std::ostream& operator<<(std::ostream& os, const DBStats& stats) + { + os << "DB " << stats.name << ", map size: " << stats.mapSize << ", num items: " << stats.numDataItems + << ", total used size: " << stats.totalUsedSize; + return os; + } +}; + +using StatsMap = std::unordered_map; + +std::ostream& operator<<(std::ostream& os, const StatsMap& stats); + /** * Creates an abstraction against a collection of LMDB databases within a single environment used to store merkle tree * data @@ -77,29 +128,58 @@ class LMDBTreeStore { WriteTransaction::Ptr create_write_transaction() const; ReadTransaction::Ptr create_read_transaction(); + void get_stats(StatsMap& stats, ReadTransaction& tx); + void write_block_data(uint64_t blockNumber, const BlockPayload& blockData, WriteTransaction& tx); bool read_block_data(uint64_t blockNumber, BlockPayload& blockData, ReadTransaction& tx); + void delete_block_data(uint64_t blockNumber, WriteTransaction& tx); + void write_meta_data(const TreeMeta& metaData, WriteTransaction& tx); bool read_meta_data(TreeMeta& metaData, ReadTransaction& tx); - bool read_leaf_indices(const fr& leafValue, Indices& indices, ReadTransaction& tx); + template bool read_leaf_indices(const fr& leafValue, Indices& indices, TxType& tx); fr find_low_leaf(const fr& leafValue, Indices& indices, std::optional sizeLimit, ReadTransaction& tx); void write_leaf_indices(const fr& leafValue, const Indices& indices, WriteTransaction& tx); + void delete_leaf_indices(const fr& leafValue, WriteTransaction& tx); + bool read_node(const fr& nodeHash, NodePayload& nodeData, ReadTransaction& tx); void write_node(const fr& nodeHash, const NodePayload& nodeData, WriteTransaction& tx); - template bool read_leaf_by_hash(const fr& leafHash, LeafType& leafData, ReadTransaction& tx); + void increment_node_reference_count(const fr& nodeHash, WriteTransaction& tx); + + void set_or_increment_node_reference_count(const fr& nodeHash, NodePayload& nodeData, WriteTransaction& tx); + + void decrement_node_reference_count(const fr& nodeHash, NodePayload& nodeData, WriteTransaction& tx); + + template + bool read_leaf_by_hash(const fr& leafHash, LeafType& leafData, TxType& tx); template void write_leaf_by_hash(const fr& leafHash, const LeafType& leafData, WriteTransaction& tx); + void delete_leaf_by_hash(const fr& leafHash, WriteTransaction& tx); + + void write_leaf_key_by_index(const fr& leafKey, const index_t& index, WriteTransaction& tx); + + template bool read_leaf_key_by_index(const index_t& index, fr& leafKey, TxType& tx); + + template + void read_all_leaf_keys_after_or_equal_index(const index_t& index, std::vector& leafKeys, TxType& tx); + + void delete_all_leaf_keys_after_or_equal_index(const index_t& index, WriteTransaction& tx); + + template + void read_all_leaf_keys_before_or_equal_index(const index_t& index, std::vector& leafKeys, TxType& tx); + + void delete_all_leaf_keys_before_or_equal_index(const index_t& index, WriteTransaction& tx); + private: std::string _name; std::string _directory; @@ -108,14 +188,28 @@ class LMDBTreeStore { LMDBDatabase::Ptr _nodeDatabase; LMDBDatabase::Ptr _leafValueToIndexDatabase; LMDBDatabase::Ptr _leafHashToPreImageDatabase; + LMDBDatabase::Ptr _leafIndexToKeyDatabase; + + template bool get_node_data(const fr& nodeHash, NodePayload& nodeData, TxType& tx); }; -template -bool LMDBTreeStore::read_leaf_by_hash(const fr& leafHash, LeafType& leafData, ReadTransaction& tx) +template bool LMDBTreeStore::read_leaf_indices(const fr& leafValue, Indices& indices, TxType& tx) +{ + FrKeyType key(leafValue); + std::vector data; + bool success = tx.template get_value(key, data, *_leafValueToIndexDatabase); + if (success) { + msgpack::unpack((const char*)data.data(), data.size()).get().convert(indices); + } + return success; +} + +template +bool LMDBTreeStore::read_leaf_by_hash(const fr& leafHash, LeafType& leafData, TxType& tx) { FrKeyType key(leafHash); std::vector data; - bool success = tx.get_value(key, data, *_leafHashToPreImageDatabase); + bool success = tx.template get_value(key, data, *_leafHashToPreImageDatabase); if (success) { msgpack::unpack((const char*)data.data(), data.size()).get().convert(leafData); } @@ -131,4 +225,54 @@ void LMDBTreeStore::write_leaf_by_hash(const fr& leafHash, const LeafType& leafD FrKeyType key(leafHash); tx.put_value(key, encoded, *_leafHashToPreImageDatabase); } + +template bool LMDBTreeStore::get_node_data(const fr& nodeHash, NodePayload& nodeData, TxType& tx) +{ + FrKeyType key(nodeHash); + std::vector data; + bool success = tx.template get_value(key, data, *_nodeDatabase); + if (success) { + msgpack::unpack((const char*)data.data(), data.size()).get().convert(nodeData); + } + return success; +} + +template bool LMDBTreeStore::read_leaf_key_by_index(const index_t& index, fr& leafKey, TxType& tx) +{ + LeafIndexKeyType key(index); + std::vector data; + bool success = tx.template get_value(key, data, *_leafIndexToKeyDatabase); + if (success) { + leafKey = from_buffer(data); + } + return success; +} + +template +void LMDBTreeStore::read_all_leaf_keys_after_or_equal_index(const index_t& index, + std::vector& leafKeys, + TxType& tx) +{ + LeafIndexKeyType key(index); + std::vector> values; + tx.get_all_values_greater_or_equal_key(key, values, *_leafIndexToKeyDatabase); + for (const auto& value : values) { + fr leafKey = from_buffer(value); + leafKeys.push_back(leafKey); + } +} + +template +void LMDBTreeStore::read_all_leaf_keys_before_or_equal_index(const index_t& index, + std::vector& leafKeys, + TxType& tx) +{ + LeafIndexKeyType key(index); + std::vector> values; + tx.get_all_values_lesser_or_equal_key(key, values, *_leafIndexToKeyDatabase); + for (const auto& value : values) { + fr leafKey = from_buffer(value); + leafKeys.push_back(leafKey); + } +} } // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp index 727f40e6f0e..f04ec2a0f39 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp @@ -82,7 +82,7 @@ TEST_F(LMDBTreeStoreTest, can_write_and_read_meta_data) metaData.initialRoot = VALUES[1]; metaData.root = VALUES[2]; metaData.depth = 40; - metaData.finalisedBlockHeight = 87; + metaData.oldestHistoricBlock = 87; metaData.unfinalisedBlockHeight = 95; metaData.name = "Note hash tree"; metaData.size = 60; @@ -177,3 +177,373 @@ TEST_F(LMDBTreeStoreTest, can_write_and_read_leaves_by_hash) EXPECT_FALSE(success); } } + +TEST_F(LMDBTreeStoreTest, can_read_write_key_by_index) +{ + bb::fr leafKey = VALUES[0]; + index_t leafIndex = 45; + LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); + { + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + store.write_leaf_key_by_index(leafKey, leafIndex, *transaction); + transaction->commit(); + } + + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + bb::fr readBack; + bool success = store.read_leaf_key_by_index(leafIndex, readBack, *transaction); + EXPECT_TRUE(success); + EXPECT_EQ(readBack, leafKey); + + success = store.read_leaf_key_by_index(leafIndex + 1, readBack, *transaction); + EXPECT_FALSE(success); + } +} + +TEST_F(LMDBTreeStoreTest, can_retrieve_all_keys_greater_than_index) +{ + std::vector values = create_values(1024); + index_t leafIndexStart = 45; + LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); + { + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + for (uint32_t i = 0; i < values.size(); i++) { + store.write_leaf_key_by_index(values[i], i + leafIndexStart, *transaction); + } + transaction->commit(); + } + + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + // Retrieve all but the first 150 keys + uint32_t offset = 150; + std::vector leafKeys; + store.read_all_leaf_keys_after_or_equal_index(leafIndexStart + offset, leafKeys, *transaction); + EXPECT_EQ(leafKeys.size(), values.size() - offset); + for (uint32_t i = offset; i < leafKeys.size(); i++) { + EXPECT_EQ(leafKeys[i], values[i + offset]); + } + } + + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + // Retrieve all keys + uint32_t offset = 0; + std::vector leafKeys; + store.read_all_leaf_keys_after_or_equal_index(leafIndexStart + offset, leafKeys, *transaction); + EXPECT_EQ(leafKeys.size(), values.size() - offset); + for (uint32_t i = offset; i < leafKeys.size(); i++) { + EXPECT_EQ(leafKeys[i], values[i + offset]); + } + } + + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + // Retrieve no keys + uint32_t offset = 10000; + std::vector leafKeys; + store.read_all_leaf_keys_after_or_equal_index(leafIndexStart + offset, leafKeys, *transaction); + EXPECT_EQ(leafKeys.size(), 0); + } +} + +TEST_F(LMDBTreeStoreTest, can_delete_all_keys_greater_than_index) +{ + std::vector values = create_values(1024); + index_t leafIndexStart = 45; + uint32_t deleteFromIndex = 150; + LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); + { + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + for (uint32_t i = 0; i < values.size(); i++) { + store.write_leaf_key_by_index(values[i], i + leafIndexStart, *transaction); + } + transaction->commit(); + } + + { + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + store.delete_all_leaf_keys_after_or_equal_index(deleteFromIndex, *transaction); + transaction->commit(); + } + + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + std::vector leafKeys; + store.read_all_leaf_keys_after_or_equal_index(leafIndexStart, leafKeys, *transaction); + EXPECT_EQ(leafKeys.size(), deleteFromIndex - leafIndexStart); + for (uint32_t i = 0; i < leafKeys.size(); i++) { + EXPECT_EQ(leafKeys[i], values[i]); + } + } + + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + for (uint32_t i = 0; i < 1024 + leafIndexStart; i++) { + bb::fr leafKey; + bool success = store.read_leaf_key_by_index(i, leafKey, *transaction); + EXPECT_EQ(success, (i >= leafIndexStart && (i < deleteFromIndex))); + if (success) { + EXPECT_EQ(leafKey, values[i - leafIndexStart]); + } + } + } +} + +TEST_F(LMDBTreeStoreTest, can_delete_all_keys_less_than_index) +{ + std::vector values = create_values(1024); + index_t leafIndexStart = 45; + uint32_t deleteFromIndex = 150; + LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); + { + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + for (uint32_t i = 0; i < values.size(); i++) { + store.write_leaf_key_by_index(values[i], i + leafIndexStart, *transaction); + } + transaction->commit(); + } + + { + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + store.delete_all_leaf_keys_before_or_equal_index(deleteFromIndex, *transaction); + transaction->commit(); + } + + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + std::vector leafKeys; + store.read_all_leaf_keys_before_or_equal_index(leafIndexStart + 1023, leafKeys, *transaction); + EXPECT_EQ(leafKeys.size(), 1024 - (deleteFromIndex - leafIndexStart + 1)); + for (uint32_t i = 0; i < leafKeys.size(); i++) { + EXPECT_EQ(leafKeys[i], values[1023 - i]); + } + } + + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + for (uint32_t i = 0; i < 1024 + leafIndexStart; i++) { + bb::fr leafKey; + bool success = store.read_leaf_key_by_index(i, leafKey, *transaction); + EXPECT_EQ(success, (i > deleteFromIndex && (i <= leafIndexStart + 1023))); + if (success) { + EXPECT_EQ(leafKey, values[i - leafIndexStart]); + } + } + } +} + +TEST_F(LMDBTreeStoreTest, can_delete_all_keys_greater_than) +{ + std::vector values = create_values(1024); + index_t leafIndexStart = 45; + uint32_t deleteFromIndex = 0; + LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); + { + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + for (uint32_t i = 0; i < values.size(); i++) { + store.write_leaf_key_by_index(values[i], i + leafIndexStart, *transaction); + } + transaction->commit(); + } + + { + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + store.delete_all_leaf_keys_after_or_equal_index(deleteFromIndex, *transaction); + transaction->commit(); + } + + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + std::vector leafKeys; + store.read_all_leaf_keys_after_or_equal_index(leafIndexStart, leafKeys, *transaction); + EXPECT_EQ(leafKeys.size(), 0); + } + + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + std::vector leafKeys; + store.read_all_leaf_keys_after_or_equal_index(0, leafKeys, *transaction); + EXPECT_EQ(leafKeys.size(), 0); + } + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + std::vector leafKeys; + store.read_all_leaf_keys_after_or_equal_index(10000, leafKeys, *transaction); + EXPECT_EQ(leafKeys.size(), 0); + } +} + +TEST_F(LMDBTreeStoreTest, can_delete_all_keys_less_than) +{ + std::vector values = create_values(1024); + index_t leafIndexStart = 45; + uint32_t deleteFromIndex = 2000; + LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); + { + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + for (uint32_t i = 0; i < values.size(); i++) { + store.write_leaf_key_by_index(values[i], i + leafIndexStart, *transaction); + } + transaction->commit(); + } + + { + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + store.delete_all_leaf_keys_before_or_equal_index(deleteFromIndex, *transaction); + transaction->commit(); + } + + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + std::vector leafKeys; + store.read_all_leaf_keys_before_or_equal_index(leafIndexStart + 1023, leafKeys, *transaction); + EXPECT_EQ(leafKeys.size(), 0); + } + + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + std::vector leafKeys; + store.read_all_leaf_keys_before_or_equal_index(2000, leafKeys, *transaction); + EXPECT_EQ(leafKeys.size(), 0); + } + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + std::vector leafKeys; + store.read_all_leaf_keys_before_or_equal_index(10, leafKeys, *transaction); + EXPECT_EQ(leafKeys.size(), 0); + } +} + +TEST_F(LMDBTreeStoreTest, can_delete_no_keys_greater_than) +{ + std::vector values = create_values(1024); + index_t leafIndexStart = 45; + uint32_t deleteFromIndex = 2000; + LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); + { + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + for (uint32_t i = 0; i < values.size(); i++) { + store.write_leaf_key_by_index(values[i], i + leafIndexStart, *transaction); + } + transaction->commit(); + } + + { + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + store.delete_all_leaf_keys_after_or_equal_index(deleteFromIndex, *transaction); + transaction->commit(); + } + + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + std::vector leafKeys; + store.read_all_leaf_keys_after_or_equal_index(leafIndexStart, leafKeys, *transaction); + EXPECT_EQ(leafKeys.size(), 1024); + for (uint32_t i = 0; i < leafKeys.size(); i++) { + EXPECT_EQ(leafKeys[i], values[i]); + } + } + + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + std::vector leafKeys; + store.read_all_leaf_keys_after_or_equal_index(0, leafKeys, *transaction); + EXPECT_EQ(leafKeys.size(), 1024); + for (uint32_t i = 0; i < leafKeys.size(); i++) { + EXPECT_EQ(leafKeys[i], values[i]); + } + } + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + std::vector leafKeys; + store.read_all_leaf_keys_after_or_equal_index(10000, leafKeys, *transaction); + EXPECT_EQ(leafKeys.size(), 0); + } +} + +TEST_F(LMDBTreeStoreTest, can_delete_no_keys_less_than) +{ + std::vector values = create_values(1024); + index_t leafIndexStart = 45; + uint32_t deleteFromIndex = 20; + LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); + { + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + for (uint32_t i = 0; i < values.size(); i++) { + store.write_leaf_key_by_index(values[i], i + leafIndexStart, *transaction); + } + transaction->commit(); + } + + { + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + store.delete_all_leaf_keys_before_or_equal_index(deleteFromIndex, *transaction); + transaction->commit(); + } + + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + std::vector leafKeys; + store.read_all_leaf_keys_before_or_equal_index(leafIndexStart + 1023, leafKeys, *transaction); + EXPECT_EQ(leafKeys.size(), 1024); + for (uint32_t i = 0; i < leafKeys.size(); i++) { + EXPECT_EQ(leafKeys[i], values[1023 - i]); + } + } + + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + std::vector leafKeys; + store.read_all_leaf_keys_before_or_equal_index(2000, leafKeys, *transaction); + EXPECT_EQ(leafKeys.size(), 1024); + for (uint32_t i = 0; i < leafKeys.size(); i++) { + EXPECT_EQ(leafKeys[i], values[1023 - i]); + } + } + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + std::vector leafKeys; + store.read_all_leaf_keys_before_or_equal_index(10, leafKeys, *transaction); + EXPECT_EQ(leafKeys.size(), 0); + } +} + +TEST_F(LMDBTreeStoreTest, can_retrieve_all_keys_when_none_are_present) +{ + std::vector values = create_values(1024); + LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); + + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + std::vector leafKeys; + store.read_all_leaf_keys_after_or_equal_index(0, leafKeys, *transaction); + EXPECT_EQ(leafKeys.size(), 0); + } + + { + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + std::vector leafKeys; + store.read_all_leaf_keys_before_or_equal_index(0, leafKeys, *transaction); + EXPECT_EQ(leafKeys.size(), 0); + } +} + +TEST_F(LMDBTreeStoreTest, can_delete_all_keys_when_none_are_present) +{ + std::vector values = create_values(1024); + LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); + + { + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + store.delete_all_leaf_keys_after_or_equal_index(0, *transaction); + transaction->commit(); + } + + { + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + store.delete_all_leaf_keys_before_or_equal_index(0, *transaction); + transaction->commit(); + } +} diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_write_transaction.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_write_transaction.cpp index c3813539eba..5065575ac69 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_write_transaction.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_write_transaction.cpp @@ -5,6 +5,7 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp" +#include "lmdb.h" #include namespace bb::crypto::merkle_tree { @@ -35,6 +36,22 @@ void LMDBTreeWriteTransaction::try_abort() LMDBTransaction::abort(); } +bool LMDBTreeWriteTransaction::get_value(std::vector& key, + std::vector& data, + const LMDBDatabase& db) const +{ + MDB_val dbKey; + dbKey.mv_size = key.size(); + dbKey.mv_data = (void*)key.data(); + + MDB_val dbVal; + if (!call_lmdb_func(mdb_get, underlying(), db.underlying(), &dbKey, &dbVal)) { + return false; + } + copy_to_vector(dbVal, data); + return true; +} + void LMDBTreeWriteTransaction::put_value(std::vector& key, std::vector& data, const LMDBDatabase& db) { MDB_val dbKey; @@ -46,4 +63,17 @@ void LMDBTreeWriteTransaction::put_value(std::vector& key, std::vector< dbVal.mv_data = (void*)data.data(); call_lmdb_func("mdb_put", mdb_put, underlying(), db.underlying(), &dbKey, &dbVal, 0U); } + +void LMDBTreeWriteTransaction::delete_value(std::vector& key, const LMDBDatabase& db) +{ + MDB_val dbKey; + dbKey.mv_size = key.size(); + dbKey.mv_data = (void*)key.data(); + + MDB_val* dbVal = nullptr; + int code = call_lmdb_func_with_return(mdb_del, underlying(), db.underlying(), &dbKey, dbVal); + if (code != 0 && code != MDB_NOTFOUND) { + throw_error("mdb_del", code); + } +} } // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_write_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_write_transaction.hpp index 77fa887d785..eb7a2f09df1 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_write_transaction.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_write_transaction.hpp @@ -4,7 +4,11 @@ #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" +#include "lmdb.h" +#include +#include namespace bb::crypto::merkle_tree { @@ -30,15 +34,80 @@ class LMDBTreeWriteTransaction : public LMDBTransaction { void put_value(std::vector& key, std::vector& data, const LMDBDatabase& db); + template void delete_value(T& key, const LMDBDatabase& db); + + void delete_value(std::vector& key, const LMDBDatabase& db); + + // There are a the following two 'getters' here copied from LMDBTreeReadTransaction + // This could be rationalised to prevent the duplication + template bool get_value(T& key, std::vector& data, const LMDBDatabase& db) const; + + template + void get_all_values_greater_or_equal_key(const T& key, + std::vector>& data, + const LMDBDatabase& db) const; + + template void delete_all_values_greater_or_equal_key(const T& key, const LMDBDatabase& db) const; + + template + void get_all_values_lesser_or_equal_key(const T& key, + std::vector>& data, + const LMDBDatabase& db) const; + + template void delete_all_values_lesser_or_equal_key(const T& key, const LMDBDatabase& db) const; + + bool get_value(std::vector& key, std::vector& data, const LMDBDatabase& db) const; + void commit(); void try_abort(); }; +template +bool LMDBTreeWriteTransaction::get_value(T& key, std::vector& data, const LMDBDatabase& db) const +{ + std::vector keyBuffer = serialise_key(key); + return get_value(keyBuffer, data, db); +} + template void LMDBTreeWriteTransaction::put_value(T& key, std::vector& data, const LMDBDatabase& db) { std::vector keyBuffer = serialise_key(key); put_value(keyBuffer, data, db); } + +template void LMDBTreeWriteTransaction::delete_value(T& key, const LMDBDatabase& db) +{ + std::vector keyBuffer = serialise_key(key); + delete_value(keyBuffer, db); +} + +template +void LMDBTreeWriteTransaction::get_all_values_greater_or_equal_key(const T& key, + std::vector>& data, + const LMDBDatabase& db) const +{ + lmdb_queries::get_all_values_greater_or_equal_key(key, data, db, *this); +} + +template +void LMDBTreeWriteTransaction::delete_all_values_greater_or_equal_key(const T& key, const LMDBDatabase& db) const +{ + lmdb_queries::delete_all_values_greater_or_equal_key(key, db, *this); +} + +template +void LMDBTreeWriteTransaction::get_all_values_lesser_or_equal_key(const T& key, + std::vector>& data, + const LMDBDatabase& db) const +{ + lmdb_queries::get_all_values_lesser_or_equal_key(key, data, db, *this); +} + +template +void LMDBTreeWriteTransaction::delete_all_values_lesser_or_equal_key(const T& key, const LMDBDatabase& db) const +{ + lmdb_queries::delete_all_values_lesser_or_equal_key(key, db, *this); +} } // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp new file mode 100644 index 00000000000..2c9efc21e0e --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp @@ -0,0 +1,397 @@ +#pragma once + +#include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_database.hpp" +#include "lmdb.h" +#include +#include +#include + +namespace bb::crypto::merkle_tree::lmdb_queries { +template +bool get_value_or_previous(TKey& key, std::vector& data, const LMDBDatabase& db, const TxType& tx) +{ + std::vector keyBuffer = serialise_key(key); + uint32_t keySize = static_cast(keyBuffer.size()); + MDB_cursor* cursor = nullptr; + bool success = false; + call_lmdb_func("mdb_cursor_open", mdb_cursor_open, tx.underlying(), db.underlying(), &cursor); + + try { + MDB_val dbKey; + dbKey.mv_size = keySize; + dbKey.mv_data = (void*)keyBuffer.data(); + MDB_val dbVal; + + // Look for the key >= to that provided + int code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_SET_RANGE); + if (code == 0) { + // we found the key, now determine if it is the exact key + std::vector temp = mdb_val_to_vector(dbKey); + if (keyBuffer == temp) { + // we have the exact key + copy_to_vector(dbVal, data); + success = true; + } else { + // We have a key of the same size but larger value OR a larger size + // either way we now need to find the previous key + code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_PREV); + if (code == 0) { + // We have found a previous key. It could be of the same size but smaller value, or smaller size + // which is equal to not found + if (dbKey.mv_size != keySize) { + // There is no previous key, do nothing + } else { + copy_to_vector(dbVal, data); + deserialise_key(dbKey.mv_data, key); + success = true; + } + } else if (code == MDB_NOTFOUND) { + // There is no previous key, do nothing + } else { + throw_error("get_value_or_previous::mdb_cursor_get", code); + } + } + } else if (code == MDB_NOTFOUND) { + // The key was not found, use the last key in the db + code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_PREV); + if (code == 0) { + // We found the last key, but we need to ensure it is the same size + if (dbKey.mv_size != keySize) { + // The key is not the same size, same as not found, do nothing + } else { + copy_to_vector(dbVal, data); + deserialise_key(dbKey.mv_data, key); + success = true; + } + } else if (code == MDB_NOTFOUND) { + // DB is empty? + } else { + throw_error("get_value_or_previous::mdb_cursor_get", code); + } + } else { + throw_error("get_value_or_previous::mdb_cursor_get", code); + } + } catch (std::exception& e) { + call_lmdb_func(mdb_cursor_close, cursor); + throw; + } + call_lmdb_func(mdb_cursor_close, cursor); + return success; +} + +template +bool get_value_or_previous(TKey& key, + std::vector& data, + const LMDBDatabase& db, + const std::function&)>& is_valid, + const TxType& tx) +{ + std::vector keyBuffer = serialise_key(key); + uint32_t keySize = static_cast(keyBuffer.size()); + MDB_cursor* cursor = nullptr; + bool success = false; + call_lmdb_func("mdb_cursor_open", mdb_cursor_open, tx.underlying(), db.underlying(), &cursor); + + try { + MDB_val dbKey; + dbKey.mv_size = keySize; + dbKey.mv_data = (void*)keyBuffer.data(); + + MDB_val dbVal; + // Look for the key >= to that provided + int code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_SET_RANGE); + if (code == 0) { + bool lower = false; + while (!success) { + // We found the key, now determine if it is the exact key + std::vector temp = mdb_val_to_vector(dbKey); + if (keyBuffer == temp || lower) { + // We have the exact key, we need to determine if it is valid + copy_to_vector(dbVal, data); + if (is_valid(data)) { + deserialise_key(dbKey.mv_data, key); + success = true; + // It's valid + break; + } + } else if (dbKey.mv_size < keySize) { + // We have a key of a smaller size, this means what we are looking for doesn't exist + break; + } + // At this point one of the following is true + // 1. We have a key of the same size but larger value + // 2. A larger size + // 3. The exact key but it is not valid + // either way we now need to find the previous key + code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_PREV); + if (code == 0) { + // Success, go round the loop again, this time we will definitely have a lower key + lower = true; + } else if (code == MDB_NOTFOUND) { + // There is no previous key, do nothing + break; + } else { + throw_error("get_value_or_previous::mdb_cursor_get", code); + } + } + } else if (code == MDB_NOTFOUND) { + while (!success) { + // The key was not found, walk down from the end of the db + code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_PREV); + if (code == 0) { + // We found the last key, but we need to ensure it is the same size + if (dbKey.mv_size != keySize) { + // The key is not the same size, same as not found, exit + break; + } + copy_to_vector(dbVal, data); + if (is_valid(data)) { + deserialise_key(dbKey.mv_data, key); + success = true; + // It's valid + break; + } + // we will need to go round again + } else if (code == MDB_NOTFOUND) { + // DB is empty? + } else { + throw_error("get_value_or_previous::mdb_cursor_get", code); + } + } + } else { + throw_error("get_value_or_previous::mdb_cursor_get", code); + } + } catch (std::exception& e) { + call_lmdb_func(mdb_cursor_close, cursor); + throw; + } + + call_lmdb_func(mdb_cursor_close, cursor); + return success; +} + +template +void get_all_values_greater_or_equal_key(const TKey& key, + std::vector>& data, + const LMDBDatabase& db, + const TxType& tx) +{ + std::vector keyBuffer = serialise_key(key); + uint32_t keySize = static_cast(keyBuffer.size()); + MDB_cursor* cursor = nullptr; + call_lmdb_func("mdb_cursor_open", mdb_cursor_open, tx.underlying(), db.underlying(), &cursor); + + try { + MDB_val dbKey; + dbKey.mv_size = keySize; + dbKey.mv_data = (void*)keyBuffer.data(); + + MDB_val dbVal; + // Look for the key >= to that provided + int code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_SET_RANGE); + while (true) { + if (code == 0) { + // found a key >= our key. if it is not the same size, it must be out of range for what we are looking + // for, this means no more data available + if (keySize != dbKey.mv_size) { + break; + } + // this is data that we need to extract + std::vector temp; + copy_to_vector(dbVal, temp); + data.emplace_back(temp); + + // move to the next key + code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_NEXT); + } else if (code == MDB_NOTFOUND) { + // no more data to extract + break; + } else { + throw_error("get_all_values_greater_or_equal_key::mdb_cursor_get", code); + } + } + } catch (std::exception& e) { + call_lmdb_func(mdb_cursor_close, cursor); + throw; + } + call_lmdb_func(mdb_cursor_close, cursor); +} + +template +void delete_all_values_greater_or_equal_key(const TKey& key, const LMDBDatabase& db, const TxType& tx) +{ + std::vector keyBuffer = serialise_key(key); + uint32_t keySize = static_cast(keyBuffer.size()); + MDB_cursor* cursor = nullptr; + call_lmdb_func("mdb_cursor_open", mdb_cursor_open, tx.underlying(), db.underlying(), &cursor); + + try { + MDB_val dbKey; + dbKey.mv_size = keySize; + dbKey.mv_data = (void*)keyBuffer.data(); + + MDB_val dbVal; + // Look for the key >= to that provided + int code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_SET_RANGE); + while (true) { + if (code == 0) { + // found a key >= our key. if it is not the same size, it must be out of range for what we are looking + // for, this means no more data available + if (keySize != dbKey.mv_size) { + break; + } + // this is data that we need to delete + code = mdb_cursor_del(cursor, 0); + + if (code != 0) { + throw_error("delete_all_values_greater_or_equal_key::mdb_cursor_del", code); + } + + // move to the next key + code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_NEXT); + } else if (code == MDB_NOTFOUND) { + // no more data to extract + break; + } else { + throw_error("delete_all_values_greater_or_equal_key::mdb_cursor_get", code); + } + } + } catch (std::exception& e) { + call_lmdb_func(mdb_cursor_close, cursor); + throw; + } + call_lmdb_func(mdb_cursor_close, cursor); +} + +template +void get_all_values_lesser_or_equal_key(const TKey& key, + std::vector>& data, + const LMDBDatabase& db, + const TxType& tx) +{ + std::vector keyBuffer = serialise_key(key); + uint32_t keySize = static_cast(keyBuffer.size()); + MDB_cursor* cursor = nullptr; + call_lmdb_func("mdb_cursor_open", mdb_cursor_open, tx.underlying(), db.underlying(), &cursor); + + try { + MDB_val dbKey; + dbKey.mv_size = keySize; + dbKey.mv_data = (void*)keyBuffer.data(); + + MDB_val dbVal; + // Look for the key >= to that provided + int code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_SET_RANGE); + if (code == 0) { + // we found the key, now determine if it is the exact key + std::vector temp = mdb_val_to_vector(dbKey); + if (keyBuffer == temp) { + // we have the exact key, copy it's data + std::vector temp; + copy_to_vector(dbVal, temp); + data.push_back(temp); + } else { + // not the exact key, either the same size but greater value or larger key size + // either way we just need to move down + } + } else if (code == MDB_NOTFOUND) { + // key not found. no key of greater size or same size and greater value, just move down + } else { + throw_error("get_all_values_lesser_or_equal_key::mdb_cursor_get", code); + } + + // we now iterate down the keys until we run out + while (true) { + code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_PREV); + + if (code == 0) { + // we have found a previous key, if it is a different size then we have reached the end of the data + if (dbKey.mv_size != keySize) { + break; + } + // the same size, grab the value and go round again + std::vector temp; + copy_to_vector(dbVal, temp); + data.push_back(temp); + + } else if (MDB_NOTFOUND) { + // we have reached the end of the db + break; + } else { + throw_error("get_all_values_lesser_or_equal_key::mdb_cursor_get", code); + } + } + } catch (std::exception& e) { + call_lmdb_func(mdb_cursor_close, cursor); + throw; + } + call_lmdb_func(mdb_cursor_close, cursor); +} + +template +void delete_all_values_lesser_or_equal_key(const TKey& key, const LMDBDatabase& db, const TxType& tx) +{ + std::vector keyBuffer = serialise_key(key); + uint32_t keySize = static_cast(keyBuffer.size()); + MDB_cursor* cursor = nullptr; + call_lmdb_func("mdb_cursor_open", mdb_cursor_open, tx.underlying(), db.underlying(), &cursor); + + try { + MDB_val dbKey; + dbKey.mv_size = keySize; + dbKey.mv_data = (void*)keyBuffer.data(); + + MDB_val dbVal; + // Look for the key >= to that provided + int code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_SET_RANGE); + if (code == 0) { + // we found the key, now determine if it is the exact key + std::vector temp = mdb_val_to_vector(dbKey); + if (keyBuffer == temp) { + // we have the exact key, delete it's data + code = mdb_cursor_del(cursor, 0); + + if (code != 0) { + throw_error("delete_all_values_lesser_or_equal_key::mdb_cursor_del", code); + } + } else { + // not the exact key, either the same size but greater value or larger key size + // either way we just need to move down + } + } else if (code == MDB_NOTFOUND) { + // key not found. no key of greater size or same size and greater value, just move down + } else { + throw_error("get_all_values_lesser_or_equal_key::mdb_cursor_get", code); + } + + // we now iterate down the keys until we run out + while (true) { + code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_PREV); + + if (code == 0) { + // we have found a previous key, if it is a different size then we have reached the end of the data + if (dbKey.mv_size != keySize) { + break; + } + // we have the exact key, delete it's data + code = mdb_cursor_del(cursor, 0); + + if (code != 0) { + throw_error("delete_all_values_lesser_or_equal_key::mdb_cursor_del", code); + } + + } else if (MDB_NOTFOUND) { + // we have reached the end of the db + break; + } else { + throw_error("get_all_values_lesser_or_equal_key::mdb_cursor_get", code); + } + } + } catch (std::exception& e) { + call_lmdb_func(mdb_cursor_close, cursor); + throw; + } + call_lmdb_func(mdb_cursor_close, cursor); +} +} // namespace bb::crypto::merkle_tree::lmdb_queries diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp index c6d1986cef7..9b113365e1a 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -32,6 +33,26 @@ template <> struct std::hash { namespace bb::crypto::merkle_tree { +template fr preimage_to_key(const LeafType& leaf) +{ + return leaf.get_key(); +} + +inline fr preimage_to_key(const fr& leaf) +{ + return leaf; +} + +template bool requires_preimage_for_key() +{ + return true; +} + +template <> inline bool requires_preimage_for_key() +{ + return false; +} + /** * @brief Serves as a key-value node store for merkle trees. Caches all changes in memory before persisting them during * a 'commit' operation. @@ -175,8 +196,14 @@ template class ContentAddressedCachedTreeStore { fr get_current_root(ReadTransaction& tx, bool includeUncommitted) const; + void remove_historical_block(const index_t& blockNumber); + + void unwind_block(const index_t& blockNumber); + std::optional get_fork_block() const; + void advance_finalised_block(index_t blockNumber); + private: std::string name_; uint32_t depth_; @@ -188,11 +215,11 @@ template class ContentAddressedCachedTreeStore { std::unordered_map nodes_; // This is a store mapping the leaf key (e.g. slot for public data or nullifier value for nullifier tree) to the - // indices in the tree For indexed tress there is only ever one index againt the key, for append-only trees there + // indices in the tree For indexed tress there is only ever one index against the key, for append-only trees there // can be multiple std::map indices_; - // This is a mapping from leaf hash to leaf pre-image. This will contai entries that need to be omitted when + // This is a mapping from leaf hash to leaf pre-image. This will contain entries that need to be omitted when // commiting updates std::unordered_map leaves_; PersistedStoreType::SharedPtr dataStore_; @@ -217,10 +244,23 @@ template class ContentAddressedCachedTreeStore { void persist_leaf_indices(WriteTransaction& tx); - void persist_leaf(const fr& hash, WriteTransaction& tx); + void persist_leaf_keys(index_t startIndex, WriteTransaction& tx); + + void persist_leaf_pre_image(const fr& hash, WriteTransaction& tx); void persist_node(const std::optional& optional_hash, uint32_t level, WriteTransaction& tx); + void remove_node(const std::optional& optional_hash, + uint32_t level, + std::optional maxIndex, + WriteTransaction& tx); + + void remove_leaf(const fr& hash, std::optional maxIndex, WriteTransaction& tx); + + void remove_leaf_indices(const fr& key, const index_t& maxIndex, WriteTransaction& tx); + + void remove_leaf_indices_after_or_equal_index(const index_t& maxIndex, WriteTransaction& tx); + index_t constrain_tree_size(const RequestContext& requestContext, ReadTransaction& tx) const; WriteTransactionPtr create_write_transaction() const { return dataStore_->create_write_transaction(); } @@ -375,22 +415,15 @@ template void ContentAddressedCachedTreeStore::set_leaf_key_at_index(const index_t& index, const IndexedLeafValueType& leaf) { - // Accessing indices under a lock - std::unique_lock lock(mtx_); - auto it = indices_.find(uint256_t(leaf.value.get_key())); - if (it == indices_.end()) { - Indices ind; - ind.indices.push_back(index); - indices_[uint256_t(leaf.value.get_key())] = ind; - return; - } - it->second.indices.push_back(index); + // std::cout << "Set leaf key at index " << index << std::endl; + update_index(index, leaf.value.get_key()); } template void ContentAddressedCachedTreeStore::update_index(const index_t& index, const fr& leaf) { - // Accessing indices_ under a lock + // std::cout << "update_index at index " << index << " leaf " << leaf << std::endl; + // Accessing indices_ under a lock std::unique_lock lock(mtx_); auto it = indices_.find(uint256_t(leaf)); if (it == indices_.end()) { @@ -425,8 +458,7 @@ std::optional ContentAddressedCachedTreeStore::find_leaf if (success) { index_t sizeLimit = constrain_tree_size(requestContext, tx); if (!committed.indices.empty()) { - for (size_t i = 0; i < committed.indices.size(); ++i) { - index_t ind = committed.indices[i]; + for (index_t ind : committed.indices) { if (ind < start_index) { continue; } @@ -446,8 +478,7 @@ std::optional ContentAddressedCachedTreeStore::find_leaf std::unique_lock lock(mtx_); auto it = indices_.find(uint256_t(leaf)); if (it != indices_.end() && !it->second.indices.empty()) { - for (size_t i = 0; i < it->second.indices.size(); ++i) { - index_t ind = it->second.indices[i]; + for (index_t ind : it->second.indices) { if (ind < start_index) { continue; } @@ -593,52 +624,66 @@ fr ContentAddressedCachedTreeStore::get_current_root(ReadTransact template void ContentAddressedCachedTreeStore::commit(bool asBlock) { - fr currentRoot = fr::zero(); bool dataPresent = false; + TreeMeta uncommittedMeta; + TreeMeta committedMeta; // We don't allow commits using images/forks if (initialised_from_block_.has_value()) { - throw std::runtime_error("Committing an image is forbidden"); + throw std::runtime_error("Committing a fork is forbidden"); } { ReadTransactionPtr tx = create_read_transaction(); - dataPresent = get_cached_node_by_index(0, 0, currentRoot); + // read both committed and uncommitted meta data + get_meta(uncommittedMeta, *tx, true); + get_meta(committedMeta, *tx, false); + + // if the meta datas are different, we have uncommitted data + bool metaToCommit = committedMeta != uncommittedMeta; + if (!metaToCommit) { + return; + } + auto currentRootIter = nodes_.find(uncommittedMeta.root); + dataPresent = currentRootIter != nodes_.end(); if (!dataPresent) { + // no uncommitted data present, if we were asked to commit as a block then we can't if (asBlock) { - return; + throw std::runtime_error("Can't commit as block if no data present"); } } else { - auto currentRootIter = nodes_.find(currentRoot); - if (currentRootIter == nodes_.end()) { - if (asBlock) { - return; - } - } else { - hydrate_indices_from_persisted_store(*tx); - } + // data is present, hydrate persisted indices + hydrate_indices_from_persisted_store(*tx); } } { WriteTransactionPtr tx = create_write_transaction(); try { if (dataPresent) { + // std::cout << "Persisting data for block " << uncommittedMeta.unfinalisedBlockHeight + 1 << std::endl; persist_leaf_indices(*tx); - persist_node(std::optional(currentRoot), 0, *tx); + persist_leaf_keys(uncommittedMeta.committedSize, *tx); + persist_node(std::optional(uncommittedMeta.root), 0, *tx); if (asBlock) { - ++meta_.unfinalisedBlockHeight; - BlockPayload block{ .size = meta_.size, - .blockNumber = meta_.unfinalisedBlockHeight, - .root = currentRoot }; - dataStore_->write_block_data(meta_.unfinalisedBlockHeight, block, *tx); + ++uncommittedMeta.unfinalisedBlockHeight; + if (uncommittedMeta.oldestHistoricBlock == 0) { + uncommittedMeta.oldestHistoricBlock = 1; + } + // std::cout << "New root " << uncommittedMeta.root << std::endl; + BlockPayload block{ .size = uncommittedMeta.size, + .blockNumber = uncommittedMeta.unfinalisedBlockHeight, + .root = uncommittedMeta.root }; + dataStore_->write_block_data(uncommittedMeta.unfinalisedBlockHeight, block, *tx); } } - meta_.committedSize = meta_.size; - persist_meta(meta_, *tx); + uncommittedMeta.committedSize = uncommittedMeta.size; + persist_meta(uncommittedMeta, *tx); tx->commit(); } catch (std::exception& e) { tx->try_abort(); throw; } } + + // rolling back destroys all cache stores and also refreshes the cached meta_ from persisted state rollback(); } @@ -652,13 +697,30 @@ void ContentAddressedCachedTreeStore::persist_leaf_indices(WriteT } template -void ContentAddressedCachedTreeStore::persist_leaf(const fr& hash, WriteTransaction& tx) +void ContentAddressedCachedTreeStore::persist_leaf_keys(index_t startIndex, WriteTransaction& tx) +{ + for (auto& idx : indices_) { + FrKeyType key = idx.first; + + // write the leaf key against the indices, this is for the pending chain store of indices + for (index_t indexForKey : idx.second.indices) { + if (indexForKey < startIndex) { + continue; + } + dataStore_->write_leaf_key_by_index(key, indexForKey, tx); + } + } +} + +template +void ContentAddressedCachedTreeStore::persist_leaf_pre_image(const fr& hash, WriteTransaction& tx) { // Now persist the leaf pre-image auto leafPreImageIter = leaves_.find(hash); if (leafPreImageIter == leaves_.end()) { return; } + // std::cout << "Persisting leaf preimage " << leafPreImageIter->second << std::endl; dataStore_->write_leaf_by_hash(hash, leafPreImageIter->second, tx); } @@ -667,6 +729,9 @@ void ContentAddressedCachedTreeStore::persist_node(const std::opt uint32_t level, WriteTransaction& tx) { + // If the optional hash does not have a value then it means it's the zero tree value at this level + // If it has a value but that value is not in our stores then it means it is referencing a node + // created in a previous block, so that will need to have it's reference count increased if (!optional_hash.has_value()) { return; } @@ -674,15 +739,24 @@ void ContentAddressedCachedTreeStore::persist_node(const std::opt if (level == depth_) { // this is a leaf - persist_leaf(hash, tx); + persist_leaf_pre_image(hash, tx); } + // std::cout << "Persisting node hash " << hash << " at level " << level << std::endl; + auto nodePayloadIter = nodes_.find(hash); if (nodePayloadIter == nodes_.end()) { - // need to reference count here + // need to increase the stored node's reference count here + dataStore_->increment_node_reference_count(hash, tx); + return; + } + NodePayload nodeData = nodePayloadIter->second; + dataStore_->set_or_increment_node_reference_count(hash, nodeData, tx); + if (nodeData.ref != 1) { + // If the node now has a ref count greater then 1, we don't continue. + // It means that the entire sub-tree underneath already exists return; } - dataStore_->write_node(hash, nodePayloadIter->second, tx); persist_node(nodePayloadIter->second.left, level + 1, tx); persist_node(nodePayloadIter->second.right, level + 1, tx); } @@ -722,6 +796,283 @@ void ContentAddressedCachedTreeStore::persist_meta(TreeMeta& m, W dataStore_->write_meta_data(m, tx); } +template +void ContentAddressedCachedTreeStore::advance_finalised_block(index_t blockNumber) +{ + TreeMeta committedMeta; + TreeMeta uncommittedMeta; + BlockPayload blockPayload; + if (blockNumber < 1) { + throw std::runtime_error("Unable to remove block"); + } + if (initialised_from_block_.has_value()) { + throw std::runtime_error("Advancing the finalised block on a fork is forbidden"); + } + { + // read both committed and uncommitted meta values + ReadTransactionPtr tx = create_read_transaction(); + get_meta(uncommittedMeta, *tx, true); + get_meta(committedMeta, *tx, false); + if (!dataStore_->read_block_data(blockNumber, blockPayload, *tx)) { + throw std::runtime_error("Failed to retrieve block data"); + } + } + // can only finalise blocks that are not finalised + if (committedMeta.finalisedBlockHeight >= blockNumber) { + std::stringstream ss; + ss << "Unable to finalise block " << blockNumber << " currently finalised block height " + << committedMeta.finalisedBlockHeight << std::endl; + throw std::runtime_error(ss.str()); + } + + // can currently only finalise up to the unfinalised block height + if (committedMeta.finalisedBlockHeight > committedMeta.unfinalisedBlockHeight) { + std::stringstream ss; + ss << "Unable to finalise block " << blockNumber << " currently unfinalised block height " + << committedMeta.unfinalisedBlockHeight << std::endl; + throw std::runtime_error(ss.str()); + } + + // commit the new finalised block + WriteTransactionPtr writeTx = create_write_transaction(); + try { + // determine where we need to prune the leaf keys store up to + index_t highestIndexToRemove = blockPayload.size - 1; + committedMeta.finalisedBlockHeight = blockNumber; + // clean up the leaf keys index table + dataStore_->delete_all_leaf_keys_before_or_equal_index(highestIndexToRemove, *writeTx); + // persist the new meta data + persist_meta(committedMeta, *writeTx); + writeTx->commit(); + } catch (std::exception& e) { + writeTx->try_abort(); + throw; + } + + // commit successful, now also update the uncommitted meta + uncommittedMeta.finalisedBlockHeight = committedMeta.finalisedBlockHeight; + put_meta(uncommittedMeta); +} + +template +void ContentAddressedCachedTreeStore::unwind_block(const index_t& blockNumber) +{ + TreeMeta uncommittedMeta; + TreeMeta committedMeta; + BlockPayload blockData; + BlockPayload previousBlockData; + if (blockNumber < 1) { + throw std::runtime_error("Unable to remove block"); + } + if (initialised_from_block_.has_value()) { + throw std::runtime_error("Removing a block on a fork is forbidden"); + } + { + ReadTransactionPtr tx = create_read_transaction(); + get_meta(uncommittedMeta, *tx, true); + get_meta(committedMeta, *tx, false); + if (committedMeta != uncommittedMeta) { + throw std::runtime_error("Can't unwind with uncommitted data, first rollback before unwinding"); + } + if (blockNumber != uncommittedMeta.unfinalisedBlockHeight) { + throw std::runtime_error("Block number is not the most recent"); + } + if (blockNumber <= uncommittedMeta.finalisedBlockHeight) { + throw std::runtime_error("Can't unwind a finalised block"); + } + + // populate the required data for the previous block + if (blockNumber == 1) { + previousBlockData.root = uncommittedMeta.initialRoot; + previousBlockData.size = uncommittedMeta.initialSize; + previousBlockData.blockNumber = 0; + } else if (!dataStore_->read_block_data(blockNumber - 1, previousBlockData, *tx)) { + throw std::runtime_error("Failed to retrieve previous block data"); + } + + // now get the root for the block we want to unwind + if (!dataStore_->read_block_data(blockNumber, blockData, *tx)) { + throw std::runtime_error("Failed to retrieve block data for block to unwind"); + } + } + WriteTransactionPtr writeTx = create_write_transaction(); + try { + // std::cout << "Removing block " << blockNumber << std::endl; + + // Remove the block's node and leaf data given the max index of the previous block + std::optional maxIndex = std::optional(previousBlockData.size); + remove_node(std::optional(blockData.root), 0, maxIndex, *writeTx); + // remove the block from the block data table + dataStore_->delete_block_data(blockNumber, *writeTx); + remove_leaf_indices_after_or_equal_index(previousBlockData.size, *writeTx); + uncommittedMeta.unfinalisedBlockHeight = previousBlockData.blockNumber; + uncommittedMeta.size = previousBlockData.size; + uncommittedMeta.committedSize = previousBlockData.size; + uncommittedMeta.root = previousBlockData.root; + // std::cout << "New block root " << previousBlockData.root << std::endl; + // commit this new meta data + persist_meta(uncommittedMeta, *writeTx); + writeTx->commit(); + } catch (std::exception& e) { + writeTx->try_abort(); + throw; + } + + // now update the uncommitted meta + put_meta(uncommittedMeta); +} + +template +void ContentAddressedCachedTreeStore::remove_historical_block(const index_t& blockNumber) +{ + TreeMeta committedMeta; + TreeMeta uncommittedMeta; + BlockPayload blockData; + if (blockNumber < 1) { + throw std::runtime_error("Unable to remove block"); + } + if (initialised_from_block_.has_value()) { + throw std::runtime_error("Removing a block on a fork is forbidden"); + } + { + // retrieve both the committed and uncommitted meta data, validate the provide block is the oldest historical + // block + ReadTransactionPtr tx = create_read_transaction(); + get_meta(uncommittedMeta, *tx, true); + get_meta(committedMeta, *tx, false); + if (blockNumber != committedMeta.oldestHistoricBlock) { + throw std::runtime_error("Block number is not the most historic"); + } + if (blockNumber >= committedMeta.finalisedBlockHeight) { + throw std::runtime_error("Can't remove current finalised block"); + } + + if (!dataStore_->read_block_data(blockNumber, blockData, *tx)) { + throw std::runtime_error("Failed to retrieve block data for historical block"); + } + } + WriteTransactionPtr writeTx = create_write_transaction(); + try { + std::optional maxIndex = std::nullopt; + // remove the historical block's node data + remove_node(std::optional(blockData.root), 0, maxIndex, *writeTx); + // remove the block's entry in the block table + dataStore_->delete_block_data(blockNumber, *writeTx); + // increment the oldest historical block number as committed data + committedMeta.oldestHistoricBlock++; + persist_meta(committedMeta, *writeTx); + writeTx->commit(); + } catch (std::exception& e) { + writeTx->try_abort(); + throw; + } + + // commit was successful, update the uncommitted meta + uncommittedMeta.oldestHistoricBlock = committedMeta.oldestHistoricBlock; + put_meta(uncommittedMeta); +} + +template +void ContentAddressedCachedTreeStore::remove_leaf_indices_after_or_equal_index(const index_t& index, + WriteTransaction& tx) +{ + std::vector leafKeys; + dataStore_->read_all_leaf_keys_after_or_equal_index(index, leafKeys, tx); + for (const fr& key : leafKeys) { + remove_leaf_indices(key, index, tx); + } + dataStore_->delete_all_leaf_keys_after_or_equal_index(index, tx); +} + +template +void ContentAddressedCachedTreeStore::remove_leaf_indices(const fr& key, + const index_t& maxIndex, + WriteTransaction& tx) +{ + // We now have the key, extract the indices + Indices indices; + // std::cout << "Reading indices for key " << key << std::endl; + dataStore_->read_leaf_indices(key, indices, tx); + // std::cout << "Indices length before removal " << indices.indices.size() << std::endl; + + size_t lengthBefore = indices.indices.size(); + + indices.indices.erase( + std::remove_if(indices.indices.begin(), indices.indices.end(), [&](index_t& ind) { return ind >= maxIndex; }), + indices.indices.end()); + + size_t lengthAfter = indices.indices.size(); + // std::cout << "Indices length after removal " << indices.indices.size() << std::endl; + + if (lengthBefore != lengthAfter) { + if (indices.indices.empty()) { + // std::cout << "Deleting indices" << std::endl; + dataStore_->delete_leaf_indices(key, tx); + } else { + // std::cout << "Writing indices" << std::endl; + dataStore_->write_leaf_indices(key, indices, tx); + } + } +} + +template +void ContentAddressedCachedTreeStore::remove_leaf(const fr& hash, + std::optional maxIndex, + WriteTransaction& tx) +{ + // std::cout << "Removing leaf " << hash << std::endl; + if (maxIndex.has_value()) { + // std::cout << "Max Index" << std::endl; + // We need to clear the entry from the leaf key to indices database as this leaf never existed + IndexedLeafValueType leaf; + fr key; + if (requires_preimage_for_key()) { + // std::cout << "Reading leaf by hash " << hash << std::endl; + if (!dataStore_->read_leaf_by_hash(hash, leaf, tx)) { + throw std::runtime_error("Failed to find leaf pre-image when attempting to delete indices"); + } + // std::cout << "Read leaf by hash " << hash << std::endl; + key = preimage_to_key(leaf.value); + } else { + key = hash; + } + remove_leaf_indices(key, maxIndex.value(), tx); + } + // std::cout << "Deleting leaf by hash " << std::endl; + dataStore_->delete_leaf_by_hash(hash, tx); +} + +template +void ContentAddressedCachedTreeStore::remove_node(const std::optional& optional_hash, + uint32_t level, + std::optional maxIndex, + WriteTransaction& tx) +{ + if (!optional_hash.has_value()) { + return; + } + fr hash = optional_hash.value(); + + // we need to retrieve the node and decrement it's reference count + // std::cout << "Decrementing ref count for node " << hash << ", level " << level << std::endl; + NodePayload nodeData; + dataStore_->decrement_node_reference_count(hash, nodeData, tx); + + if (nodeData.ref != 0) { + // node was not deleted, we don't continue the search + return; + } + + // the node was deleted, if it was a leaf then we need to remove the pre-image + if (level == depth_) { + remove_leaf(hash, maxIndex, tx); + } + + // now recursively remove the next level + remove_node(std::optional(nodeData.left), level + 1, maxIndex, tx); + remove_node(std::optional(nodeData.right), level + 1, maxIndex, tx); +} + template void ContentAddressedCachedTreeStore::initialise() { // Read the persisted meta data, if the name or depth of the tree is not consistent with what was provided during @@ -746,8 +1097,9 @@ template void ContentAddressedCachedTreeStore::initialise_from_block(const if (meta_.unfinalisedBlockHeight < blockNumber) { throw std::runtime_error("Unable to initialise from future block"); } + if (meta_.oldestHistoricBlock > blockNumber && blockNumber != 0) { + throw std::runtime_error("Unable to fork from expired historical block"); + } BlockPayload blockData; if (blockNumber == 0) { blockData.blockNumber = 0; - blockData.root = meta_.root; - blockData.size = meta_.size; + blockData.root = meta_.initialRoot; + blockData.size = meta_.initialSize; } else if (get_block_data(blockNumber, blockData, *tx) == false) { throw std::runtime_error( (std::stringstream() << "Failed to retrieve block data: " << blockNumber << ". Tree name: " << name_) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp index ba44876d8ba..164a6b254cf 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp @@ -16,18 +16,27 @@ struct TreeMeta { bb::fr root; index_t initialSize; bb::fr initialRoot; - uint64_t finalisedBlockHeight; + uint64_t oldestHistoricBlock; uint64_t unfinalisedBlockHeight; + uint64_t finalisedBlockHeight; - MSGPACK_FIELDS( - name, depth, size, committedSize, root, initialSize, initialRoot, finalisedBlockHeight, unfinalisedBlockHeight) + MSGPACK_FIELDS(name, + depth, + size, + committedSize, + root, + initialSize, + initialRoot, + oldestHistoricBlock, + unfinalisedBlockHeight, + finalisedBlockHeight) bool operator==(const TreeMeta& other) const { return name == other.name && depth == other.depth && size == other.size && committedSize == other.committedSize && root == other.root && initialRoot == other.initialRoot && initialSize == other.initialSize && unfinalisedBlockHeight == other.unfinalisedBlockHeight && - finalisedBlockHeight == other.finalisedBlockHeight; + oldestHistoricBlock == other.oldestHistoricBlock && finalisedBlockHeight == other.finalisedBlockHeight; } }; @@ -36,8 +45,8 @@ inline std::ostream& operator<<(std::ostream& os, const TreeMeta& meta) os << "TreeMeta{name: " << meta.name << ", depth: " << meta.depth << ", size: " << std::dec << (meta.size) << ", committedSize: " << std::dec << meta.committedSize << ", root: " << meta.root << ", initialSize: " << std::dec << meta.initialSize << ", initialRoot: " << meta.initialRoot - << ", finalisedBlockHeight: " << std::dec << meta.finalisedBlockHeight - << ", unfinalisedBlockHeight: " << std::dec << meta.unfinalisedBlockHeight << "}"; + << ", oldestHistoricBlock: " << std::dec << meta.oldestHistoricBlock << ", finalisedBlockHeight: " << std::dec + << meta.finalisedBlockHeight << ", unfinalisedBlockHeight: " << std::dec << meta.unfinalisedBlockHeight << "}"; return os; } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp index ffc1c6483ec..5ecf3c8c75d 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp @@ -86,6 +86,7 @@ void execute_and_report(const std::function&)>& } catch (std::exception& e) { response.success = false; response.message = e.what(); + // std::cout << "Response " << e.what() << std::endl; } try { on_completion(response); @@ -102,6 +103,7 @@ inline void execute_and_report(const std::function& f, } catch (std::exception& e) { response.success = false; response.message = e.what(); + // std::cout << "Response " << e.what() << std::endl; } try { on_completion(response); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp new file mode 100644 index 00000000000..dbf7eaa44d6 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp @@ -0,0 +1,105 @@ + +#pragma once + +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "barretenberg/ecc/curves/bn254/fr.hpp" +#include "barretenberg/numeric/random/engine.hpp" +#include +#include + +namespace bb::crypto::merkle_tree { + +void inline check_block_and_root_data(LMDBTreeStore::SharedPtr db, index_t blockNumber, fr root, bool expectedSuccess) +{ + BlockPayload blockData; + LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction(); + bool success = db->read_block_data(blockNumber, blockData, *tx); + EXPECT_EQ(success, expectedSuccess); + if (expectedSuccess) { + EXPECT_EQ(blockData.root, root); + } + NodePayload nodeData; + success = db->read_node(root, nodeData, *tx); + EXPECT_EQ(success, expectedSuccess); +} + +void inline check_block_and_root_data( + LMDBTreeStore::SharedPtr db, index_t blockNumber, fr root, bool expectedSuccess, bool expectedRootSuccess) +{ + BlockPayload blockData; + LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction(); + bool success = db->read_block_data(blockNumber, blockData, *tx); + EXPECT_EQ(success, expectedSuccess); + if (expectedSuccess) { + EXPECT_EQ(blockData.root, root); + } + NodePayload nodeData; + success = db->read_node(root, nodeData, *tx); + EXPECT_EQ(success, expectedRootSuccess); +} + +void inline check_block_and_size_data(LMDBTreeStore::SharedPtr db, + index_t blockNumber, + index_t expectedSize, + bool expectedSuccess) +{ + BlockPayload blockData; + LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction(); + bool success = db->read_block_data(blockNumber, blockData, *tx); + EXPECT_EQ(success, expectedSuccess); + if (expectedSuccess) { + EXPECT_EQ(blockData.size, expectedSize); + } +} + +void inline check_indices_data( + LMDBTreeStore::SharedPtr db, fr leaf, index_t index, bool entryShouldBePresent, bool indexShouldBePresent) +{ + Indices indices; + LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction(); + bool success = db->read_leaf_indices(leaf, indices, *tx); + EXPECT_EQ(success, entryShouldBePresent); + if (entryShouldBePresent) { + bool found = std::find(indices.indices.begin(), indices.indices.end(), index) != std::end(indices.indices); + EXPECT_EQ(found, indexShouldBePresent); + } +} + +template +void check_leaf_by_hash(LMDBTreeStore::SharedPtr db, IndexedLeaf leaf, bool shouldBePresent) +{ + LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction(); + IndexedLeaf fromStore; + bool success = db->read_leaf_by_hash(Hash::hash(leaf.get_hash_inputs()), fromStore, *tx); + EXPECT_EQ(success, shouldBePresent); + if (success) { + EXPECT_EQ(fromStore, leaf); + } +} + +void inline check_leaf_keys_are_present(LMDBTreeStore::SharedPtr db, + uint64_t startIndex, + uint64_t endIndex, + const std::vector& keys) +{ + LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction(); + for (uint64_t i = startIndex; i <= endIndex; i++) { + fr leafKey; + bool success = db->read_leaf_key_by_index(i, leafKey, *tx); + EXPECT_TRUE(success); + EXPECT_EQ(leafKey, keys[i - startIndex]); + } +} + +void inline check_leaf_keys_are_not_present(LMDBTreeStore::SharedPtr db, uint64_t startIndex, uint64_t endIndex) +{ + LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction(); + for (uint64_t i = startIndex; i < endIndex; i++) { + fr leafKey; + bool success = db->read_leaf_key_by_index(i, leafKey, *tx); + EXPECT_FALSE(success); + } +} + +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/world_state/fork.hpp b/barretenberg/cpp/src/barretenberg/world_state/fork.hpp index 255221c1124..109e8ff18d5 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/fork.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/fork.hpp @@ -29,5 +29,6 @@ struct Fork { using SharedPtr = std::shared_ptr; Id _forkId; std::unordered_map _trees; + index_t _blockNumber; }; } // namespace bb::world_state diff --git a/barretenberg/cpp/src/barretenberg/world_state/types.hpp b/barretenberg/cpp/src/barretenberg/world_state/types.hpp index 5e8c6685de0..69011a69392 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/types.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/types.hpp @@ -9,6 +9,8 @@ namespace bb::world_state { +using namespace bb::crypto::merkle_tree; + enum MerkleTreeId { NULLIFIER_TREE = 0, NOTE_HASH_TREE = 1, @@ -17,24 +19,43 @@ enum MerkleTreeId { ARCHIVE = 4, }; +const uint64_t CANONICAL_FORK_ID = 0; + std::string getMerkleTreeName(MerkleTreeId id); using TreeStateReference = std::pair; using StateReference = std::unordered_map; struct WorldStateRevision { - uint64_t forkId{ 0 }; - uint64_t blockNumber{ 0 }; + index_t forkId{ 0 }; + index_t blockNumber{ 0 }; bool includeUncommitted{ false }; MSGPACK_FIELDS(forkId, blockNumber, includeUncommitted) - // using Revision = std::variant; Revision inner; - static WorldStateRevision committed() { return WorldStateRevision{ .includeUncommitted = false }; } static WorldStateRevision uncommitted() { return WorldStateRevision{ .includeUncommitted = true }; } - // static WorldStateRevision finalised_block(uint32_t block_number) { return { WorldStateRevision{ .blockNumber = - // block_number } }; } +}; + +struct WorldStateStatus { + index_t unfinalisedBlockNumber; + index_t finalisedBlockNumber; + index_t oldestHistoricalBlock; + MSGPACK_FIELDS(unfinalisedBlockNumber, finalisedBlockNumber, oldestHistoricalBlock); + + bool operator==(const WorldStateStatus& other) const + { + return unfinalisedBlockNumber == other.unfinalisedBlockNumber && + finalisedBlockNumber == other.finalisedBlockNumber && + oldestHistoricalBlock == other.oldestHistoricalBlock; + } + + friend std::ostream& operator<<(std::ostream& os, const WorldStateStatus& status) + { + os << "unfinalisedBlockNumber: " << status.unfinalisedBlockNumber + << ", finalisedBlockNumber: " << status.finalisedBlockNumber + << ", oldestHistoricalBlock: " << status.oldestHistoricalBlock; + return os; + } }; } // namespace bb::world_state diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp index 822515b7c42..e4761843c75 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp @@ -1,6 +1,5 @@ #include "barretenberg/world_state/world_state.hpp" #include "barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" -#include "barretenberg/crypto/merkle_tree/fixtures.hpp" #include "barretenberg/crypto/merkle_tree/hash.hpp" #include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" @@ -14,6 +13,7 @@ #include "barretenberg/world_state/tree_with_store.hpp" #include "barretenberg/world_state/types.hpp" #include "barretenberg/world_state/world_state_stores.hpp" +#include "barretenberg/world_state_napi/message.hpp" #include #include #include @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -40,8 +41,8 @@ WorldState::WorldState(uint64_t thread_pool_size, : _workers(std::make_shared(thread_pool_size)) , _tree_heights(tree_heights) , _initial_tree_size(tree_prefill) - , _initial_header_generator_point(initial_header_generator_point) , _forkId(CANONICAL_FORK_ID) + , _initial_header_generator_point(initial_header_generator_point) { create_canonical_fork(data_dir, map_size, thread_pool_size); } @@ -128,7 +129,7 @@ void WorldState::create_canonical_fork(const std::string& dataDir, _forks[fork->_forkId] = fork; } -Fork::SharedPtr WorldState::retrieve_fork(uint64_t forkId) const +Fork::SharedPtr WorldState::retrieve_fork(const uint64_t& forkId) const { std::unique_lock lock(mtx); auto it = _forks.find(forkId); @@ -137,16 +138,44 @@ Fork::SharedPtr WorldState::retrieve_fork(uint64_t forkId) const } return it->second; } -uint64_t WorldState::create_fork(index_t blockNumber) +uint64_t WorldState::create_fork(const std::optional& blockNumber) { + index_t blockNumberForFork = 0; + if (!blockNumber.has_value()) { + // we are forking at latest + WorldStateStatus currentStatus; + get_status(currentStatus); + blockNumberForFork = currentStatus.unfinalisedBlockNumber; + } else { + blockNumberForFork = blockNumber.value(); + } + Fork::SharedPtr fork = create_new_fork(blockNumberForFork); std::unique_lock lock(mtx); - Fork::SharedPtr fork = create_new_fork(blockNumber); - fork->_forkId = _forkId++; - _forks[fork->_forkId] = fork; - return fork->_forkId; + uint64_t forkId = _forkId++; + fork->_forkId = forkId; + _forks[forkId] = fork; + return forkId; +} + +void WorldState::remove_forks_for_block(const index_t& blockNumber) +{ + // capture the shared pointers outside of the lock scope so we are not under the lock when the objects are destroyed + std::vector forks; + { + std::unique_lock lock(mtx); + for (auto it = _forks.begin(); it != _forks.end();) { + if (it->second->_blockNumber == blockNumber) { + forks.push_back(it->second); + it = _forks.erase(it); + + } else { + it++; + } + } + } } -void WorldState::delete_fork(uint64_t forkId) +void WorldState::delete_fork(const uint64_t& forkId) { if (forkId == 0) { throw std::runtime_error("Unable to delete canonical fork"); @@ -160,9 +189,10 @@ void WorldState::delete_fork(uint64_t forkId) } } -Fork::SharedPtr WorldState::create_new_fork(index_t blockNumber) +Fork::SharedPtr WorldState::create_new_fork(const index_t& blockNumber) { Fork::SharedPtr fork = std::make_shared(); + fork->_blockNumber = blockNumber; { uint32_t levels = _tree_heights.at(MerkleTreeId::NULLIFIER_TREE); index_t initial_size = _initial_tree_size.at(MerkleTreeId::NULLIFIER_TREE); @@ -342,18 +372,25 @@ void WorldState::update_archive(const StateReference& block_state_ref, } } -void WorldState::commit() +bool WorldState::commit() { // NOTE: the calling code is expected to ensure no other reads or writes happen during commit Fork::SharedPtr fork = retrieve_fork(CANONICAL_FORK_ID); + std::atomic_bool success = true; Signal signal(static_cast(fork->_trees.size())); for (auto& [id, tree] : fork->_trees) { std::visit( - [&signal](auto&& wrapper) { wrapper.tree->commit([&](const Response&) { signal.signal_decrement(); }); }, + [&signal, &success](auto&& wrapper) { + wrapper.tree->commit([&](const Response& response) { + success = response.success && success; + signal.signal_decrement(); + }); + }, tree); } signal.wait_for_level(0); + return success; } void WorldState::rollback() @@ -371,19 +408,23 @@ void WorldState::rollback() signal.wait_for_level(); } -bool WorldState::sync_block(const StateReference& block_state_ref, - const bb::fr& block_header_hash, - const std::vector& notes, - const std::vector& l1_to_l2_messages, - const std::vector& nullifiers, - const std::vector>& public_writes) +WorldStateStatus WorldState::sync_block( + const StateReference& block_state_ref, + const bb::fr& block_header_hash, + const std::vector& notes, + const std::vector& l1_to_l2_messages, + const std::vector& nullifiers, + const std::vector>& public_writes) { + WorldStateStatus status; if (is_same_state_reference(WorldStateRevision::uncommitted(), block_state_ref) && is_archive_tip(WorldStateRevision::uncommitted(), block_header_hash)) { - commit(); - return true; + if (!commit()) { + throw std::runtime_error("Commit failed"); + } + get_status(status); + return status; } - rollback(); Fork::SharedPtr fork = retrieve_fork(CANONICAL_FORK_ID); @@ -458,8 +499,11 @@ bool WorldState::sync_block(const StateReference& block_state_ref, throw std::runtime_error("Can't synch block: block state does not match world state"); } - commit(); - return false; + if (!commit()) { + throw std::runtime_error("Commit failed"); + } + get_status(status); + return status; } GetLowIndexedLeafResponse WorldState::find_low_leaf_index(const WorldStateRevision& revision, @@ -496,6 +540,110 @@ GetLowIndexedLeafResponse WorldState::find_low_leaf_index(const WorldStateRevisi return low_leaf_info; } +WorldStateStatus WorldState::set_finalised_blocks(const index_t& toBlockNumber) +{ + WorldStateRevision revision{ .forkId = CANONICAL_FORK_ID, .blockNumber = 0, .includeUncommitted = false }; + TreeMetaResponse archive_state = get_tree_info(revision, MerkleTreeId::ARCHIVE); + if (toBlockNumber <= archive_state.meta.finalisedBlockHeight) { + throw std::runtime_error("Unable to finalise block, already finalised"); + } + if (!set_finalised_block(toBlockNumber)) { + throw std::runtime_error("Failed to set finalised block"); + } + WorldStateStatus status; + get_status(status); + return status; +} +WorldStateStatus WorldState::unwind_blocks(const index_t& toBlockNumber) +{ + WorldStateRevision revision{ .forkId = CANONICAL_FORK_ID, .blockNumber = 0, .includeUncommitted = false }; + TreeMetaResponse archive_state = get_tree_info(revision, MerkleTreeId::ARCHIVE); + if (toBlockNumber >= archive_state.meta.unfinalisedBlockHeight) { + throw std::runtime_error("Unable to unwind block, block not found"); + } + for (index_t blockNumber = archive_state.meta.unfinalisedBlockHeight; blockNumber > toBlockNumber; blockNumber--) { + if (!unwind_block(blockNumber)) { + throw std::runtime_error("Failed to unwind block"); + } + } + WorldStateStatus status; + get_status(status); + return status; +} +WorldStateStatus WorldState::remove_historical_blocks(const index_t& toBlockNumber) +{ + WorldStateRevision revision{ .forkId = CANONICAL_FORK_ID, .blockNumber = 0, .includeUncommitted = false }; + TreeMetaResponse archive_state = get_tree_info(revision, MerkleTreeId::ARCHIVE); + if (toBlockNumber <= archive_state.meta.oldestHistoricBlock) { + throw std::runtime_error("Unable to remove historical block, block not found"); + } + for (index_t blockNumber = archive_state.meta.oldestHistoricBlock; blockNumber < toBlockNumber; blockNumber++) { + if (!remove_historical_block(blockNumber)) { + throw std::runtime_error("Failed to remove historical block"); + } + } + WorldStateStatus status; + get_status(status); + return status; +} + +bool WorldState::set_finalised_block(const index_t& blockNumber) +{ + std::atomic_bool success = true; + Fork::SharedPtr fork = retrieve_fork(CANONICAL_FORK_ID); + Signal signal(static_cast(fork->_trees.size())); + for (auto& [id, tree] : fork->_trees) { + std::visit( + [&signal, &success, blockNumber](auto&& wrapper) { + wrapper.tree->finalise_block(blockNumber, [&signal, &success](const Response& resp) { + success = success && resp.success; + signal.signal_decrement(); + }); + }, + tree); + } + signal.wait_for_level(); + return success; +} +bool WorldState::unwind_block(const index_t& blockNumber) +{ + std::atomic_bool success = true; + Fork::SharedPtr fork = retrieve_fork(CANONICAL_FORK_ID); + Signal signal(static_cast(fork->_trees.size())); + for (auto& [id, tree] : fork->_trees) { + std::visit( + [&signal, &success, blockNumber](auto&& wrapper) { + wrapper.tree->unwind_block(blockNumber, [&signal, &success](const Response& resp) { + success = success && resp.success; + signal.signal_decrement(); + }); + }, + tree); + } + signal.wait_for_level(); + remove_forks_for_block(blockNumber); + return success; +} +bool WorldState::remove_historical_block(const index_t& blockNumber) +{ + std::atomic_bool success = true; + Fork::SharedPtr fork = retrieve_fork(CANONICAL_FORK_ID); + Signal signal(static_cast(fork->_trees.size())); + for (auto& [id, tree] : fork->_trees) { + std::visit( + [&signal, &success, blockNumber](auto&& wrapper) { + wrapper.tree->remove_historic_block(blockNumber, [&signal, &success](const Response& resp) { + success = success && resp.success; + signal.signal_decrement(); + }); + }, + tree); + } + signal.wait_for_level(); + remove_forks_for_block(blockNumber); + return success; +} + bb::fr WorldState::compute_initial_archive(const StateReference& initial_state_ref, uint32_t generator_point) { // NOTE: this hash operations needs to match the one in yarn-project/circuits.js/src/structs/header.ts @@ -543,6 +691,15 @@ bool WorldState::is_archive_tip(const WorldStateRevision& revision, const bb::fr return archive_state.meta.size == leaf_index.value() + 1; } +void WorldState::get_status(WorldStateStatus& status) const +{ + WorldStateRevision revision{ .forkId = CANONICAL_FORK_ID, .blockNumber = 0, .includeUncommitted = false }; + TreeMetaResponse archive_state = get_tree_info(revision, MerkleTreeId::ARCHIVE); + status.unfinalisedBlockNumber = archive_state.meta.unfinalisedBlockHeight; + status.finalisedBlockNumber = archive_state.meta.finalisedBlockHeight; + status.oldestHistoricalBlock = archive_state.meta.oldestHistoricBlock; +} + bool WorldState::is_same_state_reference(const WorldStateRevision& revision, const StateReference& state_ref) const { return state_ref == get_state_reference(revision); diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp index 3152492cb4a..7c09ccc0606 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp @@ -18,6 +18,7 @@ #include "barretenberg/world_state/tree_with_store.hpp" #include "barretenberg/world_state/types.hpp" #include "barretenberg/world_state/world_state_stores.hpp" +#include "barretenberg/world_state_napi/message.hpp" #include #include #include @@ -41,8 +42,6 @@ template struct BatchInsertionResult { MSGPACK_FIELDS(low_leaf_witness_data, sorted_leaves, subtree_path); }; -const uint64_t CANONICAL_FORK_ID = 0; - /** * @brief Holds the Merkle trees responsible for storing the state of the Aztec protocol. * @@ -201,50 +200,59 @@ class WorldState { /** * @brief Commits the current state of the world state. */ - void commit(); + bool commit(); /** * @brief Rolls back any uncommitted changes made to the world state. */ void rollback(); - /** - * @brief Synchronizes the world state with a new block. - * - * @param block The block to synchronize with. - */ - bool sync_block(const StateReference& block_state_ref, - const bb::fr& block_header_hash, - const std::vector& notes, - const std::vector& l1_to_l2_messages, - const std::vector& nullifiers, - const std::vector>& public_writes); + uint64_t create_fork(const std::optional& blockNumber); + void delete_fork(const uint64_t& forkId); + + WorldStateStatus set_finalised_blocks(const index_t& toBlockNumber); + WorldStateStatus unwind_blocks(const index_t& toBlockNumber); + WorldStateStatus remove_historical_blocks(const index_t& toBlockNumber); - uint64_t create_fork(index_t blockNumber); - void delete_fork(Fork::Id forkId); + void get_status(WorldStateStatus& status) const; + WorldStateStatus sync_block( + const StateReference& block_state_ref, + const bb::fr& block_header_hash, + const std::vector& notes, + const std::vector& l1_to_l2_messages, + const std::vector& nullifiers, + const std::vector>& public_writes); private: std::shared_ptr _workers; WorldStateStores::Ptr _persistentStores; + std::unordered_map _tree_heights; std::unordered_map _initial_tree_size; - uint32_t _initial_header_generator_point; mutable std::mutex mtx; std::unordered_map _forks; uint64_t _forkId = 0; + uint32_t _initial_header_generator_point; TreeStateReference get_tree_snapshot(MerkleTreeId id); void create_canonical_fork(const std::string& dataDir, const std::unordered_map& dbSize, uint64_t maxReaders); - Fork::SharedPtr retrieve_fork(uint64_t forkId) const; - Fork::SharedPtr create_new_fork(index_t blockNumber); + Fork::SharedPtr retrieve_fork(const uint64_t& forkId) const; + Fork::SharedPtr create_new_fork(const index_t& blockNumber); + void remove_forks_for_block(const index_t& blockNumber); + + bool unwind_block(const index_t& blockNumber); + bool remove_historical_block(const index_t& blockNumber); + bool set_finalised_block(const index_t& blockNumber); + + static bool block_state_matches_world_state(const StateReference& block_state_ref, + const StateReference& tree_state_ref); bool is_archive_tip(const WorldStateRevision& revision, const bb::fr& block_header_hash) const; bool is_same_state_reference(const WorldStateRevision& revision, const StateReference& state_ref) const; - static bb::fr compute_initial_archive(const StateReference& initial_state_ref, uint32_t generator_point); static StateReference get_state_reference(const WorldStateRevision& revision, @@ -257,7 +265,6 @@ std::optional> WorldState::get_indexed_leaf( MerkleTreeId id, index_t leaf) const { - using namespace crypto::merkle_tree; using Store = ContentAddressedCachedTreeStore; using Tree = ContentAddressedIndexedTree; diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp index 68c98664646..f52718c7c71 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp @@ -498,9 +498,10 @@ TEST_F(WorldStateTest, SyncExternalBlockFromEmpty) { fr("0x20ea8ca97f96508aaed2d6cdc4198a41c77c640bfa8785a51bb905b9a672ba0b"), 1 } }, }; - bool sync_res = ws.sync_block( + WorldStateStatus status = ws.sync_block( block_state_ref, fr(1), { 42 }, { 43 }, { NullifierLeafValue(144) }, { { PublicDataLeafValue(145, 1) } }); - EXPECT_EQ(sync_res, false); + WorldStateStatus expected{ .unfinalisedBlockNumber = 1, .finalisedBlockNumber = 0, .oldestHistoricalBlock = 1 }; + EXPECT_EQ(status, expected); assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::NOTE_HASH_TREE, 0, fr(42)); assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::L1_TO_L2_MESSAGE_TREE, 0, fr(43)); @@ -539,9 +540,10 @@ TEST_F(WorldStateTest, SyncBlockFromDirtyState) EXPECT_NE(uncommitted_state_ref.at(tree_id), snapshot); } - bool sync_res = ws.sync_block( + WorldStateStatus status = ws.sync_block( block_state_ref, fr(1), { 42 }, { 43 }, { NullifierLeafValue(144) }, { { PublicDataLeafValue(145, 1) } }); - EXPECT_EQ(sync_res, false); + WorldStateStatus expected{ .unfinalisedBlockNumber = 1, .finalisedBlockNumber = 0, .oldestHistoricalBlock = 1 }; + EXPECT_EQ(status, expected); assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::NOTE_HASH_TREE, 0, fr(42)); assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::L1_TO_L2_MESSAGE_TREE, 0, fr(43)); @@ -582,9 +584,10 @@ TEST_F(WorldStateTest, SyncCurrentBlock) EXPECT_EQ(uncommitted_state_ref.at(tree_id), snapshot); } - bool sync_res = ws.sync_block( + WorldStateStatus status = ws.sync_block( block_state_ref, fr(1), { 42 }, { 43 }, { NullifierLeafValue(144) }, { { PublicDataLeafValue(145, 1) } }); - EXPECT_EQ(sync_res, true); + WorldStateStatus expected{ .unfinalisedBlockNumber = 1, .finalisedBlockNumber = 0, .oldestHistoricalBlock = 1 }; + EXPECT_EQ(status, expected); assert_leaf_value(ws, WorldStateRevision::uncommitted(), MerkleTreeId::ARCHIVE, 1, fr(1)); @@ -739,7 +742,8 @@ TEST_F(WorldStateTest, ForkingAtBlock0AndAdvancingCanonicalState) MerkleTreeId::ARCHIVE); // committed fork state should match the state before fork had been modified - EXPECT_EQ(fork_archive_state_after_commit.meta, fork_archive_state_before_insert.meta); + EXPECT_EQ(fork_archive_state_after_commit.meta.size, fork_archive_state_before_insert.meta.size); + EXPECT_EQ(fork_archive_state_after_commit.meta.root, fork_archive_state_before_insert.meta.root); // canonical state before commit should match state after commit // EXPECT_EQ(canonical_archive_state_after_commit.meta, canonical_archive_state_after_insert.meta); EXPECT_EQ(canonical_archive_state_after_commit.meta.root, canonical_archive_state_after_insert.meta.root); diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp index 8601ab880e8..0e6604cd58a 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -188,6 +189,21 @@ WorldStateAddon::WorldStateAddon(const Napi::CallbackInfo& info) WorldStateMessageType::DELETE_FORK, [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return delete_fork(obj, buffer); }); + _dispatcher.registerTarget( + WorldStateMessageType::FINALISE_BLOCKS, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return set_finalised(obj, buffer); }); + + _dispatcher.registerTarget(WorldStateMessageType::UNWIND_BLOCKS, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return unwind(obj, buffer); }); + + _dispatcher.registerTarget( + WorldStateMessageType::REMOVE_HISTORICAL_BLOCKS, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return remove_historical(obj, buffer); }); + + _dispatcher.registerTarget( + WorldStateMessageType::GET_STATUS, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return get_status(obj, buffer); }); + _dispatcher.registerTarget(WorldStateMessageType::CLOSE, [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return close(obj, buffer); }); } @@ -538,15 +554,15 @@ bool WorldStateAddon::sync_block(msgpack::object& obj, msgpack::sbuffer& buf) TypedMessage request; obj.convert(request); - bool is_block_ours = _ws->sync_block(request.value.blockStateRef, - request.value.blockHeaderHash, - request.value.paddedNoteHashes, - request.value.paddedL1ToL2Messages, - request.value.paddedNullifiers, - request.value.batchesOfPaddedPublicDataWrites); + WorldStateStatus status = _ws->sync_block(request.value.blockStateRef, + request.value.blockHeaderHash, + request.value.paddedNoteHashes, + request.value.paddedL1ToL2Messages, + request.value.paddedNullifiers, + request.value.batchesOfPaddedPublicDataWrites); MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg(WorldStateMessageType::SYNC_BLOCK, header, { is_block_ours }); + messaging::TypedMessage resp_msg(WorldStateMessageType::SYNC_BLOCK, header, { status }); msgpack::pack(buf, resp_msg); return true; @@ -557,7 +573,10 @@ bool WorldStateAddon::create_fork(msgpack::object& obj, msgpack::sbuffer& buf) TypedMessage request; obj.convert(request); - uint64_t forkId = _ws->create_fork(request.value.blockNumber); + std::optional blockNumber = + request.value.latest ? std::nullopt : std::optional(request.value.blockNumber); + + uint64_t forkId = _ws->create_fork(blockNumber); MsgHeader header(request.header.messageId); messaging::TypedMessage resp_msg(WorldStateMessageType::CREATE_FORK, header, { forkId }); @@ -596,6 +615,61 @@ bool WorldStateAddon::close(msgpack::object& obj, msgpack::sbuffer& buf) return true; } +bool WorldStateAddon::set_finalised(msgpack::object& obj, msgpack::sbuffer& buf) const +{ + TypedMessage request; + obj.convert(request); + WorldStateStatus status = _ws->set_finalised_blocks(request.value.toBlockNumber); + MsgHeader header(request.header.messageId); + messaging::TypedMessage resp_msg(WorldStateMessageType::FINALISE_BLOCKS, header, { status }); + msgpack::pack(buf, resp_msg); + + return true; +} + +bool WorldStateAddon::unwind(msgpack::object& obj, msgpack::sbuffer& buf) const +{ + TypedMessage request; + obj.convert(request); + + WorldStateStatus status = _ws->unwind_blocks(request.value.toBlockNumber); + + MsgHeader header(request.header.messageId); + messaging::TypedMessage resp_msg(WorldStateMessageType::UNWIND_BLOCKS, header, { status }); + msgpack::pack(buf, resp_msg); + + return true; +} + +bool WorldStateAddon::remove_historical(msgpack::object& obj, msgpack::sbuffer& buf) const +{ + TypedMessage request; + obj.convert(request); + WorldStateStatus status = _ws->remove_historical_blocks(request.value.toBlockNumber); + + MsgHeader header(request.header.messageId); + messaging::TypedMessage resp_msg( + WorldStateMessageType::REMOVE_HISTORICAL_BLOCKS, header, { status }); + msgpack::pack(buf, resp_msg); + + return true; +} + +bool WorldStateAddon::get_status(msgpack::object& obj, msgpack::sbuffer& buf) const +{ + HeaderOnlyMessage request; + obj.convert(request); + + WorldStateStatus status; + _ws->get_status(status); + + MsgHeader header(request.header.messageId); + messaging::TypedMessage resp_msg(WorldStateMessageType::GET_STATUS, header, { status }); + msgpack::pack(buf, resp_msg); + + return true; +} + Napi::Function WorldStateAddon::get_class(Napi::Env env) { return DefineClass(env, diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp index 6e10a5b8e1e..034ca9cd032 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp @@ -56,6 +56,12 @@ class WorldStateAddon : public Napi::ObjectWrap { bool delete_fork(msgpack::object& obj, msgpack::sbuffer& buffer); bool close(msgpack::object& obj, msgpack::sbuffer& buffer); + + bool set_finalised(msgpack::object& obj, msgpack::sbuffer& buffer) const; + bool unwind(msgpack::object& obj, msgpack::sbuffer& buffer) const; + bool remove_historical(msgpack::object& obj, msgpack::sbuffer& buffer) const; + + bool get_status(msgpack::object& obj, msgpack::sbuffer& buffer) const; }; } // namespace bb::world_state diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp b/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp index 25c03d82872..4868af1b473 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp @@ -4,8 +4,8 @@ #include "barretenberg/messaging/header.hpp" #include "barretenberg/serialize/msgpack.hpp" #include "barretenberg/world_state/types.hpp" -#include "barretenberg/world_state/world_state.hpp" #include +#include #include namespace bb::world_state { @@ -37,6 +37,12 @@ enum WorldStateMessageType { CREATE_FORK, DELETE_FORK, + FINALISE_BLOCKS, + UNWIND_BLOCKS, + REMOVE_HISTORICAL_BLOCKS, + + GET_STATUS, + CLOSE = 999, }; @@ -46,8 +52,9 @@ struct TreeIdOnlyRequest { }; struct CreateForkRequest { - uint64_t blockNumber; - MSGPACK_FIELDS(blockNumber); + bool latest; + index_t blockNumber; + MSGPACK_FIELDS(latest, blockNumber); }; struct CreateForkResponse { @@ -141,6 +148,11 @@ struct FindLowLeafResponse { MSGPACK_FIELDS(alreadyPresent, index); }; +struct BlockShiftRequest { + index_t toBlockNumber; + MSGPACK_FIELDS(toBlockNumber); +}; + template struct AppendLeavesRequest { MerkleTreeId treeId; std::vector leaves; @@ -181,8 +193,8 @@ struct SyncBlockRequest { }; struct SyncBlockResponse { - bool isBlockOurs; - MSGPACK_FIELDS(isBlockOurs); + WorldStateStatus status; + MSGPACK_FIELDS(status); }; } // namespace bb::world_state diff --git a/cspell.json b/cspell.json index e405fe53f0b..d886dd8733e 100644 --- a/cspell.json +++ b/cspell.json @@ -266,6 +266,7 @@ "undici", "unexclude", "unexcluded", + "unfinalised", "unprefixed", "unshield", "unshielding", diff --git a/yarn-project/circuit-types/src/interfaces/world_state.ts b/yarn-project/circuit-types/src/interfaces/world_state.ts index 600eeb4a579..4c11bfe8f97 100644 --- a/yarn-project/circuit-types/src/interfaces/world_state.ts +++ b/yarn-project/circuit-types/src/interfaces/world_state.ts @@ -13,7 +13,7 @@ export enum WorldStateRunningState { /** * Defines the status of the world state synchronizer. */ -export interface WorldStateStatus { +export interface WorldStateSynchronizerStatus { /** * The current state of the world state synchronizer. */ @@ -38,7 +38,7 @@ export interface WorldStateSynchronizer { * Returns the current status of the synchronizer. * @returns The current status of the synchronizer. */ - status(): Promise; + status(): Promise; /** * Stops the synchronizer. diff --git a/yarn-project/circuit-types/src/stats/stats.ts b/yarn-project/circuit-types/src/stats/stats.ts index 95e3da32682..c47c35154a5 100644 --- a/yarn-project/circuit-types/src/stats/stats.ts +++ b/yarn-project/circuit-types/src/stats/stats.ts @@ -197,8 +197,12 @@ export type L2BlockHandledStats = { eventName: 'l2-block-handled'; /** Total duration in ms. */ duration: number; - /** Whether the block was produced by this node. */ - isBlockOurs: boolean; + /** Pending block number. */ + unfinalisedBlockNumber: bigint; + /** Proven block number. */ + finalisedBlockNumber: bigint; + /** Oldest historic block number. */ + oldestHistoricBlock: bigint; } & L2BlockStats; /** Stats for a note processor that has caught up with the chain. */ diff --git a/yarn-project/scripts/src/benchmarks/aggregate.ts b/yarn-project/scripts/src/benchmarks/aggregate.ts index baae136653b..4e8222bd11a 100644 --- a/yarn-project/scripts/src/benchmarks/aggregate.ts +++ b/yarn-project/scripts/src/benchmarks/aggregate.ts @@ -106,9 +106,6 @@ function processRollupBlockSynced(entry: L2BlockHandledStats, results: Benchmark if (!BENCHMARK_BLOCK_SIZES.includes(bucket)) { return; } - if (entry.isBlockOurs) { - return; - } append(results, 'l2_block_processing_time_in_ms', bucket, entry.duration); } diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index 8da14b955f9..4173a03aa29 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -8,10 +8,13 @@ import { Tx, type TxHash, type TxValidator, - type WorldStateStatus, type WorldStateSynchronizer, } from '@aztec/circuit-types'; -import { type AllowedElement, BlockProofError } from '@aztec/circuit-types/interfaces'; +import { + type AllowedElement, + BlockProofError, + type WorldStateSynchronizerStatus, +} from '@aztec/circuit-types/interfaces'; import { type L2BlockBuiltStats } from '@aztec/circuit-types/stats'; import { AppendOnlyTreeSnapshot, @@ -182,7 +185,9 @@ export class Sequencer { protected async initialSync() { // TODO: Should we wait for world state to be ready, or is the caller expected to run await start? - this.lastPublishedBlock = await this.worldState.status().then((s: WorldStateStatus) => s.syncedToL2Block); + this.lastPublishedBlock = await this.worldState + .status() + .then((s: WorldStateSynchronizerStatus) => s.syncedToL2Block); } /** @@ -652,7 +657,7 @@ export class Sequencer { */ protected async isBlockSynced() { const syncedBlocks = await Promise.all([ - this.worldState.status().then((s: WorldStateStatus) => s.syncedToL2Block), + this.worldState.status().then((s: WorldStateSynchronizerStatus) => s.syncedToL2Block), this.p2pClient.getStatus().then(s => s.syncedToL2Block), this.l2BlockSource.getBlockNumber(), this.l1ToL2MessageSource.getBlockNumber(), diff --git a/yarn-project/world-state/package.json b/yarn-project/world-state/package.json index 57c2ff32a9c..246dbe0a359 100644 --- a/yarn-project/world-state/package.json +++ b/yarn-project/world-state/package.json @@ -20,7 +20,7 @@ "clean": "rm -rf ./dest ./build .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "test": "DEBUG='aztec:*' LOG_LEVEL='debug' NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests" + "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests" }, "inherits": [ "../package.common.json", diff --git a/yarn-project/world-state/package.local.json b/yarn-project/world-state/package.local.json index 950bfacd27b..181680c78a4 100644 --- a/yarn-project/world-state/package.local.json +++ b/yarn-project/world-state/package.local.json @@ -1,7 +1,7 @@ { "scripts": { "build": "yarn clean && mkdir -p build && (([ -f ../../barretenberg/cpp/build-pic/lib/world_state_napi.node ] && cp -v ../../barretenberg/cpp/build-pic/lib/world_state_napi.node build) || ([ -f ../../barretenberg/cpp/build/bin/world_state_napi.node ] && cp -v ../../barretenberg/cpp/build/bin/world_state_napi.node build) || true) && tsc -b", - "test": "DEBUG='aztec:*' LOG_LEVEL='debug' NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests", + "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests", "clean": "rm -rf ./dest ./build .tsbuildinfo" } -} +} \ No newline at end of file diff --git a/yarn-project/world-state/src/native/message.ts b/yarn-project/world-state/src/native/message.ts index 4b862f1b538..0fed942a3ad 100644 --- a/yarn-project/world-state/src/native/message.ts +++ b/yarn-project/world-state/src/native/message.ts @@ -71,6 +71,12 @@ export enum WorldStateMessageType { CREATE_FORK, DELETE_FORK, + FINALISE_BLOCKS, + UNWIND_BLOCKS, + REMOVE_HISTORICAL_BLOCKS, + + GET_STATUS, + CLOSE = 999, } @@ -78,6 +84,12 @@ interface WithTreeId { treeId: MerkleTreeId; } +export interface WorldStateStatus { + unfinalisedBlockNumber: bigint; + finalisedBlockNumber: bigint; + oldestHistoricalBlock: bigint; +} + interface WithForkId { forkId: number; } @@ -105,6 +117,10 @@ interface WithLeafValue { leaf: SerializedLeafValue; } +interface BlockShiftRequest { + toBlockNumber: bigint; +} + interface WithLeaves { leaves: SerializedLeafValue[]; } @@ -175,10 +191,11 @@ interface SyncBlockRequest { } interface SyncBlockResponse { - isBlockOurs: boolean; + status: WorldStateStatus; } interface CreateForkRequest { + latest: boolean; blockNumber: number; } @@ -190,6 +207,14 @@ interface DeleteForkRequest { forkId: number; } +interface CreateForkResponse { + forkId: number; +} + +interface DeleteForkRequest { + forkId: number; +} + export type WorldStateRequest = { [WorldStateMessageType.GET_TREE_INFO]: GetTreeInfoRequest; [WorldStateMessageType.GET_STATE_REFERENCE]: GetStateReferenceRequest; @@ -215,6 +240,12 @@ export type WorldStateRequest = { [WorldStateMessageType.CREATE_FORK]: CreateForkRequest; [WorldStateMessageType.DELETE_FORK]: DeleteForkRequest; + [WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS]: BlockShiftRequest; + [WorldStateMessageType.UNWIND_BLOCKS]: BlockShiftRequest; + [WorldStateMessageType.FINALISE_BLOCKS]: BlockShiftRequest; + + [WorldStateMessageType.GET_STATUS]: void; + [WorldStateMessageType.CLOSE]: void; }; @@ -243,6 +274,12 @@ export type WorldStateResponse = { [WorldStateMessageType.CREATE_FORK]: CreateForkResponse; [WorldStateMessageType.DELETE_FORK]: void; + [WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS]: WorldStateStatus; + [WorldStateMessageType.UNWIND_BLOCKS]: WorldStateStatus; + [WorldStateMessageType.FINALISE_BLOCKS]: WorldStateStatus; + + [WorldStateMessageType.GET_STATUS]: WorldStateStatus; + [WorldStateMessageType.CLOSE]: void; }; diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index 77601a25888..308a274a83d 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -7,7 +7,7 @@ import { tmpdir } from 'os'; import { join } from 'path'; import { NativeWorldStateService } from './native_world_state.js'; -import { assertSameState, mockBlock } from './test_util.js'; +import { assertSameState, compareChains, mockBlock } from './test_util.js'; describe('NativeWorldState', () => { let dataDir: string; @@ -68,7 +68,7 @@ describe('NativeWorldState', () => { let ws: NativeWorldStateService; beforeEach(async () => { - ws = await NativeWorldStateService.new(rollupAddress, dataDir); + ws = await NativeWorldStateService.new(EthAddress.random(), dataDir); }); afterEach(async () => { @@ -99,6 +99,288 @@ describe('NativeWorldState', () => { // initial header should still work as before expect(fork.getInitialHeader()).toEqual(initialHeader); + + await fork.close(); + }); + + it('creates a fork at a block number', async () => { + const initialFork = await ws.fork(); + for (let i = 0; i < 5; i++) { + const { block, messages } = await mockBlock(i + 1, 2, initialFork); + await ws.handleL2BlockAndMessages(block, messages); + } + + const fork = await ws.fork(3); + const stateReference = await fork.getStateReference(); + const archiveInfo = await fork.getTreeInfo(MerkleTreeId.ARCHIVE); + const header = new Header( + new AppendOnlyTreeSnapshot(new Fr(archiveInfo.root), Number(archiveInfo.size)), + makeContentCommitment(), + stateReference, + makeGlobalVariables(), + Fr.ZERO, + ); + + await fork.updateArchive(header); + + expect(await fork.getTreeInfo(MerkleTreeId.ARCHIVE)).not.toEqual(archiveInfo); + + await fork.close(); }); + + it('can create a fork at block 0 when not latest', async () => { + const fork = await ws.fork(); + const forkAtGenesis = await ws.fork(); + + for (let i = 0; i < 5; i++) { + const blockNumber = i + 1; + const { block, messages } = await mockBlock(blockNumber, 1, fork); + const status = await ws.handleL2BlockAndMessages(block, messages); + + expect(status.unfinalisedBlockNumber).toBe(blockNumber); + } + + const forkAtZero = await ws.fork(0); + await compareChains(forkAtGenesis, forkAtZero); + }, 30_000); + }); + + describe('Pending and Proven chain', () => { + let ws: NativeWorldStateService; + + beforeEach(async () => { + ws = await NativeWorldStateService.tmp(); + }); + + afterEach(async () => { + await ws.close(); + }); + + it('Tracks pending and proven chains', async () => { + const fork = await ws.fork(); + + for (let i = 0; i < 16; i++) { + const blockNumber = i + 1; + const provenBlock = blockNumber - 4; + const { block, messages } = await mockBlock(blockNumber, 1, fork); + const status = await ws.handleL2BlockAndMessages(block, messages); + + expect(status.unfinalisedBlockNumber).toBe(blockNumber); + expect(status.oldestHistoricalBlock).toBe(1); + + if (provenBlock > 0) { + const provenStatus = await ws.setFinalised(BigInt(provenBlock)); + expect(provenStatus.unfinalisedBlockNumber).toBe(blockNumber); + expect(provenStatus.finalisedBlockNumber).toBe(provenBlock); + expect(provenStatus.oldestHistoricalBlock).toBe(1); + } else { + expect(status.finalisedBlockNumber).toBe(0); + } + } + }, 30_000); + + it('Can finalise multiple blocks', async () => { + const fork = await ws.fork(); + + for (let i = 0; i < 16; i++) { + const blockNumber = i + 1; + const { block, messages } = await mockBlock(blockNumber, 1, fork); + const status = await ws.handleL2BlockAndMessages(block, messages); + + expect(status.unfinalisedBlockNumber).toBe(blockNumber); + expect(status.oldestHistoricalBlock).toBe(1); + expect(status.finalisedBlockNumber).toBe(0); + } + + const status = await ws.setFinalised(8n); + expect(status.unfinalisedBlockNumber).toBe(16); + expect(status.oldestHistoricalBlock).toBe(1); + expect(status.finalisedBlockNumber).toBe(8); + }, 30_000); + + it('Can prune historic blocks', async () => { + const fork = await ws.fork(); + const forks = []; + const provenBlockLag = 4; + const prunedBlockLag = 8; + + for (let i = 0; i < 16; i++) { + const blockNumber = i + 1; + const provenBlock = blockNumber - provenBlockLag; + const prunedBlockNumber = blockNumber - prunedBlockLag; + const { block, messages } = await mockBlock(blockNumber, 1, fork); + const status = await ws.handleL2BlockAndMessages(block, messages); + + expect(status.unfinalisedBlockNumber).toBe(blockNumber); + + const blockFork = await ws.fork(); + forks.push(blockFork); + + if (provenBlock > 0) { + const provenStatus = await ws.setFinalised(BigInt(provenBlock)); + expect(provenStatus.finalisedBlockNumber).toBe(provenBlock); + } else { + expect(status.finalisedBlockNumber).toBe(0); + } + + if (prunedBlockNumber > 0) { + const prunedStatus = await ws.removeHistoricalBlocks(BigInt(prunedBlockNumber + 1)); + expect(prunedStatus.oldestHistoricalBlock).toBe(prunedBlockNumber + 1); + } else { + expect(status.oldestHistoricalBlock).toBe(1); + } + } + + const highestPrunedBlockNumber = 16 - prunedBlockLag; + for (let i = 0; i < 16; i++) { + const blockNumber = i + 1; + if (blockNumber > highestPrunedBlockNumber) { + await expect(forks[i].getSiblingPath(MerkleTreeId.NULLIFIER_TREE, 0n)).resolves.toBeDefined(); + } else { + await expect(forks[i].getSiblingPath(MerkleTreeId.NULLIFIER_TREE, 0n)).rejects.toThrow('Fork not found'); + } + } + + //can't prune what has already been pruned + for (let i = 0; i < highestPrunedBlockNumber; i++) { + await expect(ws.removeHistoricalBlocks(BigInt(i + 1))).rejects.toThrow( + 'Unable to remove historical block, block not found', + ); + } + }, 30_000); + + it('Can re-org', async () => { + const nonReorgState = await NativeWorldStateService.tmp(); + const sequentialReorgState = await NativeWorldStateService.tmp(); + let fork = await ws.fork(); + + const blockForks = []; + const blockTreeInfos = []; + const blockStats = []; + const siblingPaths = []; + + // advance 3 chains by 8 blocks, 2 of the chains go to 16 blocks + for (let i = 0; i < 16; i++) { + const blockNumber = i + 1; + const { block, messages } = await mockBlock(blockNumber, 1, fork); + const status = await ws.handleL2BlockAndMessages(block, messages); + blockStats.push(status); + const blockFork = await ws.fork(); + blockForks.push(blockFork); + const treeInfo = await ws.getCommitted().getTreeInfo(MerkleTreeId.NULLIFIER_TREE); + blockTreeInfos.push(treeInfo); + const siblingPath = await ws.getCommitted().getSiblingPath(MerkleTreeId.NULLIFIER_TREE, 0n); + siblingPaths.push(siblingPath); + + if (blockNumber < 9) { + await nonReorgState.handleL2BlockAndMessages(block, messages); + + const statusNonReorg = await nonReorgState.handleL2BlockAndMessages(block, messages); + expect(status).toEqual(statusNonReorg); + + const treeInfoNonReorg = await nonReorgState.getCommitted().getTreeInfo(MerkleTreeId.NULLIFIER_TREE); + expect(treeInfo).toEqual(treeInfoNonReorg); + } + + await sequentialReorgState.handleL2BlockAndMessages(block, messages); + } + + // unwind 1 chain by a single block at a time + for (let blockNumber = 16; blockNumber > 8; blockNumber--) { + const unwindStatus = await sequentialReorgState.unwindBlocks(BigInt(blockNumber - 1)); + const unwindFork = await sequentialReorgState.fork(); + const unwindTreeInfo = await sequentialReorgState.getCommitted().getTreeInfo(MerkleTreeId.NULLIFIER_TREE); + const unwindSiblingPath = await sequentialReorgState + .getCommitted() + .getSiblingPath(MerkleTreeId.NULLIFIER_TREE, 0n); + + expect(unwindTreeInfo).toEqual(blockTreeInfos[blockNumber - 2]); + expect(unwindStatus).toEqual(blockStats[blockNumber - 2]); + expect(await unwindFork.getTreeInfo(MerkleTreeId.NULLIFIER_TREE)).toEqual( + await blockForks[blockNumber - 2].getTreeInfo(MerkleTreeId.NULLIFIER_TREE), + ); + expect(unwindSiblingPath).toEqual(siblingPaths[blockNumber - 2]); + } + + // unwind the other 16 block chain by a full 8 blocks in one go + await ws.unwindBlocks(8n); + + // check that it is not possible to re-org blocks that were already reorged. + await expect(ws.unwindBlocks(10n)).rejects.toThrow('Unable to unwind block, block not found'); + + await compareChains(ws.getCommitted(), sequentialReorgState.getCommitted()); + + const unwoundFork = await ws.fork(); + const unwoundTreeInfo = await ws.getCommitted().getTreeInfo(MerkleTreeId.NULLIFIER_TREE); + const unwoundStatus = await ws.getStatus(); + const unwoundSiblingPath = await ws.getCommitted().getSiblingPath(MerkleTreeId.NULLIFIER_TREE, 0n); + + expect(unwoundStatus).toEqual(blockStats[7]); + expect(unwoundTreeInfo).toEqual(blockTreeInfos[7]); + expect(await ws.getCommitted().getTreeInfo(MerkleTreeId.NULLIFIER_TREE)).toEqual(blockTreeInfos[7]); + expect(await unwoundFork.getTreeInfo(MerkleTreeId.NULLIFIER_TREE)).toEqual(blockTreeInfos[7]); + expect(unwoundSiblingPath).toEqual(siblingPaths[7]); + + fork = await ws.fork(); + + // now advance both the un-reorged chain and one of the reorged chains to 16 blocks + for (let i = 8; i < 16; i++) { + const blockNumber = i + 1; + const { block, messages } = await mockBlock(blockNumber, 1, fork); + const status = await ws.handleL2BlockAndMessages(block, messages); + blockStats[i] = status; + const blockFork = await ws.fork(); + blockForks[i] = blockFork; + const treeInfo = await ws.getCommitted().getTreeInfo(MerkleTreeId.NULLIFIER_TREE); + blockTreeInfos[i] = treeInfo; + const siblingPath = await ws.getCommitted().getSiblingPath(MerkleTreeId.NULLIFIER_TREE, 0n); + siblingPaths[i] = siblingPath; + + const statusNonReorg = await nonReorgState.handleL2BlockAndMessages(block, messages); + expect(status).toEqual(statusNonReorg); + } + + // compare snapshot across the chains + for (let i = 0; i < 16; i++) { + const blockNumber = i + 1; + const nonReorgSnapshot = nonReorgState.getSnapshot(blockNumber); + const reorgSnaphsot = ws.getSnapshot(blockNumber); + await compareChains(reorgSnaphsot, nonReorgSnapshot); + } + + await compareChains(ws.getCommitted(), nonReorgState.getCommitted()); + }, 30_000); + + it('Forks are deleted during a re-org', async () => { + const fork = await ws.fork(); + + const blockForks = []; + const blockTreeInfos = []; + const blockStats = []; + const siblingPaths = []; + + for (let i = 0; i < 16; i++) { + const blockNumber = i + 1; + const { block, messages } = await mockBlock(blockNumber, 1, fork); + const status = await ws.handleL2BlockAndMessages(block, messages); + blockStats.push(status); + const blockFork = await ws.fork(); + blockForks.push(blockFork); + const treeInfo = await ws.getCommitted().getTreeInfo(MerkleTreeId.NULLIFIER_TREE); + blockTreeInfos.push(treeInfo); + const siblingPath = await ws.getCommitted().getSiblingPath(MerkleTreeId.NULLIFIER_TREE, 0n); + siblingPaths.push(siblingPath); + } + + await ws.unwindBlocks(8n); + + for (let i = 0; i < 16; i++) { + if (i < 8) { + expect(await blockForks[i].getSiblingPath(MerkleTreeId.NULLIFIER_TREE, 0n)).toEqual(siblingPaths[i]); + } else { + await expect(blockForks[i].getSiblingPath(MerkleTreeId.NULLIFIER_TREE, 0n)).rejects.toThrow('Fork not found'); + } + } + }, 30_000); }); }); diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts index 4d94d8a1d81..8d1d83818c0 100644 --- a/yarn-project/world-state/src/native/native_world_state.ts +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -1,4 +1,5 @@ import { + type IndexedTreeId, type L2Block, MerkleTreeId, type MerkleTreeReadOperations, @@ -14,6 +15,7 @@ import { MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, NullifierLeaf, + type NullifierLeafPreimage, PartialStateReference, PublicDataTreeLeaf, StateReference, @@ -26,13 +28,11 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'fs/promises'; import { tmpdir } from 'os'; import { join } from 'path'; -import { - type HandleL2BlockAndMessagesResult, - type MerkleTreeAdminDatabase as MerkleTreeDatabase, -} from '../world-state-db/merkle_tree_db.js'; +import { type MerkleTreeAdminDatabase as MerkleTreeDatabase } from '../world-state-db/merkle_tree_db.js'; import { MerkleTreesFacade, MerkleTreesForkFacade, serializeLeaf } from './merkle_trees_facade.js'; import { WorldStateMessageType, + type WorldStateStatus, blockStateReference, treeStateReferenceToSnapshot, worldStateRevision, @@ -116,8 +116,11 @@ export class NativeWorldStateService implements MerkleTreeDatabase { return new MerkleTreesFacade(this.instance, this.initialHeader!, worldStateRevision(false, 0, blockNumber)); } - public async fork(): Promise { - const resp = await this.instance.call(WorldStateMessageType.CREATE_FORK, { blockNumber: 0 }); + public async fork(blockNumber?: number): Promise { + const resp = await this.instance.call(WorldStateMessageType.CREATE_FORK, { + latest: blockNumber === undefined, + blockNumber: blockNumber ?? 0, + }); return new MerkleTreesForkFacade(this.instance, this.initialHeader!, worldStateRevision(true, resp.forkId, 0)); } @@ -125,10 +128,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase { return this.initialHeader!; } - public async handleL2BlockAndMessages( - l2Block: L2Block, - l1ToL2Messages: Fr[], - ): Promise { + public async handleL2BlockAndMessages(l2Block: L2Block, l1ToL2Messages: Fr[]): Promise { // We have to pad both the tx effects and the values within tx effects because that's how the trees are built // by circuits. const paddedTxEffects = padArrayEnd( @@ -159,7 +159,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase { batchesOfPaddedPublicDataWrites.push(batch); } - return await this.instance.call(WorldStateMessageType.SYNC_BLOCK, { + const response = await this.instance.call(WorldStateMessageType.SYNC_BLOCK, { blockNumber: l2Block.number, blockHeaderHash: l2Block.header.hash(), paddedL1ToL2Messages: paddedL1ToL2Messages.map(serializeLeaf), @@ -168,6 +168,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase { batchesOfPaddedPublicDataWrites: batchesOfPaddedPublicDataWrites.map(batch => batch.map(serializeLeaf)), blockStateRef: blockStateReference(l2Block.header.state), }); + return response.status; } public async close(): Promise { @@ -180,6 +181,51 @@ export class NativeWorldStateService implements MerkleTreeDatabase { return Header.empty({ state }); } + /** + * Advances the finalised block number to be the number provided + * @param toBlockNumber The block number that is now the tip of the finalised chain + * @returns The new WorldStateStatus + */ + public async setFinalised(toBlockNumber: bigint) { + return await this.instance.call(WorldStateMessageType.FINALISE_BLOCKS, { + toBlockNumber, + }); + } + + /** + * Removes all historical snapshots up to but not including the given block number + * @param toBlockNumber The block number of the new oldest historical block + * @returns The new WorldStateStatus + */ + public async removeHistoricalBlocks(toBlockNumber: bigint) { + return await this.instance.call(WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS, { + toBlockNumber, + }); + } + + /** + * Removes all pending blocks down to but not including the given block number + * @param toBlockNumber The block number of the new tip of the pending chain, + * @returns The new WorldStateStatus + */ + public async unwindBlocks(toBlockNumber: bigint) { + return await this.instance.call(WorldStateMessageType.UNWIND_BLOCKS, { + toBlockNumber, + }); + } + + public async getStatus() { + return await this.instance.call(WorldStateMessageType.GET_STATUS, void 0); + } + + updateLeaf( + _treeId: ID, + _leaf: NullifierLeafPreimage | Buffer, + _index: bigint, + ): Promise { + return Promise.reject(new Error('Method not implemented')); + } + private async getInitialStateReference(): Promise { const resp = await this.instance.call(WorldStateMessageType.GET_INITIAL_STATE_REFERENCE, void 0); diff --git a/yarn-project/world-state/src/native/native_world_state_cmp.test.ts b/yarn-project/world-state/src/native/native_world_state_cmp.test.ts index 7c5cef15b3f..836dc246c8e 100644 --- a/yarn-project/world-state/src/native/native_world_state_cmp.test.ts +++ b/yarn-project/world-state/src/native/native_world_state_cmp.test.ts @@ -6,6 +6,7 @@ import { type MerkleTreeWriteOperations, } from '@aztec/circuit-types'; import { EthAddress, Fr, GENESIS_ARCHIVE_ROOT, NullifierLeaf, PublicDataTreeLeaf } from '@aztec/circuits.js'; +import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; import { elapsed } from '@aztec/foundation/timer'; import { AztecLmdbStore } from '@aztec/kv-store/lmdb'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; @@ -28,6 +29,8 @@ describe('NativeWorldState', () => { let nativeWS: NativeWorldStateService; let legacyWS: MerkleTrees; + let log: DebugLogger; + const allTrees = Object.values(MerkleTreeId) .filter((x): x is MerkleTreeId => typeof x === 'number') .map(x => [MerkleTreeId[x], x] as const); @@ -35,6 +38,8 @@ describe('NativeWorldState', () => { beforeAll(async () => { nativeDataDir = await mkdtemp(join(tmpdir(), 'native_world_state_test-')); legacyDataDir = await mkdtemp(join(tmpdir(), 'js_world_state_test-')); + + log = createDebugLogger('aztec:world-state:test:native_world_state_cmp'); }); afterAll(async () => { @@ -217,8 +222,7 @@ describe('NativeWorldState', () => { for (let i = 0; i < numBlocks; i++) { const [_nativeMs] = await elapsed(nativeWS.handleL2BlockAndMessages(blocks[i], messagesArray[i])); const [_legacyMs] = await elapsed(legacyWS.handleL2BlockAndMessages(blocks[i], messagesArray[i])); - // eslint-disable-next-line no-console - console.log(`Native: ${_nativeMs} ms, Legacy: ${_legacyMs} ms.`); + log.info(`Native: ${_nativeMs} ms, Legacy: ${_legacyMs} ms.`); } await assertSameTree(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, nativeWS.getCommitted(), legacyWS.getCommitted()); diff --git a/yarn-project/world-state/src/native/test_util.ts b/yarn-project/world-state/src/native/test_util.ts index bcf1045a00c..7c7a39c6989 100644 --- a/yarn-project/world-state/src/native/test_util.ts +++ b/yarn-project/world-state/src/native/test_util.ts @@ -89,3 +89,17 @@ export async function assertSameState(forkA: MerkleTreeReadOperations, forkB: Me expect(nativeStateRef).toEqual(legacyStateRef); expect(nativeArchive).toEqual(legacyArchive); } + +export async function compareChains(left: MerkleTreeReadOperations, right: MerkleTreeReadOperations) { + for (const treeId of [ + MerkleTreeId.ARCHIVE, + MerkleTreeId.L1_TO_L2_MESSAGE_TREE, + MerkleTreeId.NOTE_HASH_TREE, + MerkleTreeId.NULLIFIER_TREE, + MerkleTreeId.PUBLIC_DATA_TREE, + ]) { + expect(await left.getTreeInfo(treeId)).toEqual(await right.getTreeInfo(treeId)); + + expect(await left.getSiblingPath(treeId, 0n)).toEqual(await right.getSiblingPath(treeId, 0n)); + } +} diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts index 390e25684b5..ac999ecdd65 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts @@ -39,7 +39,9 @@ describe('server_world_state_synchronizer', () => { }); const merkleTreeDb = mock({ - handleL2BlockAndMessages: jest.fn(() => Promise.resolve({ isBlockOurs: false })), + handleL2BlockAndMessages: jest.fn(() => + Promise.resolve({ unfinalisedBlockNumber: 0n, finalisedBlockNumber: 0n, oldestHistoricalBlock: 0n }), + ), }); const performInitialSync = async (server: ServerWorldStateSynchronizer) => { diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts index eb176a5c5b1..70e1d2137cc 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts @@ -6,8 +6,8 @@ import { type MerkleTreeReadOperations, type MerkleTreeWriteOperations, WorldStateRunningState, - type WorldStateStatus, type WorldStateSynchronizer, + type WorldStateSynchronizerStatus, } from '@aztec/circuit-types'; import { type L2BlockHandledStats } from '@aztec/circuit-types/stats'; import { MerkleTreeCalculator } from '@aztec/circuits.js'; @@ -20,7 +20,8 @@ import { elapsed } from '@aztec/foundation/timer'; import { type AztecKVStore, type AztecSingleton } from '@aztec/kv-store'; import { SHA256Trunc } from '@aztec/merkle-tree'; -import { type HandleL2BlockAndMessagesResult, type MerkleTreeAdminDatabase } from '../world-state-db/merkle_tree_db.js'; +import { type WorldStateStatus } from '../native/message.js'; +import { type MerkleTreeAdminDatabase } from '../world-state-db/merkle_tree_db.js'; import { type WorldStateConfig } from './config.js'; /** @@ -153,11 +154,11 @@ export class ServerWorldStateSynchronizer implements WorldStateSynchronizer { } } - public status(): Promise { + public status(): Promise { const status = { syncedToL2Block: this.currentL2BlockNum, state: this.currentState, - } as WorldStateStatus; + } as WorldStateSynchronizerStatus; return Promise.resolve(status); } @@ -235,7 +236,9 @@ export class ServerWorldStateSynchronizer implements WorldStateSynchronizer { this.log.verbose(`Handled new L2 block`, { eventName: 'l2-block-handled', duration, - isBlockOurs: result.isBlockOurs, + unfinalisedBlockNumber: result.unfinalisedBlockNumber, + finalisedBlockNumber: result.finalisedBlockNumber, + oldestHistoricBlock: result.oldestHistoricalBlock, ...l2Blocks[i].getStats(), } satisfies L2BlockHandledStats); } @@ -247,10 +250,7 @@ export class ServerWorldStateSynchronizer implements WorldStateSynchronizer { * @param l1ToL2Messages - The L1 to L2 messages for the block. * @returns Whether the block handled was produced by this same node. */ - private async handleL2BlockAndMessages( - l2Block: L2Block, - l1ToL2Messages: Fr[], - ): Promise { + private async handleL2BlockAndMessages(l2Block: L2Block, l1ToL2Messages: Fr[]): Promise { // First we check that the L1 to L2 messages hash to the block inHash. // Note that we cannot optimize this check by checking the root of the subtree after inserting the messages // to the real L1_TO_L2_MESSAGE_TREE (like we do in merkleTreeDb.handleL2BlockAndMessages(...)) because that diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts index ed8c7c32e58..2f3dd68ae8a 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts @@ -3,6 +3,8 @@ import { type MerkleTreeReadOperations, type MerkleTreeWriteOperations } from '@ import { type Fr, MAX_NULLIFIERS_PER_TX, MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX } from '@aztec/circuits.js'; import { type IndexedTreeSnapshot, type TreeSnapshot } from '@aztec/merkle-tree'; +import { type WorldStateStatus } from '../native/message.js'; + /** * * @remarks Short explanation: @@ -30,18 +32,13 @@ export type TreeSnapshots = { [MerkleTreeId.ARCHIVE]: TreeSnapshot; }; -/** Return type for handleL2BlockAndMessages */ -export type HandleL2BlockAndMessagesResult = { - /** Whether the block processed was emitted by our sequencer */ isBlockOurs: boolean; -}; - export interface MerkleTreeAdminDatabase { /** * Handles a single L2 block (i.e. Inserts the new note hashes into the merkle tree). * @param block - The L2 block to handle. * @param l1ToL2Messages - The L1 to L2 messages for the block. */ - handleL2BlockAndMessages(block: L2Block, l1ToL2Messages: Fr[]): Promise; + handleL2BlockAndMessages(block: L2Block, l1ToL2Messages: Fr[]): Promise; /** * Gets a handle that allows reading the latest committed state @@ -60,11 +57,6 @@ export interface MerkleTreeAdminDatabase { */ fork(blockNumber?: number): Promise; - /** - * Forks the database at the given block number. - */ - fork(blockNumber: number): Promise; - /** Stops the database */ close(): Promise; } diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index 971aebc9c94..07349604cac 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -50,8 +50,8 @@ import { type TelemetryClient } from '@aztec/telemetry-client'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { type Hasher } from '@aztec/types/interfaces'; +import { type WorldStateStatus } from '../native/message.js'; import { - type HandleL2BlockAndMessagesResult, INITIAL_NULLIFIER_TREE_SIZE, INITIAL_PUBLIC_DATA_TREE_SIZE, type MerkleTreeAdminDatabase, @@ -447,7 +447,7 @@ export class MerkleTrees implements MerkleTreeAdminDatabase { * @param l1ToL2Messages - The L1 to L2 messages for the block. * @returns Whether the block handled was produced by this same node. */ - public async handleL2BlockAndMessages(block: L2Block, l1ToL2Messages: Fr[]): Promise { + public async handleL2BlockAndMessages(block: L2Block, l1ToL2Messages: Fr[]): Promise { return await this.synchronize(() => this.#handleL2BlockAndMessages(block, l1ToL2Messages)); } @@ -597,7 +597,7 @@ export class MerkleTrees implements MerkleTreeAdminDatabase { * @param l2Block - The L2 block to handle. * @param l1ToL2Messages - The L1 to L2 messages for the block. */ - async #handleL2BlockAndMessages(l2Block: L2Block, l1ToL2Messages: Fr[]): Promise { + async #handleL2BlockAndMessages(l2Block: L2Block, l1ToL2Messages: Fr[]): Promise { const timer = new Timer(); const treeRootWithIdPairs = [ @@ -689,8 +689,8 @@ export class MerkleTrees implements MerkleTreeAdminDatabase { await this.#snapshot(l2Block.number); this.metrics.recordDbSize(this.store.estimateSize().bytes); - this.metrics.recordSyncDuration(ourBlock ? 'commit' : 'rollback_and_update', timer); - return { isBlockOurs: ourBlock }; + this.metrics.recordSyncDuration('commit', timer); + return { unfinalisedBlockNumber: 0n, finalisedBlockNumber: 0n, oldestHistoricalBlock: 0n } as WorldStateStatus; } #isDbPopulated(): boolean {