diff --git a/barretenberg/cpp/src/barretenberg/vm/aztec_constants.hpp b/barretenberg/cpp/src/barretenberg/vm/aztec_constants.hpp index 0a934bb88af..7c6edcbe3c7 100644 --- a/barretenberg/cpp/src/barretenberg/vm/aztec_constants.hpp +++ b/barretenberg/cpp/src/barretenberg/vm/aztec_constants.hpp @@ -12,14 +12,6 @@ #define MAX_NULLIFIER_NON_EXISTENT_READ_REQUESTS_PER_CALL 16 #define MAX_L1_TO_L2_MSG_READ_REQUESTS_PER_CALL 16 #define MAX_UNENCRYPTED_LOGS_PER_CALL 4 -#define ARCHIVE_HEIGHT 16 -#define NOTE_HASH_TREE_HEIGHT 32 -#define PUBLIC_DATA_TREE_HEIGHT 40 -#define NULLIFIER_TREE_HEIGHT 20 -#define L1_TO_L2_MSG_TREE_HEIGHT 16 -#define MAX_NULLIFIERS_PER_TX 64 -#define MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX 64 -#define GENESIS_ARCHIVE_ROOT "0x1200a06aae1368abe36530b585bd7a4d2ba4de5037b82076412691a187d7621e" #define AZTEC_ADDRESS_LENGTH 1 #define GAS_FEES_LENGTH 2 #define GAS_LENGTH 2 @@ -144,4 +136,3 @@ #define AVM_EMITNULLIFIER_BASE_DA_GAS 512 #define AVM_SENDL2TOL1MSG_BASE_DA_GAS 512 #define AVM_EMITUNENCRYPTEDLOG_DYN_DA_GAS 512 -#define GENERATOR_INDEX__BLOCK_HASH 28 diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp index e45faaa87e9..822515b7c42 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp @@ -31,20 +31,29 @@ namespace bb::world_state { using namespace bb::crypto::merkle_tree; -const uint64_t INITIAL_NULLIFIER_TREE_SIZE = 2UL * MAX_NULLIFIERS_PER_TX; -const uint64_t INITIAL_PUBLIC_DATA_TREE_SIZE = 2UL * MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX; - -WorldState::WorldState(const std::string& data_dir, +WorldState::WorldState(uint64_t thread_pool_size, + const std::string& data_dir, const std::unordered_map& map_size, - uint64_t thread_pool_size) + const std::unordered_map& tree_heights, + const std::unordered_map& tree_prefill, + uint32_t initial_header_generator_point) : _workers(std::make_shared(thread_pool_size)) + , _tree_heights(tree_heights) + , _initial_tree_size(tree_prefill) + , _initial_header_generator_point(initial_header_generator_point) , _forkId(CANONICAL_FORK_ID) { create_canonical_fork(data_dir, map_size, thread_pool_size); } -WorldState::WorldState(const std::string& data_dir, uint64_t map_size, uint64_t thread_pool_size) - : WorldState(data_dir, +WorldState::WorldState(uint64_t thread_pool_size, + const std::string& data_dir, + uint64_t map_size, + const std::unordered_map& tree_heights, + const std::unordered_map& tree_prefill, + uint32_t initial_header_generator_point) + : WorldState(thread_pool_size, + data_dir, { { MerkleTreeId::NULLIFIER_TREE, map_size }, { MerkleTreeId::PUBLIC_DATA_TREE, map_size }, @@ -52,7 +61,9 @@ WorldState::WorldState(const std::string& data_dir, uint64_t map_size, uint64_t { MerkleTreeId::NOTE_HASH_TREE, map_size }, { MerkleTreeId::L1_TO_L2_MESSAGE_TREE, map_size }, }, - thread_pool_size) + tree_heights, + tree_prefill, + initial_header_generator_point) {} void WorldState::create_canonical_fork(const std::string& dataDir, @@ -76,37 +87,42 @@ void WorldState::create_canonical_fork(const std::string& dataDir, Fork::SharedPtr fork = std::make_shared(); fork->_forkId = _forkId++; { + uint32_t levels = _tree_heights.at(MerkleTreeId::NULLIFIER_TREE); + index_t initial_size = _initial_tree_size.at(MerkleTreeId::NULLIFIER_TREE); auto store = std::make_unique( - getMerkleTreeName(MerkleTreeId::NULLIFIER_TREE), NULLIFIER_TREE_HEIGHT, _persistentStores->nullifierStore); - auto tree = std::make_unique(std::move(store), _workers, INITIAL_NULLIFIER_TREE_SIZE); + getMerkleTreeName(MerkleTreeId::NULLIFIER_TREE), levels, _persistentStores->nullifierStore); + auto tree = std::make_unique(std::move(store), _workers, initial_size); fork->_trees.insert({ MerkleTreeId::NULLIFIER_TREE, TreeWithStore(std::move(tree)) }); } { + uint32_t levels = _tree_heights.at(MerkleTreeId::NOTE_HASH_TREE); auto store = std::make_unique( - getMerkleTreeName(MerkleTreeId::NOTE_HASH_TREE), NOTE_HASH_TREE_HEIGHT, _persistentStores->noteHashStore); + getMerkleTreeName(MerkleTreeId::NOTE_HASH_TREE), levels, _persistentStores->noteHashStore); auto tree = std::make_unique(std::move(store), _workers); fork->_trees.insert({ MerkleTreeId::NOTE_HASH_TREE, TreeWithStore(std::move(tree)) }); } { - auto store = std::make_unique(getMerkleTreeName(MerkleTreeId::PUBLIC_DATA_TREE), - PUBLIC_DATA_TREE_HEIGHT, - _persistentStores->publicDataStore); - auto tree = std::make_unique(std::move(store), _workers, INITIAL_PUBLIC_DATA_TREE_SIZE); + uint32_t levels = _tree_heights.at(MerkleTreeId::PUBLIC_DATA_TREE); + index_t initial_size = _initial_tree_size.at(MerkleTreeId::PUBLIC_DATA_TREE); + auto store = std::make_unique( + getMerkleTreeName(MerkleTreeId::PUBLIC_DATA_TREE), levels, _persistentStores->publicDataStore); + auto tree = std::make_unique(std::move(store), _workers, initial_size); fork->_trees.insert({ MerkleTreeId::PUBLIC_DATA_TREE, TreeWithStore(std::move(tree)) }); } { - auto store = std::make_unique(getMerkleTreeName(MerkleTreeId::L1_TO_L2_MESSAGE_TREE), - L1_TO_L2_MSG_TREE_HEIGHT, - _persistentStores->messageStore); + uint32_t levels = _tree_heights.at(MerkleTreeId::L1_TO_L2_MESSAGE_TREE); + auto store = std::make_unique( + getMerkleTreeName(MerkleTreeId::L1_TO_L2_MESSAGE_TREE), levels, _persistentStores->messageStore); auto tree = std::make_unique(std::move(store), _workers); fork->_trees.insert({ MerkleTreeId::L1_TO_L2_MESSAGE_TREE, TreeWithStore(std::move(tree)) }); } { - std::vector initial_leaves{ compute_initial_archive( - get_state_reference(WorldStateRevision::committed(), fork, true)) }; + uint32_t levels = _tree_heights.at(MerkleTreeId::ARCHIVE); + std::vector initial_values{ compute_initial_archive( + get_state_reference(WorldStateRevision::committed(), fork, true), _initial_header_generator_point) }; auto store = std::make_unique( - getMerkleTreeName(MerkleTreeId::ARCHIVE), ARCHIVE_HEIGHT, _persistentStores->archiveStore); - auto tree = std::make_unique(std::move(store), _workers, initial_leaves); + getMerkleTreeName(MerkleTreeId::ARCHIVE), levels, _persistentStores->archiveStore); + auto tree = std::make_unique(std::move(store), _workers, initial_values); fork->_trees.insert({ MerkleTreeId::ARCHIVE, TreeWithStore(std::move(tree)) }); } _forks[fork->_forkId] = fork; @@ -148,40 +164,39 @@ Fork::SharedPtr WorldState::create_new_fork(index_t blockNumber) { Fork::SharedPtr fork = std::make_shared(); { - auto store = std::make_unique(getMerkleTreeName(MerkleTreeId::NULLIFIER_TREE), - NULLIFIER_TREE_HEIGHT, - blockNumber, - _persistentStores->nullifierStore); - auto tree = std::make_unique(std::move(store), _workers, INITIAL_NULLIFIER_TREE_SIZE); + uint32_t levels = _tree_heights.at(MerkleTreeId::NULLIFIER_TREE); + index_t initial_size = _initial_tree_size.at(MerkleTreeId::NULLIFIER_TREE); + auto store = std::make_unique( + getMerkleTreeName(MerkleTreeId::NULLIFIER_TREE), levels, blockNumber, _persistentStores->nullifierStore); + auto tree = std::make_unique(std::move(store), _workers, initial_size); fork->_trees.insert({ MerkleTreeId::NULLIFIER_TREE, TreeWithStore(std::move(tree)) }); } { - auto store = std::make_unique(getMerkleTreeName(MerkleTreeId::NOTE_HASH_TREE), - NOTE_HASH_TREE_HEIGHT, - blockNumber, - _persistentStores->noteHashStore); + uint32_t levels = _tree_heights.at(MerkleTreeId::NOTE_HASH_TREE); + auto store = std::make_unique( + getMerkleTreeName(MerkleTreeId::NOTE_HASH_TREE), levels, blockNumber, _persistentStores->noteHashStore); auto tree = std::make_unique(std::move(store), _workers); fork->_trees.insert({ MerkleTreeId::NOTE_HASH_TREE, TreeWithStore(std::move(tree)) }); } { - auto store = std::make_unique(getMerkleTreeName(MerkleTreeId::PUBLIC_DATA_TREE), - PUBLIC_DATA_TREE_HEIGHT, - blockNumber, - _persistentStores->publicDataStore); - auto tree = std::make_unique(std::move(store), _workers, INITIAL_PUBLIC_DATA_TREE_SIZE); + uint32_t levels = _tree_heights.at(MerkleTreeId::PUBLIC_DATA_TREE); + index_t initial_size = _initial_tree_size.at(MerkleTreeId::PUBLIC_DATA_TREE); + auto store = std::make_unique( + getMerkleTreeName(MerkleTreeId::PUBLIC_DATA_TREE), levels, blockNumber, _persistentStores->publicDataStore); + auto tree = std::make_unique(std::move(store), _workers, initial_size); fork->_trees.insert({ MerkleTreeId::PUBLIC_DATA_TREE, TreeWithStore(std::move(tree)) }); } { - auto store = std::make_unique(getMerkleTreeName(L1_TO_L2_MESSAGE_TREE), - L1_TO_L2_MSG_TREE_HEIGHT, - blockNumber, - _persistentStores->messageStore); + uint32_t levels = _tree_heights.at(MerkleTreeId::L1_TO_L2_MESSAGE_TREE); + auto store = std::make_unique( + getMerkleTreeName(L1_TO_L2_MESSAGE_TREE), levels, blockNumber, _persistentStores->messageStore); auto tree = std::make_unique(std::move(store), _workers); fork->_trees.insert({ MerkleTreeId::L1_TO_L2_MESSAGE_TREE, TreeWithStore(std::move(tree)) }); } { + uint32_t levels = _tree_heights.at(MerkleTreeId::ARCHIVE); auto store = std::make_unique( - getMerkleTreeName(MerkleTreeId::ARCHIVE), ARCHIVE_HEIGHT, blockNumber, _persistentStores->archiveStore); + getMerkleTreeName(MerkleTreeId::ARCHIVE), levels, blockNumber, _persistentStores->archiveStore); auto tree = std::make_unique(std::move(store), _workers); fork->_trees.insert({ MerkleTreeId::ARCHIVE, TreeWithStore(std::move(tree)) }); } @@ -320,11 +335,10 @@ void WorldState::update_archive(const StateReference& block_state_ref, const bb::fr& block_header_hash, Fork::Id fork_id) { - auto world_state_ref = get_state_reference(WorldStateRevision{ .forkId = fork_id, .includeUncommitted = true }); - if (block_state_matches_world_state(block_state_ref, world_state_ref)) { + if (is_same_state_reference(WorldStateRevision{ .forkId = fork_id, .includeUncommitted = true }, block_state_ref)) { append_leaves(MerkleTreeId::ARCHIVE, { block_header_hash }, fork_id); } else { - throw std::runtime_error("Block state does not match world state"); + throw std::runtime_error("Can't update archive tree: Block state does not match world state"); } } @@ -364,9 +378,7 @@ bool WorldState::sync_block(const StateReference& block_state_ref, const std::vector& nullifiers, const std::vector>& public_writes) { - Fork::SharedPtr fork = retrieve_fork(CANONICAL_FORK_ID); - auto current_state = get_state_reference(WorldStateRevision::uncommitted()); - if (block_state_matches_world_state(block_state_ref, current_state) && + if (is_same_state_reference(WorldStateRevision::uncommitted(), block_state_ref) && is_archive_tip(WorldStateRevision::uncommitted(), block_header_hash)) { commit(); return true; @@ -374,6 +386,7 @@ bool WorldState::sync_block(const StateReference& block_state_ref, rollback(); + Fork::SharedPtr fork = retrieve_fork(CANONICAL_FORK_ID); Signal signal(static_cast(fork->_trees.size())); std::atomic_bool success = true; std::string err_message; @@ -437,16 +450,16 @@ bool WorldState::sync_block(const StateReference& block_state_ref, throw std::runtime_error("Failed to sync block: " + err_message); } - auto state_after_inserts = get_state_reference(WorldStateRevision::uncommitted()); - if (block_state_matches_world_state(block_state_ref, state_after_inserts) && - is_archive_tip(WorldStateRevision::uncommitted(), block_header_hash)) { - commit(); - return false; + if (!is_archive_tip(WorldStateRevision::uncommitted(), block_header_hash)) { + throw std::runtime_error("Can't synch block: block header hash is not the tip of the archive tree"); + } + + if (!is_same_state_reference(WorldStateRevision::uncommitted(), block_state_ref)) { + throw std::runtime_error("Can't synch block: block state does not match world state"); } - // TODO (alexg) should we rollback here? - // Potentiall not since all the changes exist only in-memory and this error will cause the process to die - throw std::runtime_error("Can't synch block: block state does not match world state"); + commit(); + return false; } GetLowIndexedLeafResponse WorldState::find_low_leaf_index(const WorldStateRevision& revision, @@ -483,11 +496,11 @@ GetLowIndexedLeafResponse WorldState::find_low_leaf_index(const WorldStateRevisi return low_leaf_info; } -bb::fr WorldState::compute_initial_archive(const StateReference& initial_state_ref) +bb::fr WorldState::compute_initial_archive(const StateReference& initial_state_ref, uint32_t generator_point) { // NOTE: this hash operations needs to match the one in yarn-project/circuits.js/src/structs/header.ts - return HashPolicy::hash({ GENERATOR_INDEX__BLOCK_HASH, // separator - // last archive - which, at genesis, is all 0s + return HashPolicy::hash({ generator_point, + // last archive - which, at genesis, is all 0s 0, 0, // content commitment - all 0s @@ -518,20 +531,6 @@ bb::fr WorldState::compute_initial_archive(const StateReference& initial_state_r 0 }); } -bool WorldState::block_state_matches_world_state(const StateReference& block_state_ref, - const StateReference& tree_state_ref) -{ - std::vector tree_ids{ - MerkleTreeId::NULLIFIER_TREE, - MerkleTreeId::NOTE_HASH_TREE, - MerkleTreeId::PUBLIC_DATA_TREE, - MerkleTreeId::L1_TO_L2_MESSAGE_TREE, - }; - - return std::all_of( - tree_ids.begin(), tree_ids.end(), [=](auto id) { return block_state_ref.at(id) == tree_state_ref.at(id); }); -} - bool WorldState::is_archive_tip(const WorldStateRevision& revision, const bb::fr& block_header_hash) const { std::optional leaf_index = find_leaf_index(revision, MerkleTreeId::ARCHIVE, block_header_hash); @@ -544,4 +543,9 @@ bool WorldState::is_archive_tip(const WorldStateRevision& revision, const bb::fr return archive_state.meta.size == leaf_index.value() + 1; } +bool WorldState::is_same_state_reference(const WorldStateRevision& revision, const StateReference& state_ref) const +{ + return state_ref == get_state_reference(revision); +} + } // namespace bb::world_state diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp index d01a19f40bb..3152492cb4a 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp @@ -51,10 +51,19 @@ const uint64_t CANONICAL_FORK_ID = 0; */ class WorldState { public: - WorldState(const std::string& data_dir, uint64_t map_size, uint64_t thread_pool_size); - WorldState(const std::string& data_dir, + WorldState(uint64_t thread_pool_size, + const std::string& data_dir, + uint64_t map_size, + const std::unordered_map& tree_heights, + const std::unordered_map& tree_prefill, + uint32_t initial_header_generator_point); + + WorldState(uint64_t thread_pool_size, + const std::string& data_dir, const std::unordered_map& map_size, - uint64_t thread_pool_size); + const std::unordered_map& tree_heights, + const std::unordered_map& tree_prefill, + uint32_t initial_header_generator_point); /** * @brief Get tree metadata for a particular tree @@ -217,6 +226,9 @@ class WorldState { private: std::shared_ptr _workers; WorldStateStores::Ptr _persistentStores; + std::unordered_map _tree_heights; + std::unordered_map _initial_tree_size; + uint32_t _initial_header_generator_point; mutable std::mutex mtx; std::unordered_map _forks; uint64_t _forkId = 0; @@ -231,14 +243,13 @@ class WorldState { bool is_archive_tip(const WorldStateRevision& revision, const bb::fr& block_header_hash) const; - static bb::fr compute_initial_archive(const StateReference& initial_state_ref); + bool is_same_state_reference(const WorldStateRevision& revision, const StateReference& state_ref) const; + + static bb::fr compute_initial_archive(const StateReference& initial_state_ref, uint32_t generator_point); static StateReference get_state_reference(const WorldStateRevision& revision, Fork::SharedPtr fork, bool initial_state = false); - - static bool block_state_matches_world_state(const StateReference& block_state_ref, - const StateReference& tree_state_ref); }; template diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp index f726fc77970..68c98664646 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp @@ -6,9 +6,11 @@ #include "barretenberg/vm/aztec_constants.hpp" #include "barretenberg/world_state/fork.hpp" #include "barretenberg/world_state/types.hpp" +#include #include #include #include +#include using namespace bb::world_state; using namespace bb::crypto::merkle_tree; @@ -26,6 +28,16 @@ class WorldStateTest : public testing::Test { static std::string data_dir; uint64_t map_size = 10240; uint64_t thread_pool_size = 1; + std::unordered_map tree_heights{ + { MerkleTreeId::NULLIFIER_TREE, 20 }, { MerkleTreeId::NOTE_HASH_TREE, 32 }, + { MerkleTreeId::PUBLIC_DATA_TREE, 40 }, { MerkleTreeId::L1_TO_L2_MESSAGE_TREE, 16 }, + { MerkleTreeId::ARCHIVE, 16 }, + }; + std::unordered_map tree_prefill{ + { MerkleTreeId::NULLIFIER_TREE, 128 }, + { MerkleTreeId::PUBLIC_DATA_TREE, 128 }, + }; + uint32_t initial_header_generator_point = 28; }; std::string WorldStateTest::data_dir; @@ -121,47 +133,48 @@ void assert_fork_state_unchanged(const WorldState& ws, TEST_F(WorldStateTest, GetInitialTreeInfoForAllTrees) { - WorldState ws(data_dir, map_size, thread_pool_size); + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); { auto info = ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::NULLIFIER_TREE); EXPECT_EQ(info.meta.size, 128); - EXPECT_EQ(info.meta.depth, NULLIFIER_TREE_HEIGHT); + EXPECT_EQ(info.meta.depth, tree_heights.at(MerkleTreeId::NULLIFIER_TREE)); EXPECT_EQ(info.meta.root, bb::fr("0x19a8c197c12bb33da6314c4ef4f8f6fcb9e25250c085df8672adf67c8f1e3dbc")); } { auto info = ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::NOTE_HASH_TREE); EXPECT_EQ(info.meta.size, 0); - EXPECT_EQ(info.meta.depth, NOTE_HASH_TREE_HEIGHT); + EXPECT_EQ(info.meta.depth, tree_heights.at(MerkleTreeId::NOTE_HASH_TREE)); EXPECT_EQ(info.meta.root, bb::fr("0x0b59baa35b9dc267744f0ccb4e3b0255c1fc512460d91130c6bc19fb2668568d")); } { auto info = ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::PUBLIC_DATA_TREE); EXPECT_EQ(info.meta.size, 128); - EXPECT_EQ(info.meta.depth, PUBLIC_DATA_TREE_HEIGHT); + EXPECT_EQ(info.meta.depth, tree_heights.at(MerkleTreeId::PUBLIC_DATA_TREE)); EXPECT_EQ(info.meta.root, bb::fr("0x23c08a6b1297210c5e24c76b9a936250a1ce2721576c26ea797c7ec35f9e46a9")); } { auto info = ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::L1_TO_L2_MESSAGE_TREE); EXPECT_EQ(info.meta.size, 0); - EXPECT_EQ(info.meta.depth, L1_TO_L2_MSG_TREE_HEIGHT); + EXPECT_EQ(info.meta.depth, tree_heights.at(MerkleTreeId::L1_TO_L2_MESSAGE_TREE)); EXPECT_EQ(info.meta.root, bb::fr("0x14f44d672eb357739e42463497f9fdac46623af863eea4d947ca00a497dcdeb3")); } { auto info = ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::ARCHIVE); EXPECT_EQ(info.meta.size, 1); - EXPECT_EQ(info.meta.depth, ARCHIVE_HEIGHT); - EXPECT_EQ(info.meta.root, bb::fr(GENESIS_ARCHIVE_ROOT)); + EXPECT_EQ(info.meta.depth, tree_heights.at(MerkleTreeId::ARCHIVE)); + // this is the expected archive tree root at genesis + EXPECT_EQ(info.meta.root, bb::fr("0x1200a06aae1368abe36530b585bd7a4d2ba4de5037b82076412691a187d7621e")); } } TEST_F(WorldStateTest, GetStateReference) { - WorldState ws(data_dir, map_size, thread_pool_size); + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); { auto state_ref = ws.get_state_reference(WorldStateRevision::committed()); @@ -230,7 +243,7 @@ TEST_F(WorldStateTest, GetStateReference) TEST_F(WorldStateTest, GetInitialStateReference) { - WorldState ws(data_dir, map_size, thread_pool_size); + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); auto before_commit = ws.get_initial_state_reference(); ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { 1 }); @@ -243,7 +256,7 @@ TEST_F(WorldStateTest, GetInitialStateReference) TEST_F(WorldStateTest, AppendOnlyTrees) { - WorldState ws(data_dir, map_size, thread_pool_size); + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); // the trees that start out empty std::vector tree_ids{ MerkleTreeId::NOTE_HASH_TREE, MerkleTreeId::L1_TO_L2_MESSAGE_TREE }; @@ -301,7 +314,7 @@ TEST_F(WorldStateTest, AppendOnlyTrees) TEST_F(WorldStateTest, AppendOnlyAllowDuplicates) { - WorldState ws(data_dir, map_size, thread_pool_size); + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); // the trees that start out empty std::vector tree_ids{ @@ -327,7 +340,7 @@ TEST_F(WorldStateTest, AppendOnlyAllowDuplicates) TEST_F(WorldStateTest, NullifierTree) { - WorldState ws(data_dir, map_size, thread_pool_size); + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); auto tree_id = MerkleTreeId::NULLIFIER_TREE; NullifierLeafValue test_nullifier(142); @@ -363,7 +376,7 @@ TEST_F(WorldStateTest, NullifierTree) TEST_F(WorldStateTest, NullifierTreeDuplicates) { - WorldState ws(data_dir, map_size, thread_pool_size); + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); auto tree_id = MerkleTreeId::NULLIFIER_TREE; NullifierLeafValue test_nullifier(142); @@ -377,7 +390,7 @@ TEST_F(WorldStateTest, NullifierTreeDuplicates) TEST_F(WorldStateTest, NullifierBatchInsert) { - WorldState ws(data_dir, map_size, thread_pool_size); + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); auto response = ws.batch_insert_indexed_leaves( MerkleTreeId::NULLIFIER_TREE, { NullifierLeafValue(150), NullifierLeafValue(142), NullifierLeafValue(180) }, 2); @@ -417,7 +430,7 @@ TEST_F(WorldStateTest, NullifierBatchInsert) TEST_F(WorldStateTest, PublicDataTree) { - WorldState ws(data_dir, map_size, thread_pool_size); + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); ws.append_leaves(MerkleTreeId::PUBLIC_DATA_TREE, std::vector{ PublicDataLeafValue(142, 0) }); assert_tree_size(ws, WorldStateRevision::uncommitted(), MerkleTreeId::PUBLIC_DATA_TREE, 129); @@ -437,7 +450,7 @@ TEST_F(WorldStateTest, PublicDataTree) TEST_F(WorldStateTest, CommitsAndRollsBackAllTrees) { - WorldState ws(data_dir, map_size, thread_pool_size); + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { fr(42) }); ws.append_leaves(MerkleTreeId::L1_TO_L2_MESSAGE_TREE, { fr(42) }); @@ -473,7 +486,7 @@ TEST_F(WorldStateTest, CommitsAndRollsBackAllTrees) TEST_F(WorldStateTest, SyncExternalBlockFromEmpty) { - WorldState ws(data_dir, map_size, thread_pool_size); + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); StateReference block_state_ref = { { MerkleTreeId::NULLIFIER_TREE, { fr("0x0342578609a7358092788d0eed7d1ee0ec8e0c596c0b1e85ba980ddd5cc79d04"), 129 } }, @@ -504,7 +517,7 @@ TEST_F(WorldStateTest, SyncExternalBlockFromEmpty) TEST_F(WorldStateTest, SyncBlockFromDirtyState) { - WorldState ws(data_dir, map_size, thread_pool_size); + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); StateReference block_state_ref = { { MerkleTreeId::NULLIFIER_TREE, { fr("0x0342578609a7358092788d0eed7d1ee0ec8e0c596c0b1e85ba980ddd5cc79d04"), 129 } }, @@ -545,7 +558,7 @@ TEST_F(WorldStateTest, SyncBlockFromDirtyState) TEST_F(WorldStateTest, SyncCurrentBlock) { - WorldState ws(data_dir, map_size, thread_pool_size); + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); bb::fr block_hash(1); StateReference block_state_ref = { { MerkleTreeId::NULLIFIER_TREE, @@ -583,7 +596,7 @@ TEST_F(WorldStateTest, SyncCurrentBlock) TEST_F(WorldStateTest, RejectSyncBlockWithBadPublicWriteBatches) { - WorldState ws(data_dir, map_size, thread_pool_size); + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); StateReference block_state_ref = { { MerkleTreeId::NULLIFIER_TREE, { fr("0x0342578609a7358092788d0eed7d1ee0ec8e0c596c0b1e85ba980ddd5cc79d04"), 129 } }, @@ -610,7 +623,7 @@ TEST_F(WorldStateTest, RejectSyncBlockWithBadPublicWriteBatches) TEST_F(WorldStateTest, RejectSyncBlockWithInvalidStateRef) { - WorldState ws(data_dir, map_size, thread_pool_size); + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); StateReference block_state_ref = { { MerkleTreeId::NULLIFIER_TREE, { fr("0x0342578609a7358092788d0eed7d1ee0ec8e0c596c0b1e85ba980ddd5cc79d04"), 129 } }, @@ -638,7 +651,7 @@ TEST_F(WorldStateTest, RejectSyncBlockWithInvalidStateRef) TEST_F(WorldStateTest, SyncEmptyBlock) { - WorldState ws(data_dir, map_size, thread_pool_size); + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); StateReference block_state_ref = ws.get_state_reference(WorldStateRevision::committed()); ws.sync_block(block_state_ref, fr(1), {}, {}, {}, {}); StateReference after_sync = ws.get_state_reference(WorldStateRevision::committed()); @@ -648,7 +661,7 @@ TEST_F(WorldStateTest, SyncEmptyBlock) TEST_F(WorldStateTest, ForkingAtBlock0SameState) { - WorldState ws(data_dir, map_size, thread_pool_size); + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); auto fork_id = ws.create_fork(0); assert_fork_state_unchanged(ws, fork_id, false); @@ -657,7 +670,7 @@ TEST_F(WorldStateTest, ForkingAtBlock0SameState) TEST_F(WorldStateTest, ForkingAtBlock0AndAdvancingFork) { - WorldState ws(data_dir, map_size, thread_pool_size); + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); auto fork_id = ws.create_fork(0); auto canonical_archive_state_before = ws.get_tree_info(WorldStateRevision::uncommitted(), MerkleTreeId::ARCHIVE); @@ -685,7 +698,7 @@ TEST_F(WorldStateTest, ForkingAtBlock0AndAdvancingFork) TEST_F(WorldStateTest, ForkingAtBlock0AndAdvancingCanonicalState) { - WorldState ws(data_dir, map_size, thread_pool_size); + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); auto fork_id = ws.create_fork(0); auto canonical_archive_state_before = ws.get_tree_info(WorldStateRevision::uncommitted(), MerkleTreeId::ARCHIVE); @@ -738,3 +751,26 @@ TEST_F(WorldStateTest, ForkingAtBlock0AndAdvancingCanonicalState) assert_leaf_value( ws, WorldStateRevision{ .forkId = fork_id, .includeUncommitted = true }, MerkleTreeId::ARCHIVE, 1, 2); } + +TEST_F(WorldStateTest, BuildsABlockInAFork) +{ + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); + auto fork_id = ws.create_fork(0); + + ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { 42 }, fork_id); + ws.append_leaves(MerkleTreeId::L1_TO_L2_MESSAGE_TREE, { 43 }, fork_id); + ws.batch_insert_indexed_leaves(MerkleTreeId::NULLIFIER_TREE, { { 129 } }, 0, fork_id); + ws.batch_insert_indexed_leaves(MerkleTreeId::PUBLIC_DATA_TREE, { { 129, 1 } }, 0, fork_id); + + auto fork_state_ref = ws.get_state_reference(WorldStateRevision{ .forkId = fork_id, .includeUncommitted = true }); + + ws.update_archive(fork_state_ref, { 1 }, fork_id); + auto fork_archive = + ws.get_tree_info(WorldStateRevision{ .forkId = fork_id, .includeUncommitted = true }, MerkleTreeId::ARCHIVE); + + ws.delete_fork(fork_id); + + ws.sync_block(fork_state_ref, { 1 }, { 42 }, { 43 }, { { 129 } }, { { { 129, 1 } } }); + + EXPECT_EQ(fork_state_ref, ws.get_state_reference(WorldStateRevision::committed())); +} diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp index 6de8a0605ed..8601ab880e8 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp @@ -2,6 +2,7 @@ #include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/messaging/header.hpp" #include "barretenberg/world_state/fork.hpp" @@ -21,6 +22,7 @@ #include #include #include +#include using namespace bb::world_state; using namespace bb::crypto::merkle_tree; @@ -31,6 +33,7 @@ const uint64_t DEFAULT_MAP_SIZE = 1024 * 1024; WorldStateAddon::WorldStateAddon(const Napi::CallbackInfo& info) : ObjectWrap(info) { + uint64_t thread_pool_size = 16; std::string data_dir; std::unordered_map map_size{ { MerkleTreeId::ARCHIVE, DEFAULT_MAP_SIZE }, @@ -39,54 +42,88 @@ WorldStateAddon::WorldStateAddon(const Napi::CallbackInfo& info) { MerkleTreeId::PUBLIC_DATA_TREE, DEFAULT_MAP_SIZE }, { MerkleTreeId::L1_TO_L2_MESSAGE_TREE, DEFAULT_MAP_SIZE }, }; - uint64_t thread_pool_size = 16; + std::unordered_map tree_height; + std::unordered_map tree_prefill; + std::vector tree_ids{ + MerkleTreeId::NULLIFIER_TREE, MerkleTreeId::NOTE_HASH_TREE, MerkleTreeId::PUBLIC_DATA_TREE, + MerkleTreeId::L1_TO_L2_MESSAGE_TREE, MerkleTreeId::ARCHIVE, + }; + uint32_t initial_header_generator_point = 0; Napi::Env env = info.Env(); - if (info.Length() < 1) { - throw Napi::TypeError::New(env, "Wrong number of arguments"); + size_t data_dir_index = 0; + if (info.Length() > data_dir_index && info[data_dir_index].IsString()) { + data_dir = info[data_dir_index].As(); + } else { + throw Napi::TypeError::New(env, "Directory needs to be a string"); } - if (!info[0].IsString()) { - throw Napi::TypeError::New(env, "Directory needs to be a string"); + size_t tree_height_index = 1; + if (info.Length() > tree_height_index && info[tree_height_index].IsObject()) { + Napi::Object obj = info[tree_height_index].As(); + + for (auto tree_id : tree_ids) { + if (obj.Has(tree_id)) { + tree_height[tree_id] = obj.Get(tree_id).As().Uint32Value(); + } + } + } else { + throw Napi::TypeError::New(env, "Tree heights must be a map"); } - data_dir = info[0].As(); + size_t tree_prefill_index = 2; + if (info.Length() > tree_prefill_index && info[tree_prefill_index].IsObject()) { + Napi::Object obj = info[tree_prefill_index].As(); + + for (auto tree_id : tree_ids) { + if (obj.Has(tree_id)) { + tree_prefill[tree_id] = obj.Get(tree_id).As().Uint32Value(); + } + } + } else { + throw Napi::TypeError::New(env, "Tree prefill must be a map"); + } + + size_t initial_header_generator_point_index = 3; + if (info.Length() > initial_header_generator_point_index && info[initial_header_generator_point_index].IsNumber()) { + initial_header_generator_point = info[initial_header_generator_point_index].As().Uint32Value(); + } else { + throw Napi::TypeError::New(env, "Header generator point needs to be a number"); + } - if (info.Length() > 1) { - if (info[1].IsObject()) { - Napi::Object obj = info[1].As(); + // optional parameters + size_t map_size_index = 4; + if (info.Length() > map_size_index) { + if (info[4].IsObject()) { + Napi::Object obj = info[map_size_index].As(); - for (auto tree_id : { MerkleTreeId::ARCHIVE, - MerkleTreeId::NULLIFIER_TREE, - MerkleTreeId::NOTE_HASH_TREE, - MerkleTreeId::PUBLIC_DATA_TREE, - MerkleTreeId::L1_TO_L2_MESSAGE_TREE }) { + for (auto tree_id : tree_ids) { if (obj.Has(tree_id)) { map_size[tree_id] = obj.Get(tree_id).As().Uint32Value(); } } - } else if (info[1].IsNumber()) { - uint64_t size = info[1].As().Uint32Value(); - for (auto tree_id : { MerkleTreeId::ARCHIVE, - MerkleTreeId::NULLIFIER_TREE, - MerkleTreeId::NOTE_HASH_TREE, - MerkleTreeId::PUBLIC_DATA_TREE, - MerkleTreeId::L1_TO_L2_MESSAGE_TREE }) { + } else if (info[map_size_index].IsNumber()) { + uint64_t size = info[map_size_index].As().Uint32Value(); + for (auto tree_id : tree_ids) { map_size[tree_id] = size; } + } else { + throw Napi::TypeError::New(env, "Map size must be a number or an object"); } } - if (info.Length() > 2) { - if (!info[2].IsNumber()) { + size_t thread_pool_size_index = 5; + if (info.Length() > thread_pool_size_index) { + if (!info[thread_pool_size_index].IsNumber()) { throw Napi::TypeError::New(env, "Thread pool size must be a number"); } - thread_pool_size = info[2].As().Uint32Value(); + thread_pool_size = info[thread_pool_size_index].As().Uint32Value(); } - _ws = std::make_unique(data_dir, map_size, thread_pool_size); + _ws = std::make_unique( + thread_pool_size, data_dir, map_size, tree_height, tree_prefill, initial_header_generator_point); _dispatcher.registerTarget( WorldStateMessageType::GET_TREE_INFO, diff --git a/yarn-project/circuits.js/src/scripts/constants.in.ts b/yarn-project/circuits.js/src/scripts/constants.in.ts index e09cd54e372..3d6191de327 100644 --- a/yarn-project/circuits.js/src/scripts/constants.in.ts +++ b/yarn-project/circuits.js/src/scripts/constants.in.ts @@ -77,18 +77,9 @@ const CPP_CONSTANTS = [ 'MEM_TAG_U64', 'MEM_TAG_U128', 'MEM_TAG_FF', - - 'NULLIFIER_TREE_HEIGHT', - 'MAX_NULLIFIERS_PER_TX', - 'NOTE_HASH_TREE_HEIGHT', - 'PUBLIC_DATA_TREE_HEIGHT', - 'MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX', - 'L1_TO_L2_MSG_TREE_HEIGHT', - 'ARCHIVE_HEIGHT', - 'GENESIS_ARCHIVE_ROOT', ]; -const CPP_GENERATORS = ['BLOCK_HASH']; +const CPP_GENERATORS: string[] = []; const PIL_CONSTANTS = [ 'MAX_NOTE_HASH_READ_REQUESTS_PER_CALL', diff --git a/yarn-project/world-state/package.json b/yarn-project/world-state/package.json index 4be33a9b3d5..57c2ff32a9c 100644 --- a/yarn-project/world-state/package.json +++ b/yarn-project/world-state/package.json @@ -14,7 +14,7 @@ "tsconfig": "./tsconfig.json" }, "scripts": { - "build": "yarn clean && mkdir -p build && (([ -f ../../barretenberg/cpp/build/bin/world_state_napi.node ] && cp -v ../../barretenberg/cpp/build/bin/world_state_napi.node build) || ([ -f ../../barretenberg/cpp/build-pic/lib/world_state_napi.node ] && cp -v ../../barretenberg/cpp/build-pic/lib/world_state_napi.node build) || true) && tsc -b", + "build": "yarn clean && mkdir -p build && (([ -f ../../barretenberg/cpp/build-pic/lib/world_state_napi.node ] && cp -v ../../barretenberg/cpp/build-pic/lib/world_state_napi.node build) || ([ -f ../../barretenberg/cpp/build/bin/world_state_napi.node ] && cp -v ../../barretenberg/cpp/build/bin/world_state_napi.node build) || true) && tsc -b", "build:cpp": "./scripts/build.sh cpp", "build:dev": "tsc -b --watch", "clean": "rm -rf ./dest ./build .tsbuildinfo", diff --git a/yarn-project/world-state/package.local.json b/yarn-project/world-state/package.local.json index 87ca839c17a..950bfacd27b 100644 --- a/yarn-project/world-state/package.local.json +++ b/yarn-project/world-state/package.local.json @@ -1,6 +1,6 @@ { "scripts": { - "build": "yarn clean && mkdir -p build && (([ -f ../../barretenberg/cpp/build/bin/world_state_napi.node ] && cp -v ../../barretenberg/cpp/build/bin/world_state_napi.node build) || ([ -f ../../barretenberg/cpp/build-pic/lib/world_state_napi.node ] && cp -v ../../barretenberg/cpp/build-pic/lib/world_state_napi.node build) || true) && tsc -b", + "build": "yarn clean && mkdir -p build && (([ -f ../../barretenberg/cpp/build-pic/lib/world_state_napi.node ] && cp -v ../../barretenberg/cpp/build-pic/lib/world_state_napi.node build) || ([ -f ../../barretenberg/cpp/build/bin/world_state_napi.node ] && cp -v ../../barretenberg/cpp/build/bin/world_state_napi.node build) || true) && tsc -b", "test": "DEBUG='aztec:*' LOG_LEVEL='debug' NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests", "clean": "rm -rf ./dest ./build .tsbuildinfo" } diff --git a/yarn-project/world-state/src/native/native_world_state_instance.ts b/yarn-project/world-state/src/native/native_world_state_instance.ts index 27fd2494bae..b5c021b8dc6 100644 --- a/yarn-project/world-state/src/native/native_world_state_instance.ts +++ b/yarn-project/world-state/src/native/native_world_state_instance.ts @@ -1,5 +1,15 @@ import { MerkleTreeId } from '@aztec/circuit-types'; -import { Fr } from '@aztec/circuits.js'; +import { + ARCHIVE_HEIGHT, + Fr, + GeneratorIndex, + L1_TO_L2_MSG_TREE_HEIGHT, + MAX_NULLIFIERS_PER_TX, + MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, + NOTE_HASH_TREE_HEIGHT, + NULLIFIER_TREE_HEIGHT, + PUBLIC_DATA_TREE_HEIGHT, +} from '@aztec/circuits.js'; import { createDebugLogger, fmtLogData } from '@aztec/foundation/log'; import { SerialQueue } from '@aztec/foundation/queue'; import { Timer } from '@aztec/foundation/timer'; @@ -7,6 +17,7 @@ import { Timer } from '@aztec/foundation/timer'; import assert from 'assert'; import bindings from 'bindings'; import { Decoder, Encoder, addExtension } from 'msgpackr'; +import { cpus } from 'os'; import { isAnyArrayBuffer } from 'util/types'; import { @@ -71,7 +82,23 @@ export class NativeWorldState implements NativeWorldStateInstance { /** Creates a new native WorldState instance */ constructor(dataDir: string, private log = createDebugLogger('aztec:world-state:database')) { - this.instance = new NATIVE_MODULE[NATIVE_CLASS_NAME](dataDir); + this.instance = new NATIVE_MODULE[NATIVE_CLASS_NAME]( + dataDir, + { + [MerkleTreeId.NULLIFIER_TREE]: NULLIFIER_TREE_HEIGHT, + [MerkleTreeId.NOTE_HASH_TREE]: NOTE_HASH_TREE_HEIGHT, + [MerkleTreeId.PUBLIC_DATA_TREE]: PUBLIC_DATA_TREE_HEIGHT, + [MerkleTreeId.L1_TO_L2_MESSAGE_TREE]: L1_TO_L2_MSG_TREE_HEIGHT, + [MerkleTreeId.ARCHIVE]: ARCHIVE_HEIGHT, + }, + { + [MerkleTreeId.NULLIFIER_TREE]: 2 * MAX_NULLIFIERS_PER_TX, + [MerkleTreeId.PUBLIC_DATA_TREE]: 2 * MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, + }, + GeneratorIndex.BLOCK_HASH, + 10 * 1024 * 1024, // 10 GB per tree (in KB) + cpus().length, + ); this.queue.start(); }