Skip to content

Commit

Permalink
fix: Correcttion of batch size calculation (#9144)
Browse files Browse the repository at this point in the history
This PR fixes a bug in the block synbc process of indexed trees
  • Loading branch information
PhilWindle authored Oct 10, 2024
1 parent d2de085 commit b0858a5
Show file tree
Hide file tree
Showing 4 changed files with 190 additions and 87 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include <filesystem>
#include <gtest/gtest.h>
#include <memory>
#include <stdexcept>
#include <vector>

using namespace bb;
Expand Down Expand Up @@ -463,13 +464,11 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, errors_are_caught_and_handle
{
// We use a deep tree with a small amount of storage (100 * 1024) bytes
constexpr size_t depth = 16;
constexpr uint32_t numDbs = 4;
std::string name = random_string();
std::string directory = random_temp_directory();
std::filesystem::create_directories(directory);

{
auto environment = std::make_unique<LMDBEnvironment>(directory, 50, numDbs, 2);
LMDBTreeStore::SharedPtr db = std::make_shared<LMDBTreeStore>(_directory, name, 50, _maxReaders);
std::unique_ptr<Store> store = std::make_unique<Store>(name, depth, db);

Expand Down Expand Up @@ -529,7 +528,6 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, errors_are_caught_and_handle
}

{
auto environment = std::make_unique<LMDBEnvironment>(directory, 500, numDbs, 2);
LMDBTreeStore::SharedPtr db = std::make_shared<LMDBTreeStore>(_directory, name, 500, _maxReaders);
std::unique_ptr<Store> store = std::make_unique<Store>(name, depth, db);

Expand Down Expand Up @@ -1548,3 +1546,47 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_advance_finalised_blocks
}
}
}

TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_not_fork_from_unwound_blocks)
{
std::string name = random_string();
uint32_t depth = 20;
LMDBTreeStore::SharedPtr db = std::make_shared<LMDBTreeStore>(_directory, name, _mapSize, _maxReaders);
std::unique_ptr<Store> store = std::make_unique<Store>(name, depth, db);
ThreadPoolPtr pool = make_thread_pool(1);
TreeType tree(std::move(store), pool);

for (uint32_t i = 0; i < 5; i++) {
std::vector<fr> values = create_values(1024);
add_values(tree, values);
commit_tree(tree);
}

unwind_block(tree, 5);
unwind_block(tree, 4);

EXPECT_THROW(Store(name, depth, 5, db), std::runtime_error);
EXPECT_THROW(Store(name, depth, 4, db), std::runtime_error);
}

TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_not_fork_from_expired_historical_blocks)
{
std::string name = random_string();
uint32_t depth = 20;
LMDBTreeStore::SharedPtr db = std::make_shared<LMDBTreeStore>(_directory, name, _mapSize, _maxReaders);
std::unique_ptr<Store> store = std::make_unique<Store>(name, depth, db);
ThreadPoolPtr pool = make_thread_pool(1);
TreeType tree(std::move(store), pool);

for (uint32_t i = 0; i < 5; i++) {
std::vector<fr> values = create_values(1024);
add_values(tree, values);
commit_tree(tree);
}

remove_historic_block(tree, 1);
remove_historic_block(tree, 2);

EXPECT_THROW(Store(name, depth, 1, db), std::runtime_error);
EXPECT_THROW(Store(name, depth, 2, db), std::runtime_error);
}
Original file line number Diff line number Diff line change
Expand Up @@ -858,14 +858,18 @@ void ContentAddressedIndexedTree<Store, HashingPolicy>::perform_insertions_witho

std::shared_ptr<Status> status = std::make_shared<Status>();

uint32_t p = static_cast<uint32_t>(std::log2(highest_index + 1)) + 1;
index_t span = static_cast<uint32_t>(std::pow(2, p));
uint64_t numBatches = workers_->num_threads();
uint32_t indexPower2Ceil = static_cast<uint32_t>(std::ceil(std::log2(highest_index + 1)));
index_t span = static_cast<uint64_t>(std::pow(2, indexPower2Ceil));
uint32_t numBatchesPower2Ceil = static_cast<uint32_t>(std::floor(std::log2(workers_->num_threads())));
uint64_t numBatches = static_cast<uint64_t>(std::pow(2, numBatchesPower2Ceil));
index_t batchSize = span / numBatches;
batchSize = std::max(batchSize, 1UL);
batchSize = std::max(batchSize, 2UL);
index_t startIndex = 0;
p = static_cast<uint32_t>(std::log2(batchSize));
uint32_t rootLevel = depth_ - p;
indexPower2Ceil = static_cast<uint32_t>(std::log2(batchSize));
uint32_t rootLevel = depth_ - indexPower2Ceil;

// std::cout << "HIGHEST INDEX " << highest_index << " SPAN " << span << " NUM BATCHES " << numBatches
// << " BATCH SIZE " << batchSize << std::endl;

struct BatchInsertResults {
std::atomic_uint32_t count;
Expand Down
Loading

0 comments on commit b0858a5

Please sign in to comment.