Skip to content

Commit

Permalink
Remove the hack
Browse files Browse the repository at this point in the history
  • Loading branch information
JaySon-Huang authored and Lloyd-Pottiger committed Aug 22, 2024
1 parent 43052df commit 4900a90
Show file tree
Hide file tree
Showing 3 changed files with 18 additions and 7 deletions.
11 changes: 7 additions & 4 deletions dbms/src/Storages/DeltaMerge/Index/VectorIndexCache.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,11 @@
#include <thread>
#include <unordered_set>

namespace DB::DM::tests
{
class VectorIndexTestUtils;
}

namespace DB::DM
{

Expand All @@ -44,18 +49,16 @@ class VectorIndexCache
std::condition_variable shutdown_cv;
std::mutex shutdown_mu;

#ifdef DBMS_PUBLIC_GTEST
public:
#else
private:
#endif
friend class ::DB::DM::tests::VectorIndexTestUtils;

// Drop the in-memory Vector Index if the on-disk file is deleted.
// mmaped file could be unmmaped so that disk space can be reclaimed.
size_t cleanOutdatedCacheEntries();

void cleanOutdatedLoop();

// TODO(vector-index): Use task on BackgroundProcessingPool instead of a raw thread
std::thread cleaner_thread;

public:
Expand Down
3 changes: 2 additions & 1 deletion dbms/src/Storages/DeltaMerge/Index/VectorIndexHNSW/Index.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ std::vector<VectorIndexBuilder::Key> VectorIndexHNSWViewer::search(
std::atomic<size_t> discarded_nodes = 0;
std::atomic<bool> has_exception_in_search = false;

// The non-valid rows should be discarded by this lambda
auto predicate = [&](typename USearchImplType::member_cref_t const & member) {
// Must catch exceptions in the predicate, because search runs on other threads.
try
Expand All @@ -180,7 +181,7 @@ std::vector<VectorIndexBuilder::Key> VectorIndexHNSWViewer::search(
}
};

// TODO: Support efSearch.
// TODO(vector-index): Support efSearch.
auto result = index.search( //
reinterpret_cast<const Float32 *>(query_info->ref_vec_f32().data() + sizeof(UInt32)),
query_info->top_k(),
Expand Down
11 changes: 9 additions & 2 deletions dbms/src/Storages/DeltaMerge/tests/gtest_dm_vector_index.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,11 @@ class VectorIndexTestUtils
// When used in read, no need to assign vector_index.
return ColumnDefine(vec_column_id, vec_column_name, tests::typeFromString("Array(Float32)"));
}

static size_t cleanVectorCacheEntries(const std::shared_ptr<VectorIndexCache> & cache)
{
return cache->cleanOutdatedCacheEntries();
}
};

class VectorIndexDMFileTest
Expand Down Expand Up @@ -1638,7 +1643,8 @@ try
{
// We should be able to clear something from the vector index cache.
auto vec_cache = TiFlashTestEnv::getGlobalContext().getVectorIndexCache();
ASSERT_EQ(1, vec_cache->cleanOutdatedCacheEntries());
ASSERT_NE(vec_cache, nullptr);
ASSERT_EQ(1, cleanVectorCacheEntries(vec_cache));
}
{
// When cache is evicted (and memory cache is dropped), the query should be fine.
Expand Down Expand Up @@ -1777,7 +1783,8 @@ try
{
// We should be able to clear something from the vector index cache.
auto vec_cache = TiFlashTestEnv::getGlobalContext().getVectorIndexCache();
ASSERT_EQ(1, vec_cache->cleanOutdatedCacheEntries());
ASSERT_NE(vec_cache, nullptr);
ASSERT_EQ(1, cleanVectorCacheEntries(vec_cache));
}
{
// Query should be fine.
Expand Down

0 comments on commit 4900a90

Please sign in to comment.