Cleaned up rebase

stellar · Sep 17, 2024 · 6bc3175 · 6bc3175
1 parent b55d6f5
commit 6bc3175
Show file tree

Hide file tree

Showing 32 changed files with 434 additions and 414 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -4,20 +4,6 @@ members = ["src/rust", "lib/tracy-client-sys"]
 
 [patch.crates-io]
 tracy-client-sys = { path = "lib/tracy-client-sys" }
-stellar-xdr = { git = "https://github.com/stellar/rs-stellar-xdr", rev = "9a31b81412712ab2dfbe32cbeaba6bdb754c264b" }
-
-#[patch."https://github.com/stellar/rs-soroban-env"]
-#soroban-env-common = { path = "../rs-soroban-env/soroban-env-common" }
-#soroban-env-host = { path = "../rs-soroban-env/soroban-env-host/" }
-#soroban-test-wasms = { path = "../rs-soroban-env/soroban-test-wasms/" }
-
-# [patch."https://github.com/stellar/rs-stellar-xdr"]
-# stellar-xdr = { path = "../rs-stellar-xdr/" }
-
-# [patch.crates-io]
-# wasmi = { package = "soroban-wasmi", git = "https://github.com/stellar/wasmi", rev = "862b32f5" }
-# soroban-env-common = { git = "https://github.com/stellar/rs-soroban-env", rev = "eda2ab70" }
-# soroban-native-sdk-macros = { git = "https://github.com/stellar/rs-soroban-env", rev = "eda2ab70" }
 
 [profile.release]
 codegen-units = 1

diff --git a/src/bucket/BucketList.cpp b/src/bucket/BucketList.cpp
@@ -26,8 +26,8 @@ namespace stellar
 
 template <> BucketListDepth BucketListBase<LiveBucket>::kNumLevels = 11;
 
-// TODO: I made this number up, do some analysis and pick a better value or
-// make this a configurable network config.
+// TODO: This is an arbitrary number. Do some analysis and pick a better value
+// or make this a configurable network config.
 template <> BucketListDepth BucketListBase<HotArchiveBucket>::kNumLevels = 9;
 
 template <typename BucketT>

diff --git a/src/bucket/BucketListSnapshot.cpp b/src/bucket/BucketListSnapshot.cpp
@@ -12,13 +12,14 @@
 
 #include "medida/timer.h"
 #include "util/GlobalChecks.h"
+#include <optional>
+#include <vector>
 
 namespace stellar
 {
 template <class BucketT>
 BucketListSnapshot<BucketT>::BucketListSnapshot(
-    BucketListBase<BucketT> const& bl,
-                                       LedgerHeader header)
+    BucketListBase<BucketT> const& bl, LedgerHeader header)
     : mHeader(std::move(header))
 {
     releaseAssert(threadIsMain());
@@ -32,7 +33,7 @@ BucketListSnapshot<BucketT>::BucketListSnapshot(
 
 template <class BucketT>
 BucketListSnapshot<BucketT>::BucketListSnapshot(
-    BucketListSnapshot const& snapshot)
+    BucketListSnapshot<BucketT> const& snapshot)
     : mLevels(snapshot.mLevels), mHeader(snapshot.mHeader)
 {
 }
@@ -54,7 +55,8 @@ BucketListSnapshot<BucketT>::getLedgerSeq() const
 template <class BucketT>
 void
 SearchableBucketListSnapshotBase<BucketT>::loopAllBuckets(
-    std::function<bool(BucketSnapshotT const&)> f, BucketListSnapshot<BucketT> const& snapshot) const
+    std::function<bool(BucketSnapshotT const&)> f,
+    BucketListSnapshot<BucketT> const& snapshot) const
 {
     for (auto const& lev : snapshot.getLevels())
     {
@@ -75,70 +77,6 @@ SearchableBucketListSnapshotBase<BucketT>::loopAllBuckets(
     }
 }
 
-// Loads bucket entry for LedgerKey k. Returns <LedgerEntry, bloomMiss>,
-// where bloomMiss is true if a bloom miss occurred during the load.
-std::pair<std::shared_ptr<LedgerEntry>, bool>
-getLedgerEntryInternal(LedgerKey const& k, BucketListSnapshot const& snapshot)
-{
-    std::shared_ptr<LedgerEntry> result{};
-    auto sawBloomMiss = false;
-
-    auto f = [&](BucketSnapshot const& b) {
-        auto [be, bloomMiss] = b.getBucketEntry(k);
-        sawBloomMiss = sawBloomMiss || bloomMiss;
-
-        if (be.has_value())
-        {
-            result =
-                be.value().type() == DEADENTRY
-                    ? nullptr
-                    : std::make_shared<LedgerEntry>(be.value().liveEntry());
-            return true;
-        }
-        else
-        {
-            return false;
-        }
-    };
-
-    loopAllBuckets(f, snapshot);
-    return {result, sawBloomMiss};
-}
-
-std::vector<LedgerEntry>
-loadKeysInternal(std::set<LedgerKey, LedgerEntryIdCmp> const& inKeys,
-                 BucketListSnapshot const& snapshot, LedgerKeyMeter* lkMeter)
-{
-    std::vector<LedgerEntry> entries;
-
-    // Make a copy of the key set, this loop is destructive
-    auto keys = inKeys;
-    auto f = [&](BucketSnapshot const& b) {
-        b.loadKeysWithLimits(keys, entries, lkMeter);
-        return keys.empty();
-    };
-
-    loopAllBuckets(f, snapshot);
-    return entries;
-}
-
-}
-
-uint32_t
-SearchableBucketListSnapshot::getLedgerSeq() const
-{
-    releaseAssert(mSnapshot);
-    return mSnapshot->getLedgerSeq();
-}
-
-LedgerHeader const&
-SearchableBucketListSnapshot::getLedgerHeader()
-{
-    releaseAssert(mSnapshot);
-    mSnapshotManager.maybeUpdateSnapshot(mSnapshot, mHistoricalSnapshots);
-    return mSnapshot->getLedgerHeader();
-}
-
 EvictionResult
 SearchableLiveBucketListSnapshot::scanForEviction(
     uint32_t ledgerSeq, EvictionCounters& counters,
@@ -204,41 +142,61 @@ SearchableLiveBucketListSnapshot::loadKeysWithLimits(
         return keys.empty();
     };
 
-    mSnapshotManager.maybeUpdateSnapshot(mSnapshot);
+    mSnapshotManager.maybeUpdateSnapshot(mSnapshot, mHistoricalSnapshots);
     if (threadIsMain())
     {
         auto timer =
             mSnapshotManager.recordBulkLoadMetrics("prefetch", inKeys.size())
                 .TimeScope();
-        loopAllBuckets(loadKeysLoop);
+        loopAllBuckets(loadKeysLoop, *mSnapshot);
     }
     else
     {
         // TODO: Background metrics
-        loopAllBuckets(loadKeysLoop);
+        loopAllBuckets(loadKeysLoop, *mSnapshot);
+    }
+
+    return entries;
+}
+
+std::optional<std::vector<LedgerEntry>>
+SearchableLiveBucketListSnapshot::loadKeysFromLedger(
+    std::set<LedgerKey, LedgerEntryIdCmp> const& inKeys, uint32_t ledgerSeq)
+{
+    ZoneScoped;
+
+    // Make a copy of the key set, this loop is destructive
+    auto keys = inKeys;
+    std::vector<LedgerEntry> entries;
+    auto loadKeysLoop = [&](auto const& b) {
+        b.loadKeys(keys, entries, /*lkMeter=*/nullptr);
+        return keys.empty();
+    };
+
+    mSnapshotManager.maybeUpdateSnapshot(mSnapshot, mHistoricalSnapshots);
+
+    if (ledgerSeq == mSnapshot->getLedgerSeq())
+    {
+        loopAllBuckets(loadKeysLoop, *mSnapshot);
+    }
+    else
+    {
+        auto iter = mHistoricalSnapshots.find(ledgerSeq);
+        if (iter == mHistoricalSnapshots.end())
+        {
+            return std::nullopt;
+        }
+
+        releaseAssert(iter->second);
+        loopAllBuckets(loadKeysLoop, *iter->second);
     }
 
     return entries;
 }
 
 std::shared_ptr<LedgerEntry>
-SearchableLiveBucketListSnapshot::getLedgerEntry(LedgerKey const& k)
+SearchableLiveBucketListSnapshot::load(LedgerKey const& k)
 {
-        // if (ledgerSeq == mSnapshot->getLedgerSeq())
-    // {
-    //     auto result = loadKeysInternal(inKeys, *mSnapshot, /*lkMeter=*/nullptr);
-    //     return {result, true};
-    // }
-
-    // auto iter = mHistoricalSnapshots.find(ledgerSeq);
-    // if (iter == mHistoricalSnapshots.end())
-    // {
-    //     return {{}, false};
-    // }
-
-    // releaseAssert(iter->second);
-    // auto result = loadKeysInternal(inKeys, *iter->second, /*lkMeter=*/nullptr);
-    // return {result, true};
     ZoneScoped;
 
     std::shared_ptr<LedgerEntry> result{};
@@ -263,18 +221,18 @@ SearchableLiveBucketListSnapshot::getLedgerEntry(LedgerKey const& k)
         }
     };
 
-    mSnapshotManager.maybeUpdateSnapshot(mSnapshot);
+    mSnapshotManager.maybeUpdateSnapshot(mSnapshot, mHistoricalSnapshots);
     if (threadIsMain())
     {
         mSnapshotManager.startPointLoadTimer();
-        loopAllBuckets(loadKeyBucketLoop);
+        loopAllBuckets(loadKeyBucketLoop, *mSnapshot);
         mSnapshotManager.endPointLoadTimer(k.type(), sawBloomMiss);
         return result;
     }
     else
     {
         // TODO: Background metrics
-        loopAllBuckets(loadKeyBucketLoop);
+        loopAllBuckets(loadKeyBucketLoop, *mSnapshot);
         return result;
     }
 }
@@ -324,7 +282,7 @@ SearchableLiveBucketListSnapshot::loadPoolShareTrustLinesByAccountAndAsset(
         return trustlinesToLoad.empty();
     };
 
-    loopAllBuckets(loadKeysLoop);
+    loopAllBuckets(loadKeysLoop, *mSnapshot);
     return result;
 }
 
@@ -433,15 +391,8 @@ SearchableBucketListSnapshotBase<BucketT>::SearchableBucketListSnapshotBase(
     BucketSnapshotManager const& snapshotManager)
     : mSnapshotManager(snapshotManager), mHistoricalSnapshots()
 {
-    // Initialize snapshot from SnapshotManager
-    if constexpr (std::is_same_v<BucketT, LiveBucket>)
-    {
-        mSnapshotManager.maybeUpdateSnapshot(mSnapshot, mHistoricalSnapshots);
-    }
-    else
-    {
-        mSnapshotManager.maybeUpdateHotArchiveSnapshot(mSnapshot);
-    }
+
+    mSnapshotManager.maybeUpdateSnapshot(mSnapshot, mHistoricalSnapshots);
 }
 
 template <class BucketT>
@@ -462,7 +413,7 @@ SearchableHotArchiveBucketListSnapshot::SearchableHotArchiveBucketListSnapshot(
 }
 
 std::shared_ptr<HotArchiveBucketEntry>
-SearchableHotArchiveBucketListSnapshot::getArchiveEntry(LedgerKey const& k)
+SearchableHotArchiveBucketListSnapshot::load(LedgerKey const& k)
 {
     ZoneScoped;
 
@@ -483,14 +434,23 @@ SearchableHotArchiveBucketListSnapshot::getArchiveEntry(LedgerKey const& k)
     };
 
     // TODO: Metrics
-    mSnapshotManager.maybeUpdateHotArchiveSnapshot(mSnapshot);
-    loopAllBuckets(loadKeyBucketLoop);
+    mSnapshotManager.maybeUpdateSnapshot(mSnapshot, mHistoricalSnapshots);
+    loopAllBuckets(loadKeyBucketLoop, *mSnapshot);
     return result;
 }
 
 std::vector<HotArchiveBucketEntry>
 SearchableHotArchiveBucketListSnapshot::loadKeys(
     std::set<LedgerKey, LedgerEntryIdCmp> const& inKeys)
+{
+    auto op = loadKeysFromLedger(inKeys, getLedgerSeq());
+    releaseAssertOrThrow(op);
+    return std::move(*op);
+}
+
+std::optional<std::vector<HotArchiveBucketEntry>>
+SearchableHotArchiveBucketListSnapshot::loadKeysFromLedger(
+    std::set<LedgerKey, LedgerEntryIdCmp> const& inKeys, uint32_t ledgerSeq)
 {
     ZoneScoped;
     std::vector<HotArchiveBucketEntry> entries;
@@ -502,10 +462,24 @@ SearchableHotArchiveBucketListSnapshot::loadKeys(
         return keys.empty();
     };
 
-    mSnapshotManager.maybeUpdateHotArchiveSnapshot(mSnapshot);
+    mSnapshotManager.maybeUpdateSnapshot(mSnapshot, mHistoricalSnapshots);
+
+    if (ledgerSeq == mSnapshot->getLedgerSeq())
+    {
+        loopAllBuckets(loadKeysLoop, *mSnapshot);
+    }
+    else
+    {
+        auto iter = mHistoricalSnapshots.find(ledgerSeq);
+        if (iter == mHistoricalSnapshots.end())
+        {
+            return std::nullopt;
+        }
+
+        releaseAssert(iter->second);
+        loopAllBuckets(loadKeysLoop, *iter->second);
+    }
 
-    // TODO: Metrics
-    loopAllBuckets(loadKeysLoop);
     return entries;
 }