Switch SyncLog to modern entrypoint format (ydb-platform#6590)

pixcc · Jul 12, 2024 · f74adc0 · f74adc0
1 parent d0aff76
commit f74adc0
Show file tree

Hide file tree

Showing 24 changed files with 25 additions and 107 deletions.
diff --git a/ydb/core/base/compile_time_flags.h b/ydb/core/base/compile_time_flags.h
diff --git a/ydb/core/base/localdb.cpp b/ydb/core/base/localdb.cpp
@@ -1,5 +1,4 @@
 #include "localdb.h"
-#include "compile_time_flags.h"
 
 #include <ydb/core/protos/resource_broker.pb.h>
 

diff --git a/ydb/core/base/statestorage_guardian.cpp b/ydb/core/base/statestorage_guardian.cpp
@@ -4,7 +4,6 @@
 #include "tablet.h"
 
 #include <ydb/core/base/appdata.h>
-#include <ydb/core/base/compile_time_flags.h>
 #include <ydb/library/services/services.pb.h>
 
 #include <ydb/library/actors/core/actor_bootstrapped.h>

diff --git a/ydb/core/base/statestorage_proxy.cpp b/ydb/core/base/statestorage_proxy.cpp
@@ -1,7 +1,6 @@
 #include "statestorage_impl.h"
 #include "tabletid.h"
 
-#include <ydb/core/base/compile_time_flags.h>
 #include <ydb/library/services/services.pb.h>
 
 #include <ydb/library/actors/core/actor_bootstrapped.h>

diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h
@@ -2,7 +2,6 @@
 #include "defs.h"
 
 #include <ydb/core/base/blobstorage.h>
-#include <ydb/core/base/compile_time_flags.h>
 #include <ydb/core/blobstorage/base/vdisk_priorities.h>
 #include <ydb/core/control/immediate_control_board_wrapper.h>
 #include <ydb/core/protos/blobstorage.pb.h>

diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_crypto.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_crypto.h
@@ -1,7 +1,6 @@
 #pragma once
 #include "defs.h"
 
-#include <ydb/core/base/compile_time_flags.h>
 #include <ydb/core/blobstorage/crypto/crypto.h>
 
 namespace NKikimr {

diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_run.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_run.h
@@ -4,7 +4,6 @@
 #include "blobstorage_pdisk_ut_config.h"
 #include "blobstorage_pdisk_ut_context.h"
 
-#include <ydb/core/base/compile_time_flags.h>
 #include <ydb/core/base/services/blobstorage_service_id.h>
 
 #include <util/folder/tempdir.h>

diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_writer.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_writer.h
@@ -9,7 +9,6 @@
 #include "blobstorage_pdisk_mon.h"
 #include "blobstorage_pdisk_request_id.h"
 
-#include <ydb/core/base/compile_time_flags.h>
 #include <ydb/core/blobstorage/crypto/crypto.h>
 
 #include <util/generic/deque.h>

diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_events.h b/ydb/core/blobstorage/vdisk/common/vdisk_events.h
@@ -15,7 +15,6 @@
 #include <ydb/core/blobstorage/storagepoolmon/storagepool_counters.h>
 #include <ydb/core/base/blobstorage_common.h>
 
-#include <ydb/core/base/compile_time_flags.h>
 #include <ydb/core/base/event_filter.h>
 #include <ydb/core/base/interconnect_channels.h>
 #include <ydb/core/protos/blobstorage_config.pb.h>

diff --git a/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclogdata.cpp b/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclogdata.cpp
@@ -431,7 +431,7 @@ namespace NKikimr {
             , ChunksToDeleteDelayed(std::move(chunksToDeleteDelayed))
         {}
 
-        void TEntryPointSerializer::Serialize(const TDeltaToDiskRecLog &delta, bool oldFormat) {
+        void TEntryPointSerializer::Serialize(const TDeltaToDiskRecLog &delta) {
             // fill in the protobuf
             NKikimrVDiskData::TSyncLogEntryPoint pb;
             pb.SetRecoveryLogConfirmedLsn(RecoveryLogConfirmedLsn);
@@ -441,48 +441,22 @@ namespace NKikimr {
             }
             const ui32 indexRecsNum = SyncLogSnap->SerializeToProto(pb, delta);
             // produce serialized data for the entry point
-            SerializedData = Serialize(pb, oldFormat);
+            SerializedData = Serialize(pb);
 
             // fill in EntryPointDbgInfo
             EntryPointDbgInfo = TEntryPointDbgInfo(SerializedData.size(),
                     ChunksToDeleteDelayed.size(),
                     indexRecsNum);
         }
 
-        TString TEntryPointSerializer::Serialize(const NKikimrVDiskData::TSyncLogEntryPoint &pb, bool oldFormat) {
-            if (oldFormat) {
-                TStringStream s;
-                // Header
-                const ui32 reservedData = 0;
-                const ui32 signature = TSyncLogHeader::SyncLogOldSignature;
-                s.Write(&signature, sizeof(signature));
-                s.Write(&reservedData, sizeof(reservedData));
-                const ui64 pdiskGuid = pb.GetPDiskGuid();
-                s.Write(&pdiskGuid, sizeof(pdiskGuid));
-                const ui64 vdiskIncarnationGuid = pb.GetVDiskIncarnationGuid();
-                s.Write(&vdiskIncarnationGuid, sizeof(vdiskIncarnationGuid));
-                // LogStartLsn
-                const ui64 logStartLsn = pb.GetLogStartLsn();
-                s.Write(&logStartLsn, sizeof(logStartLsn));
-                // chunksToDeleteDelayed
-                const ui32 delChunksSize = pb.ChunksToDeleteDelayedSize();
-                s.Write(&delChunksSize, sizeof(delChunksSize));
-                for (ui64 i = 0; i < delChunksSize; ++i) {
-                    const ui32 chunkId = pb.GetChunksToDeleteDelayed(i);
-                    s.Write(&chunkId, sizeof(chunkId));
-                }
-                s.Write(pb.GetDiskRecLogSerialized().data(), pb.GetDiskRecLogSerialized().size());
-
-                return s.Str();
-            } else {
-                // signature
-                TStringStream s;
-                s.Write(&TSyncLogHeader::SyncLogPbSignature, sizeof(ui32));
-                // pb payload
-                bool success = pb.SerializeToArcadiaStream(&s);
-                Y_ABORT_UNLESS(success);
-                return s.Str();
-            }
+        TString TEntryPointSerializer::Serialize(const NKikimrVDiskData::TSyncLogEntryPoint &pb) {
+            // signature
+            TStringStream s;
+            s.Write(&TSyncLogHeader::SyncLogPbSignature, sizeof(ui32));
+            // pb payload
+            bool success = pb.SerializeToArcadiaStream(&s);
+            Y_ABORT_UNLESS(success);
+            return s.Str();
         }
 
         ////////////////////////////////////////////////////////////////////////////

diff --git a/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclogdata.h b/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclogdata.h
@@ -280,7 +280,7 @@ namespace NKikimr {
                 TVector<ui32> &&chunksToDeleteDelayed,
                 ui64 recoveryLogConfirmedLsn);
 
-            void Serialize(const TDeltaToDiskRecLog &delta, bool oldFormat);
+            void Serialize(const TDeltaToDiskRecLog &delta);
             TString GetSerializedData() const { return SerializedData; }
             TEntryPointDbgInfo GetEntryPointDbgInfo() const { return EntryPointDbgInfo; }
 
@@ -292,7 +292,7 @@ namespace NKikimr {
             TString SerializedData;
             TEntryPointDbgInfo EntryPointDbgInfo;
 
-            static TString Serialize(const NKikimrVDiskData::TSyncLogEntryPoint &pb, bool oldFormat);
+            static TString Serialize(const NKikimrVDiskData::TSyncLogEntryPoint &pb);
         };
 
         ////////////////////////////////////////////////////////////////////////////

diff --git a/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclogdata_ut.cpp b/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclogdata_ut.cpp
@@ -43,8 +43,7 @@ namespace NKikimr {
 
 
         void TestEmptySyncLog(const TVector<ui32> &chunksToDeleteDelayed,
-                ui64 recoveryLogConfirmedLsn,
-                bool oldFormat)
+                ui64 recoveryLogConfirmedLsn)
         {
             TSyncLogPtr syncLog = CreateEmpty();
 
@@ -53,42 +52,25 @@ namespace NKikimr {
             TEntryPointSerializer serializer(syncLogSnap, std::move(copyOfChunksToDeleteDelayed),
                 recoveryLogConfirmedLsn);
             TDeltaToDiskRecLog delta(10);
-            serializer.Serialize(delta, oldFormat);
+            serializer.Serialize(delta);
             TString serializedData = serializer.GetSerializedData();
 
             auto recoveredSyncLog = Parse(serializedData);
             TEntryPointParser parser = MakeParser(serializedData);
             UNIT_ASSERT(parser.GetChunksToDelete() == chunksToDeleteDelayed);
-            if (oldFormat) {
-                UNIT_ASSERT(parser.GetRecoveryLogConfirmedLsn() == 0);
-            } else {
-                UNIT_ASSERT(parser.GetRecoveryLogConfirmedLsn() == recoveryLogConfirmedLsn);
-            }
-        }
-
-
-        Y_UNIT_TEST(SerializeParseEmpty1_Old) {
-            TVector<ui32> chunksToDeleteDelayed;
-            ui64 recoveryLogConfirmedLsn = 0;
-            TestEmptySyncLog(chunksToDeleteDelayed, recoveryLogConfirmedLsn, true);
-        }
-
-        Y_UNIT_TEST(SerializeParseEmpty2_Old) {
-            TVector<ui32> chunksToDeleteDelayed = {78, 3};
-            ui64 recoveryLogConfirmedLsn = 783246;
-            TestEmptySyncLog(chunksToDeleteDelayed, recoveryLogConfirmedLsn, true);
+            UNIT_ASSERT(parser.GetRecoveryLogConfirmedLsn() == recoveryLogConfirmedLsn);
         }
 
         Y_UNIT_TEST(SerializeParseEmpty1_Proto) {
             TVector<ui32> chunksToDeleteDelayed;
             ui64 recoveryLogConfirmedLsn = 0;
-            TestEmptySyncLog(chunksToDeleteDelayed, recoveryLogConfirmedLsn, false);
+            TestEmptySyncLog(chunksToDeleteDelayed, recoveryLogConfirmedLsn);
         }
 
         Y_UNIT_TEST(SerializeParseEmpty2_Proto) {
             TVector<ui32> chunksToDeleteDelayed = {78, 3};
             ui64 recoveryLogConfirmedLsn = 783246;
-            TestEmptySyncLog(chunksToDeleteDelayed, recoveryLogConfirmedLsn, false);
+            TestEmptySyncLog(chunksToDeleteDelayed, recoveryLogConfirmedLsn);
         }
     }
 

diff --git a/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclogkeeper_committer.cpp b/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclogkeeper_committer.cpp
@@ -3,7 +3,6 @@
 #include "blobstorage_synclog_public_events.h"
 
 #include <ydb/core/base/blobstorage_grouptype.h>
-#include <ydb/core/base/compile_time_flags.h>
 
 using namespace NKikimrServices;
 
@@ -35,8 +34,7 @@ namespace NKikimr {
 
             void GenerateCommit(const TActorContext &ctx) {
                 // serialize
-                const bool oldFormat = !KIKIMR_VDISK_SYNCLOG_ENTRY_POINT_PROTO_FORMAT;
-                EntryPointSerializer.Serialize(Delta, oldFormat);
+                EntryPointSerializer.Serialize(Delta);
 
                 // lsn
                 TLsnSeg seg = SlCtx->LsnMngr->AllocLsnForLocalUse();

diff --git a/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclogkeeper_ut.cpp b/ydb/core/blobstorage/vdisk/synclog/blobstorage_synclogkeeper_ut.cpp
@@ -55,7 +55,7 @@ namespace NKikimr {
     class TSyncLogKeeperTest {
     public:
         void CreateState(TEntryPointPair ep);
-        void Run(bool oldFormatForEntryPoint);
+        void Run();
 
         void PrintStatus(const TString &str = {}) {
             ::NKikimr::PrintStatus(State.get(), str);
@@ -130,8 +130,7 @@ namespace NKikimr {
 
     class TCommitWithNoSwapAndDelChunks {
     public:
-        TCommitWithNoSwapAndDelChunks(bool oldFormatForEntryPoint)
-            : OldFormatForEntryPoint(oldFormatForEntryPoint)
+        TCommitWithNoSwapAndDelChunks()
         {}
 
         void Start(TSyncLogKeeperState *state, ui64 recoveryLogConfirmedLsn) {
@@ -148,7 +147,7 @@ namespace NKikimr {
             TDeltaToDiskRecLog delta(10);
             TEntryPointSerializer entryPointSerializer(CommitData->SyncLogSnap,
                 std::move(CommitData->ChunksToDeleteDelayed), CommitData->RecoveryLogConfirmedLsn);
-            entryPointSerializer.Serialize(delta, OldFormatForEntryPoint);
+            entryPointSerializer.Serialize(delta);
 
             TCommitHistory commitHistory(TInstant(), commitLsn, CommitData->RecoveryLogConfirmedLsn);
             TEvSyncLogCommitDone commitDone(commitHistory, entryPointSerializer.GetEntryPointDbgInfo(),
@@ -162,11 +161,10 @@ namespace NKikimr {
             return {entryPointSerializer.GetSerializedData(), commitLsn};
         }
     private:
-        const bool OldFormatForEntryPoint;
         std::unique_ptr<TSyncLogKeeperCommitData> CommitData;
     };
 
-    void TSyncLogKeeperTest::Run(bool oldFormatForEntryPoint) {
+    void TSyncLogKeeperTest::Run() {
         TEntryPointPair entryPointPair;
         CreateState(TEntryPointPair{TString(), 0});
         // start with empty log
@@ -187,7 +185,7 @@ namespace NKikimr {
         Y_ABORT_UNLESS(commit);
 
         // start parallel commit
-        TCommitWithNoSwapAndDelChunks parallelCommit(oldFormatForEntryPoint);
+        TCommitWithNoSwapAndDelChunks parallelCommit;
         parallelCommit.Start(State.get(), 10);
 
         // write more messages during parallel commit
@@ -208,7 +206,7 @@ namespace NKikimr {
         Y_ABORT_UNLESS(commit);
 
         // start parallel commit
-        TCommitWithNoSwapAndDelChunks parallelCommit2(oldFormatForEntryPoint);
+        TCommitWithNoSwapAndDelChunks parallelCommit2;
         parallelCommit2.Start(State.get(), 31);
 
         // commit finished with lsn=33
@@ -239,14 +237,9 @@ namespace NKikimr {
     ////////////////////////////////////////////////////////////////////////////
     Y_UNIT_TEST_SUITE(TBlobStorageSyncLogKeeper) {
 
-        Y_UNIT_TEST(CutLog_EntryPointOldFormat) {
-            TSyncLogKeeperTest test;
-            test.Run(true);
-        }
-
         Y_UNIT_TEST(CutLog_EntryPointNewFormat) {
             TSyncLogKeeperTest test;
-            test.Run(false);
+            test.Run();
         }
 
     }

diff --git a/ydb/core/client/client_ut.cpp b/ydb/core/client/client_ut.cpp
@@ -1,5 +1,4 @@
 #include <ydb/core/base/appdata.h>
-#include <ydb/core/base/compile_time_flags.h>
 #include <ydb/library/services/services.pb.h>
 #include <ydb/core/tablet/tablet_impl.h>
 #include <ydb/core/testlib/test_client.h>

diff --git a/ydb/core/tablet/tablet_sys.cpp b/ydb/core/tablet/tablet_sys.cpp
@@ -1,7 +1,6 @@
 #include "tablet_sys.h"
 #include "tablet_tracing_signals.h"
 
-#include <ydb/core/base/compile_time_flags.h>
 #include <ydb/core/base/hive.h>
 #include <ydb/core/base/tablet_pipecache.h>
 #include <ydb/library/services/services.pb.h>

diff --git a/ydb/core/tablet_flat/flat_executor.cpp b/ydb/core/tablet_flat/flat_executor.cpp
@@ -27,7 +27,6 @@
 #include "util_fmt_desc.h"
 
 #include <ydb/core/base/appdata.h>
-#include <ydb/core/base/compile_time_flags.h>
 #include <ydb/core/base/hive.h>
 #include <ydb/core/base/tablet_pipecache.h>
 #include <ydb/core/control/immediate_control_board_impl.h>

diff --git a/ydb/core/tx/datashard/datashard_pipeline.cpp b/ydb/core/tx/datashard/datashard_pipeline.cpp
@@ -5,7 +5,6 @@
 #include "datashard_txs.h"
 #include "datashard_write_operation.h"
 
-#include <ydb/core/base/compile_time_flags.h>
 #include <ydb/core/base/cputime.h>
 #include <ydb/core/base/feature_flags.h>
 #include <ydb/core/tx/balance_coverage/balance_coverage_builder.h>

diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp
@@ -3,7 +3,6 @@
 #include "schemeshard_utils.h"
 
 #include <ydb/core/base/appdata.h>
-#include <ydb/core/base/compile_time_flags.h>
 #include <ydb/core/base/tx_processing.h>
 #include <ydb/core/base/channel_profiles.h>
 #include <ydb/core/engine/minikql/flat_local_tx_factory.h>

diff --git a/ydb/core/tx/schemeshard/ut_base/ut_base.cpp b/ydb/core/tx/schemeshard/ut_base/ut_base.cpp
@@ -1,8 +1,6 @@
 #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h>
 #include <ydb/core/tx/schemeshard/schemeshard_utils.h>
 
-#include <ydb/core/base/compile_time_flags.h>
-
 #include <util/generic/size_literals.h>
 #include <util/string/cast.h>
 

diff --git a/ydb/core/tx/schemeshard/ut_base/ut_info_types.cpp b/ydb/core/tx/schemeshard/ut_base/ut_info_types.cpp
@@ -1,8 +1,6 @@
 #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h>
 #include <ydb/core/tx/schemeshard/schemeshard_info_types.h>
 
-#include <ydb/core/base/compile_time_flags.h>
-
 #include <util/string/strip.h>
 
 using namespace NKikimr;

diff --git a/ydb/core/tx/schemeshard/ut_move/ut_move.cpp b/ydb/core/tx/schemeshard/ut_move/ut_move.cpp
@@ -1,7 +1,6 @@
 #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h>
 #include <ydb/core/tx/schemeshard/schemeshard_utils.h>
 
-#include <ydb/core/base/compile_time_flags.h>
 #include <ydb/core/kqp/ut/common/kqp_ut_common.h>
 #include <ydb/core/tx/datashard/change_exchange.h>
 

diff --git a/ydb/core/tx/schemeshard/ut_move_reboots/ut_move_reboots.cpp b/ydb/core/tx/schemeshard/ut_move_reboots/ut_move_reboots.cpp
@@ -1,7 +1,6 @@
 #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h>
 #include <ydb/core/tx/schemeshard/schemeshard_utils.h>
 
-#include <ydb/core/base/compile_time_flags.h>
 #include <ydb/core/kqp/ut/common/kqp_ut_common.h>
 #include <ydb/core/tx/datashard/change_exchange.h>
 

diff --git a/ydb/core/tx/schemeshard/ut_topic_splitmerge/ut_topic_splitmerge.cpp b/ydb/core/tx/schemeshard/ut_topic_splitmerge/ut_topic_splitmerge.cpp
@@ -1,7 +1,6 @@
 #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h>
 #include <ydb/core/tx/schemeshard/schemeshard_utils.h>
 
-#include <ydb/core/base/compile_time_flags.h>
 #include <ydb/services/lib/sharding/sharding.h>
 
 #include <util/generic/size_literals.h>