diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index f1fc605581bc..526fa90a08b3 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -243,6 +243,29 @@ #include +#ifndef KIKIMR_DISABLE_S3_OPS +#include +#endif + +namespace { + +#ifndef KIKIMR_DISABLE_S3_OPS +struct TAwsApiGuard { + TAwsApiGuard() { + Aws::InitAPI(Options); + } + + ~TAwsApiGuard() { + Aws::ShutdownAPI(Options); + } + +private: + Aws::SDKOptions Options; +}; +#endif + +} + namespace NKikimr { namespace NKikimrServicesInitializers { @@ -2816,5 +2839,18 @@ void TGraphServiceInitializer::InitializeServices(NActors::TActorSystemSetup* se TActorSetupCmd(NGraph::CreateGraphService(appData->TenantName), TMailboxType::HTSwap, appData->UserPoolId)); } +#ifndef KIKIMR_DISABLE_S3_OPS +TAwsApiInitializer::TAwsApiInitializer(IGlobalObjectStorage& globalObjects) + : GlobalObjects(globalObjects) +{ +} + +void TAwsApiInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { + Y_UNUSED(setup); + Y_UNUSED(appData); + GlobalObjects.AddGlobalObject(std::make_shared()); +} +#endif + } // namespace NKikimrServicesInitializers } // namespace NKikimr diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index 87d692434081..04f30522186f 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -618,5 +618,16 @@ class TGraphServiceInitializer : public IKikimrServicesInitializer { void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; +#ifndef KIKIMR_DISABLE_S3_OPS +class TAwsApiInitializer : public IServiceInitializer { + IGlobalObjectStorage& GlobalObjects; + +public: + TAwsApiInitializer(IGlobalObjectStorage& globalObjects); + + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; +}; +#endif + } // namespace NKikimrServicesInitializers } // namespace NKikimr diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index bc70d1e83bd4..e1e7ef77d0f3 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -1655,6 +1655,12 @@ TIntrusivePtr TKikimrRunner::CreateServiceInitializers sil->AddServiceInitializer(new TGraphServiceInitializer(runConfig)); } +#ifndef KIKIMR_DISABLE_S3_OPS + if (serviceMask.EnableAwsService) { + sil->AddServiceInitializer(new TAwsApiInitializer(*this)); + } +#endif + return sil; } diff --git a/ydb/core/driver_lib/run/service_mask.h b/ydb/core/driver_lib/run/service_mask.h index 3b694ce5ac84..044557229c6b 100644 --- a/ydb/core/driver_lib/run/service_mask.h +++ b/ydb/core/driver_lib/run/service_mask.h @@ -79,6 +79,7 @@ union TBasicKikimrServicesMask { bool EnableGraphService:1; bool EnableCompDiskLimiter:1; bool EnableGroupedMemoryLimiter:1; + bool EnableAwsService:1; }; struct { diff --git a/ydb/core/driver_lib/run/ya.make b/ydb/core/driver_lib/run/ya.make index 2bcc7925a9db..c4bb6a69f660 100644 --- a/ydb/core/driver_lib/run/ya.make +++ b/ydb/core/driver_lib/run/ya.make @@ -1,5 +1,15 @@ LIBRARY(run) +IF (OS_WINDOWS) + CFLAGS( + -DKIKIMR_DISABLE_S3_OPS + ) +ELSE() + PEERDIR( + contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core + ) +ENDIF() + SRCS( auto_config_initializer.cpp config.cpp diff --git a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp index 39673d15b071..deb7be3d89e9 100644 --- a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp +++ b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp @@ -17,6 +17,9 @@ #include #include +#include + +#include namespace NKikimr { @@ -32,6 +35,16 @@ enum class EInitialEviction { namespace { +Aws::SDKOptions Options; + +Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { + Aws::InitAPI(Options); +} + +Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { + Aws::ShutdownAPI(Options); +} + static const std::vector testYdbSchema = TTestSchema::YdbSchema(); static const std::vector testYdbPk = TTestSchema::YdbPkSchema(); diff --git a/ydb/core/tx/columnshard/ut_schema/ya.make b/ydb/core/tx/columnshard/ut_schema/ya.make index 35d906ee2055..d67c0d2ad5b8 100644 --- a/ydb/core/tx/columnshard/ut_schema/ya.make +++ b/ydb/core/tx/columnshard/ut_schema/ya.make @@ -18,6 +18,7 @@ PEERDIR( library/cpp/getopt library/cpp/regex/pcre library/cpp/svnversion + contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core ydb/core/testlib/default ydb/core/tx/columnshard/hooks/abstract ydb/core/tx/columnshard/hooks/testing diff --git a/ydb/core/tx/datashard/import_s3.cpp b/ydb/core/tx/datashard/import_s3.cpp index ba7227a7a74c..655ee80172ab 100644 --- a/ydb/core/tx/datashard/import_s3.cpp +++ b/ydb/core/tx/datashard/import_s3.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include diff --git a/ydb/core/tx/schemeshard/ut_backup/ut_backup.cpp b/ydb/core/tx/schemeshard/ut_backup/ut_backup.cpp index 1338f2d8f42d..771b929051c0 100644 --- a/ydb/core/tx/schemeshard/ut_backup/ut_backup.cpp +++ b/ydb/core/tx/schemeshard/ut_backup/ut_backup.cpp @@ -7,9 +7,27 @@ #include #include +#include + +#include + using namespace NSchemeShardUT_Private; using namespace NKikimr::NWrappers::NTestHelpers; +namespace { + +Aws::SDKOptions Options; + +Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { + Aws::InitAPI(Options); +} + +Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { + Aws::ShutdownAPI(Options); +} + +} + Y_UNIT_TEST_SUITE(TBackupTests) { using TFillFn = std::function; diff --git a/ydb/core/tx/schemeshard/ut_backup/ya.make b/ydb/core/tx/schemeshard/ut_backup/ya.make index d9ee6dd81405..aac9bc5f9334 100644 --- a/ydb/core/tx/schemeshard/ut_backup/ya.make +++ b/ydb/core/tx/schemeshard/ut_backup/ya.make @@ -20,6 +20,7 @@ IF (NOT OS_WINDOWS) library/cpp/getopt library/cpp/regex/pcre library/cpp/svnversion + contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core ydb/core/testlib/default ydb/core/tx ydb/core/tx/schemeshard/ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_export/ut_export.cpp b/ydb/core/tx/schemeshard/ut_export/ut_export.cpp index f2e82ce2c7d7..8d41d7bab562 100644 --- a/ydb/core/tx/schemeshard/ut_export/ut_export.cpp +++ b/ydb/core/tx/schemeshard/ut_export/ut_export.cpp @@ -13,11 +13,25 @@ #include #include +#include + +#include + using namespace NSchemeShardUT_Private; using namespace NKikimr::NWrappers::NTestHelpers; namespace { + Aws::SDKOptions Options; + + Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { + Aws::InitAPI(Options); + } + + Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { + Aws::ShutdownAPI(Options); + } + void Run(TTestBasicRuntime& runtime, TTestEnv& env, const TVector& tables, const TString& request, Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS, const TString& dbName = "/MyRoot", bool serverless = false, const TString& userSID = "", const TString& peerName = "") { @@ -1687,7 +1701,7 @@ partitioning_settings { return ev->Get()->Record .GetTransaction(0).GetOperationType() == NKikimrSchemeOp::ESchemeOpBackup; }; - + THolder delayed; auto prevObserver = runtime.SetObserverFunc([&](TAutoPtr& ev) { if (delayFunc(ev)) { @@ -2083,7 +2097,7 @@ partitioning_settings { min_partitions_count: 10 )")); } - + Y_UNIT_TEST(UserSID) { TTestBasicRuntime runtime; TTestEnv env(runtime); diff --git a/ydb/core/tx/schemeshard/ut_export/ya.make b/ydb/core/tx/schemeshard/ut_export/ya.make index 4d5bf91e2698..c62dc9ea8ebc 100644 --- a/ydb/core/tx/schemeshard/ut_export/ya.make +++ b/ydb/core/tx/schemeshard/ut_export/ya.make @@ -20,6 +20,7 @@ IF (NOT OS_WINDOWS) library/cpp/getopt library/cpp/regex/pcre library/cpp/svnversion + contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core ydb/core/testlib/default ydb/core/tx ydb/core/tx/schemeshard/ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_export_reboots_s3/ut_export_reboots_s3.cpp b/ydb/core/tx/schemeshard/ut_export_reboots_s3/ut_export_reboots_s3.cpp index 5e1e42d17ce9..97e34a6d3fa4 100644 --- a/ydb/core/tx/schemeshard/ut_export_reboots_s3/ut_export_reboots_s3.cpp +++ b/ydb/core/tx/schemeshard/ut_export_reboots_s3/ut_export_reboots_s3.cpp @@ -4,10 +4,28 @@ #include +#include + +#include + using namespace NSchemeShardUT_Private; using namespace NSchemeShardUT_Private::NExportReboots; using namespace NKikimr::NWrappers::NTestHelpers; +namespace { + +Aws::SDKOptions Options; + +Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { + Aws::InitAPI(Options); +} + +Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { + Aws::ShutdownAPI(Options); +} + +} + Y_UNIT_TEST_SUITE(TExportToS3WithRebootsTests) { using TUnderlying = std::function&, const TString&, TTestWithReboots&)>; diff --git a/ydb/core/tx/schemeshard/ut_export_reboots_s3/ya.make b/ydb/core/tx/schemeshard/ut_export_reboots_s3/ya.make index bc7ca966e0dc..caf4fb7de362 100644 --- a/ydb/core/tx/schemeshard/ut_export_reboots_s3/ya.make +++ b/ydb/core/tx/schemeshard/ut_export_reboots_s3/ya.make @@ -19,6 +19,7 @@ PEERDIR( library/cpp/getopt library/cpp/regex/pcre library/cpp/svnversion + contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core ydb/core/testlib/default ydb/core/tx ydb/core/tx/schemeshard/ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index cee6f82143e7..6007cf9619d5 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -21,8 +21,10 @@ #include +#include #include #include +#include #include #include @@ -38,6 +40,16 @@ using namespace NKikimr::NWrappers::NTestHelpers; namespace { + Aws::SDKOptions Options; + + Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { + Aws::InitAPI(Options); + } + + Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { + Aws::ShutdownAPI(Options); + } + const TString EmptyYsonStr = R"([[[[];%false]]])"; TString GenerateScheme(const NKikimrSchemeOp::TPathDescription& pathDesc) { @@ -317,7 +329,6 @@ namespace { runtime.SetObserverFunc(prevObserver); } - } // anonymous Y_UNIT_TEST_SUITE(TRestoreTests) { diff --git a/ydb/core/tx/schemeshard/ut_restore/ya.make b/ydb/core/tx/schemeshard/ut_restore/ya.make index 7044d4283b5e..d514b36b49ee 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ya.make +++ b/ydb/core/tx/schemeshard/ut_restore/ya.make @@ -14,6 +14,7 @@ ELSE() ENDIF() PEERDIR( + contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core contrib/libs/double-conversion library/cpp/string_utils/quote ydb/core/kqp/ut/common diff --git a/ydb/core/wrappers/s3_storage.h b/ydb/core/wrappers/s3_storage.h index d678f237616e..5af392c4f63c 100644 --- a/ydb/core/wrappers/s3_storage.h +++ b/ydb/core/wrappers/s3_storage.h @@ -3,7 +3,6 @@ #ifndef KIKIMR_DISABLE_S3_OPS #include "abstract.h" -#include "s3_storage_config.h" #include #include @@ -32,7 +31,7 @@ namespace NKikimr::NWrappers::NExternalStorage { -class TS3ExternalStorage: public IExternalStorageOperator, TS3User { +class TS3ExternalStorage: public IExternalStorageOperator { private: THolder Client; const Aws::Client::ClientConfiguration Config; diff --git a/ydb/core/wrappers/s3_storage_config.cpp b/ydb/core/wrappers/s3_storage_config.cpp index f0bc70e15fd0..9785eb7ed5ef 100644 --- a/ydb/core/wrappers/s3_storage_config.cpp +++ b/ydb/core/wrappers/s3_storage_config.cpp @@ -1,81 +1,15 @@ #include "s3_storage.h" #include "s3_storage_config.h" -#include -#include -#include -#include #include -#include -#include -#include #include #ifndef KIKIMR_DISABLE_S3_OPS namespace NKikimr::NWrappers::NExternalStorage { -using namespace Aws; -using namespace Aws::Auth; -using namespace Aws::Client; -using namespace Aws::S3; -using namespace Aws::S3::Model; -using namespace Aws::Utils::Stream; - namespace { -struct TCurlInitializer { - TCurlInitializer() { - curl_global_init(CURL_GLOBAL_ALL); - } - - ~TCurlInitializer() { - curl_global_cleanup(); - } -}; - -struct TApiInitializer { - TApiInitializer() { - Options.httpOptions.initAndCleanupCurl = false; - InitAPI(Options); - - Internal::CleanupEC2MetadataClient(); // speeds up config construction - } - - ~TApiInitializer() { - ShutdownAPI(Options); - } - -private: - SDKOptions Options; -}; - -class TApiOwner { -public: - void Ref() { - auto guard = Guard(Mutex); - if (!RefCount++) { - if (!CurlInitializer) { - CurlInitializer.Reset(new TCurlInitializer); - } - ApiInitializer.Reset(new TApiInitializer); - } - } - - void UnRef() { - auto guard = Guard(Mutex); - if (!--RefCount) { - ApiInitializer.Destroy(); - } - } - -private: - ui64 RefCount = 0; - TMutex Mutex; - THolder CurlInitializer; - THolder ApiInitializer; -}; - namespace NPrivate { template @@ -93,10 +27,10 @@ Aws::Client::ClientConfiguration ConfigFromSettings(const TSettings& settings) { switch (settings.scheme()) { case TSettings::HTTP: - config.scheme = Http::Scheme::HTTP; + config.scheme = Aws::Http::Scheme::HTTP; break; case TSettings::HTTPS: - config.scheme = Http::Scheme::HTTPS; + config.scheme = Aws::Http::Scheme::HTTPS; break; default: Y_ABORT("Unknown scheme"); @@ -114,22 +48,6 @@ Aws::Auth::AWSCredentials CredentialsFromSettings(const TSettings& settings) { } // anonymous -TS3User::TS3User(const TS3User& /*baseObject*/) { - Singleton()->Ref(); -} - -TS3User::TS3User(TS3User& /*baseObject*/) { - Singleton()->Ref(); -} - -TS3User::TS3User() { - Singleton()->Ref(); -} - -TS3User::~TS3User() { - Singleton()->UnRef(); -} - class TS3ThreadsPoolByEndpoint { private: diff --git a/ydb/core/wrappers/s3_storage_config.h b/ydb/core/wrappers/s3_storage_config.h index 544ccc74af82..b276bd79b363 100644 --- a/ydb/core/wrappers/s3_storage_config.h +++ b/ydb/core/wrappers/s3_storage_config.h @@ -17,14 +17,7 @@ namespace NKikimr::NWrappers::NExternalStorage { -struct TS3User { - TS3User(); - TS3User(const TS3User& baseObject); - TS3User(TS3User& baseObject); - ~TS3User(); -}; - -class TS3ExternalStorageConfig: public IExternalStorageConfig, TS3User { +class TS3ExternalStorageConfig: public IExternalStorageConfig { private: YDB_READONLY_DEF(TString, Bucket); Aws::Client::ClientConfiguration Config; diff --git a/ydb/core/wrappers/s3_wrapper_ut.cpp b/ydb/core/wrappers/s3_wrapper_ut.cpp index 4e310add8777..b8b0e185cde4 100644 --- a/ydb/core/wrappers/s3_wrapper_ut.cpp +++ b/ydb/core/wrappers/s3_wrapper_ut.cpp @@ -8,16 +8,33 @@ #include #include +#include #include #include +#include + using namespace NActors; using namespace NKikimr; using namespace NKikimr::NWrappers; using namespace Aws::S3::Model; -class TS3MockTest: public NUnitTest::TTestBase, private NExternalStorage::TS3User { +namespace { + +Aws::SDKOptions Options; + +Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { + Aws::InitAPI(Options); +} + +Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { + Aws::ShutdownAPI(Options); +} + +} + +class TS3MockTest: public NUnitTest::TTestBase { using TS3Mock = NWrappers::NTestHelpers::TS3Mock; static auto MakeClientConfig(ui16 port) { @@ -48,8 +65,8 @@ class TS3MockTest: public NUnitTest::TTestBase, private NExternalStorage::TS3Use } void TearDown() override { - S3Mock.Reset(); Runtime.Reset(); + S3Mock.Reset(); } ui16 GetPort() const { diff --git a/ydb/core/wrappers/ut/ya.make b/ydb/core/wrappers/ut/ya.make index 61e02602c881..212e65cfc54f 100644 --- a/ydb/core/wrappers/ut/ya.make +++ b/ydb/core/wrappers/ut/ya.make @@ -9,6 +9,7 @@ IF (NOT OS_WINDOWS) ydb/library/actors/core library/cpp/digest/md5 library/cpp/testing/unittest + contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core ydb/core/protos ydb/core/testlib/basics/default ydb/library/yql/minikql/comp_nodes/llvm14 diff --git a/ydb/services/ydb/backup_ut/ya.make b/ydb/services/ydb/backup_ut/ya.make new file mode 100644 index 000000000000..39732a69bd9c --- /dev/null +++ b/ydb/services/ydb/backup_ut/ya.make @@ -0,0 +1,32 @@ +UNITTEST_FOR(ydb/services/ydb) + +FORK_SUBTESTS() + +IF (SANITIZER_TYPE == "thread" OR WITH_VALGRIND) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + SIZE(MEDIUM) +ENDIF() + +SRCS( + ydb_backup_ut.cpp +) + +PEERDIR( + ydb/core/testlib/default + ydb/core/wrappers/ut_helpers + ydb/public/lib/ydb_cli/dump + ydb/public/sdk/cpp/client/ydb_export + ydb/public/sdk/cpp/client/ydb_import + ydb/public/sdk/cpp/client/ydb_operation + ydb/public/sdk/cpp/client/ydb_result + ydb/public/sdk/cpp/client/ydb_table + ydb/public/sdk/cpp/client/ydb_value + ydb/library/backup + contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp b/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp new file mode 100644 index 000000000000..8b489dbf5aba --- /dev/null +++ b/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp @@ -0,0 +1,547 @@ +#include "ydb_common_ut.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include + +using namespace NYdb; +using namespace NYdb::NTable; + +namespace NYdb::NTable { + +bool operator==(const TValue& lhs, const TValue& rhs) { + return google::protobuf::util::MessageDifferencer::Equals(lhs.GetProto(), rhs.GetProto()); +} + +bool operator==(const TKeyBound& lhs, const TKeyBound& rhs) { + return lhs.GetValue() == rhs.GetValue() && lhs.IsInclusive() == rhs.IsInclusive(); +} + +bool operator==(const TKeyRange& lhs, const TKeyRange& rhs) { + return lhs.From() == lhs.From() && lhs.To() == rhs.To(); +} + +} + +namespace { + +#define DEBUG_HINT (TStringBuilder() << "at line " << __LINE__) + +void ExecuteDataDefinitionQuery(TSession& session, const TString& script) { + const auto result = session.ExecuteSchemeQuery(script).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), "script:\n" << script << "\nissues:\n" << result.GetIssues().ToString()); +} + +TDataQueryResult ExecuteDataModificationQuery(TSession& session, + const TString& script, + const TExecDataQuerySettings& settings = {} +) { + const auto result = session.ExecuteDataQuery( + script, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + settings + ).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), "script:\n" << script << "\nissues:\n" << result.GetIssues().ToString()); + + return result; +} + +TDataQueryResult GetTableContent(TSession& session, const char* table) { + return ExecuteDataModificationQuery(session, Sprintf(R"( + SELECT * FROM `%s` ORDER BY Key; + )", table + )); +} + +void CompareResults(const TDataQueryResult& first, const TDataQueryResult& second) { + const auto& firstResults = first.GetResultSets(); + const auto& secondResults = second.GetResultSets(); + + UNIT_ASSERT_VALUES_EQUAL(firstResults.size(), secondResults.size()); + for (size_t i = 0; i < firstResults.size(); ++i) { + UNIT_ASSERT_STRINGS_EQUAL( + FormatResultSetYson(firstResults[i]), + FormatResultSetYson(secondResults[i]) + ); + } +} + +TTableDescription GetTableDescription(TSession& session, const TString& path, + const TDescribeTableSettings& settings = {} +) { + auto describeResult = session.DescribeTable(path, settings).ExtractValueSync(); + UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); + return describeResult.GetTableDescription(); +} + +auto CreateMinPartitionsChecker(ui32 expectedMinPartitions, const TString& debugHint = "") { + return [=](const TTableDescription& tableDescription) { + UNIT_ASSERT_VALUES_EQUAL_C( + tableDescription.GetPartitioningSettings().GetMinPartitionsCount(), + expectedMinPartitions, + debugHint + ); + }; +} + +void CheckTableDescription(TSession& session, const TString& path, auto&& checker, + const TDescribeTableSettings& settings = {} +) { + checker(GetTableDescription(session, path, settings)); +} + +using TBackupFunction = std::function; +using TRestoreFunction = std::function; + +void TestTableContentIsPreserved( + const char* table, TSession& session, TBackupFunction&& backup, TRestoreFunction&& restore +) { + ExecuteDataDefinitionQuery(session, Sprintf(R"( + CREATE TABLE `%s` ( + Key Uint32, + Value Utf8, + PRIMARY KEY (Key) + ); + )", + table + )); + ExecuteDataModificationQuery(session, Sprintf(R"( + UPSERT INTO `%s` ( + Key, + Value + ) + VALUES + (1, "one"), + (2, "two"), + (3, "three"), + (4, "four"), + (5, "five"); + )", + table + )); + const auto originalContent = GetTableContent(session, table); + + backup(table); + + ExecuteDataDefinitionQuery(session, Sprintf(R"( + DROP TABLE `%s`; + )", table + )); + + restore(table); + CompareResults(GetTableContent(session, table), originalContent); +} + +void TestTablePartitioningSettingsArePreserved( + const char* table, ui32 minPartitions, TSession& session, TBackupFunction&& backup, TRestoreFunction&& restore +) { + ExecuteDataDefinitionQuery(session, Sprintf(R"( + CREATE TABLE `%s` ( + Key Uint32, + Value Utf8, + PRIMARY KEY (Key) + ) + WITH ( + AUTO_PARTITIONING_BY_LOAD = ENABLED, + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = %u + ); + )", + table, minPartitions + )); + CheckTableDescription(session, table, CreateMinPartitionsChecker(minPartitions, DEBUG_HINT)); + + backup(table); + + ExecuteDataDefinitionQuery(session, Sprintf(R"( + DROP TABLE `%s`; + )", table + )); + + restore(table); + CheckTableDescription(session, table, CreateMinPartitionsChecker(minPartitions, DEBUG_HINT)); +} + +void TestIndexTablePartitioningSettingsArePreserved( + const char* table, const char* index, ui32 minIndexPartitions, TSession& session, + TBackupFunction&& backup, TRestoreFunction&& restore +) { + const TString indexTablePath = JoinFsPaths(table, index, "indexImplTable"); + + ExecuteDataDefinitionQuery(session, Sprintf(R"( + CREATE TABLE `%s` ( + Key Uint32, + Value Uint32, + PRIMARY KEY (Key), + INDEX %s GLOBAL ON (Value) + ); + )", + table, index + )); + ExecuteDataDefinitionQuery(session, Sprintf(R"( + ALTER TABLE `%s` ALTER INDEX %s SET ( + AUTO_PARTITIONING_BY_LOAD = ENABLED, + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = %u + ); + )", table, index, minIndexPartitions + )); + CheckTableDescription(session, indexTablePath, CreateMinPartitionsChecker(minIndexPartitions, DEBUG_HINT)); + + backup(table); + + ExecuteDataDefinitionQuery(session, Sprintf(R"( + DROP TABLE `%s`; + )", table + )); + + restore(table); + CheckTableDescription(session, indexTablePath, CreateMinPartitionsChecker(minIndexPartitions, DEBUG_HINT)); +} + +void TestTableSplitBoundariesArePreserved( + const char* table, ui64 partitions, TSession& session, TBackupFunction&& backup, TRestoreFunction&& restore +) { + ExecuteDataDefinitionQuery(session, Sprintf(R"( + CREATE TABLE `%s` ( + Key Uint32, + Value Utf8, + PRIMARY KEY (Key) + ) + WITH ( + PARTITION_AT_KEYS = (1, 2, 4, 8, 16, 32, 64, 128, 256) + ); + )", + table + )); + const auto describeSettings = TDescribeTableSettings() + .WithTableStatistics(true) + .WithKeyShardBoundary(true); + const auto originalTableDescription = GetTableDescription(session, table, describeSettings); + UNIT_ASSERT_VALUES_EQUAL(originalTableDescription.GetPartitionsCount(), partitions); + const auto& originalKeyRanges = originalTableDescription.GetKeyRanges(); + UNIT_ASSERT_VALUES_EQUAL(originalKeyRanges.size(), partitions); + + backup(table); + + ExecuteDataDefinitionQuery(session, Sprintf(R"( + DROP TABLE `%s`; + )", table + )); + + restore(table); + const auto restoredTableDescription = GetTableDescription(session, table, describeSettings); + UNIT_ASSERT_VALUES_EQUAL(restoredTableDescription.GetPartitionsCount(), partitions); + const auto& restoredKeyRanges = restoredTableDescription.GetKeyRanges(); + UNIT_ASSERT_VALUES_EQUAL(restoredKeyRanges.size(), partitions); + UNIT_ASSERT_EQUAL(restoredTableDescription.GetKeyRanges(), originalKeyRanges); +} + +} + +Y_UNIT_TEST_SUITE(BackupRestore) { + + void Restore(NDump::TClient& client, const TFsPath& sourceFile, const TString& dbPath) { + auto result = client.Restore(sourceFile, dbPath); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + auto CreateBackupLambda(const TDriver& driver, const TFsPath& pathToBackup, bool schemaOnly = false) { + return [&driver, &pathToBackup, schemaOnly](const char* table) { + Y_UNUSED(table); + // TO DO: implement NDump::TClient::Dump and call it instead of BackupFolder + NBackup::BackupFolder(driver, "/Root", ".", pathToBackup, {}, schemaOnly, false); + }; + } + + auto CreateRestoreLambda(const TDriver& driver, const TFsPath& pathToBackup) { + return [&driver, &pathToBackup](const char* table) { + Y_UNUSED(table); + NDump::TClient backupClient(driver); + Restore(backupClient, pathToBackup, "/Root"); + }; + } + + Y_UNIT_TEST(RestoreTableContent) { + TKikimrWithGrpcAndRootSchema server; + auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort()))); + TTableClient tableClient(driver); + auto session = tableClient.GetSession().ExtractValueSync().GetSession(); + TTempDir tempDir; + const auto& pathToBackup = tempDir.Path(); + + constexpr const char* table = "/Root/table"; + + TestTableContentIsPreserved( + table, + session, + CreateBackupLambda(driver, pathToBackup), + CreateRestoreLambda(driver, pathToBackup) + ); + } + + Y_UNIT_TEST(RestoreTablePartitioningSettings) { + TKikimrWithGrpcAndRootSchema server; + auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort()))); + TTableClient tableClient(driver); + auto session = tableClient.GetSession().ExtractValueSync().GetSession(); + TTempDir tempDir; + const auto& pathToBackup = tempDir.Path(); + + constexpr const char* table = "/Root/table"; + constexpr ui32 minPartitions = 10; + + TestTablePartitioningSettingsArePreserved( + table, + minPartitions, + session, + CreateBackupLambda(driver, pathToBackup, true), + CreateRestoreLambda(driver, pathToBackup) + ); + } + + Y_UNIT_TEST(RestoreIndexTablePartitioningSettings) { + TKikimrWithGrpcAndRootSchema server; + auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort()))); + TTableClient tableClient(driver); + auto session = tableClient.GetSession().ExtractValueSync().GetSession(); + TTempDir tempDir; + const auto& pathToBackup = tempDir.Path(); + + constexpr const char* table = "/Root/table"; + constexpr const char* index = "byValue"; + constexpr ui32 minIndexPartitions = 10; + + TestIndexTablePartitioningSettingsArePreserved( + table, + index, + minIndexPartitions, + session, + CreateBackupLambda(driver, pathToBackup, true), + CreateRestoreLambda(driver, pathToBackup) + ); + } + + Y_UNIT_TEST(RestoreTableSplitBoundaries) { + TKikimrWithGrpcAndRootSchema server; + auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort()))); + TTableClient tableClient(driver); + auto session = tableClient.GetSession().ExtractValueSync().GetSession(); + TTempDir tempDir; + const auto& pathToBackup = tempDir.Path(); + + constexpr const char* table = "/Root/table"; + constexpr ui64 partitions = 10; + + TestTableSplitBoundariesArePreserved( + table, + partitions, + session, + CreateBackupLambda(driver, pathToBackup, true), + CreateRestoreLambda(driver, pathToBackup) + ); + } + +} + +Y_UNIT_TEST_SUITE(BackupRestoreS3) { + + Aws::SDKOptions Options; + + Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { + Aws::InitAPI(Options); + } + + Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { + Aws::ShutdownAPI(Options); + } + + using NKikimr::NWrappers::NTestHelpers::TS3Mock; + + class TS3TestEnv { + TKikimrWithGrpcAndRootSchema server; + TDriver driver; + TTableClient tableClient; + TSession session; + ui16 s3Port; + TS3Mock s3Mock; + // required for exports to function + TDataShardExportFactory dataShardExportFactory; + + public: + TS3TestEnv() + : driver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort()))) + , tableClient(driver) + , session(tableClient.CreateSession().ExtractValueSync().GetSession()) + , s3Port(server.GetPortManager().GetPort()) + , s3Mock({}, TS3Mock::TSettings(s3Port)) + { + UNIT_ASSERT_C(s3Mock.Start(), s3Mock.GetError()); + + auto& runtime = *server.GetRuntime(); + runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::EPriority::PRI_DEBUG); + runtime.GetAppData().DataShardExportFactory = &dataShardExportFactory; + } + + TKikimrWithGrpcAndRootSchema& GetServer() { + return server; + } + + const TDriver& GetDriver() const { + return driver; + } + + TSession& GetSession() { + return session; + } + + ui16 GetS3Port() const { + return s3Port; + } + }; + + template + bool WaitForOperation(NOperation::TOperationClient& client, NOperationId::TOperationId id, + int retries = 10, TDuration sleepDuration = TDuration::MilliSeconds(100) + ) { + for (int retry = 0; retry <= retries; ++retry) { + auto result = client.Get(id).ExtractValueSync(); + if (result.Ready()) { + UNIT_ASSERT_VALUES_EQUAL_C( + result.Status().GetStatus(), EStatus::SUCCESS, + result.Status().GetIssues().ToString() + ); + return true; + } + Sleep(sleepDuration *= 2); + } + return false; + } + + void ExportToS3(NExport::TExportClient& exportClient, ui16 s3Port, NOperation::TOperationClient& operationClient, + const TString& source, const TString& destination + ) { + // The exact values for Bucket, AccessKey and SecretKey do not matter if the S3 backend is TS3Mock. + // Any non-empty strings should do. + const auto exportSettings = NExport::TExportToS3Settings() + .Endpoint(Sprintf("localhost:%u", s3Port)) + .Scheme(ES3Scheme::HTTP) + .Bucket("test_bucket") + .AccessKey("test_key") + .SecretKey("test_secret") + .AppendItem(NExport::TExportToS3Settings::TItem{.Src = source, .Dst = destination}); + + auto response = exportClient.ExportToS3(exportSettings).ExtractValueSync(); + UNIT_ASSERT_C(WaitForOperation(operationClient, response.Id()), + Sprintf("The export from %s to %s did not complete within the allocated time.", + source.c_str(), destination.c_str() + ) + ); + } + + void ImportFromS3(NImport::TImportClient& importClient, ui16 s3Port, NOperation::TOperationClient& operationClient, + const TString& source, const TString& destination + ) { + // The exact values for Bucket, AccessKey and SecretKey do not matter if the S3 backend is TS3Mock. + // Any non-empty strings should do. + const auto importSettings = NImport::TImportFromS3Settings() + .Endpoint(Sprintf("localhost:%u", s3Port)) + .Scheme(ES3Scheme::HTTP) + .Bucket("test_bucket") + .AccessKey("test_key") + .SecretKey("test_secret") + .AppendItem(NImport::TImportFromS3Settings::TItem{.Src = source, .Dst = destination}); + + auto response = importClient.ImportFromS3(importSettings).ExtractValueSync(); + UNIT_ASSERT_C(WaitForOperation(operationClient, response.Id()), + Sprintf("The import from %s to %s did not complete within the allocated time.", + source.c_str(), destination.c_str() + ) + ); + } + + auto CreateBackupLambda(const TDriver& driver, ui16 s3Port) { + return [&driver, s3Port](const char* table) { + NExport::TExportClient exportClient(driver); + NOperation::TOperationClient operationClient(driver); + ExportToS3(exportClient, s3Port, operationClient, table, "table"); + }; + } + + auto CreateRestoreLambda(const TDriver& driver, ui16 s3Port) { + return [&driver, s3Port](const char* table) { + NImport::TImportClient importClient(driver); + NOperation::TOperationClient operationClient(driver); + ImportFromS3(importClient, s3Port, operationClient, "table", table); + }; + } + + Y_UNIT_TEST(RestoreTableContent) { + TS3TestEnv testEnv; + constexpr const char* table = "/Root/table"; + + TestTableContentIsPreserved( + table, + testEnv.GetSession(), + CreateBackupLambda(testEnv.GetDriver(), testEnv.GetS3Port()), + CreateRestoreLambda(testEnv.GetDriver(), testEnv.GetS3Port()) + ); + } + + Y_UNIT_TEST(RestoreTablePartitioningSettings) { + TS3TestEnv testEnv; + constexpr const char* table = "/Root/table"; + constexpr ui32 minPartitions = 10; + + TestTablePartitioningSettingsArePreserved( + table, + minPartitions, + testEnv.GetSession(), + CreateBackupLambda(testEnv.GetDriver(), testEnv.GetS3Port()), + CreateRestoreLambda(testEnv.GetDriver(), testEnv.GetS3Port()) + ); + } + + Y_UNIT_TEST(RestoreIndexTablePartitioningSettings) { + TS3TestEnv testEnv; + constexpr const char* table = "/Root/table"; + constexpr const char* index = "byValue"; + constexpr ui32 minIndexPartitions = 10; + + TestIndexTablePartitioningSettingsArePreserved( + table, + index, + minIndexPartitions, + testEnv.GetSession(), + CreateBackupLambda(testEnv.GetDriver(), testEnv.GetS3Port()), + CreateRestoreLambda(testEnv.GetDriver(), testEnv.GetS3Port()) + ); + } + + Y_UNIT_TEST(RestoreTableSplitBoundaries) { + TS3TestEnv testEnv; + constexpr const char* table = "/Root/table"; + constexpr ui64 partitions = 10; + + TestTableSplitBoundariesArePreserved( + table, + partitions, + testEnv.GetSession(), + CreateBackupLambda(testEnv.GetDriver(), testEnv.GetS3Port()), + CreateRestoreLambda(testEnv.GetDriver(), testEnv.GetS3Port()) + ); + } + +} diff --git a/ydb/services/ydb/ut/ya.make b/ydb/services/ydb/ut/ya.make index 2d60535a3205..09601dd06b5e 100644 --- a/ydb/services/ydb/ut/ya.make +++ b/ydb/services/ydb/ut/ya.make @@ -42,8 +42,6 @@ PEERDIR( ydb/core/grpc_services/base ydb/core/testlib ydb/core/security - ydb/core/wrappers/ut_helpers - ydb/library/backup ydb/library/yql/minikql/dom ydb/library/yql/minikql/jsonpath ydb/library/testlib/service_mocks/ldap_mock @@ -52,7 +50,6 @@ PEERDIR( ydb/public/lib/yson_value ydb/public/lib/ut_helpers ydb/public/lib/ydb_cli/commands - ydb/public/lib/ydb_cli/dump ydb/public/sdk/cpp/client/draft ydb/public/sdk/cpp/client/ydb_coordination ydb/public/sdk/cpp/client/ydb_export diff --git a/ydb/services/ydb/ya.make b/ydb/services/ydb/ya.make index d42bbf19283c..34676794495e 100644 --- a/ydb/services/ydb/ya.make +++ b/ydb/services/ydb/ya.make @@ -37,6 +37,7 @@ PEERDIR( END() RECURSE_FOR_TESTS( + backup_ut sdk_sessions_ut sdk_sessions_pool_ut table_split_ut diff --git a/ydb/services/ydb/ydb_import_ut.cpp b/ydb/services/ydb/ydb_import_ut.cpp index 3c93e2b57d23..961cd9679c85 100644 --- a/ydb/services/ydb/ydb_import_ut.cpp +++ b/ydb/services/ydb/ydb_import_ut.cpp @@ -3,10 +3,8 @@ #include #include #include -#include #include -#include #include #include @@ -130,272 +128,3 @@ Y_UNIT_TEST_SUITE(YdbImport) { } } - -Y_UNIT_TEST_SUITE(BackupRestore) { - - using namespace NYdb::NTable; - - void ExecuteDataDefinitionQuery(TSession& session, const TString& script) { - const auto result = session.ExecuteSchemeQuery(script).ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), "script:\n" << script << "\nissues:\n" << result.GetIssues().ToString()); - } - - TDataQueryResult ExecuteDataModificationQuery(TSession& session, - const TString& script, - const TExecDataQuerySettings& settings = {} - ) { - const auto result = session.ExecuteDataQuery( - script, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - settings - ).ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), "script:\n" << script << "\nissues:\n" << result.GetIssues().ToString()); - - return result; - } - - TValue GetSingleResult(const TDataQueryResult& rawResults) { - auto resultSetParser = rawResults.GetResultSetParser(0); - UNIT_ASSERT(resultSetParser.TryNextRow()); - return resultSetParser.GetValue(0); - } - - ui64 GetUint64(const TValue& value) { - return TValueParser(value).GetUint64(); - } - - void Restore(NDump::TClient& client, const TFsPath& sourceFile, const TString& dbPath) { - auto result = client.Restore(sourceFile, dbPath); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - auto CreateMinPartitionsChecker(ui64 expectedMinPartitions) { - return [=](const TTableDescription& tableDescription) { - return tableDescription.GetPartitioningSettings().GetMinPartitionsCount() == expectedMinPartitions; - }; - } - - void CheckTableDescription(TSession& session, const TString& path, auto&& checker) { - auto describeResult = session.DescribeTable(path, TDescribeTableSettings().WithSetVal(true)).ExtractValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); - auto tableDescription = describeResult.GetTableDescription(); - Ydb::Table::CreateTableRequest descriptionProto; - // The purpose of translating to CreateTableRequest is solely to produce a clearer error message. - tableDescription.SerializeTo(descriptionProto); - UNIT_ASSERT_C( - checker(tableDescription), - descriptionProto.DebugString() - ); - } - - auto CreateHasSerialChecker(i64 nextValue, bool nextUsed) { - return [=](const TTableDescription& tableDescription) { - for (const auto& column : tableDescription.GetTableColumns()) { - if (column.Name == "Key") { - UNIT_ASSERT(column.SequenceDescription.has_value()); - UNIT_ASSERT(column.SequenceDescription->SetVal.has_value()); - UNIT_ASSERT_VALUES_EQUAL(column.SequenceDescription->SetVal->NextValue, nextValue); - UNIT_ASSERT_VALUES_EQUAL(column.SequenceDescription->SetVal->NextUsed, nextUsed); - return true; - } - } - return false; - }; - } - - Y_UNIT_TEST(Basic) { - TKikimrWithGrpcAndRootSchema server; - auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%d", server.GetPort()))); - TTableClient tableClient(driver); - auto session = tableClient.GetSession().ExtractValueSync().GetSession(); - - constexpr const char* table = "/Root/table"; - ExecuteDataDefinitionQuery(session, Sprintf(R"( - CREATE TABLE `%s` ( - Key Uint32, - Value Utf8, - PRIMARY KEY (Key) - ); - )", - table - )); - ExecuteDataModificationQuery(session, Sprintf(R"( - UPSERT INTO `%s` ( - Key, - Value - ) - VALUES - (1, "one"), - (2, "two"), - (3, "three"), - (4, "four"), - (5, "five"); - )", - table - )); - - TTempDir tempDir; - const auto& pathToBackup = tempDir.Path(); - // TO DO: implement NDump::TClient::Dump and call it instead of BackupFolder - NYdb::NBackup::BackupFolder(driver, "/Root", ".", pathToBackup, {}, false, false); - - NDump::TClient backupClient(driver); - - // restore deleted rows in an existing table - ExecuteDataModificationQuery(session, Sprintf(R"( - DELETE FROM `%s` WHERE Key > 3; - )", table - )); - Restore(backupClient, pathToBackup, "/Root"); - { - auto result = ExecuteDataModificationQuery(session, Sprintf(R"( - SELECT COUNT(*) FROM `%s`; - )", table - )); - UNIT_ASSERT_VALUES_EQUAL(GetUint64(GetSingleResult(result)), 5ull); - } - - // restore deleted table - ExecuteDataDefinitionQuery(session, Sprintf(R"( - DROP TABLE `%s`; - )", table - )); - Restore(backupClient, pathToBackup, "/Root"); - { - auto result = ExecuteDataModificationQuery(session, Sprintf(R"( - SELECT COUNT(*) FROM `%s`; - )", table - )); - UNIT_ASSERT_VALUES_EQUAL(GetUint64(GetSingleResult(result)), 5ull); - } - } - - Y_UNIT_TEST(RestoreTablePartitioningSettings) { - TKikimrWithGrpcAndRootSchema server; - auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%d", server.GetPort()))); - TTableClient tableClient(driver); - auto session = tableClient.GetSession().ExtractValueSync().GetSession(); - - constexpr const char* table = "/Root/table"; - constexpr int minPartitions = 10; - ExecuteDataDefinitionQuery(session, Sprintf(R"( - CREATE TABLE `%s` ( - Key Uint32, - Value Utf8, - PRIMARY KEY (Key) - ) - WITH ( - AUTO_PARTITIONING_BY_LOAD = ENABLED, - AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = %d - ); - )", - table, minPartitions - )); - - CheckTableDescription(session, table, CreateMinPartitionsChecker(minPartitions)); - - TTempDir tempDir; - const auto& pathToBackup = tempDir.Path(); - // TO DO: implement NDump::TClient::Dump and call it instead of BackupFolder - NYdb::NBackup::BackupFolder(driver, "/Root", ".", pathToBackup, {}, false, false); - - NDump::TClient backupClient(driver); - - // restore deleted table - ExecuteDataDefinitionQuery(session, Sprintf(R"( - DROP TABLE `%s`; - )", table - )); - Restore(backupClient, pathToBackup, "/Root"); - CheckTableDescription(session, table, CreateMinPartitionsChecker(minPartitions)); - } - - Y_UNIT_TEST(RestoreIndexTablePartitioningSettings) { - TKikimrWithGrpcAndRootSchema server; - auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%d", server.GetPort()))); - TTableClient tableClient(driver); - auto session = tableClient.GetSession().ExtractValueSync().GetSession(); - - constexpr const char* table = "/Root/table"; - constexpr const char* index = "byValue"; - const TString indexTablePath = JoinFsPaths(table, index, "indexImplTable"); - constexpr int minPartitions = 10; - ExecuteDataDefinitionQuery(session, Sprintf(R"( - CREATE TABLE `%s` ( - Key Uint32, - Value Uint32, - PRIMARY KEY (Key), - INDEX %s GLOBAL ON (Value) - ); - )", - table, index - )); - ExecuteDataDefinitionQuery(session, Sprintf(R"( - ALTER TABLE `%s` ALTER INDEX %s SET ( - AUTO_PARTITIONING_BY_LOAD = ENABLED, - AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = %d - ); - )", table, index, minPartitions - )); - - CheckTableDescription(session, indexTablePath, CreateMinPartitionsChecker(minPartitions)); - - TTempDir tempDir; - const auto& pathToBackup = tempDir.Path(); - // TO DO: implement NDump::TClient::Dump and call it instead of BackupFolder - NYdb::NBackup::BackupFolder(driver, "/Root", ".", pathToBackup, {}, false, false); - - NDump::TClient backupClient(driver); - - // restore deleted table - ExecuteDataDefinitionQuery(session, Sprintf(R"( - DROP TABLE `%s`; - )", table - )); - Restore(backupClient, pathToBackup, "/Root"); - CheckTableDescription(session, indexTablePath, CreateMinPartitionsChecker(minPartitions)); - } - - Y_UNIT_TEST(BasicRestoreTableWithSerial) { - TKikimrWithGrpcAndRootSchema server; - auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%d", server.GetPort()))); - TTableClient tableClient(driver); - auto session = tableClient.GetSession().ExtractValueSync().GetSession(); - - constexpr const char* table = "/Root/table"; - - ExecuteDataDefinitionQuery(session, Sprintf(R"( - CREATE TABLE `%s` ( - Key Serial, - Value Uint32, - PRIMARY KEY (Key) - ); - )", - table - )); - ExecuteDataModificationQuery(session, Sprintf(R"( - UPSERT INTO `%s` ( - Value - ) - VALUES (1), (2), (3), (4), (5), (6), (7); - )", - table - )); - - TTempDir tempDir; - const auto& pathToBackup = tempDir.Path(); - // TO DO: implement NDump::TClient::Dump and call it instead of BackupFolder - NYdb::NBackup::BackupFolder(driver, "/Root", ".", pathToBackup, {}, false, false); - - NDump::TClient backupClient(driver); - - // restore deleted table - ExecuteDataDefinitionQuery(session, Sprintf(R"( - DROP TABLE `%s`; - )", table - )); - Restore(backupClient, pathToBackup, "/Root"); - CheckTableDescription(session, table, CreateHasSerialChecker(8, false)); - } - -}